281 lines
13 KiB
Python
281 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""在 Debian 13 开发容器内组装 Linux x86_64 完整离线开发包。"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tomllib
|
|
|
|
|
|
SOURCE = Path(__file__).resolve().parents[1]
|
|
TRIPLE = "x86_64-unknown-linux-gnu"
|
|
PACKAGES = (
|
|
"gcc-14-x86-64-linux-gnu", "cpp-14-x86-64-linux-gnu", "libgcc-14-dev",
|
|
"libc6-dev", "linux-libc-dev", "binutils-x86-64-linux-gnu", "libc6", "libgcc-s1",
|
|
)
|
|
|
|
|
|
def native_input(path):
|
|
"""只保留原生 C 构建所需工具；Rust LTO 使用包内 LLVM，不需要 GCC LTO。"""
|
|
name = str(path)
|
|
if name.startswith("/usr/include/"):
|
|
return True
|
|
if path.parent == Path("/usr/bin"):
|
|
return path.name in {
|
|
"x86_64-linux-gnu-gcc-14", "x86_64-linux-gnu-as",
|
|
"x86_64-linux-gnu-ld", "x86_64-linux-gnu-ar", "x86_64-linux-gnu-ranlib",
|
|
}
|
|
if name.startswith("/usr/libexec/gcc/"):
|
|
return path.name in {"cc1", "collect2", "liblto_plugin.so", "lto-wrapper"}
|
|
if name.startswith("/usr/lib/gcc/"):
|
|
return "/include/" in name or path.name in {
|
|
"libgcc.a", "libgcc_eh.a", "libgcc_s.so", "crtbegin.o", "crtbeginS.o",
|
|
"crtbeginT.o", "crtend.o", "crtendS.o",
|
|
"liblto_plugin.so",
|
|
}
|
|
return path.parent in {Path("/usr/lib/x86_64-linux-gnu"), Path("/usr/lib64")}
|
|
|
|
|
|
def output(*arguments, **options):
|
|
return subprocess.check_output(arguments, text=True, **options).strip()
|
|
|
|
|
|
def digest(path):
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def copy(source, destination):
|
|
"""只解引用作者明确选定的工具文件，不复制 Cargo home 或用户配置。"""
|
|
if source.is_dir():
|
|
shutil.copytree(source, destination, dirs_exist_ok=True)
|
|
else:
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(source, destination)
|
|
destination.chmod(0o755 if os.access(source, os.X_OK) else 0o644)
|
|
|
|
|
|
def script(path, body):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text('#!/bin/sh\nset -eu\ntools=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)\n' + body)
|
|
path.chmod(0o755)
|
|
|
|
|
|
def prepare(compiler, work):
|
|
"""两种交付方式共用现有本地打包入口，不另做编译器发行格式。"""
|
|
work.mkdir()
|
|
version = tomllib.loads((SOURCE / "rust/Cargo.toml").read_text())["workspace"]["package"]["version"]
|
|
rust_version = tomllib.loads((SOURCE / "rust/rust-toolchain.toml").read_text())["toolchain"]["channel"]
|
|
author = work / "author"
|
|
for path in (
|
|
"rust/Cargo.toml", "rust/Cargo.lock", "rust/rust-toolchain.toml", "rust/src",
|
|
"lib", "skills", "LANGUAGE.md", "DOCUMENTATION.md",
|
|
):
|
|
copy(SOURCE / path, author / "source" / path)
|
|
copy(compiler, author / "bin/dever")
|
|
(author / "config").mkdir()
|
|
(author / "config/setting.json").write_text(json.dumps({
|
|
"format": "dever-local-bootstrap-input-v1", "version": version,
|
|
"target": "linux-x86_64", "source": "source",
|
|
"core": {"source": "bin/dever", "sha256": digest(compiler)},
|
|
}))
|
|
bundle = work / "dever"
|
|
subprocess.run([str(compiler), "release", str(author), "--output", str(bundle)], check=True)
|
|
return bundle, version, rust_version
|
|
|
|
|
|
def record(bundle, paths):
|
|
"""把交付附带文件纳入原有清单校验。"""
|
|
manifest = json.loads((bundle / "bootstrap.json").read_text())
|
|
for path in sorted(paths):
|
|
if path.is_symlink():
|
|
raise RuntimeError(f"staged artifact must be a regular file: {path}")
|
|
if path.is_file():
|
|
manifest["artifacts"].append({
|
|
"path": path.relative_to(bundle).as_posix(), "size": path.stat().st_size,
|
|
"sha256": digest(path), "executable": os.access(path, os.X_OK),
|
|
})
|
|
manifest["artifacts"].sort(key=lambda artifact: artifact["path"])
|
|
(bundle / "bootstrap.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
|
return manifest
|
|
|
|
|
|
def native_tools(tools):
|
|
"""保留 Debian 工具的相对安装布局；glibc 只进链接 sysroot，不覆盖宿主加载器。"""
|
|
base = {str(Path(path).resolve()) for path in output("dpkg-query", "-L", "libc6").splitlines()}
|
|
packages = set(PACKAGES)
|
|
executables = []
|
|
for package in PACKAGES:
|
|
for name in output("dpkg-query", "-L", package).splitlines():
|
|
path = Path(name)
|
|
if not path.is_file() or not native_input(path):
|
|
continue
|
|
copy(path, tools / "native" / path.relative_to("/"))
|
|
# Debian 的链接脚本使用 /lib；发行包展开 usrmerge 别名，不改写官方库文件。
|
|
if package == "libc6":
|
|
copy(path, tools / "native" / path.relative_to("/usr"))
|
|
if os.access(path, os.X_OK):
|
|
with path.open("rb") as stream:
|
|
if stream.read(4) == b"\x7fELF":
|
|
executables.append(path)
|
|
for name in ("as", "ld", "ar"):
|
|
copy(Path(f"/usr/bin/x86_64-linux-gnu-{name}"), tools / "native/usr/bin" / name)
|
|
libraries = set()
|
|
for executable in executables:
|
|
listing = subprocess.run(["ldd", str(executable)], text=True, capture_output=True)
|
|
if listing.returncode and "not a dynamic executable" not in listing.stderr + listing.stdout:
|
|
raise RuntimeError(f"ldd {executable}: {listing.stderr}{listing.stdout}")
|
|
for line in listing.stdout.splitlines():
|
|
if "=> not found" in line:
|
|
raise RuntimeError(line)
|
|
fields = line.split()
|
|
if "=>" in fields and fields[2].startswith("/"):
|
|
library = Path(fields[2])
|
|
if str(library.resolve()) not in base:
|
|
libraries.add(library)
|
|
for library in sorted(libraries):
|
|
copy(library, tools / "lib" / library.name)
|
|
owner = output("dpkg-query", "-S", str(library.resolve())).splitlines()[0].split(": ", 1)[0]
|
|
packages.add(owner)
|
|
copy(Path("/usr/lib/x86_64-linux-gnu/libgcc_s.so.1"), tools / "lib/libgcc_s.so.1")
|
|
packages.add("libgcc-s1")
|
|
for package in sorted(packages):
|
|
name = package.split(":", 1)[0]
|
|
copy(Path("/usr/share/doc") / name / "copyright", tools / "licenses" / name / "copyright")
|
|
copy(Path("/usr/share/common-licenses"), tools / "licenses/common")
|
|
(tools / "native-packages.txt").write_text(output("dpkg-query", "-W", *sorted(packages)) + "\n")
|
|
script(tools / "bin/cc", '''export LD_LIBRARY_PATH="$tools/lib"
|
|
exec "$tools/native/usr/bin/x86_64-linux-gnu-gcc-14" \\
|
|
--sysroot="$tools/native" \\
|
|
-B"$tools/native/usr/lib/gcc/x86_64-linux-gnu/14/" \\
|
|
-B"$tools/native/usr/libexec/gcc/x86_64-linux-gnu/14/" \\
|
|
-B"$tools/native/usr/bin/" \\
|
|
-B"$tools/native/usr/lib/x86_64-linux-gnu/" "$@"
|
|
''')
|
|
|
|
|
|
def trim_vendor(vendor, version):
|
|
"""开发包编译业务程序；按 Cargo 完整依赖图保留运行库的全部特性和平台依赖。"""
|
|
metadata = json.loads(output("cargo", f"+{version}", "metadata", "--locked", "--offline",
|
|
"--all-features", "--format-version", "1", cwd=SOURCE / "rust"))
|
|
dependencies = {node["id"]: node["dependencies"] for node in metadata["resolve"]["nodes"]}
|
|
pending = [package["id"] for package in metadata["packages"] if package["name"] == "dever-runtime"]
|
|
required = set()
|
|
while pending:
|
|
package = pending.pop()
|
|
if package not in required:
|
|
required.add(package)
|
|
pending.extend(dependencies[package])
|
|
for package in metadata["packages"]:
|
|
if package["source"] and package["id"] not in required:
|
|
directory = vendor / f'{package["name"]}-{package["version"]}'
|
|
# 编译器二进制仍使用这些依赖；源码可移除，第三方许可必须随包保留。
|
|
licenses = {path for path in directory.rglob("*") if path.is_file()
|
|
and path.name.upper().startswith(("LICENSE", "COPYING", "COPYRIGHT", "NOTICE", "AUTHORS"))}
|
|
if package["license_file"]:
|
|
licenses.add(directory / package["license_file"])
|
|
for path in licenses:
|
|
copy(path, vendor.parent / "licenses/cargo" / path.relative_to(vendor))
|
|
shutil.rmtree(directory)
|
|
|
|
|
|
def rust_tools(tools, version, vendor):
|
|
rust = Path(output("rustc", f"+{version}", "--print", "sysroot"))
|
|
if f"host: {TRIPLE}" not in output("rustc", f"+{version}", "-vV"):
|
|
raise RuntimeError(f"this package requires the official {TRIPLE} toolchain")
|
|
for name in ("cargo", "rustc"):
|
|
copy(rust / "bin" / name, tools / "rust/bin" / name)
|
|
for library in (rust / "lib").glob("*.so*"):
|
|
if not library.is_symlink():
|
|
copy(library, tools / "rust/lib" / library.name)
|
|
for name in ("lib", "bin"):
|
|
copy(rust / "lib/rustlib" / TRIPLE / name, tools / "rust/lib/rustlib" / TRIPLE / name)
|
|
# 只裁剪已链接的工具；标准库 rlib/rmeta 中的符号、类型元数据与位码仍用于编译。
|
|
binaries = [*(tools / "rust/bin").iterdir(), *(tools / "rust/lib").glob("*.so*")]
|
|
binaries.extend(path for path in (tools / "rust/lib/rustlib" / TRIPLE / "bin").rglob("*")
|
|
if path.is_file())
|
|
for binary in binaries:
|
|
# 官方 .so 目录也含链接脚本，按原样保留，只处理 ELF 文件。
|
|
with binary.open("rb") as stream:
|
|
if stream.read(4) == b"\x7fELF":
|
|
subprocess.run(["strip", "--strip-unneeded", str(binary)], check=True)
|
|
copy(rust / "share/doc/rust", tools / "licenses/rust")
|
|
if vendor:
|
|
copy(vendor, tools / "vendor")
|
|
else:
|
|
subprocess.run([
|
|
"cargo", f"+{version}", "vendor", "--locked", "--offline", "--versioned-dirs",
|
|
str(tools / "vendor"),
|
|
], cwd=SOURCE / "rust", check=True)
|
|
trim_vendor(tools / "vendor", version)
|
|
config = tools / ".cargo/config.toml"
|
|
config.parent.mkdir()
|
|
config.write_text('''[source.crates-io]
|
|
replace-with = "vendored-sources"
|
|
|
|
[source.vendored-sources]
|
|
directory = "vendor"
|
|
''')
|
|
script(tools / "bin/cargo", '''export CARGO_HOME="$tools/.cargo"
|
|
export RUSTC="$tools/rust/bin/rustc"
|
|
export CC="$tools/bin/cc"
|
|
export AR="$tools/native/usr/bin/x86_64-linux-gnu-ar"
|
|
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER="$CC"
|
|
export LD_LIBRARY_PATH="$tools/lib"
|
|
export PATH="$tools/native/usr/bin:/usr/bin:/bin"
|
|
exec "$tools/rust/bin/cargo" --config "$tools/.cargo/config.toml" "$@"
|
|
''')
|
|
script(tools / "bin/rustc", '''export LD_LIBRARY_PATH="$tools/lib"
|
|
exec "$tools/rust/bin/rustc" "$@"
|
|
''')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--compiler", required=True, type=Path)
|
|
parser.add_argument("--work", required=True, type=Path, help="new staging directory")
|
|
parser.add_argument("--output", required=True, type=Path, help="new .tar.xz or .tar.gz archive")
|
|
parser.add_argument("--vendor", type=Path, help="optional cargo vendor --locked --versioned-dirs result")
|
|
args = parser.parse_args()
|
|
system = Path("/etc/os-release").read_text()
|
|
if 'ID=debian\n' not in system or 'VERSION_ID="13"' not in system:
|
|
parser.error("prepare this package in the Debian 13 development container")
|
|
args.compiler = args.compiler.resolve(strict=True)
|
|
args.work = args.work.absolute()
|
|
args.output = args.output.absolute()
|
|
if not args.output.name.endswith((".tar.xz", ".tar.gz")):
|
|
parser.error("output must end in .tar.xz or .tar.gz")
|
|
if args.output.exists() or Path(str(args.output) + ".sha256").exists():
|
|
parser.error("output archive and checksum must not exist")
|
|
bundle, _, rust_version = prepare(args.compiler, args.work)
|
|
author = args.work / "author"
|
|
tools = bundle / "toolchain"
|
|
tools.mkdir()
|
|
native_tools(tools)
|
|
rust_tools(tools, rust_version, args.vendor)
|
|
# 先实际链接，避免把缺少 sysroot 文件的包花时间压缩后才发现。
|
|
smoke = author / "link.rs"
|
|
smoke.write_text('fn main() { println!("private Rust/C linker ready"); }\n')
|
|
subprocess.run([str(tools / "bin/rustc"), str(smoke), "-C",
|
|
"linker=" + str(tools / "bin/cc"), "-o", str(author / "link")], check=True)
|
|
subprocess.run([str(author / "link")], check=True)
|
|
copy(SOURCE / "build/README.md", bundle / "DEVELOPMENT.md")
|
|
manifest = record(bundle, [bundle / "DEVELOPMENT.md", *tools.rglob("*")])
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
# 归档必须先于第一次构建，不能把 Cargo 编译缓存放进发行包。
|
|
with args.output.open("xb") as stream:
|
|
compression = "xz -T4 -6" if args.output.name.endswith(".xz") else "gzip -9"
|
|
subprocess.run(["tar", "-C", str(args.work), "-I", compression, "-cf", "-", "dever"],
|
|
stdout=stream, check=True)
|
|
Path(str(args.output) + ".sha256").write_text(f"{digest(args.output)} {args.output.name}\n")
|
|
print(json.dumps({"archive": str(args.output), "bytes": args.output.stat().st_size,
|
|
"files": len(manifest["artifacts"]), "compiler_sha256": digest(args.compiler)}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|