183 lines
9.5 KiB
Python
183 lines
9.5 KiB
Python
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
|
|
|
|
MAX_BYTES = 256 * 1024 * 1024
|
|
TARGETS = {
|
|
"linux-x86_64": ("x86_64", "Advanced Micro Devices X86-64"),
|
|
"linux-aarch64": ("aarch64", "AArch64"),
|
|
}
|
|
OMIT = {"__pycache__", "site-packages", "test", "tests", "tkinter", "turtledemo", "idlelib", "ensurepip"}
|
|
PROBE = """import ctypes, hashlib, importlib.machinery, json, platform, sqlite3, ssl, sys, zlib
|
|
print(json.dumps({"version": platform.python_version(), "machine": platform.machine(),
|
|
"executable": sys.executable, "prefix": sys.prefix,
|
|
"suffixes": importlib.machinery.EXTENSION_SUFFIXES}))
|
|
"""
|
|
|
|
|
|
def checksum(path):
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def selected(name, minor):
|
|
path = PurePosixPath(name)
|
|
if path == PurePosixPath(f"python/bin/python{minor}"):
|
|
return True
|
|
if len(path.parts) == 3 and path.parts[:2] == ("python", "lib"):
|
|
return bool(re.fullmatch(r"libpython[\d.]+\.so(?:\.\d+)*", path.name))
|
|
prefix = ("python", "lib", f"python{minor}")
|
|
return (path.parts[:3] == prefix and not OMIT.intersection(path.parts[3:])
|
|
and not any(part.startswith("config-") for part in path.parts[3:])
|
|
and not path.name.startswith("_test") and path.suffix not in (".pyc", ".pyo", ".a"))
|
|
|
|
|
|
def extract(archive, output, minor):
|
|
total = 0
|
|
seen = set()
|
|
with tarfile.open(archive, "r:gz") as source:
|
|
for member in source:
|
|
name = member.name.rstrip("/")
|
|
parts = name.split("/")
|
|
if (parts[0] != "python" or any(part in ("", ".", "..") for part in parts)
|
|
or "\\" in name or name in seen or len(seen) >= 65536):
|
|
raise ValueError("invalid, repeated or excessive archive paths")
|
|
seen.add(name)
|
|
total += member.size
|
|
if total > MAX_BYTES:
|
|
raise ValueError("Python source exceeds the runtime byte budget")
|
|
if not member.isfile() or not selected(name, minor):
|
|
continue
|
|
destination = output / name
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.extractfile(member) as incoming, destination.open("xb") as outgoing:
|
|
shutil.copyfileobj(incoming, outgoing)
|
|
destination.chmod(0o755 if name == f"python/bin/python{minor}" else 0o644)
|
|
|
|
|
|
def check_elf(path, readelf, target, baseline):
|
|
with path.open("rb") as stream:
|
|
if stream.read(4) != b"\x7fELF":
|
|
return
|
|
result = subprocess.run([str(readelf), "--wide", "--file-header", "--version-info", str(path)],
|
|
check=True, capture_output=True, text=True, timeout=15)
|
|
if len(result.stdout) > 256 * 1024 or TARGETS[target][1] not in result.stdout:
|
|
raise ValueError("runtime ELF has the wrong target or excessive metadata")
|
|
if "Name: GLIBC_PRIVATE" in result.stdout:
|
|
raise ValueError("runtime ELF depends on private glibc interfaces")
|
|
versions = [tuple(map(int, value.split(".")))
|
|
for value in re.findall(r"Name: GLIBC_(\d+(?:\.\d+)+)", result.stdout)]
|
|
if versions and max(versions) > baseline:
|
|
raise ValueError(f"{path.name} requires GLIBC_{'.'.join(map(str, max(versions)))}")
|
|
|
|
|
|
def wheel_tags(minor, target, baseline):
|
|
architecture = TARGETS[target][0]
|
|
oldest = 5 if architecture == "x86_64" else 17
|
|
if baseline[0] != 2 or baseline[1] < oldest:
|
|
raise ValueError("unsupported target glibc baseline")
|
|
aliases = {17: "manylinux2014", 12: "manylinux2010", 5: "manylinux1"}
|
|
platforms = []
|
|
for glibc_minor in range(baseline[1], oldest - 1, -1):
|
|
platforms.append(f"manylinux_2_{glibc_minor}_{architecture}")
|
|
if glibc_minor in aliases:
|
|
platforms.append(f"{aliases[glibc_minor]}_{architecture}")
|
|
platforms.append(f"linux_{architecture}")
|
|
python_minor = int(minor.split(".")[1])
|
|
abi = "cp" + minor.replace(".", "")
|
|
tags = [f"{abi}-{abi}-{platform}" for platform in platforms]
|
|
tags += [f"{abi}-abi3-{platform}" for platform in platforms]
|
|
tags += [f"{abi}-none-{platform}" for platform in platforms]
|
|
tags += [f"cp3{version}-abi3-{platform}"
|
|
for version in range(python_minor - 1, 1, -1) for platform in platforms]
|
|
interpreters = [f"py3{python_minor}", "py3"]
|
|
interpreters += [f"py3{version}" for version in range(python_minor - 1, -1, -1)]
|
|
tags += [f"{interpreter}-none-{platform}" for interpreter in interpreters for platform in platforms]
|
|
tags += [f"{abi}-none-any"]
|
|
tags += [f"{interpreter}-none-any" for interpreter in interpreters]
|
|
return tags
|
|
|
|
|
|
def prepare(archive, expected, version, target, baseline, readelf, output, runner=None, sysroot=None):
|
|
if (not re.fullmatch(r"[0-9a-f]{64}", expected) or archive.stat().st_size > MAX_BYTES
|
|
or checksum(archive) != expected):
|
|
raise ValueError("Python archive SHA-256 or size mismatch")
|
|
if not re.fullmatch(r"3\.\d+\.\d+", version) or target not in TARGETS:
|
|
raise ValueError("invalid Python version or target")
|
|
if not readelf.is_absolute() or not readelf.is_file():
|
|
raise ValueError("readelf must be an explicit absolute executable")
|
|
if ((runner is None) != (sysroot is None)
|
|
or runner is not None and (not runner.is_absolute() or not runner.is_file()
|
|
or not sysroot.is_absolute() or not sysroot.is_dir())):
|
|
raise ValueError("cross probing requires an explicit QEMU runner and target sysroot")
|
|
minor = ".".join(version.split(".")[:2])
|
|
output.mkdir(mode=0o700)
|
|
try:
|
|
extract(archive, output, minor)
|
|
root = output / "python"
|
|
binary = root / "bin" / f"python{minor}"
|
|
if not binary.is_file() or not (root / "lib" / f"python{minor}" / "LICENSE.txt").is_file():
|
|
raise ValueError("Python archive lacks its interpreter or license")
|
|
for path in root.rglob("*"):
|
|
if path.is_file():
|
|
check_elf(path, readelf, target, baseline)
|
|
environment = {"HOME": str(root), "TMPDIR": str(output), "PATH": "/usr/bin:/bin"}
|
|
prefix = [str(runner), "-L", str(sysroot)] if runner else []
|
|
result = subprocess.run(prefix + [str(binary), "-I", "-S", "-B", "-c", PROBE],
|
|
check=True, capture_output=True, text=True, env=environment, timeout=30)
|
|
probe = json.loads(result.stdout)
|
|
if (probe["version"] != version or probe["machine"] != TARGETS[target][0]
|
|
or Path(probe["prefix"]) != root or Path(probe["executable"]) != binary):
|
|
raise ValueError("Python runtime is not relocatable or has the wrong identity")
|
|
tags = wheel_tags(minor, target, baseline)
|
|
markers = {"implementation_name": "cpython", "implementation_version": version,
|
|
"os_name": "posix", "platform_machine": TARGETS[target][0],
|
|
"platform_python_implementation": "CPython", "platform_release": "",
|
|
"platform_system": "Linux", "platform_version": "", "python_full_version": version,
|
|
"python_version": minor, "sys_platform": "linux"}
|
|
descriptor = {"format": "dever-worker-runtime-v1", "ecosystem": "pip", "target": target,
|
|
"executable": f"bin/python{minor}", "arguments": ["-I", "-S", "-B"],
|
|
"python_markers": markers, "python_wheel_tags": tags,
|
|
"python_extension_suffixes": probe["suffixes"]}
|
|
(root / "dever-runtime.json").write_text(json.dumps(descriptor, indent=2) + "\n")
|
|
files = [{"source": str(path.relative_to(output)), "path": str(path.relative_to(root)),
|
|
"sha256": checksum(path)} for path in sorted(root.rglob("*")) if path.is_file()]
|
|
author = {"name": "cpython", "version": version, "python_markers": markers,
|
|
"python_wheel_tags": tags, "files": files}
|
|
(output / "python-runtime.json").write_text(json.dumps(author, indent=2) + "\n")
|
|
return {"version": version, "target": target, "glibc_max": ".".join(map(str, baseline)),
|
|
"source_sha256": expected, "files": len(files)}
|
|
except BaseException:
|
|
shutil.rmtree(output)
|
|
raise
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Prepare verified portable Python inputs for native-release")
|
|
parser.add_argument("--archive", type=Path, required=True)
|
|
parser.add_argument("--sha256", required=True)
|
|
parser.add_argument("--version", required=True)
|
|
parser.add_argument("--target", choices=TARGETS, required=True)
|
|
parser.add_argument("--glibc-max", default="2.17")
|
|
parser.add_argument("--readelf", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--runner", type=Path, help="explicit QEMU user-mode executable for cross probing")
|
|
parser.add_argument("--sysroot", type=Path, help="target system libraries for the explicit QEMU runner")
|
|
arguments = parser.parse_args()
|
|
if not re.fullmatch(r"2\.\d+", arguments.glibc_max):
|
|
parser.error("glibc-max must be a 2.x baseline")
|
|
print(json.dumps(prepare(arguments.archive.resolve(), arguments.sha256, arguments.version,
|
|
arguments.target, tuple(map(int, arguments.glibc_max.split("."))),
|
|
arguments.readelf, arguments.output.absolute(), arguments.runner,
|
|
arguments.sysroot), indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|