192 lines
8.6 KiB
Python
192 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Publish base, optional extension and authenticated bootstrap assets from a signed release."""
|
|
|
|
import argparse
|
|
import ctypes
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
|
|
|
|
MAX_PACKAGE = 2 * 1024**3
|
|
|
|
|
|
def extension_suffix(extension):
|
|
kind = extension["kind"]
|
|
target = extension["target"]
|
|
if target not in ("linux-x86_64", "linux-aarch64"):
|
|
raise ValueError("unsupported extension target")
|
|
if kind == {"type": "target"}:
|
|
name = "target"
|
|
elif (set(kind) == {"type", "ecosystem"}
|
|
and kind["type"] in ("runtime", "build")
|
|
and kind["ecosystem"] in ("pip", "npm", "go")
|
|
and not (kind["type"] == "build" and kind["ecosystem"] == "go")):
|
|
name = f"{kind['type']}-{kind['ecosystem']}"
|
|
else:
|
|
raise ValueError("unsupported extension kind")
|
|
return f"ext-{name}-{target}"
|
|
|
|
|
|
def packages(manifest):
|
|
if (set(manifest) != {"format", "version", "platform", "artifacts", "extensions"}
|
|
or manifest["format"] != "dever-release-v2"
|
|
or manifest["platform"] not in ("linux-x86_64", "linux-aarch64")
|
|
or not re.fullmatch(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)", manifest["version"])):
|
|
raise ValueError("unsupported release format, version or platform")
|
|
if not isinstance(manifest["extensions"], list) or len(manifest["extensions"]) > 12:
|
|
raise ValueError("release extension count exceeds limit")
|
|
selected = {"": manifest["artifacts"]}
|
|
for extension in manifest["extensions"]:
|
|
if set(extension) != {"kind", "target", "artifacts"}:
|
|
raise ValueError("invalid extension fields")
|
|
suffix = extension_suffix(extension)
|
|
if suffix in selected:
|
|
raise ValueError("duplicate extension identity")
|
|
target = extension["target"]
|
|
kind = extension["kind"]
|
|
if kind["type"] == "target":
|
|
if target == manifest["platform"]:
|
|
raise ValueError("host native resources belong to the base package")
|
|
prefixes = (f"runtime/{target}/", f"process-runtime/{target}/")
|
|
else:
|
|
prefixes = (f"{kind['type']}/{kind['ecosystem']}/{target}/",)
|
|
if any(not artifact["path"].startswith(prefixes) for artifact in extension["artifacts"]):
|
|
raise ValueError("extension artifact is outside its resource namespace")
|
|
selected[suffix] = extension["artifacts"]
|
|
return selected
|
|
|
|
|
|
def verify_artifacts(release, groups):
|
|
names = set()
|
|
for artifacts in groups.values():
|
|
if (not 1 <= len(artifacts) <= 4096
|
|
or any(type(entry["bytes"]) is not int or entry["bytes"] < 0 for entry in artifacts)
|
|
or sum(entry["bytes"] for entry in artifacts) > MAX_PACKAGE):
|
|
raise ValueError("release exceeds the official asset limits")
|
|
for artifact in artifacts:
|
|
verify_artifact(release, artifact, names)
|
|
for name in names:
|
|
if any(str(parent) in names for parent in PurePosixPath(name).parents):
|
|
raise ValueError("release artifacts overlap as file and directory")
|
|
|
|
|
|
def verify_artifact(release, artifact, names):
|
|
name = artifact["path"]
|
|
if (set(artifact) != {"path", "bytes", "sha256"}
|
|
or not name or len(name.encode("utf-8")) > 1024
|
|
or str(PurePosixPath(name)) != name or name.startswith("/")
|
|
or any(part in ("", ".", "..") for part in name.split("/"))
|
|
or any(character in name for character in ("\\", ":", "\0"))
|
|
or name in names or name in ("manifest.json", "manifest.sig")
|
|
or not re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"])):
|
|
raise ValueError("invalid release artifact path or digest")
|
|
names.add(name)
|
|
path = release / name
|
|
for parent in (path, *path.parents):
|
|
if parent.is_symlink():
|
|
raise ValueError("release artifacts must not traverse symlinks")
|
|
if parent == release:
|
|
break
|
|
if not path.is_file() or path.stat().st_size != artifact["bytes"]:
|
|
raise ValueError("release artifact type/size mismatch")
|
|
with path.open("rb") as stream:
|
|
if hashlib.file_digest(stream, "sha256").hexdigest() != artifact["sha256"]:
|
|
raise ValueError("release artifact digest mismatch")
|
|
|
|
|
|
def compress(release, artifacts, destination, zstd):
|
|
# Compression is an explicit author tool; installation uses the signed launcher codec.
|
|
with destination.open("xb") as output:
|
|
process = subprocess.Popen([str(zstd), "-q", "-10", "--long=27", "-c"],
|
|
stdin=subprocess.PIPE, stdout=output,
|
|
stderr=subprocess.PIPE, env={})
|
|
try:
|
|
with process.stdin:
|
|
with tarfile.open(fileobj=process.stdin, mode="w|", format=tarfile.USTAR_FORMAT) as archive:
|
|
for artifact in sorted(artifacts, key=lambda entry: entry["path"]):
|
|
header = tarfile.TarInfo(artifact["path"])
|
|
header.size = artifact["bytes"]
|
|
header.mode = 0o644
|
|
with (release / artifact["path"]).open("rb") as contents:
|
|
archive.addfile(header, contents)
|
|
errors = process.stderr.read(4096)
|
|
if process.wait() != 0:
|
|
raise ValueError(f"release compression failed: {errors.decode(errors='replace')}")
|
|
finally:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
process.stderr.close()
|
|
|
|
|
|
def create(release, output, zstd):
|
|
release = release.absolute()
|
|
output = output.absolute()
|
|
if release.is_symlink() or not release.is_dir():
|
|
raise ValueError("release must be a real prepared directory")
|
|
if output.exists() or output.is_symlink():
|
|
raise ValueError("output already exists")
|
|
if not output.parent.is_dir():
|
|
raise ValueError("output parent must exist")
|
|
if not zstd.is_absolute() or not zstd.is_file():
|
|
raise ValueError("zstd must be an explicit absolute author tool path")
|
|
with (release / "manifest.json").open("rb") as stream:
|
|
manifest_bytes = stream.read(2 * 1024**2 + 1)
|
|
if len(manifest_bytes) > 2 * 1024**2:
|
|
raise ValueError("release metadata exceeds limit")
|
|
manifest = json.loads(manifest_bytes)
|
|
groups = packages(manifest)
|
|
verify_artifacts(release, groups)
|
|
with tempfile.TemporaryDirectory(prefix=".dever-release-assets-", dir=output.parent) as scratch:
|
|
prepared = Path(scratch) / "assets"
|
|
prepared.mkdir()
|
|
stem = f"dever-{manifest['platform']}"
|
|
(prepared / f"{stem}.manifest.json").write_bytes(manifest_bytes)
|
|
(prepared / f"{stem}.manifest.sig").write_bytes((release / "manifest.sig").read_bytes())
|
|
for suffix, artifacts in groups.items():
|
|
filename = f"{stem}{'.' + suffix if suffix else ''}.tar.zst"
|
|
compress(release, artifacts, prepared / filename, zstd)
|
|
for artifact in manifest["artifacts"]:
|
|
if artifact["path"] == "bootstrap/dever" or artifact["path"].startswith("bootstrap/lib/"):
|
|
blob = prepared / f"{stem}.blob-{artifact['sha256']}"
|
|
if not blob.exists():
|
|
shutil.copyfile(release / artifact["path"], blob)
|
|
publish(prepared, output)
|
|
return sorted(path.name for path in output.iterdir())
|
|
|
|
|
|
def publish(prepared, output):
|
|
# Linux renameat2 publishes the entire owned tree without replacing even an
|
|
# empty directory created by another publisher. Failure cleans only staging.
|
|
libc = ctypes.CDLL("libc.so.6", use_errno=True)
|
|
rename = libc.renameat2
|
|
rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
|
rename.restype = ctypes.c_int
|
|
if rename(-100, os.fsencode(prepared), -100, os.fsencode(output), 1) != 0:
|
|
error = ctypes.get_errno()
|
|
raise OSError(error, os.strerror(error), str(output))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("release", type=Path)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
parser.add_argument("--zstd", required=True, type=Path, help="explicit author-side zstd executable")
|
|
args = parser.parse_args()
|
|
try:
|
|
for name in create(args.release, args.output, args.zstd):
|
|
print(name)
|
|
except (OSError, ValueError, KeyError, TypeError, tarfile.TarError) as error:
|
|
parser.exit(1, f"cannot prepare release assets: {error}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|