#!/usr/bin/env python3 """Publish base, optional extension and authenticated bootstrap assets from a signed release.""" import argparse import ctypes import hashlib import json import os from pathlib import Path, PurePosixPath import re import shutil import subprocess import tarfile import tempfile import zipfile MAX_PACKAGE = 2 * 1024**3 def verify_complete(manifest, editor): groups = packages(manifest) expected = {f"ext-runtime-{ecosystem}-{target}" for ecosystem in ("pip", "npm", "go") for target in ("linux-x86_64", "linux-aarch64")} expected.update(f"ext-build-{ecosystem}-{manifest['platform']}" for ecosystem in ("pip", "npm")) expected.update(f"ext-target-{target}" for target in ("linux-x86_64", "linux-aarch64") if target != manifest["platform"]) missing = expected - groups.keys() if missing: raise ValueError("complete release is missing: " + ", ".join(sorted(missing))) if editor is None or editor.is_symlink() or editor.stat().st_size > 16 * 1024**2: raise ValueError("complete release requires a bounded editor VSIX") with zipfile.ZipFile(editor) as archive: info = archive.getinfo("extension/package.json") if info.file_size > 64 * 1024: raise ValueError("editor metadata exceeds limit") package = json.loads(archive.read(info)) if (package.get("name") != "dever-language" or package.get("publisher") != "shemic" or package.get("version") != manifest["version"]): raise ValueError("editor identity/version does not match the release") def extension_suffix(extension): kind = extension["kind"] target = extension["target"] if target not in ("linux-x86_64", "linux-aarch64"): raise ValueError("unsupported extension target") if kind == {"type": "target"}: name = "target" elif (set(kind) == {"type", "ecosystem"} and kind["type"] in ("runtime", "build") and kind["ecosystem"] in ("pip", "npm", "go") and not (kind["type"] == "build" and kind["ecosystem"] == "go")): name = f"{kind['type']}-{kind['ecosystem']}" else: raise ValueError("unsupported extension kind") return f"ext-{name}-{target}" def packages(manifest): if (set(manifest) != {"format", "version", "platform", "artifacts", "extensions"} or manifest["format"] != "dever-release-v2" or manifest["platform"] not in ("linux-x86_64", "linux-aarch64") or not re.fullmatch(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)", manifest["version"])): raise ValueError("unsupported release format, version or platform") if not isinstance(manifest["extensions"], list) or len(manifest["extensions"]) > 12: raise ValueError("release extension count exceeds limit") selected = {"": manifest["artifacts"]} for extension in manifest["extensions"]: if set(extension) != {"kind", "target", "artifacts"}: raise ValueError("invalid extension fields") suffix = extension_suffix(extension) if suffix in selected: raise ValueError("duplicate extension identity") target = extension["target"] kind = extension["kind"] if kind["type"] == "target": if target == manifest["platform"]: raise ValueError("host native resources belong to the base package") prefixes = (f"runtime/{target}/", f"process-runtime/{target}/") else: prefixes = (f"{kind['type']}/{kind['ecosystem']}/{target}/",) if any(not artifact["path"].startswith(prefixes) for artifact in extension["artifacts"]): raise ValueError("extension artifact is outside its resource namespace") selected[suffix] = extension["artifacts"] return selected def verify_artifacts(release, groups): names = set() for artifacts in groups.values(): if (not 1 <= len(artifacts) <= 4096 or any(type(entry["bytes"]) is not int or entry["bytes"] < 0 for entry in artifacts) or sum(entry["bytes"] for entry in artifacts) > MAX_PACKAGE): raise ValueError("release exceeds the official asset limits") for artifact in artifacts: verify_artifact(release, artifact, names) for name in names: if any(str(parent) in names for parent in PurePosixPath(name).parents): raise ValueError("release artifacts overlap as file and directory") def verify_artifact(release, artifact, names): name = artifact["path"] if (set(artifact) != {"path", "bytes", "sha256"} or not name or len(name.encode("utf-8")) > 1024 or str(PurePosixPath(name)) != name or name.startswith("/") or any(part in ("", ".", "..") for part in name.split("/")) or any(character in name for character in ("\\", ":", "\0")) or name in names or name in ("manifest.json", "manifest.sig") or not re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"])): raise ValueError("invalid release artifact path or digest") names.add(name) path = release / name for parent in (path, *path.parents): if parent.is_symlink(): raise ValueError("release artifacts must not traverse symlinks") if parent == release: break if not path.is_file() or path.stat().st_size != artifact["bytes"]: raise ValueError("release artifact type/size mismatch") with path.open("rb") as stream: if hashlib.file_digest(stream, "sha256").hexdigest() != artifact["sha256"]: raise ValueError("release artifact digest mismatch") def compress(release, artifacts, destination, zstd): # Compression is an explicit author tool; installation uses the signed launcher codec. with destination.open("xb") as output: process = subprocess.Popen([str(zstd), "-q", "-10", "--long=27", "-c"], stdin=subprocess.PIPE, stdout=output, stderr=subprocess.PIPE, env={}) try: with process.stdin: with tarfile.open(fileobj=process.stdin, mode="w|", format=tarfile.USTAR_FORMAT) as archive: for artifact in sorted(artifacts, key=lambda entry: entry["path"]): header = tarfile.TarInfo(artifact["path"]) header.size = artifact["bytes"] header.mode = 0o644 with (release / artifact["path"]).open("rb") as contents: archive.addfile(header, contents) errors = process.stderr.read(4096) if process.wait() != 0: raise ValueError(f"release compression failed: {errors.decode(errors='replace')}") finally: if process.poll() is None: process.kill() process.wait() process.stderr.close() def create(release, output, zstd, complete=True, editor=None): release = release.absolute() output = output.absolute() if release.is_symlink() or not release.is_dir(): raise ValueError("release must be a real prepared directory") if output.exists() or output.is_symlink(): raise ValueError("output already exists") if not output.parent.is_dir(): raise ValueError("output parent must exist") if not zstd.is_absolute() or not zstd.is_file(): raise ValueError("zstd must be an explicit absolute author tool path") with (release / "manifest.json").open("rb") as stream: manifest_bytes = stream.read(2 * 1024**2 + 1) if len(manifest_bytes) > 2 * 1024**2: raise ValueError("release metadata exceeds limit") manifest = json.loads(manifest_bytes) if complete: verify_complete(manifest, editor) groups = packages(manifest) verify_artifacts(release, groups) with tempfile.TemporaryDirectory(prefix=".dever-release-assets-", dir=output.parent) as scratch: prepared = Path(scratch) / "assets" prepared.mkdir() stem = f"dever-{manifest['platform']}" (prepared / f"{stem}.manifest.json").write_bytes(manifest_bytes) (prepared / f"{stem}.manifest.sig").write_bytes((release / "manifest.sig").read_bytes()) for suffix, artifacts in groups.items(): filename = f"{stem}{'.' + suffix if suffix else ''}.tar.zst" compress(release, artifacts, prepared / filename, zstd) for artifact in manifest["artifacts"]: if artifact["path"] == "bootstrap/dever" or artifact["path"].startswith("bootstrap/lib/"): blob = prepared / f"{stem}.blob-{artifact['sha256']}" if not blob.exists(): shutil.copyfile(release / artifact["path"], blob) if complete: shutil.copyfile(editor, prepared / f"dever-language-{manifest['version']}.vsix") publish(prepared, output) return sorted(path.name for path in output.iterdir()) def publish(prepared, output): # Linux renameat2 publishes the entire owned tree without replacing even an # empty directory created by another publisher. Failure cleans only staging. libc = ctypes.CDLL("libc.so.6", use_errno=True) rename = libc.renameat2 rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] rename.restype = ctypes.c_int if rename(-100, os.fsencode(prepared), -100, os.fsencode(output), 1) != 0: error = ctypes.get_errno() raise OSError(error, os.strerror(error), str(output)) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("release", type=Path) parser.add_argument("--output", required=True, type=Path) parser.add_argument("--zstd", required=True, type=Path, help="explicit author-side zstd executable") parser.add_argument("--partial-test-fixture", action="store_true", help="allow incomplete private test fixtures, not public releases") parser.add_argument("--editor-vsix", type=Path, help="matching editor package required for public releases") args = parser.parse_args() if args.partial_test_fixture and args.editor_vsix: parser.error("partial test fixtures must not include a public editor package") try: for name in create(args.release, args.output, args.zstd, not args.partial_test_fixture, args.editor_vsix): print(name) except (OSError, ValueError, KeyError, TypeError, tarfile.TarError, zipfile.BadZipFile) as error: parser.exit(1, f"cannot prepare release assets: {error}\n") if __name__ == "__main__": main()