feat: ship slim releases with shared on-demand extensions and locked restore
This commit is contained in:
Antigravity committed
1 parent
83b78a0db6
commit
4fda7c8c1a
58 files changed
+3774
-425
No files matched your search
+135
-40
@@ -1,18 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Turn a prepared signed release into the three official GitHub release assets."""
|
||||
"""Publish base, optional extension and authenticated bootstrap assets from a signed release."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
|
||||
def create(release, output):
|
||||
MAX_PACKAGE = 2 * 1024**3
|
||||
|
||||
|
||||
def extension_suffix(extension):
|
||||
kind = extension["kind"]
|
||||
target = extension["target"]
|
||||
if target not in ("linux-x86_64", "linux-aarch64"):
|
||||
raise ValueError("unsupported extension target")
|
||||
if kind == {"type": "target"}:
|
||||
name = "target"
|
||||
elif (set(kind) == {"type", "ecosystem"}
|
||||
and kind["type"] in ("runtime", "build")
|
||||
and kind["ecosystem"] in ("pip", "npm", "go")
|
||||
and not (kind["type"] == "build" and kind["ecosystem"] == "go")):
|
||||
name = f"{kind['type']}-{kind['ecosystem']}"
|
||||
else:
|
||||
raise ValueError("unsupported extension kind")
|
||||
return f"ext-{name}-{target}"
|
||||
|
||||
|
||||
def packages(manifest):
|
||||
if (set(manifest) != {"format", "version", "platform", "artifacts", "extensions"}
|
||||
or manifest["format"] != "dever-release-v2"
|
||||
or manifest["platform"] not in ("linux-x86_64", "linux-aarch64")
|
||||
or not re.fullmatch(r"(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)", manifest["version"])):
|
||||
raise ValueError("unsupported release format, version or platform")
|
||||
if not isinstance(manifest["extensions"], list) or len(manifest["extensions"]) > 12:
|
||||
raise ValueError("release extension count exceeds limit")
|
||||
selected = {"": manifest["artifacts"]}
|
||||
for extension in manifest["extensions"]:
|
||||
if set(extension) != {"kind", "target", "artifacts"}:
|
||||
raise ValueError("invalid extension fields")
|
||||
suffix = extension_suffix(extension)
|
||||
if suffix in selected:
|
||||
raise ValueError("duplicate extension identity")
|
||||
target = extension["target"]
|
||||
kind = extension["kind"]
|
||||
if kind["type"] == "target":
|
||||
if target == manifest["platform"]:
|
||||
raise ValueError("host native resources belong to the base package")
|
||||
prefixes = (f"runtime/{target}/", f"sandbox/{target}/")
|
||||
else:
|
||||
prefixes = (f"{kind['type']}/{kind['ecosystem']}/{target}/",)
|
||||
if any(not artifact["path"].startswith(prefixes) for artifact in extension["artifacts"]):
|
||||
raise ValueError("extension artifact is outside its resource namespace")
|
||||
selected[suffix] = extension["artifacts"]
|
||||
return selected
|
||||
|
||||
|
||||
def verify_artifacts(release, groups):
|
||||
names = set()
|
||||
for artifacts in groups.values():
|
||||
if (not 1 <= len(artifacts) <= 4096
|
||||
or any(type(entry["bytes"]) is not int or entry["bytes"] < 0 for entry in artifacts)
|
||||
or sum(entry["bytes"] for entry in artifacts) > MAX_PACKAGE):
|
||||
raise ValueError("release exceeds the official asset limits")
|
||||
for artifact in artifacts:
|
||||
verify_artifact(release, artifact, names)
|
||||
for name in names:
|
||||
if any(str(parent) in names for parent in PurePosixPath(name).parents):
|
||||
raise ValueError("release artifacts overlap as file and directory")
|
||||
|
||||
|
||||
def verify_artifact(release, artifact, names):
|
||||
name = artifact["path"]
|
||||
if (set(artifact) != {"path", "bytes", "sha256"}
|
||||
or not name or len(name.encode("utf-8")) > 1024
|
||||
or str(PurePosixPath(name)) != name or name.startswith("/")
|
||||
or any(part in ("", ".", "..") for part in name.split("/"))
|
||||
or any(character in name for character in ("\\", ":", "\0"))
|
||||
or name in names or name in ("manifest.json", "manifest.sig")
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"])):
|
||||
raise ValueError("invalid release artifact path or digest")
|
||||
names.add(name)
|
||||
path = release / name
|
||||
for parent in (path, *path.parents):
|
||||
if parent.is_symlink():
|
||||
raise ValueError("release artifacts must not traverse symlinks")
|
||||
if parent == release:
|
||||
break
|
||||
if not path.is_file() or path.stat().st_size != artifact["bytes"]:
|
||||
raise ValueError("release artifact type/size mismatch")
|
||||
with path.open("rb") as stream:
|
||||
if hashlib.file_digest(stream, "sha256").hexdigest() != artifact["sha256"]:
|
||||
raise ValueError("release artifact digest mismatch")
|
||||
|
||||
|
||||
def compress(release, artifacts, destination, zstd):
|
||||
# Compression is an explicit author tool; installation uses the signed launcher codec.
|
||||
with destination.open("xb") as output:
|
||||
process = subprocess.Popen([str(zstd), "-q", "-10", "--long=27", "-c"],
|
||||
stdin=subprocess.PIPE, stdout=output,
|
||||
stderr=subprocess.PIPE, env={})
|
||||
try:
|
||||
with process.stdin:
|
||||
with tarfile.open(fileobj=process.stdin, mode="w|", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for artifact in sorted(artifacts, key=lambda entry: entry["path"]):
|
||||
header = tarfile.TarInfo(artifact["path"])
|
||||
header.size = artifact["bytes"]
|
||||
header.mode = 0o644
|
||||
with (release / artifact["path"]).open("rb") as contents:
|
||||
archive.addfile(header, contents)
|
||||
errors = process.stderr.read(4096)
|
||||
if process.wait() != 0:
|
||||
raise ValueError(f"release compression failed: {errors.decode(errors='replace')}")
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
process.stderr.close()
|
||||
|
||||
|
||||
def create(release, output, zstd):
|
||||
release = release.absolute()
|
||||
output = output.absolute()
|
||||
if release.is_symlink() or not release.is_dir():
|
||||
@@ -21,49 +135,29 @@ def create(release, output):
|
||||
raise ValueError("output already exists")
|
||||
if not output.parent.is_dir():
|
||||
raise ValueError("output parent must exist")
|
||||
manifest_bytes = (release / "manifest.json").read_bytes()
|
||||
if not zstd.is_absolute() or not zstd.is_file():
|
||||
raise ValueError("zstd must be an explicit absolute author tool path")
|
||||
with (release / "manifest.json").open("rb") as stream:
|
||||
manifest_bytes = stream.read(2 * 1024**2 + 1)
|
||||
if len(manifest_bytes) > 2 * 1024**2:
|
||||
raise ValueError("release metadata exceeds limit")
|
||||
manifest = json.loads(manifest_bytes)
|
||||
if manifest["format"] != "dever-release-v1" or manifest["platform"] not in ("linux-x86_64", "linux-aarch64"):
|
||||
raise ValueError("unsupported release format or platform")
|
||||
artifacts = manifest["artifacts"]
|
||||
if not 1 <= len(artifacts) <= 4096 or sum(entry["bytes"] for entry in artifacts) > 2 * 1024**3:
|
||||
raise ValueError("release exceeds the official asset limits")
|
||||
names = set()
|
||||
for artifact in artifacts:
|
||||
name = artifact["path"]
|
||||
if (not name or str(PurePosixPath(name)) != name or name.startswith("/")
|
||||
or any(part in ("", ".", "..") for part in name.split("/"))
|
||||
or any(character in name for character in ("\\", ":", "\0"))
|
||||
or name in names or name in ("manifest.json", "manifest.sig")):
|
||||
raise ValueError("invalid release artifact path")
|
||||
names.add(name)
|
||||
path = release / name
|
||||
for parent in (path, *path.parents):
|
||||
if parent.is_symlink():
|
||||
raise ValueError("release artifacts must not traverse symlinks")
|
||||
if parent == release:
|
||||
break
|
||||
if not path.is_file() or path.stat().st_size != artifact["bytes"]:
|
||||
raise ValueError("release artifact type/size mismatch")
|
||||
with path.open("rb") as stream:
|
||||
if hashlib.file_digest(stream, "sha256").hexdigest() != artifact["sha256"]:
|
||||
raise ValueError("release artifact digest mismatch")
|
||||
groups = packages(manifest)
|
||||
verify_artifacts(release, groups)
|
||||
with tempfile.TemporaryDirectory(prefix=".dever-release-assets-", dir=output.parent) as scratch:
|
||||
prepared = Path(scratch) / "assets"
|
||||
prepared.mkdir()
|
||||
stem = f"dever-{manifest['platform']}"
|
||||
(prepared / f"{stem}.manifest.json").write_bytes(manifest_bytes)
|
||||
(prepared / f"{stem}.manifest.sig").write_bytes((release / "manifest.sig").read_bytes())
|
||||
with (prepared / f"{stem}.tar.gz").open("xb") as compressed:
|
||||
with gzip.GzipFile(filename="", fileobj=compressed, mode="wb", mtime=0) as stream:
|
||||
with tarfile.open(fileobj=stream, mode="w|", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for artifact in sorted(artifacts, key=lambda entry: entry["path"]):
|
||||
path = release / artifact["path"]
|
||||
header = tarfile.TarInfo(artifact["path"])
|
||||
header.size = artifact["bytes"]
|
||||
header.mode = 0o644
|
||||
with path.open("rb") as contents:
|
||||
archive.addfile(header, contents)
|
||||
for suffix, artifacts in groups.items():
|
||||
filename = f"{stem}{'.' + suffix if suffix else ''}.tar.zst"
|
||||
compress(release, artifacts, prepared / filename, zstd)
|
||||
for artifact in manifest["artifacts"]:
|
||||
if artifact["path"] == "bootstrap/dever" or artifact["path"].startswith("bootstrap/lib/"):
|
||||
blob = prepared / f"{stem}.blob-{artifact['sha256']}"
|
||||
if not blob.exists():
|
||||
shutil.copyfile(release / artifact["path"], blob)
|
||||
publish(prepared, output)
|
||||
return sorted(path.name for path in output.iterdir())
|
||||
|
||||
@@ -84,9 +178,10 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("release", type=Path)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--zstd", required=True, type=Path, help="explicit author-side zstd executable")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
for name in create(args.release, args.output):
|
||||
for name in create(args.release, args.output, args.zstd):
|
||||
print(name)
|
||||
except (OSError, ValueError, KeyError, TypeError, tarfile.TarError) as error:
|
||||
parser.exit(1, f"cannot prepare release assets: {error}\n")
|
||||
|
||||
Reference in new issue
Block a user