feat: publish Dever core with project scaffolding and versioned AI skills
This commit is contained in:
1 parent
70fa73cdc2
commit
ac458ffa7f
960 files changed
+221319
-1163
No files matched your search
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
ip link set lo up
|
||||
echo DEVER_ARM_GUEST_BEGIN
|
||||
sh /payload/run.sh
|
||||
result=$?
|
||||
echo DEVER_ARM_GUEST_EXIT=$result
|
||||
sync
|
||||
poweroff -f
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
uname -m
|
||||
test "$(uname -m)" = aarch64
|
||||
test -e /proc/self/ns/user
|
||||
echo ARM_KERNEL_READY
|
||||
check_cleanup() {
|
||||
for process in /proc/[0-9]*/exe; do
|
||||
executable=$(readlink "$process" || true)
|
||||
case "$executable" in
|
||||
/worker/*|*/workers/*|*/sandbox/bin/*|*/sandbox/lib/*)
|
||||
echo "ERROR: Worker process leaked: $executable"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
mkdir -p /tmp/apps
|
||||
for ecosystem in pip npm go; do
|
||||
cp -a "/payload/apps/$ecosystem" "/tmp/apps/$ecosystem"
|
||||
program="/tmp/apps/$ecosystem/program"
|
||||
if ! "$program" sample.worker.probe '{}' > "/tmp/$ecosystem-first.json" 2> "/tmp/$ecosystem-first.log"; then
|
||||
cat "/tmp/$ecosystem-first.json" "/tmp/$ecosystem-first.log"
|
||||
exit 1
|
||||
fi
|
||||
grep -q '"data":true' "/tmp/$ecosystem-first.json"
|
||||
check_cleanup
|
||||
echo "ARM_WORKER_CALL_PASS=$ecosystem"
|
||||
if "$program" sample.worker.reject '{}' > "/tmp/$ecosystem-reject.json" 2> "/tmp/$ecosystem-reject.log"; then
|
||||
echo "ERROR: $ecosystem Worker failure unexpectedly succeeded"
|
||||
exit 1
|
||||
fi
|
||||
grep -Fq '"level":"error"' "/tmp/$ecosystem-reject.log"
|
||||
grep -Fq 'sample/worker/app.dever:3:' "/tmp/$ecosystem-reject.log"
|
||||
grep -Fq 'cannot read component protocol frame header: early eof' "/tmp/$ecosystem-reject.log"
|
||||
check_cleanup
|
||||
echo "ARM_WORKER_ERROR_PASS=$ecosystem"
|
||||
"$program" sample.worker.probe '{}' > "/tmp/$ecosystem-repeat.json"
|
||||
grep -q '"data":true' "/tmp/$ecosystem-repeat.json"
|
||||
check_cleanup
|
||||
cat "/tmp/$ecosystem-first.json"
|
||||
echo "ARM_WORKER_PASS=$ecosystem"
|
||||
# 只清理当前 guest 用例的可写副本和缓存;原产物保留在只读 payload。
|
||||
rm -r /tmp/apps
|
||||
mkdir -p /tmp/apps
|
||||
done
|
||||
cp -a /payload/apps/http /tmp/apps/http
|
||||
/tmp/apps/http/application > /tmp/http.log 2>&1 &
|
||||
server=$!
|
||||
trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT
|
||||
attempt=0
|
||||
until wget -q -O /tmp/http.json http://127.0.0.1:18080/sample/http/hello; do
|
||||
attempt=$((attempt + 1))
|
||||
if test "$attempt" -ge 20; then
|
||||
cat /tmp/http.log
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
grep -q '"data":"arm-http"' /tmp/http.json
|
||||
cat /tmp/http.json
|
||||
kill "$server"
|
||||
wait "$server"
|
||||
trap - EXIT
|
||||
echo ARM_HTTP_PASS
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/busybox sh
|
||||
set -eu
|
||||
/bin/busybox --install -s /bin
|
||||
mount -t proc proc /proc
|
||||
mount -t sysfs sysfs /sys
|
||||
mount -t devtmpfs devtmpfs /dev
|
||||
mkdir -p /dev/pts /dev/shm /tmp /payload /run
|
||||
mount -t devpts devpts /dev/pts
|
||||
mount -t tmpfs tmpfs /tmp
|
||||
mount -t tmpfs tmpfs /dev/shm
|
||||
chmod 1777 /tmp /dev/shm
|
||||
modprobe virtio_mmio
|
||||
modprobe virtio_blk
|
||||
modprobe ext4
|
||||
mount -o ro /dev/vda /payload
|
||||
# initramfs 的 rootfs 不支持 pivot_root;先进入正常根挂载再验收嵌套沙箱。
|
||||
mkdir -p /newroot
|
||||
mount -t tmpfs tmpfs /newroot
|
||||
cp -a /bin /sbin /lib /etc /newroot/
|
||||
cp /init-stage /newroot/init-stage
|
||||
mkdir -p /newroot/proc /newroot/sys /newroot/dev /newroot/tmp /newroot/payload /newroot/run
|
||||
mount -o move /proc /newroot/proc
|
||||
mount -o move /sys /newroot/sys
|
||||
mount -o move /dev /newroot/dev
|
||||
mount -o move /tmp /newroot/tmp
|
||||
mount -o move /payload /newroot/payload
|
||||
exec /bin/busybox switch_root /newroot /bin/sh /init-stage
|
||||
@@ -0,0 +1,49 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"runtime"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func Probe(_ context.Context, _ any, _ any) (any, error) {
|
||||
identity, err := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if runtime.GOARCH != "arm64" || identity.Version() != 1 {
|
||||
return nil, errors.New("ARM target or locked UUID library differs")
|
||||
}
|
||||
message := []byte("ARM 独立依赖")
|
||||
var encoded bytes.Buffer
|
||||
writer := gzip.NewWriter(&encoded)
|
||||
if _, err := writer.Write(message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := gzip.NewReader(&encoded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decoded, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := reader.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
digest := sha256.Sum256(message)
|
||||
return map[string]any{"okay": bytes.Equal(decoded, message) && len(digest) == 32}, nil
|
||||
}
|
||||
|
||||
func Reject(_ context.Context, _ any, _ any) (any, error) {
|
||||
return nil, errors.New("expected ARM Worker failure")
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { gzipSync, gunzipSync } from 'node:zlib';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const isNumber = require('is-number');
|
||||
|
||||
export async function probe() {
|
||||
assert.equal(process.arch, 'arm64');
|
||||
const bundle = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
assert.equal(process.execPath, resolve(bundle, 'runtime/bin/node'));
|
||||
assert.equal(require('is-number/package.json').version, '7.0.0');
|
||||
assert.equal(isNumber('42'), true);
|
||||
assert.equal(isNumber('invalid'), false);
|
||||
const message = Buffer.from('ARM 独立依赖');
|
||||
assert.deepEqual(gunzipSync(gzipSync(message)), message);
|
||||
assert.equal(createHash('sha256').update(message).digest().length, 32);
|
||||
await Promise.resolve();
|
||||
return { okay: true };
|
||||
}
|
||||
|
||||
export async function reject() {
|
||||
throw new Error('expected ARM Worker failure');
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import asyncio
|
||||
import gzip
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
import six
|
||||
|
||||
|
||||
async def probe(payload, setting):
|
||||
assert platform.machine() == "aarch64"
|
||||
assert six.__version__ == "1.17.0" and six.PY3
|
||||
assert list(six.iteritems({"value": 42})) == [("value", 42)]
|
||||
runtime = Path(sys.executable).resolve().parents[1]
|
||||
bundle = runtime.parent
|
||||
assert Path(sys.prefix).resolve() == runtime
|
||||
assert all(Path(path).resolve().is_relative_to(bundle) for path in sys.path)
|
||||
assert Path(six.__file__).resolve().is_relative_to(bundle)
|
||||
message = "ARM 独立依赖".encode()
|
||||
assert gzip.decompress(gzip.compress(message)) == message
|
||||
assert len(hashlib.sha256(message).digest()) == 32
|
||||
with sqlite3.connect(":memory:") as connection:
|
||||
assert connection.execute("select 6 * 7").fetchone() == (42,)
|
||||
await asyncio.sleep(0)
|
||||
return {"okay": True}
|
||||
|
||||
|
||||
async def reject(payload, setting):
|
||||
raise RuntimeError("expected ARM Worker failure")
|
||||
@@ -0,0 +1,88 @@
|
||||
"""用显式准备的私有 ARM 内核运行独立应用,不改主机服务或沙箱策略。"""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
SOURCE = Path(__file__).resolve().parent
|
||||
ROOT = SOURCE.parents[2] / "target/arm64-cross"
|
||||
|
||||
|
||||
def prepare_images():
|
||||
init = ROOT / "initramfs"
|
||||
for source, destination in [
|
||||
("init.sh", init / "init-arm"),
|
||||
("bootstrap.sh", init / "init-stage"),
|
||||
("guest.sh", ROOT / "guest/run.sh"),
|
||||
]:
|
||||
shutil.copyfile(SOURCE / source, destination)
|
||||
destination.chmod(0o755)
|
||||
archive = ROOT / "guest-initramfs.gz"
|
||||
with archive.open("wb") as output:
|
||||
with subprocess.Popen(
|
||||
["/usr/bin/find", ".", "-print0"], cwd=init, stdout=subprocess.PIPE
|
||||
) as find:
|
||||
try:
|
||||
with subprocess.Popen(
|
||||
["/usr/bin/cpio", "--null", "-o", "--format=newc", "--quiet"],
|
||||
cwd=init, stdin=find.stdout, stdout=subprocess.PIPE,
|
||||
) as cpio:
|
||||
find.stdout.close()
|
||||
try:
|
||||
subprocess.run(["/usr/bin/gzip", "-1"], stdin=cpio.stdout, stdout=output, check=True)
|
||||
finally:
|
||||
cpio.stdout.close()
|
||||
finally:
|
||||
find.stdout.close()
|
||||
if find.returncode or cpio.returncode:
|
||||
raise RuntimeError("initramfs creation failed")
|
||||
disk = ROOT / "guest.ext4"
|
||||
with disk.open("wb") as output:
|
||||
output.truncate(1024 * 1024 * 1024)
|
||||
subprocess.run(
|
||||
["/usr/sbin/mke2fs", "-q", "-t", "ext4", "-F", "-d", str(ROOT / "guest"), str(disk)],
|
||||
check=True,
|
||||
)
|
||||
return archive, disk
|
||||
|
||||
|
||||
def main():
|
||||
archive, disk = prepare_images()
|
||||
command = [
|
||||
"/lib64/ld-linux-x86-64.so.2", "--library-path",
|
||||
str(ROOT / "tools/usr/lib/x86_64-linux-gnu"),
|
||||
str(ROOT / "tools/usr/bin/qemu-system-aarch64"),
|
||||
"-machine", "virt", "-cpu", "cortex-a72", "-smp", "2", "-m", "768",
|
||||
"-nographic", "-no-reboot", "-nic", "none",
|
||||
"-kernel", str(ROOT / "iso/boot/vmlinuz-virt"), "-initrd", str(archive),
|
||||
"-append", "console=ttyAMA0 rdinit=/init-arm panic=-1",
|
||||
"-drive", f"file={disk},format=raw,if=none,readonly=on,id=payload",
|
||||
"-device", "virtio-blk-device,drive=payload",
|
||||
]
|
||||
log = ROOT / "logs" / f"guest-{time.time_ns()}.log"
|
||||
started = time.monotonic()
|
||||
timed_out = False
|
||||
with log.open("xb") as output:
|
||||
try:
|
||||
result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, timeout=600)
|
||||
exit_code = result.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
exit_code = None
|
||||
transcript = log.read_text(errors="replace")
|
||||
required = ["DEVER_ARM_GUEST_EXIT=0", "ARM_HTTP_PASS"]
|
||||
required.extend(f"ARM_WORKER_PASS={ecosystem}" for ecosystem in ("pip", "npm", "go"))
|
||||
passed = exit_code == 0 and all(marker in transcript for marker in required)
|
||||
record = {
|
||||
"command": command, "exit_code": exit_code, "timed_out": timed_out, "passed": passed,
|
||||
"seconds": time.monotonic() - started, "log": str(log),
|
||||
}
|
||||
log.with_suffix(".json").write_text(json.dumps(record, indent=2) + "\n")
|
||||
print(json.dumps(record), flush=True)
|
||||
if not passed:
|
||||
raise RuntimeError("ARM guest acceptance failed; see serial log")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Explicit Linux author SDK leaves for offline Python/npm build acceptance.
|
||||
|
||||
This prepares test inputs, not a trusted public distribution. System files are
|
||||
declared here and hashed; fixed Python/Node headers and npm come from the same
|
||||
verified upstream archives used by prepare.py. No PATH lookup or downloads.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
|
||||
from prepare import SOURCES, digest, materialize
|
||||
|
||||
|
||||
TARGET = "linux-x86_64"
|
||||
MULTIARCH = "x86_64-linux-gnu"
|
||||
GCC_VERSION = "13"
|
||||
NPM_NODE_GYP_SHIM = "npm/frontend/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin/node-gyp"
|
||||
TOOLS = {
|
||||
"sh": "dash", "cc": "x86_64-linux-gnu-gcc-13", "gcc": "x86_64-linux-gnu-gcc-13",
|
||||
"c++": "x86_64-linux-gnu-g++-13", "g++": "x86_64-linux-gnu-g++-13",
|
||||
"as": "x86_64-linux-gnu-as", "ld": "x86_64-linux-gnu-ld.bfd",
|
||||
"ar": "x86_64-linux-gnu-ar", "ranlib": "x86_64-linux-gnu-ranlib",
|
||||
"nm": "x86_64-linux-gnu-nm", "strip": "x86_64-linux-gnu-strip",
|
||||
**{name: name for name in (
|
||||
"make", "env", "basename", "dirname", "cat", "cp", "mv", "rm", "mkdir",
|
||||
"rmdir", "ln", "touch", "chmod", "pwd", "printf", "test", "true", "false",
|
||||
"sed", "grep", "find", "sort", "tr", "cut", "head", "tail", "wc", "uname",
|
||||
)},
|
||||
}
|
||||
LIBRARIES = (
|
||||
"libc.so.6", "libm.so.6", "libdl.so.2", "libpthread.so.0", "librt.so.1",
|
||||
"libgcc_s.so.1", "libstdc++.so.6", "libisl.so.23", "libmpc.so.3",
|
||||
"libmpfr.so.6", "libgmp.so.10", "libz.so.1", "libzstd.so.1",
|
||||
"libbfd-2.42-system.so", "libctf.so.0", "libjansson.so.4",
|
||||
"libsframe.so.1", "libpcre2-8.so.0", "libselinux.so.1", "libacl.so.1",
|
||||
"libattr.so.1", "libtinfo.so.6", "libcrypto.so.3", "libssl.so.3",
|
||||
"libc.so", "libc_nonshared.a", "libm.so", "libm-2.39.a", "libmvec.so.1",
|
||||
"libpthread.a", "libdl.a", "librt.a", "libutil.a",
|
||||
"crt1.o", "Scrt1.o", "crti.o", "crtn.o",
|
||||
)
|
||||
|
||||
|
||||
def signed_library_paths(content):
|
||||
# 仅迁移以 /lib 开头的绝对 token,不能再次改写已有的 /usr/lib。
|
||||
return re.sub(r"(?<!\S)/lib/" + re.escape(MULTIARCH) + "/",
|
||||
f"/usr/lib/{MULTIARCH}/", content)
|
||||
|
||||
|
||||
def sdk_leaf(source, destination, changes):
|
||||
original = source
|
||||
source = source.resolve(strict=True)
|
||||
if not source.is_file() or not source.is_relative_to("/usr"):
|
||||
raise ValueError(f"SDK leaf must resolve to an explicit /usr file: {original}")
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if original.name in ("libc.so", "libm.so", "libgcc_s.so"):
|
||||
content = source.read_text()
|
||||
rewritten = signed_library_paths(content)
|
||||
if rewritten != content:
|
||||
destination.write_text(rewritten)
|
||||
changes.append({"source": str(original), "source_sha256": digest(source),
|
||||
"transform": "linker script uses signed /usr/lib multiarch leaves"})
|
||||
return
|
||||
# These are read-only author inputs. Never chmod a hardlink to a host file.
|
||||
os.link(source, destination)
|
||||
|
||||
|
||||
def sdk_tree(source, destination, changes):
|
||||
for current, directories, names in os.walk(source, followlinks=False):
|
||||
current = Path(current)
|
||||
if any((current / name).is_symlink() for name in directories):
|
||||
raise ValueError(f"unexpected directory alias in explicit SDK input: {current}")
|
||||
for name in sorted(names):
|
||||
sdk_leaf(current / name, destination / (current / name).relative_to(source), changes)
|
||||
|
||||
|
||||
def leaves(root, sources):
|
||||
entries = []
|
||||
executables = []
|
||||
for source, prefix in sources:
|
||||
for path in sorted(source.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
name = (Path(prefix) / path.relative_to(source)).as_posix()
|
||||
entries.append({"source": path.relative_to(root).as_posix(),
|
||||
"path": name, "sha256": digest(path)})
|
||||
if path.stat().st_mode & 0o111:
|
||||
executables.append(name)
|
||||
if len(entries) > 65_536 or sum((root / entry["source"]).stat().st_size for entry in entries) > 512 * 1024 * 1024:
|
||||
raise ValueError("author SDK exceeds the build-only pack budget")
|
||||
return {"files": entries, "executables": executables}
|
||||
|
||||
|
||||
def prepare(output):
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
changes = []
|
||||
try:
|
||||
rootfs = output / "rootfs"
|
||||
for name, source in TOOLS.items():
|
||||
sdk_leaf(Path("/usr/bin") / source, rootfs / "usr/bin" / name, changes)
|
||||
for source in (Path("/usr/include"),
|
||||
Path(f"/usr/lib/gcc/{MULTIARCH}/{GCC_VERSION}"),
|
||||
Path(f"/usr/libexec/gcc/{MULTIARCH}/{GCC_VERSION}")):
|
||||
sdk_tree(source, rootfs / source.relative_to("/"), changes)
|
||||
for name in LIBRARIES:
|
||||
source = Path("/usr/lib") / MULTIARCH / name
|
||||
sdk_leaf(source, rootfs / source.relative_to("/"), changes)
|
||||
python = output / "python"
|
||||
node = output / "node"
|
||||
materialize("pip", python, selector=lambda name: name.startswith("include/"))
|
||||
materialize("npm", node, selector=lambda name: name.startswith(("include/node/", "lib/node_modules/npm/")))
|
||||
npm = leaves(output, ((rootfs, "rootfs"), (node / "include/node", "npm/include/node"),
|
||||
(node / "lib/node_modules/npm", "npm/frontend/node_modules/npm")))
|
||||
shim = next(entry for entry in npm["files"] if entry["path"] == NPM_NODE_GYP_SHIM)
|
||||
if not (output / shim["source"]).read_bytes().startswith(b"#!/usr/bin/env sh\n"):
|
||||
raise ValueError("pinned npm node-gyp shim changed its interpreter")
|
||||
# run-script executes this upstream shell entry; the remaining frontend
|
||||
# JavaScript is loaded by the exact packaged Node interpreter.
|
||||
npm["executables"] = sorted(set(npm["executables"]) | {NPM_NODE_GYP_SHIM})
|
||||
manifest = {
|
||||
"format": "dever-build-author-leaves-v1", "target": TARGET,
|
||||
"provenance": {"kind": "private Ubuntu GNU author SDK",
|
||||
"gcc": GCC_VERSION, "system_linker_script_changes": changes,
|
||||
"upstream_archives": {name: {"path": str(value[0]), "sha256": value[1]}
|
||||
for name, value in SOURCES.items() if name != "go"}},
|
||||
"pip": leaves(output, ((rootfs, "rootfs"), (python / "include", "runtime/include"))),
|
||||
"npm": npm,
|
||||
}
|
||||
(output / "inputs.json").write_text(json.dumps(manifest, indent=2) + "\n")
|
||||
except BaseException:
|
||||
shutil.rmtree(output)
|
||||
raise
|
||||
print(output / "inputs.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("output", type=Path)
|
||||
prepare(parser.parse_args().output.absolute())
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Prepare pinned, offline upstream inputs for the explicit release acceptance.
|
||||
|
||||
This test author tool never discovers interpreters or downloads anything. It
|
||||
materializes selected distribution files and resolves only in-archive aliases.
|
||||
The production release maker subsequently verifies every prepared input again.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import posixpath
|
||||
import shutil
|
||||
import tarfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
WORKSPACE = Path(__file__).resolve().parents[2]
|
||||
INPUTS = WORKSPACE / "target/ecosystem-release-inputs"
|
||||
SOURCES = {
|
||||
"pip": (
|
||||
INPUTS / "downloads/cpython-3.12.14.tar.gz",
|
||||
"72748da13197c1fb161e3afeef20a6a385ff24f2165e6e2758e47008e7faba4c",
|
||||
"python/",
|
||||
),
|
||||
"npm": (
|
||||
INPUTS / "downloads/node-v24.15.0-linux-x64.tar.xz",
|
||||
"472655581fb851559730c48763e0c9d3bc25975c59d518003fc0849d3e4ba0f6",
|
||||
"node-v24.15.0-linux-x64/",
|
||||
),
|
||||
"go": (
|
||||
WORKSPACE / "target/go-managed/runtime.pack",
|
||||
"8818a1eccb510948a55f63088c01678391a5e0c2236e6eacadfe45d0a5edbb43",
|
||||
"",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def relative_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or path.as_posix() != name or ".." in path.parts:
|
||||
raise ValueError(f"unsafe upstream archive path: {name}")
|
||||
return name
|
||||
|
||||
|
||||
def selected(ecosystem, path):
|
||||
if ecosystem == "npm":
|
||||
return path in ("bin/node", "LICENSE")
|
||||
if ecosystem == "pip":
|
||||
# Keep the complete standard library and its native/data dependencies.
|
||||
# pip, user site-packages, development headers and command aliases are
|
||||
# not needed by the isolated runtime (-I -S -B).
|
||||
return path == "bin/python3.12" or (
|
||||
path.startswith(("lib/", "share/"))
|
||||
and not path.startswith("lib/python3.12/site-packages/")
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def materialize(ecosystem, destination, *, selector=None, source_descriptor=None):
|
||||
source, expected, prefix = source_descriptor or SOURCES[ecosystem]
|
||||
if digest(source) != expected:
|
||||
raise ValueError(f"{ecosystem} upstream input digest differs")
|
||||
aliases = {}
|
||||
seen = set()
|
||||
with tarfile.open(source, "r:*") as archive:
|
||||
for member in archive:
|
||||
name = relative_path(member.name.rstrip("/"))
|
||||
if not name.startswith(prefix) or member.isdir():
|
||||
continue
|
||||
path = name[len(prefix):]
|
||||
if not (selector(path) if selector is not None else selected(ecosystem, path)):
|
||||
continue
|
||||
if path in seen:
|
||||
raise ValueError(f"duplicate upstream path: {path}")
|
||||
seen.add(path)
|
||||
output = destination / path
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
if member.issym():
|
||||
target = posixpath.normpath(posixpath.join(posixpath.dirname(path), member.linkname))
|
||||
aliases[path] = relative_path(target)
|
||||
elif member.isfile():
|
||||
with archive.extractfile(member) as payload, output.open("xb") as target:
|
||||
shutil.copyfileobj(payload, target)
|
||||
output.chmod(0o644)
|
||||
else:
|
||||
raise ValueError(f"unsupported upstream entry: {path}")
|
||||
while aliases:
|
||||
resolved = []
|
||||
for path, target in aliases.items():
|
||||
if (destination / target).is_file():
|
||||
shutil.copyfile(destination / target, destination / path)
|
||||
resolved.append(path)
|
||||
if not resolved:
|
||||
raise ValueError("upstream aliases escape the selected tree or contain a cycle")
|
||||
for path in resolved:
|
||||
del aliases[path]
|
||||
return {"sha256": expected, "bytes": source.stat().st_size}
|
||||
|
||||
|
||||
def runtime_settings(root, ecosystem, *, target="linux-x86_64"):
|
||||
architecture = {"linux-x86_64": "x86_64", "linux-aarch64": "aarch64"}[target]
|
||||
destination = root / ecosystem
|
||||
versions = {"pip": ("cpython", "3.12.14"), "npm": ("node", "24.15.0"), "go": ("go", "1.26.3")}
|
||||
name, version = versions[ecosystem]
|
||||
python_markers = {
|
||||
"implementation_name": "cpython", "implementation_version": version,
|
||||
"os_name": "posix", "platform_machine": architecture,
|
||||
"platform_python_implementation": "CPython", "platform_release": "",
|
||||
"platform_system": "Linux", "platform_version": "",
|
||||
"python_full_version": version, "python_version": "3.12", "sys_platform": "linux",
|
||||
} if ecosystem == "pip" else None
|
||||
# This fixed CPython 3.12 author fixture uses a conservative glibc 2.36
|
||||
# compatibility ceiling. Its separately signed Ubuntu OS ABI pack currently
|
||||
# supplies glibc 2.39; admission checks actual ELF version requirements too.
|
||||
oldest_glibc = 4 if architecture == "x86_64" else 16
|
||||
platforms = [f"manylinux_2_{minor}_{architecture}" for minor in range(36, oldest_glibc, -1)]
|
||||
platforms += [f"manylinux2014_{architecture}"]
|
||||
if architecture == "x86_64":
|
||||
platforms += ["manylinux2010_x86_64", "manylinux1_x86_64"]
|
||||
platforms += [f"linux_{architecture}"]
|
||||
python_tags = [f"cp312-cp312-{platform}" for platform in platforms]
|
||||
python_tags += [f"cp3{minor}-abi3-{platform}" for minor in range(12, 1, -1) for platform in platforms]
|
||||
python_tags += [f"py3-none-{platform}" for platform in platforms]
|
||||
python_tags += ["py312-none-any", "py3-none-any"]
|
||||
if ecosystem != "go":
|
||||
manifest = {
|
||||
"format": "dever-worker-runtime-v1",
|
||||
"ecosystem": ecosystem,
|
||||
"target": target,
|
||||
"executable": "bin/python3.12" if ecosystem == "pip" else "bin/node",
|
||||
"arguments": ["-I", "-S", "-B"] if ecosystem == "pip" else [],
|
||||
}
|
||||
if ecosystem == "pip":
|
||||
manifest["python_wheel_tags"] = python_tags
|
||||
manifest["python_extension_suffixes"] = [f".cpython-312-{architecture}-linux-gnu.so", ".abi3.so", ".so"]
|
||||
manifest["python_markers"] = python_markers
|
||||
(destination / "dever-runtime.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
files = [
|
||||
{"source": path.relative_to(root).as_posix(), "path": path.relative_to(destination).as_posix(), "sha256": digest(path)}
|
||||
for path in sorted(destination.rglob("*")) if path.is_file()
|
||||
]
|
||||
settings = {"name": name, "version": version, "files": files}
|
||||
if ecosystem == "npm":
|
||||
settings["npm_libc"] = "glibc"
|
||||
if ecosystem == "pip":
|
||||
settings["python_markers"] = python_markers
|
||||
settings["python_wheel_tags"] = python_tags
|
||||
return settings
|
||||
|
||||
|
||||
def worker_runtime(root, output, ecosystem):
|
||||
"""Make one explicit fixture pack from verified leaves, without another tree copy."""
|
||||
settings = json.loads((root / "config/setting.json").read_text(encoding="utf-8"))["runtimes"][ecosystem]
|
||||
inputs = sorted(settings["files"], key=lambda entry: entry["path"])
|
||||
for entry in inputs:
|
||||
source = root / relative_path(entry["source"])
|
||||
if source.is_symlink() or not source.is_file() or digest(source) != entry["sha256"]:
|
||||
raise ValueError(f"existing {ecosystem} author input changed: {entry['source']}")
|
||||
descriptor = next(entry for entry in inputs if entry["path"] == "dever-runtime.json")
|
||||
manifest = json.loads((root / descriptor["source"]).read_text(encoding="utf-8"))
|
||||
if manifest["ecosystem"] != ecosystem:
|
||||
raise ValueError("runtime ecosystem differs from the selected author input")
|
||||
if ecosystem == "pip" and (manifest["python_wheel_tags"] != settings["python_wheel_tags"] or manifest["python_markers"] != settings["python_markers"]):
|
||||
raise ValueError("Python runtime and registry descriptors disagree")
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
try:
|
||||
pack = output / "runtime.pack"
|
||||
with pack.open("xb") as target, gzip.GzipFile(filename="", mode="wb", fileobj=target, mtime=0) as compressed, tarfile.open(fileobj=compressed, mode="w|") as archive:
|
||||
for entry in inputs:
|
||||
source = root / entry["source"]
|
||||
info = tarfile.TarInfo(relative_path(entry["path"]))
|
||||
info.size = source.stat().st_size
|
||||
info.mode = 0o755 if entry["path"] == manifest["executable"] else 0o644
|
||||
with source.open("rb") as payload:
|
||||
archive.addfile(info, payload)
|
||||
if digest(source) != entry["sha256"]:
|
||||
raise ValueError(f"{ecosystem} author input changed during archive creation: {entry['source']}")
|
||||
runtime = {
|
||||
"pack": {"name": settings["name"], "version": settings["version"], "sha256": digest(pack)},
|
||||
"target": manifest["target"],
|
||||
"python_markers": None,
|
||||
"python_wheel_tags": [],
|
||||
}
|
||||
metadata = ("python_markers", "python_wheel_tags") if ecosystem == "pip" else ("npm_libc",)
|
||||
runtime.update({field: settings[field] for field in metadata})
|
||||
(output / "config").mkdir()
|
||||
(output / "config/setting.json").write_text(json.dumps({"runtime": runtime}, indent=2) + "\n", encoding="utf-8")
|
||||
except BaseException:
|
||||
shutil.rmtree(output)
|
||||
raise
|
||||
print(output)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("output", type=Path)
|
||||
action = parser.add_mutually_exclusive_group()
|
||||
action.add_argument("--refresh", action="store_true", help="refresh descriptors of existing verified inputs without extracting a second copy")
|
||||
action.add_argument("--python-worker", type=Path, help="write a fresh explicit Python Worker fixture pack from already prepared inputs")
|
||||
action.add_argument("--npm-worker", type=Path, help="write a fresh explicit Node Worker fixture pack from already prepared inputs")
|
||||
arguments = parser.parse_args()
|
||||
root = arguments.output.absolute()
|
||||
for ecosystem, destination in (("pip", arguments.python_worker), ("npm", arguments.npm_worker)):
|
||||
if destination is not None:
|
||||
worker_runtime(root, destination.absolute(), ecosystem)
|
||||
return
|
||||
if arguments.refresh:
|
||||
setting_path = root / "config/setting.json"
|
||||
settings = json.loads(setting_path.read_text(encoding="utf-8"))
|
||||
for runtime in settings["runtimes"].values():
|
||||
for entry in runtime["files"]:
|
||||
if digest(root / relative_path(entry["source"])) != entry["sha256"]:
|
||||
raise ValueError(f"existing author input changed: {entry['source']}")
|
||||
settings["runtimes"] = {ecosystem: runtime_settings(root, ecosystem) for ecosystem in SOURCES}
|
||||
setting_path.write_text(json.dumps(settings, indent=2) + "\n", encoding="utf-8")
|
||||
print(root)
|
||||
return
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
try:
|
||||
provenance = {}
|
||||
runtimes = {}
|
||||
for ecosystem in SOURCES:
|
||||
destination = root / ecosystem
|
||||
destination.mkdir()
|
||||
provenance[ecosystem] = materialize(ecosystem, destination)
|
||||
runtimes[ecosystem] = runtime_settings(root, ecosystem)
|
||||
(root / "config").mkdir()
|
||||
(root / "config/setting.json").write_text(json.dumps({"runtimes": runtimes}, indent=2) + "\n", encoding="utf-8")
|
||||
(root / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8")
|
||||
except BaseException:
|
||||
shutil.rmtree(root)
|
||||
raise
|
||||
print(root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
)
|
||||
|
||||
func Probe(_ context.Context, _ any, _ any) (any, error) {
|
||||
message := []byte("独立标准库")
|
||||
var encoded bytes.Buffer
|
||||
writer := gzip.NewWriter(&encoded)
|
||||
if _, err := writer.Write(message); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := gzip.NewReader(&encoded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decoded, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := reader.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, okay := new(big.Int).SetString("9007199254740993", 10)
|
||||
if !okay || value.Add(value, big.NewInt(1)).String() != "9007199254740994" || !bytes.Equal(decoded, message) {
|
||||
return nil, errors.New("packaged standard library result differs")
|
||||
}
|
||||
digest := sha256.Sum256(message)
|
||||
return map[string]any{"okay": len(digest) == 32}, nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { gzipSync, gunzipSync } from 'node:zlib';
|
||||
import assert from 'node:assert/strict';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
const addon = createRequire(import.meta.url)('bufferutil/build/Release/bufferutil.node');
|
||||
|
||||
export async function probe() {
|
||||
const bundle = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
assert.equal(process.execPath, resolve(bundle, 'runtime/bin/node'));
|
||||
const message = Buffer.from('独立标准库');
|
||||
const mask = Buffer.from([1, 2, 3, 4]);
|
||||
const masked = Buffer.alloc(message.length);
|
||||
addon.mask(message, mask, masked, 0, message.length);
|
||||
assert.notDeepEqual(masked, message);
|
||||
addon.unmask(masked, mask);
|
||||
assert.deepEqual(masked, message);
|
||||
assert.deepEqual(gunzipSync(gzipSync(message)), message);
|
||||
assert.equal(createHash('sha256').update(message).digest().length, 32);
|
||||
assert.equal(9007199254740993n + 1n, 9007199254740994n);
|
||||
await Promise.resolve();
|
||||
return { okay: true };
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import asyncio
|
||||
import bz2
|
||||
import ctypes
|
||||
import decimal
|
||||
import gzip
|
||||
import hashlib
|
||||
import lzma
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import ssl
|
||||
import sys
|
||||
import simplejson
|
||||
from simplejson import _speedups
|
||||
|
||||
|
||||
async def probe(payload, setting):
|
||||
assert _speedups.__file__.endswith(".so")
|
||||
assert _speedups.scanstring('"native"', 1) == ("native", 8)
|
||||
assert simplejson.loads(simplejson.dumps({"native": 42})) == {"native": 42}
|
||||
runtime = Path(sys.executable).resolve().parents[1]
|
||||
bundle = runtime.parent
|
||||
assert Path(sys.prefix).resolve() == runtime
|
||||
assert all(Path(path).resolve().is_relative_to(bundle) for path in sys.path)
|
||||
for module in tuple(sys.modules.values()):
|
||||
path = getattr(module, "__file__", None)
|
||||
if path and not path.startswith("<"):
|
||||
assert Path(path).resolve().is_relative_to(bundle), path
|
||||
await asyncio.sleep(0)
|
||||
message = "独立标准库".encode()
|
||||
for compress, decompress in ((gzip.compress, gzip.decompress), (bz2.compress, bz2.decompress), (lzma.compress, lzma.decompress)):
|
||||
assert decompress(compress(message)) == message
|
||||
assert len(hashlib.sha256(message).digest()) == 32
|
||||
assert ssl.OPENSSL_VERSION.startswith("OpenSSL ")
|
||||
assert ctypes.sizeof(ctypes.c_uint64) == 8
|
||||
assert decimal.Decimal("0.1") + decimal.Decimal("0.2") == decimal.Decimal("0.3")
|
||||
with sqlite3.connect(":memory:") as connection:
|
||||
assert connection.execute("select 6 * 7").fetchone() == (42,)
|
||||
return {"okay": True}
|
||||
@@ -0,0 +1,23 @@
|
||||
"""作者工具的 GNU 链接脚本迁移合同;不读取或修改宿主工具。"""
|
||||
|
||||
import unittest
|
||||
|
||||
from build_inputs import MULTIARCH, signed_library_paths
|
||||
|
||||
|
||||
class LinkerScriptTests(unittest.TestCase):
|
||||
def test_rewrites_only_root_library_tokens(self):
|
||||
original = (f"GROUP ( /lib/{MULTIARCH}/libc.so.6 "
|
||||
f"/usr/lib/{MULTIARCH}/libc_nonshared.a "
|
||||
"AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) )")
|
||||
expected = original.replace(f"( /lib/{MULTIARCH}/", f"( /usr/lib/{MULTIARCH}/")
|
||||
self.assertEqual(signed_library_paths(original), expected)
|
||||
self.assertEqual(signed_library_paths(expected), expected)
|
||||
|
||||
def test_keeps_relative_and_other_target_paths(self):
|
||||
original = "GROUP ( libgcc_s.so.1 -lgcc /lib/other-target/libc.so )"
|
||||
self.assertEqual(signed_library_paths(original), original)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user