"""Compare current Dever collection kernels with independent Go and Rust programs.""" import argparse import errno import hashlib import json import os from pathlib import Path import shutil import statistics import subprocess import tempfile import time FIXTURE = Path(__file__).parent / "fixtures/kernel" CORE = Path(__file__).resolve().parents[2] CASES = {"integer": (16384, 65536), "float": (16384, 65536), "list": (512, 1024, 2048, 4096), "map": (512, 1024, 2048, 4096), "list_read": (512, 1024, 2048, 4096), "list_record": (512, 1024, 2048, 4096), "list_record_read": (512, 1024, 2048, 4096), "record_update": (512, 1024, 2048, 4096), "record_inspect": (512, 1024, 2048, 4096), "call_read": (512, 1024, 2048, 4096), "map_read": (512, 1024, 2048, 4096), "map_lookup": (512, 1024, 2048, 4096), "map_text_lookup": (512, 1024, 2048, 4096), "text_equal": (512, 1024, 2048, 4096), "text_contains": (512, 1024, 2048, 4096), "text_concat": (512, 2048, 8192, 32768), "bytes_concat": (512, 2048, 8192, 32768)} def fingerprint(path): with path.open("rb") as source: digest = hashlib.file_digest(source, "sha256").hexdigest() return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest} def source_identity(): files = [CORE / "rust" / name for name in ("Cargo.toml", "Cargo.lock", "rust-toolchain.toml")] for directory in ("rust/src", "lib", "sdk"): files.extend(path for path in (CORE / directory).rglob("*") if path.is_file() and path.suffix in {".rs", ".toml", ".md", ".dever", ".py", ".c", ".h", ".js", ".mjs", ".json", ".go"}) digest = hashlib.sha256() for path in sorted(files): digest.update(str(path.relative_to(CORE)).encode() + b"\0") digest.update(fingerprint(path)["sha256"].encode() + b"\n") return {"sha256": digest.hexdigest(), "files": len(files)} def build(arguments): tree = source_identity() output = arguments.output.resolve() output.mkdir(parents=True, exist_ok=False) module = output / "module/benchmark/kernel" module.mkdir(parents=True) for name in ("app.dever", "api.dever"): shutil.copy2(FIXTURE / name, module / name) (output / "config").mkdir() (output / "config/setting.json").write_text("{}\n") artifacts = {} for label, compiler in (("dever", arguments.compiler), ("dever_before", arguments.baseline_compiler)): if compiler is None: continue binary = output / label with (output / (label + ".build.log")).open("w") as log: subprocess.run([compiler, "build", str(output), "--output", str(binary)], stdout=log, stderr=subprocess.STDOUT, check=True, timeout=300) artifacts[label] = fingerprint(binary) artifacts[label]["compiler"] = fingerprint(Path(compiler).resolve()) artifacts[label]["source_tree"] = tree generated = output / (label + ".rs") with generated.open("w") as source: subprocess.run([compiler, "emit", str(output)], stdout=source, check=True, timeout=120) artifacts[label]["generated"] = fingerprint(generated) binary = output / "go" subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", str(binary), str(FIXTURE.resolve() / "peer.go")], check=True, timeout=120) artifacts["go"] = fingerprint(binary) artifacts["go"]["compiler"] = subprocess.check_output(["go", "version"], text=True).strip() peer = output / "rust-peer" peer.mkdir() shutil.copy2(FIXTURE / "peer.rs", peer / "main.rs") core = FIXTURE.resolve().parents[3] shutil.copy2(core / "rust/rust-toolchain.toml", peer / "rust-toolchain.toml") (peer / "Cargo.toml").write_text( '[package]\nname = "kernel-peer"\nversion = "0.0.1"\nedition = "2024"\n' '[workspace]\n[[bin]]\nname = "kernel-peer"\npath = "main.rs"\n' '[dependencies]\nserde_json = "=1.0.145"\nindexmap = "=2.14.0"\n' '[profile.release]\nopt-level = 3\nlto = "fat"\ncodegen-units = 1\n' 'overflow-checks = true\nstrip = "symbols"\n') subprocess.run(["cargo", "generate-lockfile", "--offline"], cwd=peer, check=True, timeout=120) command = ["cargo", "build", "--offline", "--locked", "--release", "--bin", "kernel-peer", "--message-format=json"] process = subprocess.run(command, cwd=peer, capture_output=True, text=True, check=True, timeout=300) (output / "rust.build.log").write_text(process.stderr) executables = [message["executable"] for line in process.stdout.splitlines() if (message := json.loads(line)).get("reason") == "compiler-artifact" and message.get("executable") and message["target"]["name"] == "kernel-peer"] if len(executables) != 1: raise RuntimeError("expected exactly one native Rust peer") binary = output / "rust" shutil.copy2(executables[0], binary) artifacts["rust"] = fingerprint(binary) artifacts["rust"]["compiler"] = subprocess.check_output( ["rustc", "--version"], cwd=peer, text=True).strip() artifacts["rust"]["profile"] = "release: opt-level=3, fat LTO, codegen-units=1" artifacts["rust"]["source"] = fingerprint(peer / "main.rs") artifacts["rust"]["lock"] = fingerprint(peer / "Cargo.lock") artifacts["rust"]["map"] = "ordered IndexMap with owned keys, matching Dever" if source_identity() != tree: raise RuntimeError("sources changed during build; rebuild the comparison") (output / "manifest.json").write_text(json.dumps(artifacts, indent=2) + "\n") print(output / "manifest.json") def run(arguments): if arguments.output.exists(): raise FileExistsError(arguments.output) if arguments.cpu is not None: if arguments.cpu not in os.sched_getaffinity(0): raise ValueError("requested CPU is outside the allowed affinity") os.sched_setaffinity(0, {arguments.cpu}) manifest = json.loads(arguments.manifest.read_text()) if arguments.candidate not in manifest: raise ValueError("candidate is missing from the manifest") if arguments.parity_with is not None and arguments.parity_with not in manifest: raise ValueError("parity comparator is missing from the manifest") for artifact in manifest.values(): actual = fingerprint(Path(artifact["path"])) if any(actual[field] != artifact[field] for field in ("bytes", "sha256")): raise ValueError("benchmark artifact changed after build") identities = [artifact["source_tree"] for artifact in manifest.values() if "source_tree" in artifact] if identities and any(identity != source_identity() for identity in identities): raise ValueError("current sources differ from the build; rebuild before measuring") report = {"artifacts": manifest, "cpu_affinity": sorted(os.sched_getaffinity(0)), "measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "manifest": fingerprint(arguments.manifest), "repeats": arguments.repeats, "samples_per_process": 10 if arguments.continuous else 1, "measurement": "continuous" if arguments.continuous else "cold", "samples": [], "growth": {}} operations = getattr(arguments, "operations", None) or list(CASES) if len(set(operations)) != len(operations) or any(operation not in CASES for operation in operations): raise ValueError("kernel selection contains duplicate or unknown operations") if arguments.check_growth and not any(operation in operations for operation in ("list", "map", "list_record")): raise ValueError("growth checking requires a collection construction operation") report["operations"] = operations with tempfile.TemporaryDirectory(prefix="kernel-run-", dir=arguments.output.resolve().parent) as temporary: execution = stage_execution(manifest, Path(temporary)) report["execution"] = execution measure(arguments, report) def stage_execution(manifest, directory): for label, artifact in manifest.items(): if label.startswith("dever"): configuration = Path(artifact["path"]).parent / "config/setting.json" if json.loads(configuration.read_text()) != {}: raise ValueError("kernel benchmark requires an empty application configuration") configuration = directory / "config/setting.json" configuration.parent.mkdir() configuration.write_text("{}\n") artifacts = {} for index, (label, artifact) in enumerate(manifest.items()): binary = directory / f"program-{index:04d}" method = "hardlink" try: os.link(artifact["path"], binary) except OSError as error: if error.errno != errno.EXDEV: raise shutil.copy2(artifact["path"], binary) method = "copy" actual = fingerprint(binary) if any(actual[field] != artifact[field] for field in ("bytes", "sha256")): raise ValueError("benchmark artifact changed during staging") artifacts[label] = {**actual, "method": method} return {"artifacts": artifacts, "configuration": fingerprint(configuration)} def measure(arguments, report): manifest = report["artifacts"] operations = report["operations"] executables = report["execution"]["artifacts"] for operation in operations: sizes = CASES[operation] for count in sizes: ticks = ",".join(["x"] * count) if operation == "text_concat" else "x" * count continuous = arguments.continuous payload = json.dumps({"ticks": ticks}) timings = {label: [] for label in manifest} process_timings = {label: [] for label in manifest} process_elapsed = {label: [] for label in manifest} for repeat in range(arguments.repeats): labels = list(manifest) offset = repeat % len(labels) labels = labels[offset:] + labels[:offset] for label in labels: action = (("benchmark.kernel." if label.startswith("dever") else "") + ("continuous_" if continuous else "") + operation) started = time.perf_counter_ns() process = subprocess.run([executables[label]["path"], action, payload], capture_output=True, text=True, timeout=120, check=True) process_elapsed[label].append((time.perf_counter_ns() - started) / 1_000_000) if process.stderr: raise RuntimeError(process.stderr) value = json.loads(process.stdout) if label.startswith("dever"): if value["code"] != 0: raise RuntimeError("Dever benchmark returned failure") value = value["data"] values = value if continuous else [value] if not isinstance(values, list) or len(values) != (10 if continuous else 1): raise AssertionError("kernel sample count mismatch") if any(sample["checksum"] != count * 120 or sample["elapsed_ns"] <= 0 for sample in values): raise AssertionError("kernel output or checksum mismatch") measured = [sample["elapsed_ns"] / 1_000_000 for sample in values] process_timings[label].append(measured) timings[label].extend(measured[1:] if continuous else measured) item = {"operation": operation, "count": count, "samples_ms": timings, "process_samples_ms": process_timings, "process_elapsed_ms": process_elapsed, "first_sample_median_ms": { label: statistics.median(values[0] for values in processes) for label, processes in process_timings.items()}, "median_ms": {label: statistics.median(values) for label, values in timings.items()}} report["samples"].append(item) print(json.dumps(item), flush=True) for operation in ("list", "map", "list_record"): if operation not in operations: continue samples = [item for item in report["samples"] if item["operation"] == operation] report["growth"][operation] = { label: samples[-1]["median_ms"][label] / samples[0]["median_ms"][label] for label in manifest} if arguments.parity_with is not None: cases = [{ "operation": item["operation"], "count": item["count"], "ratio": item["median_ms"][arguments.candidate] / item["median_ms"][arguments.parity_with], "passed": item["median_ms"][arguments.candidate] <= item["median_ms"][arguments.parity_with], } for item in report["samples"]] report["parity"] = { "candidate": arguments.candidate, "comparator": arguments.parity_with, "passed": all(item["passed"] for item in cases), "cases": cases, } with arguments.output.open("x") as output: json.dump(report, output, indent=2) output.write("\n") if arguments.check_growth and any(values[arguments.candidate] > 24 for values in report["growth"].values()): raise AssertionError("8x input exceeded 24x collection construction time; inspect raw samples") if arguments.parity_with is not None and not report["parity"]["passed"]: failures = [item for item in report["parity"]["cases"] if not item["passed"]] raise AssertionError( f"{len(failures)} of {len(cases)} cases did not reach {arguments.parity_with} parity; " f"full samples and ratios saved to {arguments.output}" ) def repeats(value): parsed = int(value) if not 3 <= parsed <= 25: raise argparse.ArgumentTypeError("expected 3..25 repetitions") return parsed def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) building = commands.add_parser("build") building.add_argument("--compiler", required=True) building.add_argument("--baseline-compiler") building.add_argument("--output", type=Path, required=True) running = commands.add_parser("run") running.add_argument("--manifest", type=Path, required=True) running.add_argument("--output", type=Path, required=True) running.add_argument("--repeats", type=repeats, default=5) running.add_argument("--continuous", action="store_true", help="retain ten calls per process; compare calls 2..10 separately from the first") running.add_argument("--cpu", type=int) running.add_argument("--check-growth", action="store_true") running.add_argument("--operations", nargs="+", choices=tuple(CASES), help="select diagnostic kernels; omitted runs the complete matrix") running.add_argument("--parity-with", choices=("go", "rust")) running.add_argument("--candidate", default="dever") arguments = parser.parse_args() (build if arguments.command == "build" else run)(arguments) if __name__ == "__main__": main()