305 lines
15 KiB
Python
305 lines
15 KiB
Python
"""Compare current Dever collection kernels with independent Go and Rust programs."""
|
|
|
|
import argparse
|
|
import errno
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
FIXTURE = Path(__file__).parent / "fixtures/kernel"
|
|
CORE = Path(__file__).resolve().parents[2]
|
|
CASES = {"integer": (16384, 65536), "float": (16384, 65536),
|
|
"list": (512, 1024, 2048, 4096),
|
|
"map": (512, 1024, 2048, 4096),
|
|
"list_read": (512, 1024, 2048, 4096),
|
|
"list_record": (512, 1024, 2048, 4096),
|
|
"list_record_read": (512, 1024, 2048, 4096),
|
|
"record_update": (512, 1024, 2048, 4096),
|
|
"record_inspect": (512, 1024, 2048, 4096),
|
|
"call_read": (512, 1024, 2048, 4096),
|
|
"map_read": (512, 1024, 2048, 4096),
|
|
"map_lookup": (512, 1024, 2048, 4096),
|
|
"map_text_lookup": (512, 1024, 2048, 4096),
|
|
"text_equal": (512, 1024, 2048, 4096),
|
|
"text_contains": (512, 1024, 2048, 4096),
|
|
"text_concat": (512, 2048, 8192, 32768),
|
|
"bytes_concat": (512, 2048, 8192, 32768)}
|
|
|
|
|
|
def fingerprint(path):
|
|
with path.open("rb") as source:
|
|
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
|
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest}
|
|
|
|
|
|
def source_identity():
|
|
files = [CORE / "rust" / name for name in
|
|
("Cargo.toml", "Cargo.lock", "rust-toolchain.toml")]
|
|
for directory in ("rust/src", "lib", "sdk"):
|
|
files.extend(path for path in (CORE / directory).rglob("*")
|
|
if path.is_file() and path.suffix in
|
|
{".rs", ".toml", ".md", ".dever", ".py", ".c", ".h", ".js", ".mjs", ".json", ".go"})
|
|
digest = hashlib.sha256()
|
|
for path in sorted(files):
|
|
digest.update(str(path.relative_to(CORE)).encode() + b"\0")
|
|
digest.update(fingerprint(path)["sha256"].encode() + b"\n")
|
|
return {"sha256": digest.hexdigest(), "files": len(files)}
|
|
|
|
|
|
def build(arguments):
|
|
tree = source_identity()
|
|
output = arguments.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
module = output / "module/benchmark/kernel"
|
|
module.mkdir(parents=True)
|
|
for name in ("app.dever", "api.dever"):
|
|
shutil.copy2(FIXTURE / name, module / name)
|
|
(output / "config").mkdir()
|
|
(output / "config/setting.json").write_text("{}\n")
|
|
artifacts = {}
|
|
for label, compiler in (("dever", arguments.compiler),
|
|
("dever_before", arguments.baseline_compiler)):
|
|
if compiler is None:
|
|
continue
|
|
binary = output / label
|
|
with (output / (label + ".build.log")).open("w") as log:
|
|
subprocess.run([compiler, "build", str(output), "--output", str(binary)],
|
|
stdout=log, stderr=subprocess.STDOUT, check=True, timeout=300)
|
|
artifacts[label] = fingerprint(binary)
|
|
artifacts[label]["compiler"] = fingerprint(Path(compiler).resolve())
|
|
artifacts[label]["source_tree"] = tree
|
|
generated = output / (label + ".rs")
|
|
with generated.open("w") as source:
|
|
subprocess.run([compiler, "emit", str(output)], stdout=source,
|
|
check=True, timeout=120)
|
|
artifacts[label]["generated"] = fingerprint(generated)
|
|
binary = output / "go"
|
|
subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", str(binary),
|
|
str(FIXTURE.resolve() / "peer.go")], check=True, timeout=120)
|
|
artifacts["go"] = fingerprint(binary)
|
|
artifacts["go"]["compiler"] = subprocess.check_output(["go", "version"], text=True).strip()
|
|
peer = output / "rust-peer"
|
|
peer.mkdir()
|
|
shutil.copy2(FIXTURE / "peer.rs", peer / "main.rs")
|
|
core = FIXTURE.resolve().parents[3]
|
|
shutil.copy2(core / "rust/rust-toolchain.toml", peer / "rust-toolchain.toml")
|
|
(peer / "Cargo.toml").write_text(
|
|
'[package]\nname = "kernel-peer"\nversion = "0.0.1"\nedition = "2024"\n'
|
|
'[workspace]\n[[bin]]\nname = "kernel-peer"\npath = "main.rs"\n'
|
|
'[dependencies]\nserde_json = "=1.0.145"\nindexmap = "=2.14.0"\n'
|
|
'[profile.release]\nopt-level = 3\nlto = "fat"\ncodegen-units = 1\n'
|
|
'overflow-checks = true\nstrip = "symbols"\n')
|
|
subprocess.run(["cargo", "generate-lockfile", "--offline"], cwd=peer,
|
|
check=True, timeout=120)
|
|
command = ["cargo", "build", "--offline", "--locked", "--release",
|
|
"--bin", "kernel-peer", "--message-format=json"]
|
|
process = subprocess.run(command, cwd=peer,
|
|
capture_output=True, text=True, check=True, timeout=300)
|
|
(output / "rust.build.log").write_text(process.stderr)
|
|
executables = [message["executable"] for line in process.stdout.splitlines()
|
|
if (message := json.loads(line)).get("reason") == "compiler-artifact"
|
|
and message.get("executable") and message["target"]["name"] == "kernel-peer"]
|
|
if len(executables) != 1:
|
|
raise RuntimeError("expected exactly one native Rust peer")
|
|
binary = output / "rust"
|
|
shutil.copy2(executables[0], binary)
|
|
artifacts["rust"] = fingerprint(binary)
|
|
artifacts["rust"]["compiler"] = subprocess.check_output(
|
|
["rustc", "--version"], cwd=peer, text=True).strip()
|
|
artifacts["rust"]["profile"] = "release: opt-level=3, fat LTO, codegen-units=1"
|
|
artifacts["rust"]["source"] = fingerprint(peer / "main.rs")
|
|
artifacts["rust"]["lock"] = fingerprint(peer / "Cargo.lock")
|
|
artifacts["rust"]["map"] = "ordered IndexMap with owned keys, matching Dever"
|
|
if source_identity() != tree:
|
|
raise RuntimeError("sources changed during build; rebuild the comparison")
|
|
(output / "manifest.json").write_text(json.dumps(artifacts, indent=2) + "\n")
|
|
print(output / "manifest.json")
|
|
|
|
|
|
def run(arguments):
|
|
if arguments.output.exists():
|
|
raise FileExistsError(arguments.output)
|
|
if arguments.cpu is not None:
|
|
if arguments.cpu not in os.sched_getaffinity(0):
|
|
raise ValueError("requested CPU is outside the allowed affinity")
|
|
os.sched_setaffinity(0, {arguments.cpu})
|
|
manifest = json.loads(arguments.manifest.read_text())
|
|
if arguments.candidate not in manifest:
|
|
raise ValueError("candidate is missing from the manifest")
|
|
if arguments.parity_with is not None and arguments.parity_with not in manifest:
|
|
raise ValueError("parity comparator is missing from the manifest")
|
|
for artifact in manifest.values():
|
|
actual = fingerprint(Path(artifact["path"]))
|
|
if any(actual[field] != artifact[field] for field in ("bytes", "sha256")):
|
|
raise ValueError("benchmark artifact changed after build")
|
|
identities = [artifact["source_tree"] for artifact in manifest.values()
|
|
if "source_tree" in artifact]
|
|
if identities and any(identity != source_identity() for identity in identities):
|
|
raise ValueError("current sources differ from the build; rebuild before measuring")
|
|
report = {"artifacts": manifest, "cpu_affinity": sorted(os.sched_getaffinity(0)),
|
|
"measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"manifest": fingerprint(arguments.manifest), "repeats": arguments.repeats,
|
|
"samples_per_process": 10 if arguments.continuous else 1,
|
|
"measurement": "continuous" if arguments.continuous else "cold",
|
|
"samples": [], "growth": {}}
|
|
operations = getattr(arguments, "operations", None) or list(CASES)
|
|
if len(set(operations)) != len(operations) or any(operation not in CASES for operation in operations):
|
|
raise ValueError("kernel selection contains duplicate or unknown operations")
|
|
if arguments.check_growth and not any(operation in operations for operation in ("list", "map", "list_record")):
|
|
raise ValueError("growth checking requires a collection construction operation")
|
|
report["operations"] = operations
|
|
with tempfile.TemporaryDirectory(prefix="kernel-run-", dir=arguments.output.resolve().parent) as temporary:
|
|
execution = stage_execution(manifest, Path(temporary))
|
|
report["execution"] = execution
|
|
measure(arguments, report)
|
|
|
|
|
|
def stage_execution(manifest, directory):
|
|
for label, artifact in manifest.items():
|
|
if label.startswith("dever"):
|
|
configuration = Path(artifact["path"]).parent / "config/setting.json"
|
|
if json.loads(configuration.read_text()) != {}:
|
|
raise ValueError("kernel benchmark requires an empty application configuration")
|
|
configuration = directory / "config/setting.json"
|
|
configuration.parent.mkdir()
|
|
configuration.write_text("{}\n")
|
|
artifacts = {}
|
|
for index, (label, artifact) in enumerate(manifest.items()):
|
|
binary = directory / f"program-{index:04d}"
|
|
method = "hardlink"
|
|
try:
|
|
os.link(artifact["path"], binary)
|
|
except OSError as error:
|
|
if error.errno != errno.EXDEV:
|
|
raise
|
|
shutil.copy2(artifact["path"], binary)
|
|
method = "copy"
|
|
actual = fingerprint(binary)
|
|
if any(actual[field] != artifact[field] for field in ("bytes", "sha256")):
|
|
raise ValueError("benchmark artifact changed during staging")
|
|
artifacts[label] = {**actual, "method": method}
|
|
return {"artifacts": artifacts, "configuration": fingerprint(configuration)}
|
|
|
|
|
|
def measure(arguments, report):
|
|
manifest = report["artifacts"]
|
|
operations = report["operations"]
|
|
executables = report["execution"]["artifacts"]
|
|
for operation in operations:
|
|
sizes = CASES[operation]
|
|
for count in sizes:
|
|
ticks = ",".join(["x"] * count) if operation == "text_concat" else "x" * count
|
|
continuous = arguments.continuous
|
|
payload = json.dumps({"ticks": ticks})
|
|
timings = {label: [] for label in manifest}
|
|
process_timings = {label: [] for label in manifest}
|
|
process_elapsed = {label: [] for label in manifest}
|
|
for repeat in range(arguments.repeats):
|
|
labels = list(manifest)
|
|
offset = repeat % len(labels)
|
|
labels = labels[offset:] + labels[:offset]
|
|
for label in labels:
|
|
action = (("benchmark.kernel." if label.startswith("dever") else "")
|
|
+ ("continuous_" if continuous else "") + operation)
|
|
started = time.perf_counter_ns()
|
|
process = subprocess.run([executables[label]["path"], action, payload],
|
|
capture_output=True, text=True, timeout=120, check=True)
|
|
process_elapsed[label].append((time.perf_counter_ns() - started) / 1_000_000)
|
|
if process.stderr:
|
|
raise RuntimeError(process.stderr)
|
|
value = json.loads(process.stdout)
|
|
if label.startswith("dever"):
|
|
if value["code"] != 0:
|
|
raise RuntimeError("Dever benchmark returned failure")
|
|
value = value["data"]
|
|
values = value if continuous else [value]
|
|
if not isinstance(values, list) or len(values) != (10 if continuous else 1):
|
|
raise AssertionError("kernel sample count mismatch")
|
|
if any(sample["checksum"] != count * 120 or sample["elapsed_ns"] <= 0
|
|
for sample in values):
|
|
raise AssertionError("kernel output or checksum mismatch")
|
|
measured = [sample["elapsed_ns"] / 1_000_000 for sample in values]
|
|
process_timings[label].append(measured)
|
|
timings[label].extend(measured[1:] if continuous else measured)
|
|
item = {"operation": operation, "count": count, "samples_ms": timings,
|
|
"process_samples_ms": process_timings,
|
|
"process_elapsed_ms": process_elapsed,
|
|
"first_sample_median_ms": {
|
|
label: statistics.median(values[0] for values in processes)
|
|
for label, processes in process_timings.items()},
|
|
"median_ms": {label: statistics.median(values) for label, values in timings.items()}}
|
|
report["samples"].append(item)
|
|
print(json.dumps(item), flush=True)
|
|
for operation in ("list", "map", "list_record"):
|
|
if operation not in operations:
|
|
continue
|
|
samples = [item for item in report["samples"] if item["operation"] == operation]
|
|
report["growth"][operation] = {
|
|
label: samples[-1]["median_ms"][label] / samples[0]["median_ms"][label]
|
|
for label in manifest}
|
|
if arguments.parity_with is not None:
|
|
cases = [{
|
|
"operation": item["operation"],
|
|
"count": item["count"],
|
|
"ratio": item["median_ms"][arguments.candidate] / item["median_ms"][arguments.parity_with],
|
|
"passed": item["median_ms"][arguments.candidate] <= item["median_ms"][arguments.parity_with],
|
|
} for item in report["samples"]]
|
|
report["parity"] = {
|
|
"candidate": arguments.candidate,
|
|
"comparator": arguments.parity_with,
|
|
"passed": all(item["passed"] for item in cases),
|
|
"cases": cases,
|
|
}
|
|
with arguments.output.open("x") as output:
|
|
json.dump(report, output, indent=2)
|
|
output.write("\n")
|
|
if arguments.check_growth and any(values[arguments.candidate] > 24 for values in report["growth"].values()):
|
|
raise AssertionError("8x input exceeded 24x collection construction time; inspect raw samples")
|
|
if arguments.parity_with is not None and not report["parity"]["passed"]:
|
|
failures = [item for item in report["parity"]["cases"] if not item["passed"]]
|
|
raise AssertionError(
|
|
f"{len(failures)} of {len(cases)} cases did not reach {arguments.parity_with} parity; "
|
|
f"full samples and ratios saved to {arguments.output}"
|
|
)
|
|
|
|
|
|
def repeats(value):
|
|
parsed = int(value)
|
|
if not 3 <= parsed <= 25:
|
|
raise argparse.ArgumentTypeError("expected 3..25 repetitions")
|
|
return parsed
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
building = commands.add_parser("build")
|
|
building.add_argument("--compiler", required=True)
|
|
building.add_argument("--baseline-compiler")
|
|
building.add_argument("--output", type=Path, required=True)
|
|
running = commands.add_parser("run")
|
|
running.add_argument("--manifest", type=Path, required=True)
|
|
running.add_argument("--output", type=Path, required=True)
|
|
running.add_argument("--repeats", type=repeats, default=5)
|
|
running.add_argument("--continuous", action="store_true",
|
|
help="retain ten calls per process; compare calls 2..10 separately from the first")
|
|
running.add_argument("--cpu", type=int)
|
|
running.add_argument("--check-growth", action="store_true")
|
|
running.add_argument("--operations", nargs="+", choices=tuple(CASES),
|
|
help="select diagnostic kernels; omitted runs the complete matrix")
|
|
running.add_argument("--parity-with", choices=("go", "rust"))
|
|
running.add_argument("--candidate", default="dever")
|
|
arguments = parser.parse_args()
|
|
(build if arguments.command == "build" else run)(arguments)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|