197 lines
9.2 KiB
Python
197 lines
9.2 KiB
Python
"""Compare native Dever JSON operations with Go and Rust; build separately."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
FIXTURE = Path(__file__).parent / "fixtures/json"
|
|
|
|
|
|
def fingerprint(path):
|
|
with path.open("rb") as source:
|
|
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
|
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest}
|
|
|
|
|
|
def build(arguments):
|
|
output = arguments.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
module = output / "module/benchmark/json"
|
|
module.mkdir(parents=True)
|
|
for name in ("app.dever", "api.dever"):
|
|
shutil.copy2(FIXTURE / name, module / name)
|
|
(output / "config").mkdir()
|
|
(output / "config/setting.json").write_text("{}\n")
|
|
artifacts = {}
|
|
for label, compiler in (("dever", arguments.compiler),
|
|
("dever_before", arguments.baseline_compiler)):
|
|
if compiler is None:
|
|
continue
|
|
binary = output / label
|
|
with (output / (label + ".build.log")).open("w") as log:
|
|
subprocess.run([compiler, "build", str(output), "--output", str(binary)],
|
|
stdout=log, stderr=subprocess.STDOUT, check=True)
|
|
artifacts[label] = fingerprint(binary)
|
|
artifacts[label]["compiler"] = fingerprint(Path(compiler).resolve())
|
|
binary = output / "go"
|
|
subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", str(binary),
|
|
str(FIXTURE.resolve() / "peer.go")], check=True)
|
|
artifacts["go"] = fingerprint(binary)
|
|
artifacts["go"]["compiler"] = subprocess.check_output(["go", "version"], text=True).strip()
|
|
command = ["cargo", "build", "--offline", "--locked", "--release", "-p", "dever-tests",
|
|
"--example", "json-peer", "--message-format=json"]
|
|
process = subprocess.run(command, cwd=FIXTURE.resolve().parents[3],
|
|
capture_output=True, text=True, check=True, timeout=300)
|
|
(output / "rust.build.log").write_text(process.stderr)
|
|
executables = [message["executable"] for line in process.stdout.splitlines()
|
|
if (message := json.loads(line)).get("reason") == "compiler-artifact"
|
|
and message.get("executable") and message["target"]["name"] == "json-peer"]
|
|
if len(executables) != 1:
|
|
raise RuntimeError("expected exactly one native Rust JSON peer")
|
|
binary = output / "rust"
|
|
shutil.copy2(executables[0], binary)
|
|
artifacts["rust"] = fingerprint(binary)
|
|
artifacts["rust"]["compiler"] = subprocess.check_output(["rustc", "--version"], text=True).strip()
|
|
artifacts["rust"]["source"] = fingerprint(FIXTURE.resolve() / "peer.rs")
|
|
artifacts["rust"]["profile"] = "release: opt-level=3, fat LTO, codegen-units=1"
|
|
(output / "manifest.json").write_text(json.dumps(artifacts, indent=2) + "\n")
|
|
print(output / "manifest.json")
|
|
|
|
|
|
def measure(command):
|
|
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
|
|
process = subprocess.Popen(command, stdout=stdout, stderr=stderr, close_fds=False)
|
|
deadline = time.monotonic() + 120
|
|
high_water = 0
|
|
try:
|
|
while True:
|
|
try:
|
|
process_root = Path("/proc") / str(process.pid)
|
|
if (process_root / "exe").resolve() == Path(command[0]).resolve():
|
|
for line in (process_root / "status").read_text().splitlines():
|
|
if line.startswith("VmHWM:"):
|
|
high_water = max(high_water, int(line.split()[1]))
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
waited, status, _ = os.wait4(process.pid, os.WNOHANG)
|
|
if waited:
|
|
process.returncode = os.waitstatus_to_exitcode(status)
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError("benchmark exceeded 120 seconds")
|
|
time.sleep(0.005)
|
|
finally:
|
|
if process.returncode is None:
|
|
process.kill()
|
|
process.wait()
|
|
stdout.seek(0)
|
|
stderr.seek(0)
|
|
errors = stderr.read().decode()
|
|
if process.returncode or errors:
|
|
raise RuntimeError(errors or "benchmark failed")
|
|
result = json.loads(stdout.read())
|
|
if "code" in result:
|
|
if result["code"] != 0:
|
|
raise RuntimeError("Dever benchmark returned failure")
|
|
result = result["data"]
|
|
if high_water == 0:
|
|
raise RuntimeError("process ended before native RSS could be sampled")
|
|
result["sampled_hwm_kib"] = high_water
|
|
return result
|
|
|
|
|
|
def run(arguments):
|
|
if arguments.output.exists():
|
|
raise FileExistsError(arguments.output)
|
|
if arguments.cpu is not None:
|
|
if arguments.cpu not in os.sched_getaffinity(0):
|
|
raise ValueError("requested CPU is outside the allowed affinity")
|
|
os.sched_setaffinity(0, {arguments.cpu})
|
|
manifest = json.loads(arguments.manifest.read_text())
|
|
for artifact in manifest.values():
|
|
actual = fingerprint(Path(artifact["path"]))
|
|
if any(actual[field] != artifact[field] for field in ("bytes", "sha256")):
|
|
raise ValueError("benchmark artifact changed after build")
|
|
report = {"artifacts": manifest, "cpu_affinity": sorted(os.sched_getaffinity(0)),
|
|
"order": "rotate implementations each repeat within each workload",
|
|
"samples": [], "summary": []}
|
|
row = {"zeta": "field", "id": 9007199254740993, "alpha": "x" * 80,
|
|
"enabled": True, "values": [1, None, "escaped\nvalue"]}
|
|
for rows in (4, 40, 160):
|
|
source = {"zeta": {"caption": "form"}, "data": {"rows": [row] * rows},
|
|
"active": False, "alpha": "end"}
|
|
payload_bytes = len(json.dumps(source, separators=(",", ":")).encode())
|
|
for update in (False, True):
|
|
expected = dict(source, active=True) if update else source["data"]
|
|
groups = {label: [] for label in manifest}
|
|
labels = list(manifest)
|
|
for repeat in range(arguments.repeats):
|
|
offset = repeat % len(labels)
|
|
for label in labels[offset:] + labels[:offset]:
|
|
artifact = manifest[label]
|
|
iterations = (arguments.baseline_iterations if label == "dever_before"
|
|
else arguments.iterations)
|
|
payload = json.dumps({"source": source, "update": update,
|
|
"ticks": "x" * iterations}, separators=(",", ":"))
|
|
command = [artifact["path"]]
|
|
if label.startswith("dever"):
|
|
command.append("benchmark.json.run")
|
|
sample = measure(command + [payload])
|
|
if (json.loads(sample["output"]) != expected or
|
|
sample["checksum"] != len(sample["output"].encode()) * iterations or
|
|
sample["elapsed_ns"] <= 0):
|
|
raise AssertionError("JSON output or checksum mismatch")
|
|
sample.pop("output")
|
|
sample.update(label=label, update=update, bytes=payload_bytes,
|
|
iterations=iterations, repeat=repeat,
|
|
us_per_operation=sample["elapsed_ns"] / iterations / 1000)
|
|
groups[label].append(sample)
|
|
report["samples"].append(sample)
|
|
for label, samples in groups.items():
|
|
summary = {"label": label, "operation": "set" if update else "get",
|
|
"bytes": payload_bytes,
|
|
"median_us": statistics.median(item["us_per_operation"] for item in samples),
|
|
"sampled_hwm_kib": max(item["sampled_hwm_kib"] for item in samples)}
|
|
report["summary"].append(summary)
|
|
print(json.dumps(summary), flush=True)
|
|
with arguments.output.open("x") as output:
|
|
json.dump(report, output, indent=2)
|
|
output.write("\n")
|
|
|
|
|
|
def positive(value):
|
|
parsed = int(value)
|
|
if not 1 <= parsed <= 10000:
|
|
raise argparse.ArgumentTypeError("expected 1..10000")
|
|
return parsed
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
building = commands.add_parser("build")
|
|
building.add_argument("--compiler", required=True)
|
|
building.add_argument("--baseline-compiler")
|
|
building.add_argument("--output", type=Path, required=True)
|
|
running = commands.add_parser("run")
|
|
running.add_argument("--manifest", type=Path, required=True)
|
|
running.add_argument("--output", type=Path, required=True)
|
|
running.add_argument("--iterations", type=positive, default=2000)
|
|
running.add_argument("--baseline-iterations", type=positive, default=1)
|
|
running.add_argument("--repeats", type=positive, default=5)
|
|
running.add_argument("--cpu", type=int)
|
|
arguments = parser.parse_args()
|
|
(build if arguments.command == "build" else run)(arguments)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|