Initial Dever 0.0.1
This commit is contained in:
Codex committed
commit
81233d098c
1479 files changed
+163782
No files matched your search
@@ -0,0 +1,196 @@
|
||||
"""Compare native Dever JSON operations with Go and Rust; build separately."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures/json"
|
||||
|
||||
|
||||
def fingerprint(path):
|
||||
with path.open("rb") as source:
|
||||
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
||||
return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest}
|
||||
|
||||
|
||||
def build(arguments):
|
||||
output = arguments.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
module = output / "module/benchmark/json"
|
||||
module.mkdir(parents=True)
|
||||
for name in ("app.dever", "api.dever"):
|
||||
shutil.copy2(FIXTURE / name, module / name)
|
||||
(output / "config").mkdir()
|
||||
(output / "config/setting.json").write_text("{}\n")
|
||||
artifacts = {}
|
||||
for label, compiler in (("dever", arguments.compiler),
|
||||
("dever_before", arguments.baseline_compiler)):
|
||||
if compiler is None:
|
||||
continue
|
||||
binary = output / label
|
||||
with (output / (label + ".build.log")).open("w") as log:
|
||||
subprocess.run([compiler, "build", str(output), "--output", str(binary)],
|
||||
stdout=log, stderr=subprocess.STDOUT, check=True)
|
||||
artifacts[label] = fingerprint(binary)
|
||||
artifacts[label]["compiler"] = fingerprint(Path(compiler).resolve())
|
||||
binary = output / "go"
|
||||
subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", str(binary),
|
||||
str(FIXTURE.resolve() / "peer.go")], check=True)
|
||||
artifacts["go"] = fingerprint(binary)
|
||||
artifacts["go"]["compiler"] = subprocess.check_output(["go", "version"], text=True).strip()
|
||||
command = ["cargo", "build", "--offline", "--locked", "--release", "-p", "dever-tests",
|
||||
"--example", "json-peer", "--message-format=json"]
|
||||
process = subprocess.run(command, cwd=FIXTURE.resolve().parents[3],
|
||||
capture_output=True, text=True, check=True, timeout=300)
|
||||
(output / "rust.build.log").write_text(process.stderr)
|
||||
executables = [message["executable"] for line in process.stdout.splitlines()
|
||||
if (message := json.loads(line)).get("reason") == "compiler-artifact"
|
||||
and message.get("executable") and message["target"]["name"] == "json-peer"]
|
||||
if len(executables) != 1:
|
||||
raise RuntimeError("expected exactly one native Rust JSON peer")
|
||||
binary = output / "rust"
|
||||
shutil.copy2(executables[0], binary)
|
||||
artifacts["rust"] = fingerprint(binary)
|
||||
artifacts["rust"]["compiler"] = subprocess.check_output(["rustc", "--version"], text=True).strip()
|
||||
artifacts["rust"]["source"] = fingerprint(FIXTURE.resolve() / "peer.rs")
|
||||
artifacts["rust"]["profile"] = "release: opt-level=3, fat LTO, codegen-units=1"
|
||||
(output / "manifest.json").write_text(json.dumps(artifacts, indent=2) + "\n")
|
||||
print(output / "manifest.json")
|
||||
|
||||
|
||||
def measure(command):
|
||||
with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr:
|
||||
process = subprocess.Popen(command, stdout=stdout, stderr=stderr, close_fds=False)
|
||||
deadline = time.monotonic() + 120
|
||||
high_water = 0
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
process_root = Path("/proc") / str(process.pid)
|
||||
if (process_root / "exe").resolve() == Path(command[0]).resolve():
|
||||
for line in (process_root / "status").read_text().splitlines():
|
||||
if line.startswith("VmHWM:"):
|
||||
high_water = max(high_water, int(line.split()[1]))
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
pass
|
||||
waited, status, _ = os.wait4(process.pid, os.WNOHANG)
|
||||
if waited:
|
||||
process.returncode = os.waitstatus_to_exitcode(status)
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("benchmark exceeded 120 seconds")
|
||||
time.sleep(0.005)
|
||||
finally:
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
stdout.seek(0)
|
||||
stderr.seek(0)
|
||||
errors = stderr.read().decode()
|
||||
if process.returncode or errors:
|
||||
raise RuntimeError(errors or "benchmark failed")
|
||||
result = json.loads(stdout.read())
|
||||
if "code" in result:
|
||||
if result["code"] != 0:
|
||||
raise RuntimeError("Dever benchmark returned failure")
|
||||
result = result["data"]
|
||||
if high_water == 0:
|
||||
raise RuntimeError("process ended before native RSS could be sampled")
|
||||
result["sampled_hwm_kib"] = high_water
|
||||
return result
|
||||
|
||||
|
||||
def run(arguments):
|
||||
if arguments.output.exists():
|
||||
raise FileExistsError(arguments.output)
|
||||
if arguments.cpu is not None:
|
||||
if arguments.cpu not in os.sched_getaffinity(0):
|
||||
raise ValueError("requested CPU is outside the allowed affinity")
|
||||
os.sched_setaffinity(0, {arguments.cpu})
|
||||
manifest = json.loads(arguments.manifest.read_text())
|
||||
for artifact in manifest.values():
|
||||
actual = fingerprint(Path(artifact["path"]))
|
||||
if any(actual[field] != artifact[field] for field in ("bytes", "sha256")):
|
||||
raise ValueError("benchmark artifact changed after build")
|
||||
report = {"artifacts": manifest, "cpu_affinity": sorted(os.sched_getaffinity(0)),
|
||||
"order": "rotate implementations each repeat within each workload",
|
||||
"samples": [], "summary": []}
|
||||
row = {"zeta": "field", "id": 9007199254740993, "alpha": "x" * 80,
|
||||
"enabled": True, "values": [1, None, "escaped\nvalue"]}
|
||||
for rows in (4, 40, 160):
|
||||
source = {"zeta": {"caption": "form"}, "data": {"rows": [row] * rows},
|
||||
"active": False, "alpha": "end"}
|
||||
payload_bytes = len(json.dumps(source, separators=(",", ":")).encode())
|
||||
for update in (False, True):
|
||||
expected = dict(source, active=True) if update else source["data"]
|
||||
groups = {label: [] for label in manifest}
|
||||
labels = list(manifest)
|
||||
for repeat in range(arguments.repeats):
|
||||
offset = repeat % len(labels)
|
||||
for label in labels[offset:] + labels[:offset]:
|
||||
artifact = manifest[label]
|
||||
iterations = (arguments.baseline_iterations if label == "dever_before"
|
||||
else arguments.iterations)
|
||||
payload = json.dumps({"source": source, "update": update,
|
||||
"ticks": "x" * iterations}, separators=(",", ":"))
|
||||
command = [artifact["path"]]
|
||||
if label.startswith("dever"):
|
||||
command.append("benchmark.json.run")
|
||||
sample = measure(command + [payload])
|
||||
if (json.loads(sample["output"]) != expected or
|
||||
sample["checksum"] != len(sample["output"].encode()) * iterations or
|
||||
sample["elapsed_ns"] <= 0):
|
||||
raise AssertionError("JSON output or checksum mismatch")
|
||||
sample.pop("output")
|
||||
sample.update(label=label, update=update, bytes=payload_bytes,
|
||||
iterations=iterations, repeat=repeat,
|
||||
us_per_operation=sample["elapsed_ns"] / iterations / 1000)
|
||||
groups[label].append(sample)
|
||||
report["samples"].append(sample)
|
||||
for label, samples in groups.items():
|
||||
summary = {"label": label, "operation": "set" if update else "get",
|
||||
"bytes": payload_bytes,
|
||||
"median_us": statistics.median(item["us_per_operation"] for item in samples),
|
||||
"sampled_hwm_kib": max(item["sampled_hwm_kib"] for item in samples)}
|
||||
report["summary"].append(summary)
|
||||
print(json.dumps(summary), flush=True)
|
||||
with arguments.output.open("x") as output:
|
||||
json.dump(report, output, indent=2)
|
||||
output.write("\n")
|
||||
|
||||
|
||||
def positive(value):
|
||||
parsed = int(value)
|
||||
if not 1 <= parsed <= 10000:
|
||||
raise argparse.ArgumentTypeError("expected 1..10000")
|
||||
return parsed
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
building = commands.add_parser("build")
|
||||
building.add_argument("--compiler", required=True)
|
||||
building.add_argument("--baseline-compiler")
|
||||
building.add_argument("--output", type=Path, required=True)
|
||||
running = commands.add_parser("run")
|
||||
running.add_argument("--manifest", type=Path, required=True)
|
||||
running.add_argument("--output", type=Path, required=True)
|
||||
running.add_argument("--iterations", type=positive, default=2000)
|
||||
running.add_argument("--baseline-iterations", type=positive, default=1)
|
||||
running.add_argument("--repeats", type=positive, default=5)
|
||||
running.add_argument("--cpu", type=int)
|
||||
arguments = parser.parse_args()
|
||||
(build if arguments.command == "build" else run)(arguments)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user