"""Measure real typed/raw JSON API requests with isolated native applications.""" import argparse import http.client import json import os from pathlib import Path import platform import secrets import shutil import signal import socket import statistics import subprocess import tempfile import time from build import fingerprint from process import process_sample from settings import write_private_json FIXTURE = Path(__file__).parent / "fixtures/api-json" def configuration(port): return { "http": {"host": "127.0.0.1", "port": port}, "runtime": {"mode": "api", "shutdown_ms": 2000}, "log": {"level": "error"}, "database": {"default": {"type": "sqlite", "path": "data/control.db", "max_connections": 2}}, "auth": {"providers": {"session": { "verify": "benchmark.auth.verify", "jwtSecret": secrets.token_hex(32), "cookie": "benchmark_session"}}}, "sites": {"test": {"path": "", "auth": "session", "hosts": ["localhost"], "origin": "https://localhost"}}, } def build(arguments): output = arguments.output.resolve() output.mkdir(parents=True, exist_ok=False) project = output / "project" shutil.copytree(FIXTURE / "benchmark", project / "module/benchmark") (project / "config").mkdir() write_private_json(project / "config/setting.json", configuration(8080)) artifacts = {} for label, compiler in (("before", arguments.baseline_compiler), ("after", arguments.compiler)): compiler = compiler.resolve() executable = output / label started = time.monotonic() with (output / f"{label}.build.log").open("x") as log: subprocess.run([compiler, "build", project, "--output", executable], stdout=log, stderr=subprocess.STDOUT, check=True, timeout=900) artifacts[label] = {"path": str(executable), **fingerprint(executable), "compiler": {"path": str(compiler), **fingerprint(compiler)}, "build_seconds": time.monotonic() - started, "runtime_manifests": { str(path.relative_to(compiler.parent)): fingerprint(path) for path in sorted((compiler.parent / "runtime").glob("*/manifest.json"))}} if arguments.with_peers: go_binary = output / "go" subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", go_binary, FIXTURE.resolve() / "peer.go"], check=True, timeout=180) artifacts["go"] = {"path": str(go_binary), **fingerprint(go_binary), "compiler": subprocess.check_output(["go", "version"], text=True).strip(), "source": fingerprint(FIXTURE / "peer.go"), "scope": "native codec/transport baseline; not full Dever API semantics"} built = subprocess.run(["cargo", "build", "--offline", "--locked", "--release", "-p", "dever-tests", "--example", "api-json-peer", "--message-format=json"], capture_output=True, text=True, timeout=600) (output / "rust.build.log").write_text(built.stderr) built.check_returncode() executables = [message["executable"] for line in built.stdout.splitlines() if (message := json.loads(line)).get("reason") == "compiler-artifact" and message.get("executable") and message["target"]["name"] == "api-json-peer"] if len(executables) != 1: raise RuntimeError("expected one native Rust API peer") rust_binary = output / "rust" shutil.copy2(executables[0], rust_binary) artifacts["rust"] = {"path": str(rust_binary), **fingerprint(rust_binary), "compiler": subprocess.check_output(["rustc", "--version"], text=True).strip(), "source": fingerprint(FIXTURE / "peer.rs"), "profile": "release: opt-level=3, fat LTO, codegen-units=1", "scope": "native codec/transport baseline; not full Dever API semantics"} if arguments.with_runtime_peer: runtime = arguments.compiler.resolve().parent / "runtime/linux-x86_64" if not (runtime / "libc.a").is_file() or not (runtime / "crt1.o").is_file(): raise ValueError("Rust runtime peer requires the compiler's Linux x86_64 runtime inputs") command = ["cargo", "rustc", "--offline", "--locked", "--profile", "runtime-pack", "-p", "dever-runtime", "--features", "api,crypto,external,sqlite", "--example", "api-json-runtime-peer", "--message-format=json", "--", "-C", "target-feature=+crt-static", "-C", "relocation-model=static", "-C", f"link-arg=-B{runtime}/", "-C", f"link-arg=-L{runtime}/", "-C", f"link-arg=-Wl,-Map,{output / 'rust-runtime.link.map'}"] built = subprocess.run(command, capture_output=True, text=True, timeout=600) (output / "rust-runtime.build.log").write_text(built.stderr) built.check_returncode() executables = [message["executable"] for line in built.stdout.splitlines() if (message := json.loads(line)).get("reason") == "compiler-artifact" and message.get("executable") and message["target"]["name"] == "api-json-runtime-peer"] if len(executables) != 1: raise RuntimeError("expected one Rust runtime API peer") runtime_binary = output / "rust-runtime" shutil.copy2(executables[0], runtime_binary) artifacts["rust_runtime"] = { "path": str(runtime_binary), **fingerprint(runtime_binary), "compiler": subprocess.check_output(["rustc", "--version"], text=True).strip(), "source": fingerprint(FIXTURE / "runtime-peer.rs"), "command": command, "runtime_inputs": {name: fingerprint(runtime / name) for name in ["crt1.o", "crti.o", "crtbeginT.o", "crtend.o", "crtn.o", "libc.a", "libm.a", "libgcc.a", "libgcc_eh.a"]}, "profile": "runtime-pack: opt-level=3, fat LTO, static executable, codegen-units=1", "scope": "shared Dever runtime contract with native Rust values and codecs; not an independent runtime", } manifest = { "artifacts": artifacts, "sources": {str(path.relative_to(FIXTURE)): fingerprint(path) for path in sorted(FIXTURE.rglob("*.dever"))}, } write_private_json(output / "manifest.json", manifest) print(output / "manifest.json", flush=True) def request(client, route, body, expected): started = time.perf_counter_ns() client.request("POST", f"/benchmark/echo/{route}", body, {"Host": "localhost", "Content-Type": "application/json"}) response = client.getresponse() encoded = response.read() elapsed = time.perf_counter_ns() - started if response.status != 200 or json.loads(encoded) != { "code": 0, "message": "ok", "data": expected}: raise AssertionError(f"API response mismatch: HTTP {response.status}") return elapsed def percentile(values, fraction): return sorted(values)[min(len(values) - 1, int(len(values) * fraction))] / 1000 def measure(artifact, route, payload, arguments): expected = payload["payload" if route == "echo" else "source"] body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode() with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) port = listener.getsockname()[1] with tempfile.TemporaryDirectory(prefix="api-json-", dir=arguments.output) as temporary: directory = Path(temporary) executable = directory / "app" shutil.copy2(artifact["path"], executable) (directory / "config").mkdir() write_private_json(directory / "config/setting.json", configuration(port)) with (directory / "stdout").open("w+") as stdout, \ (directory / "stderr").open("w+") as stderr: process = subprocess.Popen(["taskset", "-c", str(arguments.server_cpu), executable], cwd=directory, stdout=stdout, stderr=stderr, start_new_session=True) client = http.client.HTTPConnection("127.0.0.1", port, timeout=10) try: deadline = time.monotonic() + 20 while True: if process.poll() is not None: raise RuntimeError(f"API exited before readiness: {process.returncode}") try: request(client, route, body, expected) break except (ConnectionError, http.client.HTTPException): client.close() if time.monotonic() >= deadline: raise TimeoutError("API readiness timeout") time.sleep(0.02) for _ in range(50): request(client, route, body, expected) before = process_sample(process.pid) started = time.monotonic() latencies = [request(client, route, body, expected) for _ in range(arguments.requests)] elapsed = time.monotonic() - started after = process_sample(process.pid) result = { "requests": len(latencies), "request_bytes": len(body), "elapsed_seconds": elapsed, "p50_us": percentile(latencies, 0.5), "p95_us": percentile(latencies, 0.95), "p99_us": percentile(latencies, 0.99), "server_cpu_us_per_request": (after["cpu_seconds"] - before["cpu_seconds"]) * 1e6 / len(latencies), "rss_bytes": after["rss_bytes"], "rss_growth_bytes": after["rss_bytes"] - before["rss_bytes"], } except BaseException as error: stderr.flush() stderr.seek(0) error.add_note(stderr.read()[-4000:]) raise finally: client.close() if process.poll() is None: os.killpg(process.pid, signal.SIGTERM) try: process.wait(timeout=10) except subprocess.TimeoutExpired: os.killpg(process.pid, signal.SIGKILL) process.wait(timeout=5) raise TimeoutError("API did not drain after SIGTERM") if process.returncode: raise RuntimeError(f"API shutdown failed: {process.returncode}") return result def run(arguments): allowed = os.sched_getaffinity(0) if (arguments.server_cpu == arguments.client_cpu or not {arguments.server_cpu, arguments.client_cpu} <= allowed): raise ValueError("server and client require distinct allowed CPUs") if arguments.requests < 1 or arguments.repeats < 1: raise ValueError("requests and repeats must be positive") manifest = json.loads(arguments.manifest.read_text()) for artifact in manifest["artifacts"].values(): if fingerprint(Path(artifact["path"])) != { key: artifact[key] for key in ("bytes", "sha256")}: raise ValueError("API benchmark binary changed after build") arguments.output.mkdir(parents=True, exist_ok=False) os.sched_setaffinity(0, {arguments.client_cpu}) labels = list(manifest["artifacts"]) report = {"manifest": manifest, "server_cpu": arguments.server_cpu, "client_cpu": arguments.client_cpu, "samples": [], "summary": [], "environment": {"kernel": platform.release(), "machine": platform.machine(), "allowed_cpus": sorted(allowed), "python": platform.python_version()}, "runner": fingerprint(Path(__file__)), "requests_per_process": arguments.requests, "repeats": arguments.repeats} try: for count in (1, 16, 128): row = {"id": 9007199254740993, "text": "雪😀\n\"" + "x" * 80, "weight": 1.25, "note": None} value = {"rows": [row] * count, "raw": {"large": 123456789012345678901234567890}} for route, argument in (("echo", "payload"), ("raw", "source")): for repeat in range(arguments.repeats): offset = repeat % len(labels) order = labels[offset:] + labels[:offset] for label in order: sample = measure(manifest["artifacts"][label], route, {argument: value}, arguments) report["samples"].append({"label": label, "route": route, "rows": count, "repeat": repeat, **sample}) for label in labels: samples = [row for row in report["samples"] if row["label"] == label and row["route"] == route and row["rows"] == count] summary = {"label": label, "route": route, "rows": count} for metric in ("p50_us", "p95_us", "p99_us", "server_cpu_us_per_request", "rss_bytes"): summary[metric] = statistics.median(sample[metric] for sample in samples) report["summary"].append(summary) print(json.dumps(summary), flush=True) for artifact in manifest["artifacts"].values(): if fingerprint(Path(artifact["path"])) != { key: artifact[key] for key in ("bytes", "sha256")}: raise ValueError("API benchmark binary changed during measurement") report["passed"] = True finally: write_private_json(arguments.output / "report.json", report) def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) prepare = commands.add_parser("build") prepare.add_argument("--compiler", type=Path, required=True) prepare.add_argument("--baseline-compiler", type=Path, required=True) prepare.add_argument("--output", type=Path, required=True) prepare.add_argument("--with-peers", action="store_true") prepare.add_argument("--with-runtime-peer", action="store_true") prepare.set_defaults(action=build) measure = commands.add_parser("run") measure.add_argument("--manifest", type=Path, required=True) measure.add_argument("--output", type=Path, required=True) measure.add_argument("--server-cpu", type=int, required=True) measure.add_argument("--client-cpu", type=int, required=True) measure.add_argument("--requests", type=int, default=1000) measure.add_argument("--repeats", type=int, default=7) measure.set_defaults(action=run) arguments = parser.parse_args() arguments.output = arguments.output.resolve() arguments.action(arguments) if __name__ == "__main__": main()