293 lines
15 KiB
Python
293 lines
15 KiB
Python
"""Measure real typed/raw JSON API requests with isolated native applications."""
|
|
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import secrets
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import statistics
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
from build import fingerprint
|
|
from process import process_sample
|
|
from settings import write_private_json
|
|
|
|
|
|
FIXTURE = Path(__file__).parent / "fixtures/api-json"
|
|
|
|
|
|
def configuration(port):
|
|
return {
|
|
"http": {"host": "127.0.0.1", "port": port},
|
|
"runtime": {"mode": "api", "shutdown_ms": 2000},
|
|
"log": {"level": "error"},
|
|
"database": {"default": {"type": "sqlite", "path": "data/control.db",
|
|
"max_connections": 2}},
|
|
"auth": {"providers": {"session": {
|
|
"verify": "benchmark.auth.verify", "jwtSecret": secrets.token_hex(32),
|
|
"cookie": "benchmark_session"}}},
|
|
"sites": {"test": {"path": "", "auth": "session", "hosts": ["localhost"],
|
|
"origin": "https://localhost"}},
|
|
}
|
|
|
|
|
|
def build(arguments):
|
|
output = arguments.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
project = output / "project"
|
|
shutil.copytree(FIXTURE / "benchmark", project / "module/benchmark")
|
|
(project / "config").mkdir()
|
|
write_private_json(project / "config/setting.json", configuration(8080))
|
|
artifacts = {}
|
|
for label, compiler in (("before", arguments.baseline_compiler),
|
|
("after", arguments.compiler)):
|
|
compiler = compiler.resolve()
|
|
executable = output / label
|
|
started = time.monotonic()
|
|
with (output / f"{label}.build.log").open("x") as log:
|
|
subprocess.run([compiler, "build", project, "--output", executable],
|
|
stdout=log, stderr=subprocess.STDOUT, check=True, timeout=900)
|
|
artifacts[label] = {"path": str(executable), **fingerprint(executable),
|
|
"compiler": {"path": str(compiler), **fingerprint(compiler)},
|
|
"build_seconds": time.monotonic() - started,
|
|
"runtime_manifests": {
|
|
str(path.relative_to(compiler.parent)): fingerprint(path)
|
|
for path in sorted((compiler.parent / "runtime").glob("*/manifest.json"))}}
|
|
if arguments.with_peers:
|
|
go_binary = output / "go"
|
|
subprocess.run(["go", "build", "-trimpath", "-ldflags=-s -w", "-o", go_binary,
|
|
FIXTURE.resolve() / "peer.go"], check=True, timeout=180)
|
|
artifacts["go"] = {"path": str(go_binary), **fingerprint(go_binary),
|
|
"compiler": subprocess.check_output(["go", "version"], text=True).strip(),
|
|
"source": fingerprint(FIXTURE / "peer.go"),
|
|
"scope": "native codec/transport baseline; not full Dever API semantics"}
|
|
built = subprocess.run(["cargo", "build", "--offline", "--locked", "--release",
|
|
"-p", "dever-tests", "--example", "api-json-peer",
|
|
"--message-format=json"], capture_output=True, text=True,
|
|
timeout=600)
|
|
(output / "rust.build.log").write_text(built.stderr)
|
|
built.check_returncode()
|
|
executables = [message["executable"] for line in built.stdout.splitlines()
|
|
if (message := json.loads(line)).get("reason") == "compiler-artifact"
|
|
and message.get("executable") and message["target"]["name"] == "api-json-peer"]
|
|
if len(executables) != 1:
|
|
raise RuntimeError("expected one native Rust API peer")
|
|
rust_binary = output / "rust"
|
|
shutil.copy2(executables[0], rust_binary)
|
|
artifacts["rust"] = {"path": str(rust_binary), **fingerprint(rust_binary),
|
|
"compiler": subprocess.check_output(["rustc", "--version"], text=True).strip(),
|
|
"source": fingerprint(FIXTURE / "peer.rs"),
|
|
"profile": "release: opt-level=3, fat LTO, codegen-units=1",
|
|
"scope": "native codec/transport baseline; not full Dever API semantics"}
|
|
if arguments.with_runtime_peer:
|
|
runtime = arguments.compiler.resolve().parent / "runtime/linux-x86_64"
|
|
if not (runtime / "libc.a").is_file() or not (runtime / "crt1.o").is_file():
|
|
raise ValueError("Rust runtime peer requires the compiler's Linux x86_64 runtime inputs")
|
|
command = ["cargo", "rustc", "--offline", "--locked", "--profile", "runtime-pack",
|
|
"-p", "dever-runtime", "--features", "api,crypto,external,sqlite",
|
|
"--example", "api-json-runtime-peer", "--message-format=json", "--",
|
|
"-C", "target-feature=+crt-static", "-C", "relocation-model=static",
|
|
"-C", f"link-arg=-B{runtime}/", "-C", f"link-arg=-L{runtime}/",
|
|
"-C", f"link-arg=-Wl,-Map,{output / 'rust-runtime.link.map'}"]
|
|
built = subprocess.run(command, capture_output=True, text=True, timeout=600)
|
|
(output / "rust-runtime.build.log").write_text(built.stderr)
|
|
built.check_returncode()
|
|
executables = [message["executable"] for line in built.stdout.splitlines()
|
|
if (message := json.loads(line)).get("reason") == "compiler-artifact"
|
|
and message.get("executable") and message["target"]["name"] == "api-json-runtime-peer"]
|
|
if len(executables) != 1:
|
|
raise RuntimeError("expected one Rust runtime API peer")
|
|
runtime_binary = output / "rust-runtime"
|
|
shutil.copy2(executables[0], runtime_binary)
|
|
artifacts["rust_runtime"] = {
|
|
"path": str(runtime_binary), **fingerprint(runtime_binary),
|
|
"compiler": subprocess.check_output(["rustc", "--version"], text=True).strip(),
|
|
"source": fingerprint(FIXTURE / "runtime-peer.rs"), "command": command,
|
|
"runtime_inputs": {name: fingerprint(runtime / name) for name in
|
|
["crt1.o", "crti.o", "crtbeginT.o", "crtend.o", "crtn.o", "libc.a", "libm.a", "libgcc.a", "libgcc_eh.a"]},
|
|
"profile": "runtime-pack: opt-level=3, fat LTO, static executable, codegen-units=1",
|
|
"scope": "shared Dever runtime contract with native Rust values and codecs; not an independent runtime",
|
|
}
|
|
manifest = {
|
|
"artifacts": artifacts,
|
|
"sources": {str(path.relative_to(FIXTURE)): fingerprint(path)
|
|
for path in sorted(FIXTURE.rglob("*.dever"))},
|
|
}
|
|
write_private_json(output / "manifest.json", manifest)
|
|
print(output / "manifest.json", flush=True)
|
|
|
|
|
|
def request(client, route, body, expected):
|
|
started = time.perf_counter_ns()
|
|
client.request("POST", f"/benchmark/echo/{route}", body,
|
|
{"Host": "localhost", "Content-Type": "application/json"})
|
|
response = client.getresponse()
|
|
encoded = response.read()
|
|
elapsed = time.perf_counter_ns() - started
|
|
if response.status != 200 or json.loads(encoded) != {
|
|
"code": 0, "message": "ok", "data": expected}:
|
|
raise AssertionError(f"API response mismatch: HTTP {response.status}")
|
|
return elapsed
|
|
|
|
|
|
def percentile(values, fraction):
|
|
return sorted(values)[min(len(values) - 1, int(len(values) * fraction))] / 1000
|
|
|
|
|
|
def measure(artifact, route, payload, arguments):
|
|
expected = payload["payload" if route == "echo" else "source"]
|
|
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = listener.getsockname()[1]
|
|
with tempfile.TemporaryDirectory(prefix="api-json-", dir=arguments.output) as temporary:
|
|
directory = Path(temporary)
|
|
executable = directory / "app"
|
|
shutil.copy2(artifact["path"], executable)
|
|
(directory / "config").mkdir()
|
|
write_private_json(directory / "config/setting.json", configuration(port))
|
|
with (directory / "stdout").open("w+") as stdout, \
|
|
(directory / "stderr").open("w+") as stderr:
|
|
process = subprocess.Popen(["taskset", "-c", str(arguments.server_cpu), executable],
|
|
cwd=directory, stdout=stdout, stderr=stderr,
|
|
start_new_session=True)
|
|
client = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
|
try:
|
|
deadline = time.monotonic() + 20
|
|
while True:
|
|
if process.poll() is not None:
|
|
raise RuntimeError(f"API exited before readiness: {process.returncode}")
|
|
try:
|
|
request(client, route, body, expected)
|
|
break
|
|
except (ConnectionError, http.client.HTTPException):
|
|
client.close()
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError("API readiness timeout")
|
|
time.sleep(0.02)
|
|
for _ in range(50):
|
|
request(client, route, body, expected)
|
|
before = process_sample(process.pid)
|
|
started = time.monotonic()
|
|
latencies = [request(client, route, body, expected)
|
|
for _ in range(arguments.requests)]
|
|
elapsed = time.monotonic() - started
|
|
after = process_sample(process.pid)
|
|
result = {
|
|
"requests": len(latencies), "request_bytes": len(body),
|
|
"elapsed_seconds": elapsed,
|
|
"p50_us": percentile(latencies, 0.5),
|
|
"p95_us": percentile(latencies, 0.95),
|
|
"p99_us": percentile(latencies, 0.99),
|
|
"server_cpu_us_per_request":
|
|
(after["cpu_seconds"] - before["cpu_seconds"]) * 1e6 / len(latencies),
|
|
"rss_bytes": after["rss_bytes"],
|
|
"rss_growth_bytes": after["rss_bytes"] - before["rss_bytes"],
|
|
}
|
|
except BaseException as error:
|
|
stderr.flush()
|
|
stderr.seek(0)
|
|
error.add_note(stderr.read()[-4000:])
|
|
raise
|
|
finally:
|
|
client.close()
|
|
if process.poll() is None:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait(timeout=5)
|
|
raise TimeoutError("API did not drain after SIGTERM")
|
|
if process.returncode:
|
|
raise RuntimeError(f"API shutdown failed: {process.returncode}")
|
|
return result
|
|
|
|
|
|
def run(arguments):
|
|
allowed = os.sched_getaffinity(0)
|
|
if (arguments.server_cpu == arguments.client_cpu or
|
|
not {arguments.server_cpu, arguments.client_cpu} <= allowed):
|
|
raise ValueError("server and client require distinct allowed CPUs")
|
|
if arguments.requests < 1 or arguments.repeats < 1:
|
|
raise ValueError("requests and repeats must be positive")
|
|
manifest = json.loads(arguments.manifest.read_text())
|
|
for artifact in manifest["artifacts"].values():
|
|
if fingerprint(Path(artifact["path"])) != {
|
|
key: artifact[key] for key in ("bytes", "sha256")}:
|
|
raise ValueError("API benchmark binary changed after build")
|
|
arguments.output.mkdir(parents=True, exist_ok=False)
|
|
os.sched_setaffinity(0, {arguments.client_cpu})
|
|
labels = list(manifest["artifacts"])
|
|
report = {"manifest": manifest, "server_cpu": arguments.server_cpu,
|
|
"client_cpu": arguments.client_cpu, "samples": [], "summary": [],
|
|
"environment": {"kernel": platform.release(), "machine": platform.machine(),
|
|
"allowed_cpus": sorted(allowed), "python": platform.python_version()},
|
|
"runner": fingerprint(Path(__file__)),
|
|
"requests_per_process": arguments.requests, "repeats": arguments.repeats}
|
|
try:
|
|
for count in (1, 16, 128):
|
|
row = {"id": 9007199254740993, "text": "雪😀\n\"" + "x" * 80,
|
|
"weight": 1.25, "note": None}
|
|
value = {"rows": [row] * count, "raw": {"large": 123456789012345678901234567890}}
|
|
for route, argument in (("echo", "payload"), ("raw", "source")):
|
|
for repeat in range(arguments.repeats):
|
|
offset = repeat % len(labels)
|
|
order = labels[offset:] + labels[:offset]
|
|
for label in order:
|
|
sample = measure(manifest["artifacts"][label], route,
|
|
{argument: value}, arguments)
|
|
report["samples"].append({"label": label, "route": route,
|
|
"rows": count, "repeat": repeat, **sample})
|
|
for label in labels:
|
|
samples = [row for row in report["samples"]
|
|
if row["label"] == label and row["route"] == route
|
|
and row["rows"] == count]
|
|
summary = {"label": label, "route": route, "rows": count}
|
|
for metric in ("p50_us", "p95_us", "p99_us",
|
|
"server_cpu_us_per_request", "rss_bytes"):
|
|
summary[metric] = statistics.median(sample[metric] for sample in samples)
|
|
report["summary"].append(summary)
|
|
print(json.dumps(summary), flush=True)
|
|
for artifact in manifest["artifacts"].values():
|
|
if fingerprint(Path(artifact["path"])) != {
|
|
key: artifact[key] for key in ("bytes", "sha256")}:
|
|
raise ValueError("API benchmark binary changed during measurement")
|
|
report["passed"] = True
|
|
finally:
|
|
write_private_json(arguments.output / "report.json", report)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
prepare = commands.add_parser("build")
|
|
prepare.add_argument("--compiler", type=Path, required=True)
|
|
prepare.add_argument("--baseline-compiler", type=Path, required=True)
|
|
prepare.add_argument("--output", type=Path, required=True)
|
|
prepare.add_argument("--with-peers", action="store_true")
|
|
prepare.add_argument("--with-runtime-peer", action="store_true")
|
|
prepare.set_defaults(action=build)
|
|
measure = commands.add_parser("run")
|
|
measure.add_argument("--manifest", type=Path, required=True)
|
|
measure.add_argument("--output", type=Path, required=True)
|
|
measure.add_argument("--server-cpu", type=int, required=True)
|
|
measure.add_argument("--client-cpu", type=int, required=True)
|
|
measure.add_argument("--requests", type=int, default=1000)
|
|
measure.add_argument("--repeats", type=int, default=7)
|
|
measure.set_defaults(action=run)
|
|
arguments = parser.parse_args()
|
|
arguments.output = arguments.output.resolve()
|
|
arguments.action(arguments)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|