Initial Dever 0.0.1

test/performance/api_json_contract.py

@@ -0,0 +1,174 @@
+"""Verify the API JSON fixture's observable contract before runtime comparison."""
+
+import argparse
+import base64
+from contextlib import ExitStack, contextmanager
+from copy import deepcopy
+import hashlib
+import hmac
+import http.client
+import json
+from pathlib import Path
+import shutil
+import signal
+import socket
+import subprocess
+import tempfile
+import time
+
+from api_json import configuration, request, write_private_json
+from build import fingerprint
+
+
+@contextmanager
+def server(binary, output, settings):
+    with tempfile.TemporaryDirectory(prefix="api-contract-", dir=output) as temporary:
+        directory = Path(temporary)
+        executable = directory / "app"
+        shutil.copy2(binary, executable)
+        with socket.socket() as listener:
+            listener.bind(("127.0.0.1", 0))
+            port = listener.getsockname()[1]
+        settings = deepcopy(settings)
+        settings["http"]["port"] = port
+        (directory / "config").mkdir()
+        write_private_json(directory / "config/setting.json", settings)
+        with (directory / "stdout").open("w+") as stdout, (directory / "stderr").open("w+") as stderr:
+            process = subprocess.Popen([executable], cwd=directory, stdout=stdout,
+                                       stderr=stderr, start_new_session=True)
+            client = http.client.HTTPConnection("127.0.0.1", port, timeout=20)
+            try:
+                deadline = time.monotonic() + 30
+                while True:
+                    if process.poll() is not None:
+                        raise RuntimeError(f"API contract server exited: {process.returncode}")
+                    try:
+                        request(client, "raw", b'{"source":0}', 0)
+                        break
+                    except (ConnectionError, http.client.HTTPException):
+                        client.close()
+                        if time.monotonic() >= deadline:
+                            raise TimeoutError("API contract server readiness timeout")
+                        time.sleep(0.05)
+                yield client
+            finally:
+                client.close()
+                if process.poll() is None:
+                    process.send_signal(signal.SIGTERM)
+                try:
+                    process.wait(timeout=10)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                    process.wait(timeout=5)
+                    raise
+            if process.returncode:
+                raise RuntimeError(f"API contract server shutdown failed: {process.returncode}")
+
+
+def credential(settings):
+    def encode(value):
+        return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).rstrip(b"=")
+
+    now = int(time.time())
+    payload = encode({"alg": "HS256", "typ": "JWT"}) + b"." + encode({
+        "iss": "session", "sub": "fixture", "sid": "fixture-session", "aud": "test",
+        "tenant": None, "iat": now, "exp": now + 600,
+    })
+    key = settings["auth"]["providers"]["session"]["jwtSecret"].encode()
+    signature = base64.urlsafe_b64encode(hmac.digest(key, payload, "sha256")).rstrip(b"=")
+    return (payload + b"." + signature).decode()
+
+
+def cases(settings):
+    result = []
+    row = '{"id":9007199254740993,"text":"雪😀\\n\\\"","weight":1.25,"note":null}'
+    typed = '{"payload":{"rows":[' + row + '],"raw":{"large":123456789012345678901234567890}}}'
+
+    def add(name, body, status=200, route="raw", headers=None, method="POST", query=""):
+        result.append({"name": name, "body": body.encode() if isinstance(body, str) else body,
+                       "status": status, "path": "/benchmark/echo/" + route + query,
+                       "headers": headers or [("Host", "localhost"), ("Content-Type", "application/json")],
+                       "method": method})
+
+    add("typed-unicode-exact-number", typed, route="echo")
+    add("typed-optional-absent", typed.replace(',"note":null', ""), route="echo")
+    add("typed-optional-text", typed.replace('"note":null', '"note":"雪"'), route="echo")
+    for spelling in ["-0", "-9223372036854775808", "9223372036854775807"]:
+        add("typed-int-" + spelling, typed.replace("9007199254740993", spelling), route="echo")
+    for spelling in ["1.0", "1e0", "9223372036854775808", "-9223372036854775809", "true"]:
+        add("typed-invalid-int-" + spelling, typed.replace("9007199254740993", spelling), 400, "echo")
+    add("typed-nonfinite", typed.replace("1.25", "1e999"), 400, "echo")
+    add("typed-unknown-field", typed.replace('"weight":', '"extra":0,"weight":'), 400, "echo")
+    add("typed-missing-field", typed.replace('"weight":1.25,', ""), 400, "echo")
+    add("typed-duplicate-escaped-field", typed.replace('"weight":', '"we\\u0069ght":0,"weight":'), 400, "echo")
+    add("typed-invalid-optional", typed.replace('"note":null', '"note":false'), 400, "echo")
+    for name, value in [("number", "1e999"), ("null", "null"), ("array", "[1,true,null]"),
+                        ("unicode", '"\\ud83d\\ude00"'), ("object", '{"large":123456789012345678901234567890}')]:
+        add("raw-" + name, '{"source":' + value + '}')
+    for name, body in [("duplicate", '{"source":{"key":1,"k\\u0065y":2}}'),
+                       ("surrogate", '{"source":"\\ud800"}'),
+                       ("trailing-comma", '{"source":[0,]}'),
+                       ("root-duplicate", '{"source":0,"source":1}'),
+                       ("root-unknown", '{"source":0,"extra":1}'),
+                       ("root-missing", '{}'), ("trailing-root", '{"source":0}[]'),
+                       ("invalid-utf8", b'{"source":"\xff"}')]:
+        add(name, body, 400)
+    for depth, status in [(63, 200), (64, 400)]:
+        add(f"depth-{depth}", '{"source":' + '[' * depth + '0' + ']' * depth + '}', status)
+    for count, status in [(65534, 200), (65535, 400)]:
+        add(f"elements-{count}", '{"source":[' + ','.join(['0'] * count) + ']}', status)
+    add("wrong-method", '{}', 405, method="GET")
+    add("unknown-route", '{}', 404, route="missing")
+    add("query-on-post", '{"source":0}', 400, query="?value=1")
+    add("wrong-content-type", '{}', 400, headers=[("Host", "localhost"), ("Content-Type", "text/plain")])
+    add("duplicate-content-type", '{}', 400, headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Content-Type", "application/json")])
+    add("wrong-host", '{}', 401, headers=[("Host", "other.invalid"), ("Content-Type", "application/json")])
+    add("invalid-bearer", '{}', 401, headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Authorization", "Bearer invalid")])
+    add("valid-bearer", typed, route="echo", headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Authorization", "Bearer " + credential(settings))])
+    add("public-expired-cookie", '{"source":0}', headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Cookie", "benchmark_session=invalid")])
+    return result
+
+
+def verify(dever, rust, output):
+    output.mkdir(parents=True, exist_ok=False)
+    artifacts = {label: {"path": str(binary.resolve()), **fingerprint(binary)}
+                 for label, binary in [("dever", dever), ("rust_runtime", rust)]}
+    settings = configuration(8080)
+    report = {"artifacts": artifacts, "cases": [], "passed": False}
+    try:
+        with ExitStack() as stack:
+            clients = {label: stack.enter_context(server(Path(artifact["path"]), output, settings))
+                       for label, artifact in artifacts.items()}
+            for case in cases(settings):
+                observed = {}
+                for label, client in clients.items():
+                    client.putrequest(case["method"], case["path"], skip_host=True, skip_accept_encoding=True)
+                    for name, value in case["headers"]:
+                        client.putheader(name, value)
+                    client.putheader("Content-Length", str(len(case["body"])))
+                    client.endheaders(case["body"])
+                    response = client.getresponse()
+                    body = response.read()
+                    observed[label] = (response.status, response.getheader("Content-Type"), response.getheader("Allow"), body)
+                    client.close()
+                passed = all(value[0] == case["status"] for value in observed.values()) and len(set(observed.values())) == 1
+                report["cases"].append({"name": case["name"], "expected_status": case["status"], "passed": passed,
+                    "observed": {label: {"status": value[0], "body_sha256": hashlib.sha256(value[3]).hexdigest(), "bytes": len(value[3])}
+                                 for label, value in observed.items()}})
+                if not passed:
+                    raise AssertionError(f"API contract differs for {case['name']}: " + repr({label: (value[0], value[3][:240]) for label, value in observed.items()}))
+        for artifact in artifacts.values():
+            assert fingerprint(Path(artifact["path"])) == {key: artifact[key] for key in ["bytes", "sha256"]}
+        report["passed"] = True
+    finally:
+        write_private_json(output / "report.json", report)
+    print(json.dumps({"passed": True, "cases": len(report["cases"])}), flush=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--dever", type=Path, required=True)
+    parser.add_argument("--rust", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    arguments = parser.parse_args()
+    verify(arguments.dever, arguments.rust, arguments.output)