175 lines
9.4 KiB
Python
175 lines
9.4 KiB
Python
"""Verify the API JSON fixture's observable contract before runtime comparison."""
|
|
|
|
import argparse
|
|
import base64
|
|
from contextlib import ExitStack, contextmanager
|
|
from copy import deepcopy
|
|
import hashlib
|
|
import hmac
|
|
import http.client
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
from api_json import configuration, request, write_private_json
|
|
from build import fingerprint
|
|
|
|
|
|
@contextmanager
|
|
def server(binary, output, settings):
|
|
with tempfile.TemporaryDirectory(prefix="api-contract-", dir=output) as temporary:
|
|
directory = Path(temporary)
|
|
executable = directory / "app"
|
|
shutil.copy2(binary, executable)
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = listener.getsockname()[1]
|
|
settings = deepcopy(settings)
|
|
settings["http"]["port"] = port
|
|
(directory / "config").mkdir()
|
|
write_private_json(directory / "config/setting.json", settings)
|
|
with (directory / "stdout").open("w+") as stdout, (directory / "stderr").open("w+") as stderr:
|
|
process = subprocess.Popen([executable], cwd=directory, stdout=stdout,
|
|
stderr=stderr, start_new_session=True)
|
|
client = http.client.HTTPConnection("127.0.0.1", port, timeout=20)
|
|
try:
|
|
deadline = time.monotonic() + 30
|
|
while True:
|
|
if process.poll() is not None:
|
|
raise RuntimeError(f"API contract server exited: {process.returncode}")
|
|
try:
|
|
request(client, "raw", b'{"source":0}', 0)
|
|
break
|
|
except (ConnectionError, http.client.HTTPException):
|
|
client.close()
|
|
if time.monotonic() >= deadline:
|
|
raise TimeoutError("API contract server readiness timeout")
|
|
time.sleep(0.05)
|
|
yield client
|
|
finally:
|
|
client.close()
|
|
if process.poll() is None:
|
|
process.send_signal(signal.SIGTERM)
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait(timeout=5)
|
|
raise
|
|
if process.returncode:
|
|
raise RuntimeError(f"API contract server shutdown failed: {process.returncode}")
|
|
|
|
|
|
def credential(settings):
|
|
def encode(value):
|
|
return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).rstrip(b"=")
|
|
|
|
now = int(time.time())
|
|
payload = encode({"alg": "HS256", "typ": "JWT"}) + b"." + encode({
|
|
"iss": "session", "sub": "fixture", "sid": "fixture-session", "aud": "test",
|
|
"tenant": None, "iat": now, "exp": now + 600,
|
|
})
|
|
key = settings["auth"]["providers"]["session"]["jwtSecret"].encode()
|
|
signature = base64.urlsafe_b64encode(hmac.digest(key, payload, "sha256")).rstrip(b"=")
|
|
return (payload + b"." + signature).decode()
|
|
|
|
|
|
def cases(settings):
|
|
result = []
|
|
row = '{"id":9007199254740993,"text":"雪😀\\n\\\"","weight":1.25,"note":null}'
|
|
typed = '{"payload":{"rows":[' + row + '],"raw":{"large":123456789012345678901234567890}}}'
|
|
|
|
def add(name, body, status=200, route="raw", headers=None, method="POST", query=""):
|
|
result.append({"name": name, "body": body.encode() if isinstance(body, str) else body,
|
|
"status": status, "path": "/benchmark/echo/" + route + query,
|
|
"headers": headers or [("Host", "localhost"), ("Content-Type", "application/json")],
|
|
"method": method})
|
|
|
|
add("typed-unicode-exact-number", typed, route="echo")
|
|
add("typed-optional-absent", typed.replace(',"note":null', ""), route="echo")
|
|
add("typed-optional-text", typed.replace('"note":null', '"note":"雪"'), route="echo")
|
|
for spelling in ["-0", "-9223372036854775808", "9223372036854775807"]:
|
|
add("typed-int-" + spelling, typed.replace("9007199254740993", spelling), route="echo")
|
|
for spelling in ["1.0", "1e0", "9223372036854775808", "-9223372036854775809", "true"]:
|
|
add("typed-invalid-int-" + spelling, typed.replace("9007199254740993", spelling), 400, "echo")
|
|
add("typed-nonfinite", typed.replace("1.25", "1e999"), 400, "echo")
|
|
add("typed-unknown-field", typed.replace('"weight":', '"extra":0,"weight":'), 400, "echo")
|
|
add("typed-missing-field", typed.replace('"weight":1.25,', ""), 400, "echo")
|
|
add("typed-duplicate-escaped-field", typed.replace('"weight":', '"we\\u0069ght":0,"weight":'), 400, "echo")
|
|
add("typed-invalid-optional", typed.replace('"note":null', '"note":false'), 400, "echo")
|
|
for name, value in [("number", "1e999"), ("null", "null"), ("array", "[1,true,null]"),
|
|
("unicode", '"\\ud83d\\ude00"'), ("object", '{"large":123456789012345678901234567890}')]:
|
|
add("raw-" + name, '{"source":' + value + '}')
|
|
for name, body in [("duplicate", '{"source":{"key":1,"k\\u0065y":2}}'),
|
|
("surrogate", '{"source":"\\ud800"}'),
|
|
("trailing-comma", '{"source":[0,]}'),
|
|
("root-duplicate", '{"source":0,"source":1}'),
|
|
("root-unknown", '{"source":0,"extra":1}'),
|
|
("root-missing", '{}'), ("trailing-root", '{"source":0}[]'),
|
|
("invalid-utf8", b'{"source":"\xff"}')]:
|
|
add(name, body, 400)
|
|
for depth, status in [(63, 200), (64, 400)]:
|
|
add(f"depth-{depth}", '{"source":' + '[' * depth + '0' + ']' * depth + '}', status)
|
|
for count, status in [(65534, 200), (65535, 400)]:
|
|
add(f"elements-{count}", '{"source":[' + ','.join(['0'] * count) + ']}', status)
|
|
add("wrong-method", '{}', 405, method="GET")
|
|
add("unknown-route", '{}', 404, route="missing")
|
|
add("query-on-post", '{"source":0}', 400, query="?value=1")
|
|
add("wrong-content-type", '{}', 400, headers=[("Host", "localhost"), ("Content-Type", "text/plain")])
|
|
add("duplicate-content-type", '{}', 400, headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Content-Type", "application/json")])
|
|
add("wrong-host", '{}', 401, headers=[("Host", "other.invalid"), ("Content-Type", "application/json")])
|
|
add("invalid-bearer", '{}', 401, headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Authorization", "Bearer invalid")])
|
|
add("valid-bearer", typed, route="echo", headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Authorization", "Bearer " + credential(settings))])
|
|
add("public-expired-cookie", '{"source":0}', headers=[("Host", "localhost"), ("Content-Type", "application/json"), ("Cookie", "benchmark_session=invalid")])
|
|
return result
|
|
|
|
|
|
def verify(dever, rust, output):
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
artifacts = {label: {"path": str(binary.resolve()), **fingerprint(binary)}
|
|
for label, binary in [("dever", dever), ("rust_runtime", rust)]}
|
|
settings = configuration(8080)
|
|
report = {"artifacts": artifacts, "cases": [], "passed": False}
|
|
try:
|
|
with ExitStack() as stack:
|
|
clients = {label: stack.enter_context(server(Path(artifact["path"]), output, settings))
|
|
for label, artifact in artifacts.items()}
|
|
for case in cases(settings):
|
|
observed = {}
|
|
for label, client in clients.items():
|
|
client.putrequest(case["method"], case["path"], skip_host=True, skip_accept_encoding=True)
|
|
for name, value in case["headers"]:
|
|
client.putheader(name, value)
|
|
client.putheader("Content-Length", str(len(case["body"])))
|
|
client.endheaders(case["body"])
|
|
response = client.getresponse()
|
|
body = response.read()
|
|
observed[label] = (response.status, response.getheader("Content-Type"), response.getheader("Allow"), body)
|
|
client.close()
|
|
passed = all(value[0] == case["status"] for value in observed.values()) and len(set(observed.values())) == 1
|
|
report["cases"].append({"name": case["name"], "expected_status": case["status"], "passed": passed,
|
|
"observed": {label: {"status": value[0], "body_sha256": hashlib.sha256(value[3]).hexdigest(), "bytes": len(value[3])}
|
|
for label, value in observed.items()}})
|
|
if not passed:
|
|
raise AssertionError(f"API contract differs for {case['name']}: " + repr({label: (value[0], value[3][:240]) for label, value in observed.items()}))
|
|
for artifact in artifacts.values():
|
|
assert fingerprint(Path(artifact["path"])) == {key: artifact[key] for key in ["bytes", "sha256"]}
|
|
report["passed"] = True
|
|
finally:
|
|
write_private_json(output / "report.json", report)
|
|
print(json.dumps({"passed": True, "cases": len(report["cases"])}), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dever", type=Path, required=True)
|
|
parser.add_argument("--rust", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
arguments = parser.parse_args()
|
|
verify(arguments.dever, arguments.rust, arguments.output)
|