299 lines
17 KiB
Python
299 lines
17 KiB
Python
"""显式 ignored 验收与现有长测运行器的 Linux gate 接线。"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from urllib.parse import parse_qsl, unquote, urlsplit
|
|
|
|
from linux_gate import (ROOT, MissingInput, absolute_file, run_command,
|
|
target_command, workspace_targets, remove_executables, input_fingerprint)
|
|
from settings import write_private_json
|
|
from process import resource_trend
|
|
from cms import validate_read_descriptor_recovery
|
|
|
|
|
|
def execute(command, config, directory, result, *, single_harness=False):
|
|
record, content = run_command(command, directory, len(result["commands"]),
|
|
config.get("command_timeout_seconds", 14400),
|
|
single_harness=single_harness)
|
|
result["commands"].append(record)
|
|
return record, content
|
|
|
|
|
|
def classify_cases(names, manifest, target):
|
|
by_name = {}
|
|
for row in manifest:
|
|
key = (row["package"], row["target"], row["name"])
|
|
if key in by_name:
|
|
raise RuntimeError(f"duplicate ignored classification: {key}")
|
|
by_name[key] = row
|
|
classified = []
|
|
for name in names:
|
|
leaf = name.rsplit("::", 1)[-1]
|
|
key = (target["package"], target["target"], leaf)
|
|
if key not in by_name:
|
|
raise RuntimeError(f"unclassified ignored test: {name}")
|
|
classified.append({**by_name[key], "exact": name})
|
|
if len({case["name"] for case in classified}) != len(classified):
|
|
raise RuntimeError("same ignored leaf appears in multiple modules; classify by a distinct source name")
|
|
return classified
|
|
|
|
|
|
def audit_classification():
|
|
manifest = json.loads((ROOT / "test/ci/ignored.json").read_text())
|
|
stages = {"probe", "author", "native", "postgres", "process-runtime", "bounded-perf"}
|
|
if any(row["stage"] not in stages for row in manifest):
|
|
raise RuntimeError("ignored classification contains an unknown execution stage")
|
|
expected = {(row["source"], row["name"], row["reason"]) for row in manifest}
|
|
actual = set()
|
|
pattern = re.compile(r'#\[ignore(?:\s*=\s*"([^"]*)")?\]\s*(?:#\[[^\]]*\]\s*)*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)')
|
|
for directory in ("test", "crates"):
|
|
for path in (ROOT / directory).rglob("*.rs"):
|
|
for match in pattern.finditer(path.read_text()):
|
|
actual.add((str(path.relative_to(ROOT)), match[2], match[1] or ""))
|
|
if actual != expected:
|
|
raise RuntimeError(f"ignored classification drift: new={sorted(actual - expected)}, removed={sorted(expected - actual)}")
|
|
return manifest
|
|
|
|
|
|
def test_executable(content, target):
|
|
artifacts = []
|
|
for line in content.splitlines():
|
|
if not line.startswith("{"):
|
|
continue
|
|
message = json.loads(line)
|
|
if (message.get("reason") == "compiler-artifact" and message.get("profile", {}).get("test")
|
|
and message.get("target", {}).get("name") == target["target"] and message.get("executable")):
|
|
artifacts.append(message["executable"])
|
|
if len(set(artifacts)) != 1:
|
|
raise RuntimeError(f"expected one libtest executable for {target}")
|
|
return artifacts[0]
|
|
|
|
|
|
def inventory(config, directory, result):
|
|
manifest = audit_classification()
|
|
targets, target_directory = workspace_targets(config, directory, result, True)
|
|
cases = []
|
|
for target in targets:
|
|
if target["selector"] == ["--doc"]:
|
|
continue
|
|
_, content = execute([*target_command(config, target), "--no-run", "--message-format=json"], config, directory, result)
|
|
executable = test_executable(content, target)
|
|
_, listing = execute([executable, "--ignored", "--list", "--format", "terse"], config, directory, result)
|
|
names = [line.removesuffix(": test") for line in listing.splitlines() if line.endswith(": test")]
|
|
cases.extend({**case, "target": target, "target_directory": str(target_directory)}
|
|
for case in classify_cases(names, manifest, target))
|
|
result.setdefault("removed_test_executables", []).extend(
|
|
remove_executables(content, target_directory, config.get("remove_test_executables", False)))
|
|
write_private_json(directory / "inventory.json", cases, replace=(directory / "inventory.json").exists())
|
|
missing = {(row["package"], row["target"], row["name"]) for row in manifest} - {
|
|
(case["target"]["package"], case["target"]["target"], case["name"]) for case in cases}
|
|
if missing:
|
|
raise RuntimeError(f"ignored source cases missing from Linux all-feature libtest inventory: {sorted(missing)}")
|
|
result["classified_cases"] = len(cases)
|
|
result["internal_probes"] = [{"target": case["target"], "exact": case["exact"],
|
|
"status": "parent-only", "reason": case["reason"]}
|
|
for case in cases if case["stage"] == "probe"]
|
|
|
|
|
|
def current_inventory(document, config, result):
|
|
paths = sorted((Path(config.get("output", ROOT / "target/linux-ci")) / "inventory").glob("*/result.json"))
|
|
if paths:
|
|
path = paths[-1]
|
|
record = json.loads(path.read_text())
|
|
if (record.get("source"), record.get("inputs"), record.get("status")) == (result["source"], input_fingerprint(document, config, "inventory"), "passed"):
|
|
return json.loads((path.parent / "inventory.json").read_text())
|
|
raise MissingInput("run inventory for the current source and input fingerprints first")
|
|
|
|
|
|
def validate_postgres(document):
|
|
setting = document.get("database", {}).get("postgres_test", {})
|
|
url = urlsplit(setting.get("url", ""))
|
|
database = unquote(url.path.removeprefix("/"))
|
|
if (setting.get("type") != "postgres" or url.scheme not in ("postgres", "postgresql")
|
|
or url.hostname not in ("127.0.0.1", "::1") or url.port is None
|
|
or database.count("{case}") != 1
|
|
or any(key.lower() in ("dbname", "host", "hostaddr", "port", "service") for key, _ in parse_qsl(url.query))):
|
|
raise MissingInput("database.postgres_test must select the explicitly owned loopback cluster and one {case} database template")
|
|
|
|
|
|
def require_inputs(stage, config):
|
|
required = config.get("required_inputs", {}).get(stage)
|
|
if not required:
|
|
raise MissingInput(f"performance.linux_ci.required_inputs.{stage} must declare the prepared author/fixture input paths")
|
|
declared = [Path(value).resolve() for value in config.get("inputs", [])]
|
|
for value in required:
|
|
path = Path(value)
|
|
if not path.is_absolute() or not path.exists():
|
|
raise MissingInput(f"missing {stage} input: {value}")
|
|
if not any(path.resolve() == root or path.resolve().is_relative_to(root) for root in declared):
|
|
raise MissingInput(f"{stage} input is not fingerprinted by performance.linux_ci.inputs: {value}")
|
|
|
|
|
|
def ignored_stage(stage, document, config, directory, result):
|
|
require_inputs(stage, config)
|
|
if stage == "postgres":
|
|
validate_postgres(document)
|
|
if not config.get("owned_postgres"):
|
|
raise MissingInput("performance.linux_ci.owned_postgres must record the owned cluster data_directory and port")
|
|
owned = config["owned_postgres"]
|
|
if urlsplit(document["database"]["postgres_test"]["url"]).port != owned.get("port"):
|
|
raise MissingInput("PostgreSQL URL does not match performance.linux_ci.owned_postgres.port")
|
|
if not Path(owned.get("data_directory", "")).is_absolute():
|
|
raise MissingInput("owned_postgres.data_directory must be absolute")
|
|
if stage == "author" and config.get("author_network") is not True:
|
|
raise MissingInput("author stage fetches fixed upstream inputs; explicitly set performance.linux_ci.author_network=true")
|
|
if stage == "author" and not config.get("author_outputs"):
|
|
raise MissingInput("declare performance.linux_ci.author_outputs separately from immutable raw author inputs")
|
|
cases = [case for case in current_inventory(document, config, result) if case["stage"] == stage]
|
|
if not cases:
|
|
raise RuntimeError(f"no classified cases for {stage}")
|
|
# 固定作者先准备工具，再执行真实源码包构建；其他用例按 target 复用链接产物。
|
|
cases.sort(key=lambda case: (0 if case["name"].startswith("prepare_") else 1,
|
|
case["target"]["package"], case["target"]["target"], case["exact"]))
|
|
target_key = None
|
|
build_content = None
|
|
target_directory = ROOT / "target"
|
|
for case in cases:
|
|
target = case["target"]
|
|
target_directory = Path(case["target_directory"])
|
|
result["target_directory"] = str(target_directory)
|
|
key = json.dumps(target, sort_keys=True)
|
|
if key != target_key:
|
|
if build_content is not None:
|
|
result.setdefault("removed_test_executables", []).extend(
|
|
remove_executables(build_content, target_directory, config.get("remove_test_executables", False)))
|
|
_, build_content = execute([*target_command(config, target), "--no-run", "--message-format=json"], config, directory, result)
|
|
executable = test_executable(build_content, target)
|
|
target_key = key
|
|
record, _ = execute([executable, "--ignored", "--exact", case["exact"], "--test-threads=1", "--nocapture"],
|
|
config, directory, result, single_harness=True)
|
|
if record.get("rust_tests") != {"passed": 1, "failed": 0, "ignored": 0}:
|
|
raise RuntimeError(f"exact ignored selection did not execute one case: {case['exact']}")
|
|
result.setdefault("cases", []).append({"exact": case["exact"], "target": target, "status": "passed"})
|
|
result.setdefault("removed_test_executables", []).extend(
|
|
remove_executables(build_content, target_directory, config.get("remove_test_executables", False)))
|
|
|
|
|
|
def validate_budget(config):
|
|
parent = Path(config.get("cgroup_parent", ""))
|
|
if not parent.is_absolute() or not (parent / "cgroup.controllers").is_file():
|
|
raise MissingInput("performance.linux_ci.cgroup_parent must select the owned empty cgroup v2 parent")
|
|
if (parent / "cgroup.procs").read_text().strip():
|
|
raise MissingInput("CI runner must be in a control child, not the cgroup parent")
|
|
if not {"cpu", "memory"} <= set((parent / "cgroup.subtree_control").read_text().split()):
|
|
raise MissingInput("owned parent requires enabled memory/cpu child controllers")
|
|
quota, period = (parent / "cpu.max").read_text().split()
|
|
memory = (parent / "memory.max").read_text().strip()
|
|
if quota == "max" or int(quota) / int(period) > 2 or memory == "max" or int(memory) > 4 * 1024 ** 3:
|
|
raise MissingInput("owned parent must bound total CPU <=2 and memory <=4GiB")
|
|
if (parent / "memory.swap.max").read_text().strip() != "0":
|
|
raise MissingInput("owned parent must disable swap")
|
|
membership = Path("/proc/self/cgroup").read_text().strip().split("::", 1)[1]
|
|
current = Path("/sys/fs/cgroup") / membership.lstrip("/")
|
|
if current == parent or not current.is_relative_to(parent):
|
|
raise MissingInput("CI runner is outside the configured owned parent control subgroup")
|
|
return parent
|
|
|
|
|
|
def validate_soak(report, suite, seconds, growth_limit, memory_mib):
|
|
if report.get("status") not in ("passed", "complete"):
|
|
raise RuntimeError("soak runner did not complete")
|
|
cases = list(report["cases"].values()) if suite == "cms" else report["cases"]
|
|
expected = 3 if suite == "live" else 2
|
|
if len(cases) != expected:
|
|
raise RuntimeError(f"expected {expected} {suite} cases, received {len(cases)}")
|
|
if suite == "cms":
|
|
coverage = set(report["cases"])
|
|
required = {"dever", "md"}
|
|
else:
|
|
coverage = {case["protocol" if suite == "live" else "transport"] for case in cases}
|
|
required = {"tcp", "ws", "sse"} if suite == "live" else {"http", "https"}
|
|
if any(case["suite"] != suite or (suite != "live" and case["implementation"] != "dever") for case in cases):
|
|
raise RuntimeError("soak report contains a different suite or implementation")
|
|
if coverage != required:
|
|
raise RuntimeError("soak report does not cover every required source or transport")
|
|
for case in cases:
|
|
budget = case.get("cgroup") if suite == "cms" else case.get("budget_after_stop")
|
|
if budget is None or any(budget["events"][key] for key in ("oom", "oom_kill")):
|
|
raise RuntimeError("missing memory-budget evidence or nonzero OOM")
|
|
if budget["max"] != str(memory_mib * 1024 ** 2):
|
|
raise RuntimeError("soak used a different memory limit from the selected stage")
|
|
if suite == "cms":
|
|
load = case["concurrent_reads"]
|
|
if load["elapsed_seconds"] < seconds or load["errors"] or load["succeeded"] == 0:
|
|
raise RuntimeError("CMS duration, business load or FD recovery failed")
|
|
validate_read_descriptor_recovery(load)
|
|
trend = load["trend"]
|
|
elif suite == "live":
|
|
if (case["result"]["elapsed_ms"] < seconds * 1000 or case["result"]["errors"]
|
|
or any(case["fd_delta_after_cycles"])):
|
|
raise RuntimeError("live duration, protocol or FD recovery failed")
|
|
samples = [phase["last"] for phase in case["phases"] if phase["phase"] == "disconnected"]
|
|
trend = resource_trend(samples)
|
|
else:
|
|
loads = [case["load"]] if suite == "http2" else [cycle["load"] for cycle in case["cycles"]]
|
|
disconnected = [case["disconnected"]] if suite == "http2" else [cycle["disconnected"] for cycle in case["cycles"]]
|
|
if any(sample["fd_delta"] != 0 for sample in disconnected):
|
|
raise RuntimeError("HTTP/2 did not restore file descriptors after disconnect")
|
|
if sum(load["elapsed_seconds"] for load in loads) < seconds:
|
|
raise RuntimeError("HTTP/2 measured load is shorter than the required duration")
|
|
for load in loads:
|
|
if load["errors"] or load["succeeded"] <= 0 or load["succeeded"] < .99 * load["scheduled"]:
|
|
raise RuntimeError("HTTP/2 did not achieve 99% scheduled successful replies with zero errors")
|
|
trend = resource_trend(case["raw"])
|
|
if trend is None or trend["rss_late_growth_bytes"] > growth_limit:
|
|
raise RuntimeError("late RSS growth exceeds the configured budget; inspect raw resource samples")
|
|
case["gate_resource_trend"] = trend
|
|
|
|
|
|
def soak_stage(stage, config, directory, result):
|
|
suite, memory = stage.rsplit("-", 1)
|
|
suite = {"recovery": "http2-recovery"}.get(suite, suite)
|
|
require_inputs("soak", config)
|
|
parent = validate_budget(config)
|
|
settings = config.get("soak", {})
|
|
seconds = settings.get("seconds", 1800)
|
|
if not 1800 <= seconds <= 2700:
|
|
raise ValueError("performance.linux_ci.soak.seconds must be 1800..2700")
|
|
python = str(absolute_file(config["tools"].get("python"), "performance.linux_ci.tools.python"))
|
|
output = directory / "measurement"
|
|
if suite == "cms":
|
|
manifest = str(absolute_file(settings.get("cms_manifest"), "performance.linux_ci.soak.cms_manifest"))
|
|
command = [python, "-B", "test/performance/cms.py", "run", "--manifest", manifest,
|
|
"--concurrency", str(settings.get("cms_concurrency", 16)), "--read-seconds", str(seconds)]
|
|
else:
|
|
artifacts = Path(settings.get("artifacts", ""))
|
|
if not artifacts.is_absolute() or not (artifacts / "manifest.json").is_file():
|
|
raise MissingInput("performance.linux_ci.soak.artifacts requires prepared protocol artifacts")
|
|
command = [python, "-B", "test/performance/run.py", "run", "--artifacts", str(artifacts),
|
|
"--suite", suite, "--implementations", "dever", "--repeats", "1", "--interval", "0.1"]
|
|
if suite == "http2":
|
|
command += ["--duration", str(seconds), "--warmup", "5", "--paths", "/json",
|
|
"--rates", str(settings.get("http2_rate", 5000)), "--http2-connections", "4"]
|
|
else:
|
|
if seconds % 30:
|
|
raise ValueError("cyclic soak seconds must be divisible by 30")
|
|
command += ["--duration", "30", "--cycles", str(seconds // 30)]
|
|
command += (["--connection-tiers", "32"] if suite == "live" else
|
|
["--recovery-rate", str(settings.get("http2_rate", 5000)),
|
|
"--recovery-path", "/json", "--recovery-connections", "4"])
|
|
if "server_cpu" in settings:
|
|
command += ["--server-cpu", str(settings["server_cpu"]), "--client-cpus", str(settings["client_cpu"])]
|
|
command += ["--output", str(output), "--memory-mib", memory, "--cgroup-parent", str(parent)]
|
|
execute(command, config, directory, result)
|
|
report = json.loads((output / "report.json").read_text())
|
|
validate_soak(report, suite, seconds, settings.get("rss_late_growth_mib", 8) * 1024 ** 2, int(memory))
|
|
write_private_json(directory / "validated-report.json", report)
|
|
result["measurement"] = str(output / "report.json")
|
|
result["coverage"] = "CMS dever/md; HTTP/2 h2c/TLS JSON; recovery h2c/TLS; live TCP/WS/SSE, each selected stage only"
|
|
|
|
|
|
def run_stage(stage, document, config, directory, result):
|
|
if stage == "inventory":
|
|
inventory(config, directory, result)
|
|
elif stage in ("author", "native", "postgres", "process-runtime", "bounded-perf"):
|
|
ignored_stage(stage, document, config, directory, result)
|
|
else:
|
|
soak_stage(stage, config, directory, result)
|