feat: publish Dever core with project scaffolding and versioned AI skills

test/performance/process.py

@@ -0,0 +1,231 @@
+"""Linux 进程与可选专属 cgroup 的生命周期和采样。"""
+
+import os
+from pathlib import Path
+import signal
+import statistics
+import subprocess
+import time
+import uuid
+
+from settings import stage_peer
+
+
+def key_values(path):
+    return {fields[0].rstrip(":"): fields[1] for line in path.read_text().splitlines()
+            if len(fields := line.split()) >= 2}
+
+
+def process_sample(pid):
+    root = Path("/proc") / str(pid)
+    status = key_values(root / "status")
+    if "VmRSS" not in status:
+        raise ProcessLookupError(f"process {pid} exited during sampling")
+    memory = key_values(root / "smaps_rollup")
+    # comm 字段可含空格和括号；从最后一个右括号之后解析固定 stat 字段。
+    stat = (root / "stat").read_text().rsplit(")", 1)[1].split()
+    switches = {"voluntary_ctxt_switches": 0, "nonvoluntary_ctxt_switches": 0}
+    for thread in (root / "task").iterdir():
+        try:
+            thread_status = key_values(thread / "status")
+        except FileNotFoundError:
+            continue  # A completed blocking thread can disappear during a sample.
+        for name in switches:
+            switches[name] += int(thread_status[name])
+    return {
+        "time": time.monotonic(),
+        "rss_bytes": int(status["VmRSS"]) * 1024,
+        "rss_high_water_bytes": int(status["VmHWM"]) * 1024,
+        "pss_bytes": int(memory["Pss"]) * 1024,
+        "threads": int(status["Threads"]),
+        "fds": len(list((root / "fd").iterdir())),
+        "cpu_seconds": (int(stat[11]) + int(stat[12])) / os.sysconf("SC_CLK_TCK"),
+        **switches,
+    }
+
+
+class Budget:
+    def __init__(self, parent, memory_mib, cpu_quota):
+        self.path = None
+        self.final = None
+        if parent is None:
+            if memory_mib is not None or cpu_quota is not None:
+                raise ValueError("memory/cpu quotas require --cgroup-parent")
+            return
+        self.path = parent.resolve() / ("dever-perf-" + uuid.uuid4().hex)
+        self.path.mkdir()
+        try:
+            if memory_mib is not None:
+                (self.path / "memory.max").write_text(str(memory_mib * 1024 * 1024))
+                (self.path / "memory.swap.max").write_text("0")
+            if cpu_quota is not None:
+                (self.path / "cpu.max").write_text(f"{round(cpu_quota * 100000)} 100000")
+        except BaseException:
+            self.path.rmdir()
+            raise
+
+    def sample(self):
+        if self.path is None:
+            return None
+        return {
+            "current_bytes": int((self.path / "memory.current").read_text()),
+            "peak_bytes": int((self.path / "memory.peak").read_text()),
+            "events": {k: int(v) for k, v in key_values(self.path / "memory.events").items()},
+            "cpu": {k: int(v) for k, v in key_values(self.path / "cpu.stat").items()},
+            "max": (self.path / "memory.max").read_text().strip(),
+            "cpu_max": (self.path / "cpu.max").read_text().strip(),
+        }
+
+    def close(self):
+        if self.path is not None:
+            try:
+                self.final = self.sample()
+            except OSError as error:
+                try:
+                    self.path.rmdir()
+                except OSError as cleanup_error:
+                    error.add_note(f"cannot remove owned cgroup {self.path}: {cleanup_error}")
+                raise
+            else:
+                self.path.rmdir()
+
+
+class Process:
+    def __init__(self, command, directory, *, peer_settings=None, cpus=None, budget=None):
+        self.command = [str(arg) for arg in command]
+        self.directory = directory
+        self.budget = budget
+        self.samples = []
+        self.directory.mkdir(parents=True, exist_ok=False)
+        if peer_settings is not None:
+            self.command[0] = str(stage_peer(command[0], directory, peer_settings))
+        self.stdout = (directory / "stdout.log").open("w")
+        self.stderr = (directory / "stderr.log").open("w")
+        self.started = time.monotonic()
+
+        def prepare_child():
+            # 运行器单线程；只对即将 exec 的自有子进程设置 CPU/cgroup。
+            if cpus is not None:
+                os.sched_setaffinity(0, cpus)
+            if budget is not None and budget.path is not None:
+                (budget.path / "cgroup.procs").write_text(str(os.getpid()))
+
+        try:
+            self.child = subprocess.Popen(self.command, stdout=self.stdout, stderr=self.stderr,
+                                          preexec_fn=prepare_child, start_new_session=True)
+        except BaseException:
+            self.stdout.close()
+            self.stderr.close()
+            raise
+
+    def text(self):
+        return (self.directory / "stdout.log").read_text()
+
+    def descriptors(self):
+        descriptors = {}
+        for path in (Path("/proc") / str(self.child.pid) / "fd").iterdir():
+            try:
+                descriptors[path.name] = os.readlink(path)
+            except FileNotFoundError:
+                continue  # A descriptor may close while /proc is being read.
+        return descriptors
+
+    def sample(self):
+        if self.child.poll() is not None:
+            return
+        try:
+            sample = process_sample(self.child.pid)
+        except (FileNotFoundError, ProcessLookupError):
+            # Linux can release /proc memory fields before waitpid reports exit.
+            # This observation is unavailable; check() still owns exit status.
+            return
+        sample["elapsed_seconds"] = sample.pop("time") - self.started
+        if self.budget is not None:
+            sample["cgroup"] = self.budget.sample()
+        self.samples.append(sample)
+
+    def ready(self, timeout=10):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            self.sample()
+            for line in self.text().splitlines():
+                if line.startswith("ERROR|"):
+                    raise RuntimeError(line)
+                if line.startswith("READY|"):
+                    self.ready_seconds = time.monotonic() - self.started
+                    return int(line.split("|")[1])
+            if self.child.poll() is not None:
+                raise RuntimeError(f"process exited before readiness: {self.directory}")
+            time.sleep(0.005)
+        raise TimeoutError(f"readiness timeout: {self.directory}")
+
+    def check(self):
+        code = self.child.poll()
+        if code is not None and code != 0:
+            raise RuntimeError(f"process exited {code}: {self.directory}")
+        if "ERROR|" in self.text() or (self.directory / "stderr.log").stat().st_size:
+            raise RuntimeError(f"benchmark reported an error: {self.directory}")
+
+    def close(self):
+        try:
+            if self.child.poll() is None:
+                try:
+                    os.killpg(self.child.pid, signal.SIGTERM)
+                except ProcessLookupError:
+                    # 子进程可能恰好在 poll 与 signal 之间自然退出。
+                    pass
+                try:
+                    self.child.wait(timeout=3)
+                except subprocess.TimeoutExpired:
+                    os.killpg(self.child.pid, signal.SIGKILL)
+                    self.child.wait(timeout=3)
+        finally:
+            self.stdout.close()
+            self.stderr.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self.close()
+
+
+def observe(processes, seconds, interval):
+    deadline = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        for process in processes:
+            process.sample()
+            process.check()
+        time.sleep(min(interval, max(0, deadline - time.monotonic())))
+
+
+def summarize(samples):
+    if not samples:
+        raise ValueError("process ended before any resource sample")
+    result = {"samples": len(samples)}
+    for metric in ("rss_bytes", "pss_bytes", "threads", "fds"):
+        values = sorted(sample[metric] for sample in samples)
+        result[metric + "_median"] = values[len(values) // 2]
+        result[metric + "_max"] = max(values)
+    result["rss_high_water_bytes"] = max(sample["rss_high_water_bytes"] for sample in samples)
+    elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"]
+    cpu = samples[-1]["cpu_seconds"] - samples[0]["cpu_seconds"]
+    result["cpu_seconds_sampled"] = cpu
+    result["cpu_cores_sampled"] = cpu / elapsed if elapsed > 0 else None
+    for name in ("voluntary_ctxt_switches", "nonvoluntary_ctxt_switches"):
+        result[name + "_sampled"] = samples[-1][name] - samples[0][name]
+    return result
+
+
+def resource_trend(samples):
+    """报告四段中位数与后半程增长，避免将启动分配误记为长期泄漏。"""
+    if len(samples) < 4:
+        raise ValueError("resource trend requires at least four observations")
+    windows = [samples[len(samples) * index // 4:len(samples) * (index + 1) // 4]
+               for index in range(4)]
+    medians = [statistics.median(sample["rss_bytes"] for sample in window) for window in windows]
+    elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"]
+    return {"observed_seconds": elapsed, "rss_quarter_medians_bytes": medians,
+            "rss_late_growth_bytes": medians[-1] - medians[2],
+            "rss_late_growth_bytes_per_second": (medians[-1] - medians[2]) / (elapsed / 4) if elapsed > 0 else None,
+            "fds_first": samples[0]["fds"], "fds_last": samples[-1]["fds"]}