232 lines
9.2 KiB
Python
232 lines
9.2 KiB
Python
"""Linux 进程与可选专属 cgroup 的生命周期和采样。"""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import statistics
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
from settings import stage_peer
|
|
|
|
|
|
def key_values(path):
|
|
return {fields[0].rstrip(":"): fields[1] for line in path.read_text().splitlines()
|
|
if len(fields := line.split()) >= 2}
|
|
|
|
|
|
def process_sample(pid):
|
|
root = Path("/proc") / str(pid)
|
|
status = key_values(root / "status")
|
|
if "VmRSS" not in status:
|
|
raise ProcessLookupError(f"process {pid} exited during sampling")
|
|
memory = key_values(root / "smaps_rollup")
|
|
# comm 字段可含空格和括号；从最后一个右括号之后解析固定 stat 字段。
|
|
stat = (root / "stat").read_text().rsplit(")", 1)[1].split()
|
|
switches = {"voluntary_ctxt_switches": 0, "nonvoluntary_ctxt_switches": 0}
|
|
for thread in (root / "task").iterdir():
|
|
try:
|
|
thread_status = key_values(thread / "status")
|
|
except FileNotFoundError:
|
|
continue # A completed blocking thread can disappear during a sample.
|
|
for name in switches:
|
|
switches[name] += int(thread_status[name])
|
|
return {
|
|
"time": time.monotonic(),
|
|
"rss_bytes": int(status["VmRSS"]) * 1024,
|
|
"rss_high_water_bytes": int(status["VmHWM"]) * 1024,
|
|
"pss_bytes": int(memory["Pss"]) * 1024,
|
|
"threads": int(status["Threads"]),
|
|
"fds": len(list((root / "fd").iterdir())),
|
|
"cpu_seconds": (int(stat[11]) + int(stat[12])) / os.sysconf("SC_CLK_TCK"),
|
|
**switches,
|
|
}
|
|
|
|
|
|
class Budget:
|
|
def __init__(self, parent, memory_mib, cpu_quota):
|
|
self.path = None
|
|
self.final = None
|
|
if parent is None:
|
|
if memory_mib is not None or cpu_quota is not None:
|
|
raise ValueError("memory/cpu quotas require --cgroup-parent")
|
|
return
|
|
self.path = parent.resolve() / ("dever-perf-" + uuid.uuid4().hex)
|
|
self.path.mkdir()
|
|
try:
|
|
if memory_mib is not None:
|
|
(self.path / "memory.max").write_text(str(memory_mib * 1024 * 1024))
|
|
(self.path / "memory.swap.max").write_text("0")
|
|
if cpu_quota is not None:
|
|
(self.path / "cpu.max").write_text(f"{round(cpu_quota * 100000)} 100000")
|
|
except BaseException:
|
|
self.path.rmdir()
|
|
raise
|
|
|
|
def sample(self):
|
|
if self.path is None:
|
|
return None
|
|
return {
|
|
"current_bytes": int((self.path / "memory.current").read_text()),
|
|
"peak_bytes": int((self.path / "memory.peak").read_text()),
|
|
"events": {k: int(v) for k, v in key_values(self.path / "memory.events").items()},
|
|
"cpu": {k: int(v) for k, v in key_values(self.path / "cpu.stat").items()},
|
|
"max": (self.path / "memory.max").read_text().strip(),
|
|
"cpu_max": (self.path / "cpu.max").read_text().strip(),
|
|
}
|
|
|
|
def close(self):
|
|
if self.path is not None:
|
|
try:
|
|
self.final = self.sample()
|
|
except OSError as error:
|
|
try:
|
|
self.path.rmdir()
|
|
except OSError as cleanup_error:
|
|
error.add_note(f"cannot remove owned cgroup {self.path}: {cleanup_error}")
|
|
raise
|
|
else:
|
|
self.path.rmdir()
|
|
|
|
|
|
class Process:
|
|
def __init__(self, command, directory, *, peer_settings=None, cpus=None, budget=None):
|
|
self.command = [str(arg) for arg in command]
|
|
self.directory = directory
|
|
self.budget = budget
|
|
self.samples = []
|
|
self.directory.mkdir(parents=True, exist_ok=False)
|
|
if peer_settings is not None:
|
|
self.command[0] = str(stage_peer(command[0], directory, peer_settings))
|
|
self.stdout = (directory / "stdout.log").open("w")
|
|
self.stderr = (directory / "stderr.log").open("w")
|
|
self.started = time.monotonic()
|
|
|
|
def prepare_child():
|
|
# 运行器单线程；只对即将 exec 的自有子进程设置 CPU/cgroup。
|
|
if cpus is not None:
|
|
os.sched_setaffinity(0, cpus)
|
|
if budget is not None and budget.path is not None:
|
|
(budget.path / "cgroup.procs").write_text(str(os.getpid()))
|
|
|
|
try:
|
|
self.child = subprocess.Popen(self.command, stdout=self.stdout, stderr=self.stderr,
|
|
preexec_fn=prepare_child, start_new_session=True)
|
|
except BaseException:
|
|
self.stdout.close()
|
|
self.stderr.close()
|
|
raise
|
|
|
|
def text(self):
|
|
return (self.directory / "stdout.log").read_text()
|
|
|
|
def descriptors(self):
|
|
descriptors = {}
|
|
for path in (Path("/proc") / str(self.child.pid) / "fd").iterdir():
|
|
try:
|
|
descriptors[path.name] = os.readlink(path)
|
|
except FileNotFoundError:
|
|
continue # A descriptor may close while /proc is being read.
|
|
return descriptors
|
|
|
|
def sample(self):
|
|
if self.child.poll() is not None:
|
|
return
|
|
try:
|
|
sample = process_sample(self.child.pid)
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
# Linux can release /proc memory fields before waitpid reports exit.
|
|
# This observation is unavailable; check() still owns exit status.
|
|
return
|
|
sample["elapsed_seconds"] = sample.pop("time") - self.started
|
|
if self.budget is not None:
|
|
sample["cgroup"] = self.budget.sample()
|
|
self.samples.append(sample)
|
|
|
|
def ready(self, timeout=10):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
self.sample()
|
|
for line in self.text().splitlines():
|
|
if line.startswith("ERROR|"):
|
|
raise RuntimeError(line)
|
|
if line.startswith("READY|"):
|
|
self.ready_seconds = time.monotonic() - self.started
|
|
return int(line.split("|")[1])
|
|
if self.child.poll() is not None:
|
|
raise RuntimeError(f"process exited before readiness: {self.directory}")
|
|
time.sleep(0.005)
|
|
raise TimeoutError(f"readiness timeout: {self.directory}")
|
|
|
|
def check(self):
|
|
code = self.child.poll()
|
|
if code is not None and code != 0:
|
|
raise RuntimeError(f"process exited {code}: {self.directory}")
|
|
if "ERROR|" in self.text() or (self.directory / "stderr.log").stat().st_size:
|
|
raise RuntimeError(f"benchmark reported an error: {self.directory}")
|
|
|
|
def close(self):
|
|
try:
|
|
if self.child.poll() is None:
|
|
try:
|
|
os.killpg(self.child.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
# 子进程可能恰好在 poll 与 signal 之间自然退出。
|
|
pass
|
|
try:
|
|
self.child.wait(timeout=3)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(self.child.pid, signal.SIGKILL)
|
|
self.child.wait(timeout=3)
|
|
finally:
|
|
self.stdout.close()
|
|
self.stderr.close()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
self.close()
|
|
|
|
|
|
def observe(processes, seconds, interval):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
for process in processes:
|
|
process.sample()
|
|
process.check()
|
|
time.sleep(min(interval, max(0, deadline - time.monotonic())))
|
|
|
|
|
|
def summarize(samples):
|
|
if not samples:
|
|
raise ValueError("process ended before any resource sample")
|
|
result = {"samples": len(samples)}
|
|
for metric in ("rss_bytes", "pss_bytes", "threads", "fds"):
|
|
values = sorted(sample[metric] for sample in samples)
|
|
result[metric + "_median"] = values[len(values) // 2]
|
|
result[metric + "_max"] = max(values)
|
|
result["rss_high_water_bytes"] = max(sample["rss_high_water_bytes"] for sample in samples)
|
|
elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"]
|
|
cpu = samples[-1]["cpu_seconds"] - samples[0]["cpu_seconds"]
|
|
result["cpu_seconds_sampled"] = cpu
|
|
result["cpu_cores_sampled"] = cpu / elapsed if elapsed > 0 else None
|
|
for name in ("voluntary_ctxt_switches", "nonvoluntary_ctxt_switches"):
|
|
result[name + "_sampled"] = samples[-1][name] - samples[0][name]
|
|
return result
|
|
|
|
|
|
def resource_trend(samples):
|
|
"""报告四段中位数与后半程增长，避免将启动分配误记为长期泄漏。"""
|
|
if len(samples) < 4:
|
|
raise ValueError("resource trend requires at least four observations")
|
|
windows = [samples[len(samples) * index // 4:len(samples) * (index + 1) // 4]
|
|
for index in range(4)]
|
|
medians = [statistics.median(sample["rss_bytes"] for sample in window) for window in windows]
|
|
elapsed = samples[-1]["elapsed_seconds"] - samples[0]["elapsed_seconds"]
|
|
return {"observed_seconds": elapsed, "rss_quarter_medians_bytes": medians,
|
|
"rss_late_growth_bytes": medians[-1] - medians[2],
|
|
"rss_late_growth_bytes_per_second": (medians[-1] - medians[2]) / (elapsed / 4) if elapsed > 0 else None,
|
|
"fds_first": samples[0]["fds"], "fds_last": samples[-1]["fds"]}
|