Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/scripts/run_dataset_acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
"tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_attach_and_scratch_routes",
"tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_over_limit_respects_disabled_scratch_budget",
"tests/core/test_dataset_route_scale_acceptance.py::test_million_lifecycles_real_subprocess_deep_offset_and_budget_precedence",
"tests/core/test_export_validation_resources.py::test_million_event_export_validation_memory_is_bounded",
"tests/core/test_export_validation_resources.py::test_first_500k_dataset_cli_query_startup_trend",
)


Expand Down
156 changes: 156 additions & 0 deletions benchmarks/export_validation_memory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
"""Measure only msinsight export preflight in fresh subprocesses (#241/#253).

Build synthetic finalized SQLite shards first, then invoke ``measure`` in a NEW
process for each metric. No pickle load, replay, shard construction, conversion
or publication is included in the measurement. tracemalloc is Python allocator
peak only. Linux RSS uses /proc/self/status VmHWM for the current address space,
with VmRSS snapshots alongside it. ru_maxrss can retain pre-exec parent history,
so it is reported separately and never used for the bound. The VmHWM increment
is relative to the post-import baseline, not a sampled instantaneous phase peak.

Example (from the repository root, with PYTHONPATH=src):
python benchmarks/export_validation_memory.py build /tmp/export-check --events 1000000
python benchmarks/export_validation_memory.py measure /tmp/export-check --metric python
python benchmarks/export_validation_memory.py measure /tmp/export-check --metric rss
"""

from __future__ import annotations

import argparse
import gc
import json
import platform
import sqlite3
import sys
import time
import tracemalloc
from contextlib import closing
from pathlib import Path

from pt_snap_cli.core.msinsight_export import DEFAULT_CAPACITY, _validate_shards
from pt_snap_cli.core.sharded_replay_service import ShardedReplayResult
from pt_snap_cli.snapshot.tools.adaptors.sharded_replay import ReplaySlice


def build(root: Path, events: int) -> dict:
"""Stream synthetic rows; do not retain an event-sized list during setup."""
root.mkdir()
(root / "device_0").mkdir()
slices = []
for index, low in enumerate(range(0, events, DEFAULT_CAPACITY)):
high = min(events, low + DEFAULT_CAPACITY) - 1
path = root / f"device_0/slice_{index:05d}.db"
with closing(sqlite3.connect(path)) as conn, conn:
conn.execute(
"CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, action INTEGER, "
"address INTEGER, size INTEGER, stream INTEGER, allocated INTEGER, "
"active INTEGER, reserved INTEGER, callstackId INTEGER)"
)
conn.executemany(
"INSERT INTO trace_entry_0 VALUES (?,7,0,0,0,0,0,0,0)",
((event,) for event in range(low, high + 1)),
)
conn.execute("INSERT INTO trace_entry_0 VALUES (-1,2,0,0,0,0,0,0,0)")
count = conn.execute("SELECT COUNT(*) FROM trace_entry_0 WHERE id>=0").fetchone()[0]
assert count == high - low + 1
slices.append(
{"index": index, "low": low, "high": high, "file": str(path.relative_to(root))}
)
result = {"events": events, "capacity": DEFAULT_CAPACITY, "slices": slices}
(root / "probe.json").write_text(json.dumps(result), encoding="utf-8")
return result


def _parse_linux_rss(status: str) -> dict[str, int]:
"""Read Linux's current-address-space RSS/high-water values, in bytes."""
values = {}
for line in status.splitlines():
fields = line.split()
if fields and fields[0] in ("VmRSS:", "VmHWM:"):
if len(fields) != 3 or fields[2] != "kB" or not fields[1].isdigit():
raise RuntimeError(f"Unsupported /proc RSS record: {line!r}")
key = fields[0].removesuffix(":")
if key in values:
raise RuntimeError(f"Duplicate /proc RSS record: {key}")
values[key] = int(fields[1]) * 1024
if set(values) != {"VmRSS", "VmHWM"}:
raise RuntimeError("RSS probe requires Linux /proc/self/status VmRSS and VmHWM")
return values


def _linux_rss_snapshot() -> dict[str, int]:
if not sys.platform.startswith("linux"):
raise RuntimeError("RSS probe requires Linux /proc; scheduled acceptance uses Ubuntu")
return _parse_linux_rss(Path("/proc/self/status").read_text())


def measure(root: Path, metric: str) -> dict:
specification = json.loads((root / "probe.json").read_text())
items = tuple(
ReplaySlice(
0, row["index"], row["low"], row["high"], row["low"], row["high"], root / row["file"]
)
for row in specification["slices"]
)
staged = ShardedReplayResult(root, items, ())
# Import and setup high-water are separate from the measured validation.
gc.collect()
evidence = {
"phase": "msinsight_export._validate_shards",
"events": specification["events"],
"shards": len(items),
"events_per_slice": specification["capacity"],
"metric": metric,
"python": platform.python_version(),
"platform": platform.platform(),
"tracemalloc_enabled": metric == "python",
}
if metric == "python":
tracemalloc.start()
started = time.perf_counter()
_validate_shards(staged)
evidence["validation_seconds"] = time.perf_counter() - started
evidence["python_current_bytes"], evidence["python_peak_bytes"] = (
tracemalloc.get_traced_memory()
)
tracemalloc.stop()
else:
before = _linux_rss_snapshot()
import resource

historical_before = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
started = time.perf_counter()
_validate_shards(staged)
evidence["validation_seconds"] = time.perf_counter() - started
after = _linux_rss_snapshot()
historical_after = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
evidence.update(
rss_source="Linux /proc/self/status: current-address-space VmHWM and VmRSS",
rss_current_before_bytes=before["VmRSS"],
rss_current_after_bytes=after["VmRSS"],
rss_high_water_before_bytes=before["VmHWM"],
rss_high_water_after_bytes=after["VmHWM"],
rss_high_water_increase_bytes=max(0, after["VmHWM"] - before["VmHWM"]),
ru_maxrss_before_bytes=historical_before,
ru_maxrss_after_bytes=historical_after,
ru_maxrss_scope="diagnostic only; may retain pre-exec parent high-water history",
)
return evidence


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("operation", choices=("build", "measure"))
parser.add_argument("directory", type=Path)
parser.add_argument("--events", type=int, default=1_000_000)
parser.add_argument("--metric", choices=("python", "rss"), default="python")
args = parser.parse_args()
if args.events < 1:
parser.error("--events must be positive")
root = args.directory.resolve()
evidence = build(root, args.events) if args.operation == "build" else measure(root, args.metric)
print(json.dumps(evidence, sort_keys=True))


if __name__ == "__main__":
main()
136 changes: 136 additions & 0 deletions tests/core/test_export_validation_resources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
"""Nightly million-event validation-only memory acceptance (#253)."""

import json
import os
import platform
import sqlite3
import subprocess
import sys
import time
from pathlib import Path

import pytest

import pt_snap_cli

PROBE = Path(__file__).resolve().parents[2] / "benchmarks/export_validation_memory.py"
MIB = 1024 * 1024


def run_probe(tmp_path, *arguments):
env = os.environ.copy()
env["PYTHONPATH"] = str(Path(pt_snap_cli.__file__).resolve().parents[1])
env["HOME"] = env["USERPROFILE"] = str(tmp_path)
result = subprocess.run(
[sys.executable, str(PROBE), *map(str, arguments)],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
timeout=180,
check=False,
)
assert result.returncode == 0, result.stderr
return json.loads(result.stdout)


@pytest.mark.nightly
@pytest.mark.slow
def test_million_event_export_validation_memory_is_bounded(tmp_path, record_property):
# fcntl/resource are supported on the Linux scheduled runner. Keep the
# expensive gate out of every-PR platform jobs; do not silently skip it.
import fcntl

evidence = []
with Path("/tmp/ptsnap-large-acceptance.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
for events in (10_000, 1_000_000):
root = tmp_path / f"native-{events}"
built = run_probe(tmp_path, "build", root, "--events", events)
assert built["events"] == events and built["capacity"] == 500_000
assert len(built["slices"]) == (1 if events == 10_000 else 2)
# Setup and both instruments each get a fresh process. RSS never
# includes tracemalloc overhead or the synthetic fixture builder.
for metric in ("python", "rss"):
result = run_probe(tmp_path, "measure", root, "--metric", metric)
assert result["events"] == events
assert result["tracemalloc_enabled"] == (metric == "python")
evidence.append(result)
record_property("export_validation_memory", json.dumps(evidence, sort_keys=True))
destination = os.environ.get("PT_SNAP_ACCEPTANCE_EVIDENCE_DIR")
if destination:
output = Path(destination)
output.mkdir(parents=True, exist_ok=True)
(output / "export-validation-memory.json").write_text(json.dumps(evidence, indent=2) + "\n")
python_results = [row for row in evidence if row["metric"] == "python"]
rss_results = [row for row in evidence if row["metric"] == "rss"]
# A Python list of one million integer IDs needs tens of MiB. Leave ample
# runtime headroom while rejecting the old O(events) validation regression.
assert all(row["python_peak_bytes"] < 8 * MIB for row in python_results), evidence
assert python_results[1]["python_peak_bytes"] - python_results[0]["python_peak_bytes"] < 2 * MIB
# Linux VmHWM belongs to the current address space. ru_maxrss can carry
# the previous producer/pytest parent's peak across exec; keep it diagnostic.
assert all("/proc/self/status" in row["rss_source"] for row in rss_results)
assert all(row["rss_high_water_increase_bytes"] < 32 * MIB for row in rss_results), evidence
assert all(row["rss_high_water_after_bytes"] < 128 * MIB for row in rss_results), evidence


@pytest.mark.nightly
@pytest.mark.slow
def test_first_500k_dataset_cli_query_startup_trend(tmp_path, record_property):
"""Fresh CLI/process cache, with no latency threshold or OS-cold claim."""
import fcntl

from tests.core.test_dataset_scale_acceptance import scale_dataset

with Path("/tmp/ptsnap-large-acceptance.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
root = scale_dataset(tmp_path / "dataset", 500_000, 500_000, "synthetic stack")
env = os.environ.copy()
env["PYTHONPATH"] = str(Path(pt_snap_cli.__file__).resolve().parents[1])
env["HOME"] = env["USERPROFILE"] = str(tmp_path)
for key in ("PT_SNAP_DB_PATH", "PT_SNAP_QUERY_TIMEOUT", "PT_SNAP_DATASET_CACHE"):
env.pop(key, None)
command = [
sys.executable,
"-m",
"pt_snap_cli.cli",
"query",
str(root),
"--template-use",
"memory_peak",
"--json",
]
started = time.perf_counter()
result = subprocess.run(
command,
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
timeout=180,
check=False,
)
seconds = time.perf_counter() - started
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout)
assert payload["rows"][0]["peak_active"] == 16
evidence = {
"events": 500_000,
"shards": 1,
"python": platform.python_version(),
"sqlite": sqlite3.sqlite_version,
"platform": platform.platform(),
"runtime_module": str(Path(pt_snap_cli.__file__).resolve()),
"command": command,
"seconds": seconds,
"cache_contract": "default strong; fresh CLI process and empty process-local cache",
"os_cache": "uncontrolled; fixture construction may warm the OS page cache",
"timing_gate": False,
}
record_property("dataset_cold_start_trend", json.dumps(evidence, sort_keys=True))
destination = os.environ.get("PT_SNAP_ACCEPTANCE_EVIDENCE_DIR")
if destination:
output = Path(destination)
output.mkdir(parents=True, exist_ok=True)
(output / "dataset-cold-start-trend.json").write_text(json.dumps(evidence, indent=2) + "\n")
Loading
Loading