Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 82 additions & 18 deletions benchmarks/baseline_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
import logging
import os
import pstats
import re
import sqlite3
import statistics
import subprocess
import sys
import tempfile
import time
from dataclasses import dataclass, field
from pathlib import Path

Expand All @@ -46,12 +46,21 @@ class RunResult:
db_size_mb: float


@dataclass
class FrameMetrics:
events: int
total_frames: int
average_depth: float
max_depth: int


@dataclass
class SampleReport:
name: str
path: Path
formal_runs: int = 0
runs: list[RunResult] = field(default_factory=list)
frames: FrameMetrics | None = None

@property
def median_wall(self) -> float:
Expand All @@ -63,6 +72,13 @@ def format_table(self) -> str:
"",
f"正式测量 {self.formal_runs} 次,另有 1 次 warmup",
"",
(
f"Events: {self.frames.events}; frame references: {self.frames.total_frames}; "
f"average/max depth: {self.frames.average_depth:.1f}/{self.frames.max_depth}"
if self.frames
else "Frame depth metrics unavailable"
),
"",
"| Run | Wall (s) | User (s) | Sys (s) | Max RSS (MB) | DB (MB) |",
"|-----|----------|----------|---------|--------------|---------|",
]
Expand All @@ -76,32 +92,46 @@ def format_table(self) -> str:
return "\n".join(lines)


def _parse_time_output(stderr: str) -> dict[str, float]:
wall_match = re.search(r"(\d+\.\d+) real", stderr)
user_match = re.search(r"(\d+\.\d+) user", stderr)
sys_match = re.search(r"(\d+\.\d+) sys", stderr)
rss_match = re.search(r"(\d+)\s+maximum resident set size", stderr)
def _parse_resource_output(stderr: str) -> dict[str, float]:
prefix = "PT_SNAP_RESOURCE "
line = next((line for line in stderr.splitlines() if line.startswith(prefix)), None)
if line is None:
raise RuntimeError("benchmark subprocess did not report resource usage")
try:
values = {
key: float(value)
for key, value in (part.split("=", 1) for part in line.removeprefix(prefix).split())
}
return {
"user_s": values["user"],
"sys_s": values["sys"],
"max_rss_kb": values["max_rss_kb"],
}
except (KeyError, ValueError) as exc:
raise RuntimeError(f"invalid benchmark resource output: {line}") from exc

return {
"wall_s": float(wall_match.group(1)) if wall_match else 0.0,
"user_s": float(user_match.group(1)) if user_match else 0.0,
"sys_s": float(sys_match.group(1)) if sys_match else 0.0,
"max_rss_bytes": int(rss_match.group(1)) if rss_match else 0,
}

def _max_rss_kb(max_rss: float, platform: str) -> float:
return max_rss / 1024 if platform == "darwin" else max_rss


def _run_import_once(snapshot: Path, output_dir: Path, device: int) -> RunResult:
env = {**os.environ, "PYTHONPATH": "src"}
started_at = time.perf_counter()
result = subprocess.run(
[
"/usr/bin/time",
"-l",
sys.executable,
"-c",
f"""
import logging; logging.disable(logging.CRITICAL)
import pt_snap_cli.core.import_metadata as import_metadata
import resource
import sys
from benchmarks.baseline_import import _max_rss_kb
from pt_snap_cli.core.import_service import ImportOptions, ImportService
from pathlib import Path
if not import_metadata.__version__:
import_metadata.__version__ = "benchmark"
options = ImportOptions(
snapshot_file=Path({str(snapshot)!r}),
output_dir=Path({str(output_dir)!r}),
Expand All @@ -110,28 +140,36 @@ def _run_import_once(snapshot: Path, output_dir: Path, device: int) -> RunResult
force=True,
)
ImportService().import_snapshot(options)
usage = resource.getrusage(resource.RUSAGE_SELF)
max_rss_kb = _max_rss_kb(usage.ru_maxrss, sys.platform)
print(
f"PT_SNAP_RESOURCE user={{usage.ru_utime}} sys={{usage.ru_stime}} "
f"max_rss_kb={{max_rss_kb}}",
file=sys.stderr,
)
""",
],
capture_output=True,
text=True,
env=env,
)
wall_s = time.perf_counter() - started_at

if result.returncode != 0:
print(f"STDOUT: {result.stdout}", file=sys.stderr)
print(f"STDERR: {result.stderr}", file=sys.stderr)
raise RuntimeError(f"import failed: {result.returncode}")

metrics = _parse_time_output(result.stderr)
metrics = _parse_resource_output(result.stderr)

db_files = list(output_dir.glob("*.db"))
db_size = sum(f.stat().st_size for f in db_files) if db_files else 0

return RunResult(
wall_s=metrics["wall_s"],
wall_s=wall_s,
user_s=metrics["user_s"],
sys_s=metrics["sys_s"],
max_rss_mb=metrics["max_rss_bytes"] / (1024 * 1024),
max_rss_mb=metrics["max_rss_kb"] / 1024,
db_size_mb=db_size / (1024 * 1024),
)

Expand Down Expand Up @@ -184,6 +222,22 @@ def _representative_queries(db_path: Path, device: int) -> dict:
return results


def _frame_metrics(snapshot: Path, device: int) -> FrameMetrics:
from pt_snap_cli.snapshot.representation import load_snapshot_representation

representation = load_snapshot_representation(snapshot)
device_traces = representation.get("device_traces", [])
traces = device_traces[device] if 0 <= device < len(device_traces) else []
depths = [len(trace.get("frames", [])) for trace in traces]
total = sum(depths)
return FrameMetrics(
events=len(depths),
total_frames=total,
average_depth=total / len(depths) if depths else 0.0,
max_depth=max(depths, default=0),
)


def _correctness_summary(output_dir: Path, device: int) -> dict:
db_files = list(output_dir.glob("*.db"))
if not db_files:
Expand Down Expand Up @@ -239,6 +293,7 @@ def _profile_import(snapshot: Path, output_dir: Path, device: int, top_n: int =
stats = pstats.Stats(profiler, stream=stream)
stats.sort_stats("cumulative")
stats.print_stats(top_n)
stats.print_stats("from_dict|get_callstack|replay|_do_insert")
return stream.getvalue()


Expand Down Expand Up @@ -270,11 +325,20 @@ def main() -> None:
continue

n_runs = run_counts.get(name, 3)
report = SampleReport(name=name, path=snapshot, formal_runs=n_runs)
report = SampleReport(
name=name,
path=snapshot,
formal_runs=n_runs,
frames=_frame_metrics(snapshot, DEVICE),
)

print(f"\n{'='*60}")
print(f"Sample: {name} ({snapshot.name})")
print(f"{'='*60}")
print(
f" Frames: total={report.frames.total_frames} "
f"avg_depth={report.frames.average_depth:.1f} max_depth={report.frames.max_depth}"
)

print(" Warmup run...")
with tempfile.TemporaryDirectory(prefix=f"baseline_{name}_warmup_") as tmp:
Expand Down
6 changes: 6 additions & 0 deletions src/pt_snap_cli/snapshot/PROVENANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,12 @@ The upstream standalone split frontend is deliberately excluded, not deferred:
segment lookup by reusing discovered indices and standard-library keyed
binary search. This is a local performance change against the first-party
runtime; audited upstream source mappings and license terms are unchanged.
- On 2026-08-07, issue #84 added an internal database-import-only raw frame
path and generator-based callstack formatting. Public snapshot construction
remains eager, while database replay avoids reconstructing per-event `Frame`
objects and preserves source frame dictionaries through synthetic segments.
This is a local performance change; audited upstream mappings and licensing
evidence are unchanged.

## Migration toolchain

Expand Down
Loading
Loading