diff --git a/README.md b/README.md index f4a8471..751670c 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,10 @@ pytest # Run all tests black --check . && ruff check . # Check formatting and lint ``` +For the regular suite use `pytest -m 'not slow'`. Default-capacity scale and +non-editable installed exporter tests are described in the +[dataset acceptance guide](docs/en/dataset-acceptance.md). + ### Building distributions Build from a fresh checkout or a new worktree at the intended commit, with no diff --git a/README_zh.md b/README_zh.md index b7c1ddc..66abca3 100644 --- a/README_zh.md +++ b/README_zh.md @@ -129,6 +129,9 @@ pytest # 运行所有测试 black --check . && ruff check . # 检查格式和 lint ``` +常规套件可用 `pytest -m 'not slow'`。默认容量规模及非 editable 安装包导出测试见 +[数据集验收指南](docs/zh/dataset-acceptance.md)。 + ### 构建分发包 请从目标提交的全新 checkout 或新 worktree 构建,确保没有已有的 `build/` 和 diff --git a/docs/README.md b/docs/README.md index 7ee95a6..0449598 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ | Importing snapshots | [en/quickstart.md](en/quickstart.md) | | Focus Management | [en/focus-management.md](en/focus-management.md) | | Querying | [en/querying.md](en/querying.md) | +| Dataset correctness and default-scale acceptance | [en/dataset-acceptance.md](en/dataset-acceptance.md) | | Reports | [en/querying.md](en/querying.md) | | Splitting Snapshots | [en/splitting.md](en/splitting.md) | | Agent skills | [en/skills.md](en/skills.md) | @@ -26,6 +27,7 @@ | 导入快照 | [zh/quickstart.md](zh/quickstart.md) | | Focus 管理 | [zh/focus-management.md](zh/focus-management.md) | | 运行查询 | [zh/querying.md](zh/querying.md) | +| 数据集正确性与默认规模验收 | [zh/dataset-acceptance.md](zh/dataset-acceptance.md) | | 分析报告 | [zh/querying.md](zh/querying.md) | | 拆分快照 | [zh/splitting.md](zh/splitting.md) | | Agent skill | [zh/skills.md](zh/skills.md) | diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md new file mode 100644 index 0000000..d3bccba --- /dev/null +++ b/docs/en/dataset-acceptance.md @@ -0,0 +1,95 @@ +# Dataset correctness and default-scale acceptance + +Dataset tests distinguish three sources of evidence. Passing a synthetic SQL +fixture is not evidence that a producer, collector, or exporter ran. + +## Regular correctness gate + +`tests/core/test_dataset_realistic.py` runs eight fixed seeds and twenty query +shapes per seed against a standalone reference. Each shard contains only blocks +allocated within its window and blocks still alive on entry. Completed lifetimes +do not appear in later windows; global allocation/free-completion IDs remain +unchanged. Cases include address reuse, a pending free crossing a window, +preexisting/static blocks, NULL/empty/whitespace stacks, a literal missing-stack +label, sorting, pagination, filters and dynamic `top_n` truncation. + +Comparisons are exact, including four-decimal percentages. The minimal 1-byte / +127-byte case requires `0.7813`; an 8-MiB scale variant also checks six-decimal +GiB rounding. The contract is exact parity with the active SQLite runtime's +arithmetic and `ROUND`, not a mathematical half-up approximation. Near-half +3/2,000,000 and 7/2,000,000 proportions, nearby values and large GiB conversions +are compared with standalone SQL on that same runtime, since SQLite versions +can differ in how they format binary doubles. Scalar rounding uses short-lived, +explicitly closed in-memory connections without source reads or merge tables; +timeouts during either rounding pass also verify connection/cursor cleanup. Dataset-only provenance is excluded +from common-schema comparisons. Point-event stored `state` is an observation, +not state at that event; global block comparisons use the latest containing +observation for both sides. + +A separate two-device native-v2 case executes actual replay/import, then attaches +an explicitly constructed ordered-frame extension. Equal local stack IDs in +different members must not merge different ordered arrays. Order, repeats and +extra raw-frame fields are checked against input arrays. Common event/block +fields are compared with standalone SQL; the optional extension does not claim +that the native writer emits ordered frames. + +```bash +pytest tests/test_fixture_provenance.py +pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution.py tests/core/test_dataset_global.py +``` + +## Slow scale and installed export gates + +`tests/core/test_dataset_scale_acceptance.py` is marked `slow` and contains: + +- Synthetic compatibility-v1 SQLite: 1,000,000 real events in two default-sized + 500,000-event members, with window-correct lifecycle membership. Global peak, + gap, event/allocation pages, block pages, terminal leak candidates and stack + aggregation must succeed + using the real default work budgets. Unbounded materialization must still fail + at the genuine row ceiling. Budgets are not monkeypatched down or disabled. + Ordered-frame aggregation may still need the bounded raw-frame fallback; this + gate does not claim all large ordered-frame or unlimited-output queries succeed. +- A separate 10,000-event / 8-KiB-inline-stack dataset checks aggregate/small-page + success and genuine 64-MiB full-output rejection without creating an 8-GB file. +- A trusted, test-owned snapshot with 500,002 alloc/free-request/free-completion + events goes through a non-editable installed `pt-snap import --format msinsight` + with **no capacity override**. The resulting 500,000 + 2 member layout and + cross-boundary lifecycle membership are checked before installed CLI/API peak, + page, terminal-candidate, point-event and report queries. Artifact hashes must + remain unchanged during queries. + +Build the wheel from the intended clean checkout using the +[distribution instructions](../../README.md#building-distributions), then run: + +```bash +pytest tests/test_fixture_provenance.py +PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ + pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml +``` + +Replace `` with the actual wheel name. The test installs that exact +wheel into its temporary directory using `--no-deps --no-index`; subprocesses +clear source `PYTHONPATH` and verify that `pt_snap_cli` resolves inside the +installation. Runtime dependencies are reused from the test interpreter: this +is not a fresh dependency-resolution test. Without `PT_SNAP_ACCEPTANCE_WHEEL`, +only the installed test is explicitly skipped; that skip is not E2E acceptance. +Use `pytest -m 'not slow'` for the regular suite. + +The installed test writes `acceptance.json` under its pytest temporary directory +and a `dataset_acceptance` JUnit property containing the wheel/source SHA-256, +package path, event counts, import duration and query assertions. Duration is +observational; there is no fixed timing assertion or RSS promise. + +## Evidence limits + +The [historical performance baseline](../dataset-performance-baseline.json) +uses a 581,140-byte reviewed snapshot and capacity 2,000. Its sharded run has +8,092 real events across five members. It remains useful historical evidence, +but cannot demonstrate default-capacity scalability. + +The new exporter gate exercises the real first-party replay/export/query chain +on inspectable test-owned input. It does not execute live PyTorch/NPU collection, +the upstream msinsight producer, or its GUI. Those acceptance steps remain +separate. No new committed pickle is introduced. The provenance gate runs before +collection; the test-owned temporary input is written during the test. diff --git a/docs/en/querying.md b/docs/en/querying.md index 5d0f807..1730893 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -161,6 +161,13 @@ only when the caller guarantees immutable finalized files and reliable filesyste change times; see [dataset validation and cache contracts](sharded-snapshotdb.md). No temporary merge DB, repair or artifact write. +The compatible exporter defaults to **500000 real events per device shard**. +That storage capacity is separate from the **100000-row / 64-MiB query work +budget**: even one default-sized shard can exceed it when a result is materialized +without a sufficient filter or page. Use supported aggregates, bounded pages or +narrower event ranges; a small requested output does not waive source-work limits. +See [default-scale acceptance and evidence limits](dataset-acceptance.md). + Global peak queries fetch scalar range evidence and at most three counter rows per shard. Event/allocation pages apply SQL filters and ordering per shard, merge at most `offset + limit` candidates per shard, and count matches separately. Range evidence diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md new file mode 100644 index 0000000..4a247a7 --- /dev/null +++ b/docs/zh/dataset-acceptance.md @@ -0,0 +1,73 @@ +# 数据集正确性及默认规模验收 + +数据集测试区分三类证据。合成 SQL 夹具通过,不代表生产端、采集器或导出器已运行。 + +## 常规正确性门禁 + +`tests/core/test_dataset_realistic.py` 对八个固定种子各运行二十种查询形态,并与单库 +参考结果比较。每片仅包含窗口内新分配及进入窗口时仍存活的 block;已完成释放的 +生命周期不进入后续窗口,全局分配/完成释放 ID 保持不变。覆盖地址复用、跨窗口 +pending free、preexisting/static、NULL/空/空白栈、字面量缺失栈标签、排序、分页、 +筛选及动态组 `top_n` 截断。 + +所有比较严格相等,包括四位小数百分比。1-byte / 127-byte 最小用例要求 `0.7813`, +8-MiB 倍率另检查六位小数 GiB 舍入。契约是与当前 SQLite 运行时的运算及 `ROUND` +严格一致,不是数学上的 half-up 近似。3/2,000,000、7/2,000,000 比例、临近值及大数 +GiB 转换都以同一运行时的单库 SQL 为参考,因为 SQLite 版本间的二进制浮点格式化 +可能不同。标量舍入使用短生命周期且显式关闭的内存连接,不读取来源或创建合并表; +两个舍入阶段的超时测试也验证连接/cursor 关闭。公共 schema 比较排除 +数据集独有来源元数据。定点 block 的存储 `state` 是观察值,不是该事件时刻状态; +全局 block 两侧都采用最后一个包含该生命周期的窗口观察。 + +另一个双设备原生 v2 用例执行真实 replay/import,再附加明确构造的有序 frame 扩展。 +不同成员中的相同局部栈 ID 不得合并不同有序数组;顺序、重复项及额外原始 frame +字段都与输入数组核对。公共事件/block 字段与单库 SQL 比较;该扩展夹具不声称原生 +writer 会生成有序 frames。 + +```bash +pytest tests/test_fixture_provenance.py +pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution.py tests/core/test_dataset_global.py +``` + +## slow 规模及安装包导出门禁 + +`tests/core/test_dataset_scale_acceptance.py` 标记为 `slow`,包括: + +- 合成兼容 v1 SQLite:1,000,000 个真实事件,两个默认大小的 500,000-event 成员, + block 成员符合生命周期窗口。全局 peak、gap、event/allocation 分页、block 分页及 + 终片泄漏候选及栈聚合必须在真实默认工作预算内成功;无界物化仍需触发真实行数上限。测试不调低 + 或关闭预算。有序 frame 聚合可能仍需有界原始 frame 回退路径;此门禁不声称所有 + 大规模有序 frame 或无界输出查询均可成功。 +- 独立 10,000-event / 8-KiB 内联栈数据集验证聚合/小页成功,以及完整输出触发真实 + 64-MiB 上限,无需创建 8-GB 数据文件。 +- 受信、测试自行构造的 500,002 个 alloc/free-request/free-completion 事件快照, + 经非 editable 安装的 `pt-snap import --format msinsight`,**不覆盖容量默认值**。 + 先核对 500,000 + 2 成员布局及跨界生命周期,再通过该安装包的 CLI/API 验证峰值、 + 分页、终片候选、定点事件及报告。查询前后产物 hash 必须不变。 + +按照[分发包构建说明](../../README_zh.md#构建分发包),从目标干净 checkout 构建 wheel: + +```bash +pytest tests/test_fixture_provenance.py +PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ + pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml +``` + +将 `` 替换为实际名称。测试以 `--no-deps --no-index` 将该确切 wheel 安装到 +临时目录,子进程清除源码 `PYTHONPATH` 并核对 `pt_snap_cli` 来自该安装目录。运行依赖 +复用测试解释器环境,不代表全新依赖解析测试。未设置 `PT_SNAP_ACCEPTANCE_WHEEL` 时, +仅安装包用例明确 skip;skip 不算 E2E 通过。常规套件可用 `pytest -m 'not slow'`。 + +安装包用例在 pytest 临时目录写 `acceptance.json`,并记录 `dataset_acceptance` JUnit +属性,包括 wheel/source SHA-256、包路径、事件数量、导入耗时及查询断言。耗时只是 +单次观察,不设固定时限断言,也不承诺进程 RSS。 + +## 证据边界 + +[历史性能基线](../dataset-performance-baseline.json) 使用 581,140-byte 已审核快照及 +容量 2,000;分片模式仅 8,092 个真实事件、五片。保留此历史证据,但它不能证明默认 +容量可扩展性。 + +新增导出门禁在可审查的测试输入上执行真实第一方 replay/export/query 链路;不执行 +实时 PyTorch/NPU 采集、上游 msinsight 生产端或 GUI,这些验收仍独立。未增加任何提交 +到仓库的 pickle。provenance 门禁在收集测试前运行;临时受信输入在测试期间自行写入。 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 15964d7..104cf01 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -130,13 +130,18 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 数据集读取先分离批次再查外片来源,使用借用的有界 Context LRU。所有片、来源、分组、 精确总数共用递减 deadline;报告的峰值选择与归因明确共用**一个**预算。累计取回/输出工作上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 -结论。Python 物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ +结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ SQLite、一个批次及单个 cell 有额外开销。默认仍对 manifest/成员执行完整校验和 hash, 并共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 单个文件系统调用不提供 OS 级硬抢占。仅当调用方保证最终产物不可变且文件系统变更时间 可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用进程内校验复用;参见 [数据集校验与缓存合同](sharded-snapshotdb.md)。不生成合并临时 DB、不修复或写产物。 +兼容导出器默认**每设备每片 500000 个真实事件**。该存储容量与 **100000 行 / 64 MiB +查询工作预算**独立:即使只有一个默认大小的分片,无足够筛选或分页的结果物化也可能 +超限。请使用受支持的聚合、有界分页或较窄事件范围;请求输出较少不代表可忽略来源 +读取预算。详见[默认规模验收及证据边界](dataset-acceptance.md)。 + 全局峰值查询每片只取范围汇总及最多三条计数器记录。event/allocation 在各片 SQL 中筛选、 排序,每片最多取 `offset + limit` 个候选后全局归并,并单独统计匹配总数;实际范围证据在 非范围筛选前计算。普通文本栈统计先合并各片完整分组,再应用全局阈值,平均值由总大小 diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index 0fcc5a0..1d806f3 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -2,9 +2,13 @@ from __future__ import annotations +import sqlite3 +from collections.abc import Iterator +from contextlib import closing, contextmanager from typing import cast from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, EventSource +from pt_snap_cli.core.errors import QueryExecutionError from pt_snap_cli.core.models import QueryResult from pt_snap_cli.core.stack_summary import summarize_stacks @@ -139,12 +143,16 @@ def event_attribution( ) selected.sort(key=_rank) denominator = sum(cast(int, row["size_bytes"]) for row in selected) - for row in selected: - row["percent_of_active_blocks"] = ( - round(cast(int, row["size_bytes"]) * 100.0 / denominator, 4) - if denominator - else None - ) + # Use the same SQLite arithmetic/ROUND implementation as standalone SQL. + # A Decimal conversion of a binary float changes some near-half values. + # This connection evaluates scalars only: no source reads or merge tables. + with _rounding_cursor(sources) as cursor: + for row in selected: + sources.budget.remaining() + row["percent_of_active_blocks"] = cursor.execute( + "SELECT ROUND(? * 100.0 / NULLIF(?, 0), 4)", + (row["size_bytes"], denominator), + ).fetchone()[0] offset = 0 has_more = len(selected) < len(all_rows) - offset or rank_full if max_rows is not None and max_rows > 0: @@ -243,9 +251,16 @@ def _groups( cast(int, group["stack_event_id"]), cast(int, source.event["id"]) ) rows = list(grouped.values()) - for row in rows: - row["size_gib"] = round(cast(int, row["size_bytes"]) / 1073741824.0, 6) - row["requested_gib"] = round(cast(int, row["requested_bytes"]) / 1073741824.0, 6) + # Reuse one scalar-only engine for this pass and close it on timeout as well + # as success. Derived values do not add source IO; final output is still + # charged by event_attribution's existing consume(selected) boundary. + with _rounding_cursor(sources) as cursor: + for row in rows: + sources.budget.remaining() + row["size_gib"], row["requested_gib"] = cursor.execute( + "SELECT ROUND(? / 1073741824.0, 6), ROUND(? / 1073741824.0, 6)", + (row["size_bytes"], row["requested_bytes"]), + ).fetchone() rows.sort(key=_rank) return rows @@ -261,3 +276,17 @@ def summarize_dataset_stacks(rows: list[dict[str, object]], budget: int) -> None for row, (identity, kind) in zip(rows, identities, strict=True): row["stack_id"] = identity row["stack_kind"] = kind + + +@contextmanager +def _rounding_cursor(sources: DatasetSourceResolver) -> Iterator[sqlite3.Cursor]: + """Keep scalar arithmetic failures on the same domain boundary as shard SQL.""" + try: + with ( + closing(sqlite3.connect(":memory:")) as connection, + closing(connection.cursor()) as cursor, + ): + yield cursor + except (sqlite3.Error, OverflowError) as exc: + sources.budget.remaining() # Preserve a deadline that expired during SQLite work. + raise QueryExecutionError(str(exc)) from exc diff --git a/tests/core/dataset_lifecycle_fixtures.py b/tests/core/dataset_lifecycle_fixtures.py new file mode 100644 index 0000000..fd945f6 --- /dev/null +++ b/tests/core/dataset_lifecycle_fixtures.py @@ -0,0 +1,276 @@ +"""Deterministic synthetic v1 fixtures with replay-faithful window membership. + +These fixtures exercise SQL/query contracts, not the snapshot producer/exporter. +The independent model retains full lifecycle IDs across windows, includes only +carry-in/new allocations, and drops lifetimes completed before a window starts. +""" + +import json +import random +import sqlite3 +from contextlib import closing + +from pt_snap_cli.core.dataset_contract import ACTION_NAMES, BLOCK_STATES, validate_dataset + +EVENT_COUNT = 80 +CAPACITY = 10 +SCHEMA = """CREATE TABLE dictionary ("table" TEXT,"column" TEXT,"key" TEXT,"value" TEXT); +CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, action INTEGER,address INTEGER,size INTEGER,stream INTEGER,allocated INTEGER,active INTEGER,reserved INTEGER,callstack TEXT); +CREATE TABLE block_0 (id INTEGER PRIMARY KEY,address INTEGER,size INTEGER,requestedSize INTEGER,state INTEGER,allocEventId INTEGER,freeEventId INTEGER);""" + + +def timeline(seed): + rng = random.Random(seed) + blocks = { + -2: { + "id": -2, + "address": 900, + "size": 48, + "requestedSize": 40, + "alloc": -1, + "free": -1, + "request": -1, + "stack": None, + }, + -1: { + "id": -1, + "address": 800, + "size": 96, + "requestedSize": 80, + "alloc": -1, + "free": -1, + "request": -1, + "stack": None, + }, + } + trace = [] + nextaddr = 2000 + fixed = { + 0: ("a", 1000, "shared"), + 1: ("a", 1100, None), + 2: ("a", 1200, ""), + 3: ("a", 1300, "[missing callstack]"), + 4: ("a", 1400, " \n"), + 17: ("r", -1), + 18: ("f", -1), + 22: ("r", 0), + 23: ("f", 0), + 24: ("a", 1000, "reuse"), + 60: ("r", 24), + 61: ("f", 24), + } + protected = {-2, -1, 0, 1, 2, 3, 4, 24} + for event in range(EVENT_COUNT): + if event in fixed: + op = fixed[event] + else: + live = [ + b["id"] for b in blocks.values() if b["free"] == -1 and b["id"] not in protected + ] + pending = [i for i in live if blocks[i]["request"] >= 0] + ready = [i for i in live if blocks[i]["request"] == -1] + coin = rng.random() + if coin < 0.23 and pending: + op = ("f", rng.choice(pending)) + elif coin < 0.48 and ready: + op = ("r", rng.choice(ready)) + elif coin < 0.87: + available = [ + b["address"] + for b in blocks.values() + if b["free"] >= 0 + and b["address"] >= 2000 + and all(c["address"] != b["address"] or c["free"] >= 0 for c in blocks.values()) + ] + address = rng.choice(available) if available and rng.random() < 0.6 else nextaddr + if address == nextaddr: + nextaddr += 100 + op = ( + "a", + address, + rng.choice( + [ + "shared", + "stack-A", + "stack-B", + None, + "", + "[missing callstack]", + " \n", + "长栈", + ] + ), + ) + else: + op = ("w",) + kind = op[0] + if kind == "a": + size = rng.choice([16, 32, 64, 128, 256]) + req = size - rng.randrange(0, size // 4 + 1) + blocks[event] = { + "id": event, + "address": op[1], + "size": size, + "requestedSize": req, + "alloc": event, + "free": -1, + "request": -1, + "stack": op[2], + } + action, address, stack = 4, op[1], op[2] + elif kind in ("r", "f"): + b = blocks[op[1]] + size, address = b["size"], b["address"] + stack = rng.choice(["free-A", "free-B", None, ""]) + if kind == "r": + b["request"] = event + action = 5 + else: + b["free"] = event + action = 6 + else: + action, address, size, stack = 7, 0, 0, rng.choice(["workspace", None, ""]) + alive = [b for b in blocks.values() if b["free"] == -1] + active = sum(b["size"] for b in alive) + allocated = sum(b["size"] for b in alive if b["request"] == -1) + reserved = ((active + 255) // 256) * 256 + 1024 + trace.append( + (event, action, address, size, rng.choice([0, 1]), allocated, active, reserved, stack) + ) + return trace, list(blocks.values()) + + +def member(b, lo, hi): + return (b["alloc"] == -1 or b["alloc"] <= hi) and (b["free"] == -1 or b["free"] >= lo) + + +def state_at_left(b, lo): + return 0 if b["request"] >= 0 and b["request"] < lo else 1 + + +def row(b, state): + return (b["id"], b["address"], b["size"], b["requestedSize"], state, b["alloc"], b["free"]) + + +def db(path, trace, rows, boundary=True): + with closing(sqlite3.connect(path)) as c, c: + c.executescript(SCHEMA) + c.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + [("trace_entry_0", "action", str(i), n) for i, n in enumerate(ACTION_NAMES)] + + [("block_0", "state", str(i), n) for i, n in BLOCK_STATES], + ) + c.executemany("INSERT INTO trace_entry_0 VALUES (?,?,?,?,?,?,?,?,?)", trace) + if boundary: + c.execute( + "INSERT INTO trace_entry_0 VALUES (-1,2,0,1048576,0,999999,999999,999999,'boundary-excluded')" + ) + c.executemany("INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", rows) + + +def build(out, seed): + root = out / f"seed-{seed:04d}" + root.mkdir() + dataset = root / "dataset" + (dataset / "device_0").mkdir(parents=True) + trace, blocks = timeline(seed) + slices = [] + layout = [] + for i, lo in enumerate(range(0, EVENT_COUNT, CAPACITY)): + hi = lo + CAPACITY - 1 + rows = [row(b, state_at_left(b, lo)) for b in blocks if member(b, lo, hi)] + assert all(b[5] == -1 or b[5] <= hi for b in rows) + assert all(b[6] == -1 or b[6] >= lo for b in rows) + file = f"device_0/slice_{i:05d}.db" + db(dataset / file, trace[lo : hi + 1], rows) + slices.append( + {"index": i, "startEventId": lo, "endEventId": hi, "file": file, "ready": True} + ) + layout.append({"slice": i, "lo": lo, "hi": hi, "block_ids": [r[0] for r in rows]}) + manifest = { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/synthetic/no-pickle", + "cacheHash": "", + "eventsPerSlice": CAPACITY, + "devices": { + "0": { + "eventCount": EVENT_COUNT, + "sliceCount": len(slices), + "readySlices": list(range(len(slices))), + "slices": slices, + } + }, + } + (dataset / "manifest.json").write_text(json.dumps(manifest)) + single = root / "single.db" + # Global block uses latest containing observation, matching its documented state_scope; + # point-event comparisons exclude stored state, which is not state at requested E. + finalrows = [ + row( + b, + state_at_left( + b, + max( + i * CAPACITY + for i in range(EVENT_COUNT // CAPACITY) + if member(b, i * CAPACITY, (i + 1) * CAPACITY - 1) + ), + ), + ) + for b in blocks + ] + db(single, trace, finalrows) + (root / "timeline.json").write_text( + json.dumps( + {"seed": seed, "trace": trace, "blocks": blocks, "layout": layout}, + ensure_ascii=False, + indent=2, + ) + ) + validate_dataset(dataset) + return root, dataset, single, trace, blocks + + +def shapes(seed): + event = random.Random(seed + 10000).choice( + [0, 9, 10, 17, 18, 19, 20, 22, 23, 24, 29, 30, 39, 40, 59, 60, 61, 69, 70, 79] + ) + return [ + ("memory_peak", {}), + ("memory_peak", {"start_id": 10, "end_id": 61}), + ("allocator_gap", {}), + ("allocator_gap", {"start_id": 10, "end_id": 61}), + ("allocation", {"min_id": 0}), + ( + "allocation", + { + "min_id": 10, + "max_id": 61, + "order_by": "active", + "order_dir": "DESC", + "limit": 7, + "offset": 2, + }, + ), + ("event", {"min_id": 0}), + ( + "event", + {"min_id": 0, "order_by": "callstack", "order_dir": "ASC", "limit": 9, "offset": 2}, + ), + ("event", {"min_id": 0, "action": 4}), + ("event", {"id": event}), + ("block", {}), + ("block", {"address": 1000}), + ("block", {"min_size": 32, "order_by": "id", "order_dir": "ASC", "limit": 6, "offset": 2}), + ("leak_detection", {}), + ("freed_block_lifetime", {}), + ("callstack_analysis", {"min_count": 1}), + ("preexisting_live", {"event_id": event}), + ("active_blocks_at_event", {"event_id": event}), + ("active_memory_callstack_at_event", {"event_id": event, "top_n": -1}), + ( + "active_memory_callstack_at_event", + {"event_id": event, "top_n": 1, "include_static": False, "min_size": 16}, + ), + ] diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py index e797a9f..92e7515 100644 --- a/tests/core/test_dataset_attribution.py +++ b/tests/core/test_dataset_attribution.py @@ -50,7 +50,15 @@ def case(tmp_path, devices=(0, 2)): closing(sqlite3.connect(root / f"device_{device}/slice_{index:05d}.db")) as conn, conn, ): - conn.executemany(f"INSERT INTO block_{device} VALUES (?,?,?,?,?,?,?)", blocks) + # Carry-in and new allocations only. Future allocations and completed + # lifetimes from earlier windows are not member observations. + lo, hi = index * 2, index * 2 + 1 + members = [ + (*b[:4], 0 if b[0] == 0 and lo > 2 else 1, *b[5:]) + for b in blocks + if (b[5] == -1 or b[5] <= hi) and (b[6] == -1 or b[6] >= lo) + ] + conn.executemany(f"INSERT INTO block_{device} VALUES (?,?,?,?,?,?,?)", members) for event in range(index * 2, index * 2 + 2): text = { 0: "shared:" + "长" * 100, @@ -104,10 +112,12 @@ def standalone(root, tmp_path, device=0): "INSERT INTO dictionary VALUES (?,?,?,?)", conn.execute("SELECT * FROM dictionary").fetchall(), ) - out.executemany( - f"INSERT INTO block_{device} VALUES (?,?,?,?,?,?,?)", - conn.execute(f"SELECT * FROM block_{device}").fetchall(), - ) + # Use the same latest-containing observation contract as global + # block queries; never take only the first shard's block table. + out.executemany( + f"INSERT OR REPLACE INTO block_{device} VALUES (?,?,?,?,?,?,?)", + conn.execute(f"SELECT * FROM block_{device}").fetchall(), + ) return target diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 86332b2..46d158d 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -231,7 +231,9 @@ def test_lifecycles_terminal_survival_completed_free_reuse_and_unknown(tmp_path) conn.execute("UPDATE trace_entry_0 SET action=2 WHERE id=0") with SnapshotAnalyzer(root) as analyzer: result = analyzer.execute_query("block") - assert result["scope"]["source_coverage"]["unproved_identities"] == 8 + assert ( + result["scope"]["source_coverage"]["unproved_identities"] == 6 + ) # Present through E10. assert not result["scope"]["source_coverage"]["allocation_source_complete"] assert all(r["id"] != 0 for r in analyzer.execute_query("leak_detection")["rows"]) diff --git a/tests/core/test_dataset_realistic.py b/tests/core/test_dataset_realistic.py new file mode 100644 index 0000000..2a38823 --- /dev/null +++ b/tests/core/test_dataset_realistic.py @@ -0,0 +1,388 @@ +"""Small, deterministic differential contracts; scale/export acceptance is separate.""" + +import copy +import dataclasses +import hashlib +import json +import sqlite3 +import time +from collections import Counter +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core import dataset_attribution +from pt_snap_cli.core.dataset_contract import validate_dataset +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.core.query_service import QueryService +from tests.core.dataset_lifecycle_fixtures import CAPACITY, build, db, member, shapes +from tests.core.test_dataset_attribution import case +from tests.snapshot.test_sharded_replay import block, event, segment, write_source +from tests.test_dataset_focus import hashes + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def common_rows(actual, expected, *, omit=()): + """Ignore only dataset provenance and explicitly non-comparable observations.""" + fields = set(expected[0]) - set(omit) if expected else set() + return [[{key: row[key] for key in fields} for row in rows] for rows in (actual, expected)] + + +def test_existing_attribution_fixture_has_only_actual_window_members(tmp_path): + root = case(tmp_path, devices=(0,)) + observed = [] + for index in range(8): + lo, hi = index * 2, index * 2 + 1 + with closing(sqlite3.connect(root / f"device_0/slice_{index:05d}.db")) as conn: + blocks = conn.execute("SELECT id,allocEventId,freeEventId FROM block_0").fetchall() + assert all(alloc == -1 or alloc <= hi for _, alloc, _ in blocks) + assert all(free == -1 or free >= lo for _, _, free in blocks) + observed.append({row[0] for row in blocks}) + assert 11 not in observed[0] and 11 in observed[5] + assert 0 in observed[5] and 0 not in observed[6] + assert -21 in set.intersection(*observed) + + +@pytest.mark.parametrize("seed", [0, 7, 31, 67, 136, 196, 2026, 8191]) +def test_seeded_lifecycle_windows_match_single_database_strictly(tmp_path, seed): + root, dataset, single, traces, blocks = build(tmp_path, seed) + before = hashes(root) + # Every seed contains these cases, rather than relying on probability. + assert {b["stack"] for b in blocks} >= {None, "", "[missing callstack]", " \n"} + assert sum(b["address"] == 1000 for b in blocks) == 2 + assert any(b["alloc"] == -1 and b["free"] == -1 for b in blocks) + assert any(b["alloc"] == -1 and b["free"] >= 0 for b in blocks) + assert any( + b["alloc"] >= 0 and b["free"] >= 0 and b["alloc"] // CAPACITY != b["free"] // CAPACITY + for b in blocks + ) + for entry in json.loads((root / "timeline.json").read_text())["layout"]: + expected = {b["id"] for b in blocks if member(b, entry["lo"], entry["hi"])} + assert set(entry["block_ids"]) == expected + with SnapshotAnalyzer(dataset) as actual, SnapshotAnalyzer(single) as reference: + for template, params in shapes(seed): + got = actual.execute_query(template, params, exact_total=True) + want = reference.execute_query(template, params, exact_total=True) + omitted = ("state",) if template == "active_blocks_at_event" else () + got_rows, want_rows = common_rows(got["rows"], want["rows"], omit=omitted) + assert got_rows == want_rows, (seed, template, params) + for key in ("total", "returned", "total_is_exact", "has_more", "truncated"): + assert got[key] == want[key], (seed, template, params, key) + if template == "active_blocks_at_event": + at = params["event_id"] + expected = { + b["id"] + for b in blocks + if (b["alloc"] == -1 or b["alloc"] <= at) + and (b["free"] == -1 or b["free"] > at) + } + assert {r["id"] for r in got["rows"]} == expected + assert sum(r["size"] for r in got["rows"]) == traces[at][6] + assert got["scope"]["source_coverage"]["allocation_source_complete"] + assert hashes(root) == before + assert not list(root.rglob("*-wal")) + + +def rounding_case(tmp_path, sizes): + root = tmp_path / "dataset" + (root / "device_0").mkdir(parents=True) + first, second = sizes + total = first + second + traces = [ + (0, 4, 100, first, 0, first, first, total, "first"), + (1, 4, 200, second, 0, total, total, total, "second"), + ] + blocks = [(0, 100, first, first, 1, 0, -1), (1, 200, second, second, 1, 1, -1)] + slices = [] + for index in range(2): + file = f"device_0/slice_{index:05d}.db" + db(root / file, [traces[index]], blocks[: index + 1]) + slices.append( + { + "index": index, + "startEventId": index, + "endEventId": index, + "file": file, + "ready": True, + } + ) + (root / "manifest.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/synthetic/no-pickle", + "cacheHash": "", + "eventsPerSlice": 1, + "devices": { + "0": {"eventCount": 2, "sliceCount": 2, "readySlices": [0, 1], "slices": slices} + }, + } + ) + ) + single = tmp_path / "single.db" + db(single, traces, blocks) + validate_dataset(root) + return root, single + + +@pytest.mark.parametrize( + "sizes", + [ + (1, 127), + (8 * 1024 * 1024, 127 * 8 * 1024 * 1024), + (3, 1_999_997), + (7, 1_999_993), + (3, 1_999_996), + (3, 1_999_998), + (7, 1_999_992), + (7, 1_999_994), + (914_708_987_140_385, 1), + (685_251_271_850_673, 1), + (2**53 - 1, 128), + (2**53 + 1, 128), + (8 * 1024 * 1024 - 1, 1), + (8 * 1024 * 1024 + 1, 1), + ], +) +def test_half_near_half_and_large_values_match_actual_sqlite_rounding(tmp_path, sizes): + root, single = rounding_case(tmp_path, sizes) + before = hashes(tmp_path) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + params = {"event_id": 1, "top_n": -1} + actual = dataset.execute_query("active_memory_callstack_at_event", params)["rows"] + expected = reference.execute_query("active_memory_callstack_at_event", params)["rows"] + left, right = common_rows(actual, expected) + # SQLite versions can differ for decimal near-halves. The actual + # standalone query is the oracle, never a decimal/string approximation. + assert left == right + if sizes == (1, 127): + assert ( + next(r for r in actual if r["size_bytes"] == 1)["percent_of_active_blocks"] + == 0.7813 + ) + assert ( + next(r for r in actual if r["size_bytes"] == 127)["percent_of_active_blocks"] + == 99.2188 + ) + if sizes[0] == 8 * 1024 * 1024: + first = next(r for r in actual if r["size_bytes"] == sizes[0]) + assert first["size_gib"] == first["requested_gib"] == 0.007813 + assert hashes(tmp_path) == before + + +@pytest.mark.parametrize("same_stack", [True, False], ids=["gib", "percentages"]) +def test_rounding_int64_aggregate_overflow_is_a_domain_error(tmp_path, same_stack): + root, single = rounding_case(tmp_path, (1, 1)) + for path in [single, *root.glob("device_0/*.db")]: + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute("UPDATE block_0 SET size=?, requestedSize=?", (2**62, 2**62)) + if same_stack: + connection.execute("UPDATE trace_entry_0 SET callstack='same'") + validate_dataset(root) + before = hashes(tmp_path) + for path in (root, single): + with SnapshotAnalyzer(path) as analyzer, pytest.raises(QueryExecutionError): + analyzer.execute_query("active_memory_callstack_at_event", {"event_id": 1, "top_n": -1}) + result = CliRunner().invoke( + app, + [ + "query", + str(path), + "--template-use", + "active_memory_callstack_at_event", + "--params", + '{"event_id": 1, "top_n": -1}', + "--json", + ], + ) + assert result.exit_code == 1 and result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "QUERY_FAILED" + assert "Traceback" not in result.stderr + assert hashes(tmp_path) == before + + +@pytest.mark.parametrize("rounding_pass", [1, 2], ids=["gib", "percentages"]) +def test_rounding_timeout_closes_scalar_connection_and_cursor(tmp_path, monkeypatch, rounding_pass): + root, _ = rounding_case(tmp_path, (1, 127)) + connections, cursors = [], [] + connect = sqlite3.connect + + class TrackedCursor(sqlite3.Cursor): + calls = 0 + closed = False + + def execute(self, *args, **kwargs): + self.calls += 1 + return super().execute(*args, **kwargs) + + def close(self): + self.closed = True + super().close() + + class TrackedConnection(sqlite3.Connection): + closed = False + + def cursor(self, *args, **kwargs): + cursor = super().cursor(*args, factory=TrackedCursor, **kwargs) + cursors.append(cursor) + return cursor + + def close(self): + self.closed = True + super().close() + + def scalar_connect(database, *args, **kwargs): + if database == ":memory:": + connection = connect(database, *args, factory=TrackedConnection, **kwargs) + connections.append(connection) + return connection + return connect(database, *args, **kwargs) + + budget = QueryBudget(None, time.monotonic()) + remaining = budget.remaining + + def expire_during_rounding(): + if len(connections) == rounding_pass and cursors[-1].calls == 1: + raise QueryTimeoutError("rounding deadline expired") + return remaining() + + monkeypatch.setattr(dataset_attribution.sqlite3, "connect", scalar_connect) + monkeypatch.setattr(budget, "remaining", expire_during_rounding) + service = QueryService() + try: + with pytest.raises(QueryTimeoutError, match="rounding deadline expired"): + service.execute_query( + "active_memory_callstack_at_event", + {"event_id": 1, "top_n": -1}, + db_path=root, + _budget=budget, + ) + finally: + service.close() + assert len(connections) == len(cursors) == rounding_pass + assert all(connection.closed for connection in connections) + assert all(cursor.closed for cursor in cursors) + for connection in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + for cursor in cursors: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + cursor.fetchone() + + +def test_native_ordered_frames_and_local_stack_collisions_use_full_identity(tmp_path): + """Real native replay plus an explicitly attached optional-frame fixture. + + Standalone SQL supplies the common event/block oracle. Ordered group identity + is checked independently against input frames, since standalone SQL is text-only. + """ + first = {"filename": "a.py", "line": 1, "name": "a"} + second = {"filename": "b.py", "line": 2, "name": "b"} + frames = [ + [first, second, first], + [second, first, second], + [first, second, first], + [{**first, "extra": {"native": True}}, second, first], + ] + sizes = [16, 32, 16, 64] + addresses = [1000, 1016, 1048, 1064] + events = [ + event("alloc", addr=addr, size=size) for addr, size in zip(addresses, sizes, strict=True) + ] + for trace, stack in zip(events, frames, strict=True): + trace["frames"] = stack + data = { + "segments": [ + segment(0, 128, [block(a, n, n) for a, n in zip(addresses, sizes, strict=True)]) + ], + "device_traces": [events, copy.deepcopy(events)], + } + data["segments"] += [{**copy.deepcopy(s), "device": 1} for s in data["segments"]] + request = ImportOptions( + write_source(tmp_path, data), tmp_path / "output", set_focus=False, events_per_slice=1 + ) + service = ImportService() + root = service.import_snapshot(request).db_path + single = service.import_snapshot(dataclasses.replace(request, events_per_slice=None)).db_path + manifest = json.loads((root / "manifest.json").read_text()) + manifest["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + for device in (0, 1): + for index, item in enumerate(manifest["devices"][str(device)]["slices"]): + path = root / item["file"] + with closing(sqlite3.connect(path)) as conn, conn: + local_id = conn.execute( + f"SELECT callstackId FROM trace_entry_{device} WHERE id=?", (index,) + ).fetchone()[0] + conn.execute("UPDATE callstack SET id=42 WHERE id=?", (local_id,)) + conn.execute( + f"UPDATE trace_entry_{device} SET callstackId=42 WHERE callstackId=?", + (local_id,), + ) + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT,PRIMARY KEY(eventId,frameIndex));" + ) + conn.execute("INSERT INTO pt_snap_frame_coverage VALUES (?,3)", (index,)) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,?,?)", + [(index, i, json.dumps(f)) for i, f in enumerate(frames[index])], + ) + item["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + (root / "manifest.json").write_text(json.dumps(manifest)) + before = hashes(root) + identities = [] + expected_groups = Counter() + for stack, size in zip(frames, sizes, strict=True): + expected_groups[json.dumps(stack, sort_keys=True)] += size + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in (0, 1): + for at in range(4): + actual_event = dataset.execute_query("event", {"id": at}, device_id=device)["rows"] + expected_event = reference.execute_query("event", {"id": at}, device_id=device)[ + "rows" + ] + left, right = common_rows(actual_event, expected_event) + assert left == right + params = {"event_id": at} + got = dataset.execute_query("active_blocks_at_event", params, device_id=device)[ + "rows" + ] + want = reference.execute_query("active_blocks_at_event", params, device_id=device)[ + "rows" + ] + left, right = common_rows(got, want, omit=("state",)) + assert left == right + for row in got: + source = row["allocation_source"] + assert source["local_stack_id"] == 42 + assert source["frames"] == frames[row["allocEventId"]] + assert source["frames_status"] == "ordered" + params["top_n"] = -1 + grouped = dataset.execute_query( + "active_memory_callstack_at_event", params, device_id=device + ) + assert grouped["scope"]["source_coverage"]["ordered_frames_complete"] + rows = grouped["rows"] + assert { + json.dumps(r["frames"], sort_keys=True): r["size_bytes"] for r in rows + } == expected_groups + assert sorted(r["block_count"] for r in rows) == [1, 1, 2] + identities.append({row["lifecycle_id"] for row in got}) + assert not identities[0] & identities[1] + assert hashes(root) == before diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py new file mode 100644 index 0000000..e2d1b0d --- /dev/null +++ b/tests/core/test_dataset_scale_acceptance.py @@ -0,0 +1,349 @@ +"""Default-budget scale gates, separated from the small differential suite. + +Synthetic SQL tests do not claim producer coverage. The final test imports a +trusted test-owned snapshot through a non-editable installed CLI/exporter, then +uses that installation's CLI and Python API. Supply PT_SNAP_ACCEPTANCE_WHEEL to +run it; this is not live PyTorch/NPU capture or upstream msinsight GUI acceptance. +""" + +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +import time +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.dataset_contract import ACTION_NAMES, BLOCK_STATES +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.msinsight_export import DEFAULT_CAPACITY +from tests.core.dataset_lifecycle_fixtures import SCHEMA +from tests.snapshot.test_sharded_replay import block, event, segment, write_source +from tests.test_dataset_focus import hashes + +pytestmark = pytest.mark.slow + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def scale_dataset(root, count, capacity, stack): + """Bounded occupancy, repeated addresses, real lifecycle-window membership.""" + (root / "device_0").mkdir(parents=True) + slices = [] + for index, lo in enumerate(range(0, count, capacity)): + hi = min(count, lo + capacity) - 1 + file = f"device_0/slice_{index:05d}.db" + with closing(sqlite3.connect(root / file)) as conn, conn: + conn.executescript(SCHEMA) + conn.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + [("trace_entry_0", "action", str(i), name) for i, name in enumerate(ACTION_NAMES)] + + [("block_0", "state", str(i), name) for i, name in BLOCK_STATES], + ) + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,?,?,?,?,?,?,?,?)", + ( + ( + at, + (4, 5, 6, 7)[at % 4], + 1000 if at % 4 < 3 else 0, + 16 if at % 4 < 3 else 0, + 0, + 16 if at % 4 == 0 else 0, + 16 if at % 4 < 2 else 0, + 4096, + stack, + ) + for at in range(lo, hi + 1) + ), + ) + # Include a crossing lifecycle even when capacity is not a multiple of four. + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", + ( + ( + alloc, + 1000, + 16, + 16, + 0 if alloc + 1 < lo else 1, + alloc, + alloc + 2 if alloc + 2 < count else -1, + ) + for alloc in range(lo - lo % 4, hi + 1, 4) + if alloc + 2 >= lo + ), + ) + conn.execute( + "INSERT INTO trace_entry_0 VALUES (-1,2,1000,4096,0,999999,999999,999999,'boundary')" + ) + for metric in ("allocated", "active", "reserved"): + conn.execute(f"CREATE INDEX trace_{metric} ON trace_entry_0 ({metric})") + slices.append( + {"index": index, "startEventId": lo, "endEventId": hi, "file": file, "ready": True} + ) + (root / "manifest.json").write_text( + json.dumps( + { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/synthetic/no-pickle", + "cacheHash": "", + "eventsPerSlice": capacity, + "devices": { + "0": { + "eventCount": count, + "sliceCount": len(slices), + "readySlices": list(range(len(slices))), + "slices": slices, + } + }, + } + ) + ) + return root + + +@pytest.fixture(scope="module") +def default_scale(tmp_path_factory): + assert DEFAULT_CAPACITY == 500_000 + budget = QueryBudget(None, time.monotonic()) + assert budget.max_work_rows == 100_000 and budget.max_work_bytes == 64 * 1024 * 1024 + return scale_dataset( + tmp_path_factory.mktemp("default-scale") / "dataset", + 2 * DEFAULT_CAPACITY, + DEFAULT_CAPACITY, + "short", + ) + + +@pytest.mark.parametrize( + "template,params,total", + [ + ("memory_peak", {}, 1), + ("allocator_gap", {}, 1), + ("event", {"min_id": 0, "limit": 5}, 1_000_000), + ("allocation", {"min_id": 0, "limit": 5}, 1_000_000), + ("block", {"limit": 5}, 250_000), + ("leak_detection", {"limit": 5}, 0), + ("callstack_analysis", {"min_count": 1}, 1), + ], +) +def test_two_default_shards_return_global_results_with_real_budgets( + default_scale, template, params, total +): + before = hashes(default_scale) + with SnapshotAnalyzer(default_scale) as analyzer: + result = analyzer.execute_query(template, params, exact_total=True) + assert result["total"] == total and result["total_is_exact"] + assert result["returned"] == min(total, 5) + if template == "memory_peak": + assert result["rows"][0]["peak_active"] == 16 + assert result["rows"][0]["peak_active_event_id"] == 0 + elif template == "callstack_analysis": + assert result["rows"][0]["alloc_count"] == 1_000_000 + assert result["rows"][0]["total_size"] == 12_000_000 + assert hashes(default_scale) == before + + +def test_real_default_row_budget_still_rejects_unbounded_materialization(default_scale): + with ( + SnapshotAnalyzer(default_scale) as analyzer, + pytest.raises(QueryExecutionError, match="budget"), + ): + analyzer.execute_query("event", {"min_id": 0, "limit": 100_001}) + + +def test_long_stack_aggregate_and_small_page_succeed_but_full_output_is_byte_bounded(tmp_path): + root = scale_dataset(tmp_path / "long-stacks", 10_000, 5_000, "x" * 8192) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + peak = analyzer.execute_query("memory_peak")["rows"][0] + assert peak["peak_active"] == 16 + grouped = analyzer.execute_query("callstack_analysis", {"min_count": 1}) + assert grouped["rows"][0]["alloc_count"] == 10_000 + page = analyzer.execute_query("event", {"min_id": 0, "limit": 5}, exact_total=True) + assert page["returned"] == 5 and page["total"] == 10_000 + with pytest.raises(QueryExecutionError, match="budget"): + analyzer.execute_query("event", {"min_id": 0, "limit": -1}) + assert hashes(root) == before + + +def run(command, root, env): + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, check=False) + assert result.returncode == 0, f"{command}\n{result.stdout}\n{result.stderr}" + return result + + +@pytest.fixture +def installed_runtime(tmp_path): + wheel_value = os.environ.get("PT_SNAP_ACCEPTANCE_WHEEL") + if not wheel_value: + pytest.skip("Set PT_SNAP_ACCEPTANCE_WHEEL to test a non-editable installed artifact") + wheel = Path(wheel_value).resolve() + assert wheel.is_file() and wheel.suffix == ".whl" + target = tmp_path / "installed" + env = os.environ.copy() + env.pop("PYTHONHOME", None) + env.pop("PT_SNAP_DB_PATH", None) + env.pop("PT_SNAP_QUERY_TIMEOUT", None) + env["PYTHONPATH"] = str(target) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-deps", + "--no-index", + "--target", + str(target), + str(wheel), + ], + tmp_path, + env, + ) + probe = run( + [sys.executable, "-c", "import pt_snap_cli; print(pt_snap_cli.__file__)"], tmp_path, env + ) + module = Path(probe.stdout.strip()).resolve() + assert module.is_relative_to(target) + cli = target / "bin" / "pt-snap" + assert cli.is_file() + return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() + + +def test_installed_trusted_producer_default_export_cli_and_api( + tmp_path, installed_runtime, record_property +): + cli, env, module, wheel_hash = installed_runtime + # Original, inspectable test-owned data. No committed or external pickle is loaded. + # 500000 ends after free_requested; free_completed is in the next shard. + count = DEFAULT_CAPACITY + 2 + traces = [event(("alloc", "free_requested", "free_completed")[i % 3]) for i in range(count)] + data = {"segments": [segment(0, 64, [block(1000, 16, 16)])], "device_traces": [traces]} + source = write_source(tmp_path, data) + # The installed importer owns its own full input load; do not retain a second + # 500002-event graph in the test process while it runs. + del traces, data + source_hash = hashlib.sha256(source.read_bytes()).hexdigest() + started = time.perf_counter() + imported = json.loads( + run( + [ + sys.executable, + str(cli), + "import", + str(source), + "--format", + "msinsight", + "--output-dir", + str(tmp_path / "exported"), + "--no-focus", + "--json", + ], + tmp_path, + env, + ).stdout + ) + import_seconds = time.perf_counter() - started + root = Path(imported["db_path"]) + manifest = json.loads((root / "manifest.json").read_text()) + assert manifest["eventsPerSlice"] == DEFAULT_CAPACITY + assert manifest["devices"]["0"]["eventCount"] == count + slices = manifest["devices"]["0"]["slices"] + assert [(s["startEventId"], s["endEventId"]) for s in slices] == [ + (0, 499_999), + (500_000, 500_001), + ] + before = hashes(root) + # Inspect actual producer membership before running any query. + for item in slices: + with closing(sqlite3.connect(root / item["file"])) as conn: + assert ( + conn.execute( + "SELECT COUNT(*) FROM block_0 WHERE allocEventId>?", (item["endEventId"],) + ).fetchone()[0] + == 0 + ) + assert ( + conn.execute( + "SELECT COUNT(*) FROM block_0 WHERE freeEventId>=0 AND freeEventId installed pt-snap replay/export -> installed CLI/API; no live collector or upstream GUI", + "module": str(module), + "wheel_sha256": wheel_hash, + "source_sha256": source_hash, + "events": count, + "events_per_slice": DEFAULT_CAPACITY, + "slices": 2, + "import_seconds": import_seconds, + "result": result, + } + record_property("dataset_acceptance", json.dumps(evidence)) + (tmp_path / "acceptance.json").write_text(json.dumps(evidence, indent=2)) diff --git a/tests/test_interop_acceptance.py b/tests/test_interop_acceptance.py index ef87be7..6c7d796 100644 --- a/tests/test_interop_acceptance.py +++ b/tests/test_interop_acceptance.py @@ -45,7 +45,7 @@ def test_all_eleven_contracts_against_independent_physical_reference(tmp_path, d result = compare_artifact(root, device, tmp_path / "reference.db") assert set(result["templates"]) == set(DATASET_SUPPORT) assert result["real_events"] == 16 and result["canonical_lifecycles"] == 10 - assert result["raw_block_rows"] == 80 and result["repeated_block_rows"] == 70 + assert result["raw_block_rows"] == 61 and result["repeated_block_rows"] == 51 assert result["GUI"] == "human-deferred/pending/not-run" assert result["original_producer"] == "not-run by this runner" @@ -73,8 +73,10 @@ def test_reference_is_exclusive_and_ambiguous_address_is_not_identity(tmp_path): assert reference.read_bytes() == b"do not replace" oracle = physical_oracle(root, 0) assert len([b for b in oracle["blocks"].values() if b["address"] == 100]) == 2 - with closing(sqlite3.connect(root / "device_0/slice_00007.db")) as conn, conn: - conn.execute("UPDATE block_0 SET size=size+1 WHERE id=0") + # Lifecycle 0 completes at event 10: its last real observation is slice 5. + with closing(sqlite3.connect(root / "device_0/slice_00005.db")) as conn, conn: + changed = conn.execute("UPDATE block_0 SET size=size+1 WHERE id=0") + assert changed.rowcount == 1 with pytest.raises(ValueError, match="Ambiguous"): physical_oracle(root, 0)