From ff2392f381f405a259fbb158f914769473739e7f Mon Sep 17 00:00:00 2001 From: codeasier Date: Tue, 6 Oct 2026 01:14:39 +0800 Subject: [PATCH] fix: resolve batched cross-shard event attribution (#203) --- README.md | 6 +- README_zh.md | 5 +- docs/en/querying.md | 85 +++ docs/en/quickstart.md | 9 +- docs/en/sharded-snapshotdb.md | 22 +- docs/zh/querying.md | 66 ++ docs/zh/quickstart.md | 8 +- docs/zh/sharded-snapshotdb.md | 16 +- src/pt_snap_cli/core/dataset_attribution.py | 263 +++++++ src/pt_snap_cli/core/dataset_resolver.py | 17 +- src/pt_snap_cli/core/dataset_sources.py | 310 ++++++++ src/pt_snap_cli/core/models.py | 1 + src/pt_snap_cli/core/query_service.py | 84 ++- src/pt_snap_cli/core/report_service.py | 49 +- .../query/templates/basic/event.yaml | 12 + .../active_memory_callstack_at_event.yaml | 16 +- .../statistical/active_blocks_at_event.yaml | 20 + tests/core/test_dataset_attribution.py | 700 ++++++++++++++++++ tests/test_contract_cli_api.py | 53 ++ 19 files changed, 1705 insertions(+), 37 deletions(-) create mode 100644 src/pt_snap_cli/core/dataset_attribution.py create mode 100644 src/pt_snap_cli/core/dataset_sources.py create mode 100644 tests/core/test_dataset_attribution.py diff --git a/README.md b/README.md index 04b29db..13e038b 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,11 @@ Nonmatching recognized targets require `--force`; unknown targets are preserved. Publication/focus failures are compensated, with explicit recovery evidence if rollback fails (force is not a single crash-atomic swap). See the [native import guide](docs/en/quickstart.md#optional-import-a-native-sharded-dataset). -Only bounded dataset `event` queries are currently supported. +Dataset `event` addressing and point-event `active_blocks_at_event` / +`active_memory_callstack_at_event` share batched cross-shard alloc/free sources, +including original artifacts without extensions. Dataset-global peak/list/leak +queries and `report peak-memory` remain explicitly unsupported. See +[dataset attribution and coverage](docs/en/querying.md#dataset-point-event-attribution). Explicit `pt-snap import snapshot.pkl --format msinsight --json` produces `snapshot.pkl.msinsight/` for the fixed msinsight revision's **same original diff --git a/README_zh.md b/README_zh.md index cf2dd8e..7eaf3bd 100644 --- a/README_zh.md +++ b/README_zh.md @@ -56,7 +56,10 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices 默认单库导入不变;复用校验全部片及内容/选项/语义版本。不匹配的已识别目标须 `--force`,未知目标保留。发布/focus 失败会补偿,回滚自身失败则保留并报告恢复证据 (force 不是单次崩溃原子 swap)。详见 -[原生分库导入](docs/zh/quickstart.md#可选导入原生分库数据集);当前仅支持有界数据集 `event` 查询。 +[原生分库导入](docs/zh/quickstart.md#可选导入原生分库数据集)。数据集 `event` 寻址及定点 +`active_blocks_at_event` / `active_memory_callstack_at_event` 共用跨片 alloc/free 批量来源, +原版无扩展产物也适用。数据集全局 peak/list/leak 及 `report peak-memory` 仍明确不支持。 +详见[数据集定点归因与覆盖](docs/zh/querying.md#数据集定点事件归因)。 显式 `pt-snap import snapshot.pkl --format msinsight --json` 生成 `snapshot.pkl.msinsight/`,对齐固定 msinsight 版本的**同一原始 pickle + 邻接缓存**入口。 diff --git a/docs/en/querying.md b/docs/en/querying.md index 074496f..7dc54be 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -78,6 +78,91 @@ group identity. Missing/static/preexisting groups have no captured allocation stack to retrieve; omit `stack_bytes` on the original query to see their full display labels. Results are read-only and summaries do not change the database. +## Dataset point-event attribution + +A complete compatibility-v1 or native-v2 directory/manifest supports `event` +addressing plus `active_blocks_at_event` and `active_memory_callstack_at_event` +with a real `event_id`. The latter two route one device's actual event and use +its shard's complete lifecycle observations **after** that forward event: +`allocEventId <= E` and completed `freeEventId > E` (or `-1`). `free_requested` +does not remove active memory. Negative boundary rows represent the instant +before a shard's first event and are not query endpoints. Native sparse IDs are +not renumbered: `event(id=gap)` may return no rows inside an addressed slice; +active attribution at a nonexistent ID fails instead of inventing event 0. + +```bash +pt-snap query '' --device 0 --template-use active_memory_callstack_at_event --params '{"event_id":700,"top_n":20}' --json +pt-snap query '' --device 0 --template-use active_blocks_at_event --params '{"event_id":700,"limit":10}' --exact-total --json +``` + +Replace the path/device/event with actual values. Foreign alloc/free events and +full v1 inline or v2 local-ID stacks are resolved **before** grouping/top_n, in +batches of at most 256 distinct event IDs per owning shard. No foreign real rows +are inserted. Original artifacts need neither metadata nor reference tables. +Lifetimes use dataset generation/device/alloc-event or stable negative identity, +not addresses; `state_scope=slice_observation` warns that stored `state` is not +necessarily the pending-free state at E. Block rows expose `allocation_source`, +`free_source`, their statuses and `lifecycle_id`. Missing dynamic sources retain +bytes and dynamic category, separate from static/preexisting unknown history. +`free=-1` means live **or unknown**, not a proven leak. + +Dataset groups always expose full `source_stack_id`/`stack_id`, `stack_kind` and +`stack_event_id`. Default no-extension `event` rows keep their original nine +values; source/frame row fields appear with `stack_bytes>=0` or a recognized +ordered-frame declaration. `scope.source_coverage` is still supplied. Both dataset +layouts use full-text canonical identity; v2 looks up the owning slice's actual +local ID, returned as source `local_stack_id`/`slice_index` provenance, not a +cross-shard grouping key. Covered ordered arrays use only exact raw-frame identity, +independent of formatted text availability; `text_kind` reports that availability +separately. Explicitly covered arrays (including empty arrays) are captured raw +evidence. Groups prefer an available captured-text representative for retrieval. Nonempty whitespace and a captured literal `[missing callstack]` +are not missing. Never merge by local IDs or shortened display labels. `stack_bytes` +shortens display only; use `event(id=stack_event_id)` on the same dataset/device +for full text. Ordered raw frames, when explicitly covered, also participate in +identity; no text parsing reconstructs frames. + +`scope.source_coverage` is independent of `has_more`/`truncated`/exact row totals: +active-query coverage describes **all filtered blocks before top_n/paging**, with +active/dynamic bytes, resolved allocation/free counts, captured dynamic bytes, +unknown/preexisting counts, ordered-frame coverage and degradation. For `event`, +its coverage scope is only `returned_events`. A complete manifest/range does not +prove complete historical sources or frames. Percentages still use included +bytes **after dynamic top_n but before max_rows**, not the dataset active counter. +Exact totals ignore row/rank caps, but do not change that percentage denominator. + +One call shares its deadline across source batches, grouping and exact totals. +Manifest validation/hash and Context setup elapsed time are charged, but synchronous +filesystem I/O/validation is checked at phase boundaries, not hard-preempted; no +OS-level time guarantee is claimed. SQL uses cleared-on-exit progress handlers. +The Context LRU remains bounded (default four) and borrowed caches remain caller-owned. +No merged temporary database is created. `ReportService.event_attribution(E, ...)` +reuses this same query path for a supplied event. Dataset-global peak selection, +`report peak-memory`, cross-slice event ranges, arbitrary SQL/global list/leak/group +merging remain explicitly unsupported until their own contracts are implemented. + +### Optional ordered-frame reader contract + +This reader-only pt-snap extension is new, not an existing original-producer +feature. Current native/compatible writers emit **no** ordered-frame tables. +Declare `extensions.ptSnapOrderedFrames={"version":1}` in the manifest and use +isolated actual tables in each covered source shard: + +- `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` +- `pt_snap_frame(eventId INTEGER, frameIndex INTEGER, frameJson TEXT, PRIMARY KEY(eventId,frameIndex))` + +Raw objects must be finite standard JSON, with nesting depth at most 128; deeper +or nonfinite/unparseable input degrades safely, including recursion failures. +Each covered real event needs nonnegative integer `frameCount`, exactly indices +`0..frameCount-1`, and JSON **objects** preserving all raw typed fields. Order and +repeated frames are retained; empty arrays require an explicit count zero row. +Only recognized version/schema and valid per-event coverage produce `frames` +with `frames_status=ordered`. Unknown version, missing schema/rows, invalid JSON +or incomplete coverage degrade to `frames=null`, `frames_status=text_only`; +formatted text is never split/reversed into raw frames. P0/overview's dataset-wide +`structured_frames=false` remains a conservative validation capability, not a +claim about the scoped source payload. Tree rendering and GUI acceptance are not +provided by this extension interface. + ## Query Templates Templates are organized into three categories. Use `pt-snap capabilities --json` for the full catalog (CLI version, every template contract, and bundled skills), or `pt-snap query --list` to see names and descriptions. Filter with `--category`. diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index aa90d08..89c2ea1 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -96,8 +96,10 @@ itself fails, recovery paths are preserved and explicitly reported. Do not delet them before inspecting the evidence. Arbitrary I/O failures, concurrent writers and crashes do not have unconditional atomic restoration guarantees. -Dataset focus currently supports only bounded `event` addressing; it does not -silently run peak/lifecycle/aggregate queries over one shard. See the +Dataset focus supports bounded `event` addressing and point-event active block/ +callstack attribution with shared cross-shard sources. Dataset-global peak/list/leak +aggregates remain explicit errors, not one-shard approximations. See +[point-event coverage](querying.md#dataset-point-event-attribution) and the [native manifest contract](sharded-snapshotdb.md#published-native-dataset-p2). `SnapshotAnalyzer` remains analysis-only, without import/split methods. @@ -123,7 +125,8 @@ Base tables are physical inline-v1 tables; metadata has import format `1`. Callstack text is copied once **per event**, not once per distinct stack, so disk and temporary conversion space can grow substantially. Native interning remains unchanged. No ordered structured-frame capability is invented from formatted text. -Only bounded `event` dataset queries are currently available. +Bounded `event` and point-event active block/callstack dataset queries are +available; global aggregate/report selection remains unsupported. Compatible publication is **no-replace**: external/original/user caches are never overwritten or deleted, even under `--force`. A complete, attested, identical diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index d24c1f9..9397bec 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -170,7 +170,8 @@ which P0 does not implement. Text stacks cannot substitute for these guarantees. There is no global event order across devices. Boundary IDs cannot be range endpoints. P0 `DatasetValidation.query_execution=False` describes validation, not execution. P1 accepts complete manifests/directories through the resolver -below; cross-slice execution remains deferred. +below; general cross-slice aggregation remains deferred; point-event source +resolution is described in the [querying guide](querying.md#dataset-point-event-attribution). ```python from pt_snap_cli.core.dataset_contract import QueryScope, validate_dataset @@ -218,16 +219,22 @@ with SnapshotAnalyzer(Path("/capture/snapshot.pkl.msinsight"), device_id=0) as a print(event["scope"], page["total_is_exact"]) ``` -Only the `event` template is currently supported through dataset focus: a real ID +The `event` template supports bounded addressing through dataset focus: a real ID routes to its containing slice; an inclusive `min_id/max_id` range must fit one slice; `--slice`/`slice_index` lists that slice's real events. Negative IDs are not whole-dataset selectors. Output `scope` identifies the actual DB/device/slice, restricted real interval, dataset fingerprint and boundary exclusion. Totals and pagination apply to this scope, **not** to an implicitly truncated full dataset. Caller parameter defaults in CLI `effective_params` are additionally constrained -by `scope`. Cross-slice ranges, unbounded multi-slice queries and aggregate or -lifecycle templates fail explicitly; they never silently choose the first/latest -slice. Direct single-DB v1/v2 queries retain their existing semantics. +by `scope`. Point-event `active_blocks_at_event` and +`active_memory_callstack_at_event` now resolve alloc/free/full stack sources in +bounded batches before grouping/ranking, including original no-extension artifacts. +See [point-event attribution](querying.md#dataset-point-event-attribution) for +lifecycle identity, separate source/frame coverage, and the optional versioned +ordered-frame reader interface. Dataset-global aggregates, peak/list/leak queries, +cross-slice event ranges and unbounded multi-slice event queries still fail +explicitly; they never silently choose the first/latest slice. Direct single-DB +v1/v2 queries retain their existing semantics. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` @@ -302,8 +309,9 @@ alias/sidecar/persistent-WAL checks and bounded borrowed-cache ownership. Focus, metadata, overview and single-slice `event` addressing work without the source pickle. Native metadata is available dataset-wide; external compatibility metadata remains unavailable unless the recognized compatible attestation below is present. -Native mode implies no migration, repair, cross-slice aggregation/lifecycle execution, -compatible export, salted hash or GUI acceptance. P0 schema/manifest semantics are +Native mode implies no migration, repair, dataset-global aggregation/peak/list/leak, +compatible export, salted hash or GUI acceptance. Point-event source resolution is +available through the shared query path described above. P0 schema/manifest semantics are unchanged; the compatibility-only finalized-WAL transport is described above. Publication and force/focus compensation are described in the diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 6ad5ad8..62c4a54 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -71,6 +71,72 @@ pt-snap query '' --device 0 --template-use event --params '{"id":123}' 分组没有可取回的采集分配栈;在原查询中省略 `stack_bytes` 可查看完整显示标签。 查询保持只读,摘要不会改动数据库。 +## 数据集定点事件归因 + +完整兼容 v1 或原生 v2 目录/manifest 支持 `event` 寻址,以及接受真实 `event_id` 的 +`active_blocks_at_event`、`active_memory_callstack_at_event`。后两者按设备路由实际事件, +从所在片完整生命周期观察计算正向事件**之后**的集合:`allocEventId <= E` 且完成释放 +`freeEventId > E`(或 `-1`)。`free_requested` 不移除 active 内存。负边界行表示片内 +首事件之前,不是定点端点。原生稀疏 ID 不改号:片内 `event(id=gap)` 可返回空行, +不存在事件的 active 归因则明确失败,不虚构 event 0。 + +```bash +pt-snap query '' --device 0 --template-use active_memory_callstack_at_event --params '{"event_id":700,"top_n":20}' --json +pt-snap query '' --device 0 --template-use active_blocks_at_event --params '{"event_id":700,"limit":10}' --exact-total --json +``` + +请替换实际路径/设备/事件。片外 alloc/free 事件及完整 v1 内联或 v2 局部 ID 栈在 +分组/top_n **之前**解析,按实际来源片每批至多 256 个不同事件 ID。不补入外片真实行。 +原版产物无需 metadata 或引用表。生命周期用数据集 generation/设备/分配事件或稳定 +负身份,不用地址;`state_scope=slice_observation` 提醒存储 `state` 不一定是 E 时的 +pending-free 状态。block 行提供 `allocation_source`、`free_source`、状态和 +`lifecycle_id`。缺失动态来源仍保留字节及动态类别,与 static/preexisting 未知历史分开; +`free=-1` 表示存活**或未知**,不证明泄漏。 + +数据集分组始终提供完整 `source_stack_id`/`stack_id`、`stack_kind`、`stack_event_id`。 +无扩展的默认 `event` 行保留原始九个值;`stack_bytes>=0` 或已识别有序 frame 声明时 +才增加来源/frame 行字段,`scope.source_coverage` 仍返回。 +两种数据集布局均按完整文本的 canonical 身份分组;v2 仍按来源片真实局部 ID 查栈, +来源 `local_stack_id`/`slice_index` 是 provenance,不是跨片分组 key。有覆盖的有序数组 +只使用精确原始 frame 身份,与格式化文本是否存在无关;`text_kind` 独立描述文本 +可用性。明确覆盖的数组(含空数组)是已采集原始证据;分组优先选择有文本的代表 +事件以便取回全文。非空 whitespace 和真实采集的字面量 +`[missing callstack]` 不是 missing。禁止按局部 ID 或缩略显示标签合并。`stack_bytes` +只缩短显示;在同一数据集/设备用 `event(id=stack_event_id)` 取回全文。明确覆盖的原始 +有序 frames 也参与身份;不从文本重建 frames。 + +`scope.source_coverage` 独立于 `has_more`/`truncated`/精确行数:active 查询覆盖的是 +**top_n/分页之前全部筛选 block**,包括 active/dynamic 字节、已解析 alloc/free 数、采集 +动态字节、未知/预存数、有序 frame 覆盖及降级。`event` 覆盖仅为 `returned_events`。 +完整 manifest/范围不证明历史来源或 frame 完整。百分比分母仍为动态 top_n **之后、 +max_rows 之前**所含字节,不是数据集 active 计数器。精确总数忽略行数/排名上限,但 +不改变百分比分母。 + +一次调用的来源批次、分组和精确总数共用 deadline。manifest 校验/hash 及 Context +设置耗时计入预算,但同步文件 I/O/校验只在阶段边界检查,不硬抢占,不保证 OS 级 +时间上限。SQL progress handler 退出即清除。Context LRU 有界(默认四个),借用 +cache 仍由调用者所有。不生成合并临时数据库。`ReportService.event_attribution(E, ...)` +对给定事件复用同一路径。数据集全局峰值选择、`report peak-memory`、跨片事件范围、 +任意 SQL/全局 list/leak/group 合并仍明确不支持,等待各自合同实现。 + +### 可选有序 frame reader 合同 + +这是新增的仅 reader pt-snap 扩展,不是原版 producer 已有功能。当前原生/兼容 writer +**不生成**有序 frame 表。manifest 声明 `extensions.ptSnapOrderedFrames={"version":1}`, +并在有覆盖的来源片使用隔离的实际表: + +- `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` +- `pt_snap_frame(eventId INTEGER, frameIndex INTEGER, frameJson TEXT, PRIMARY KEY(eventId,frameIndex))` + +原始 object 必须是有限数字的标准 JSON,嵌套深度至多 128;超深、非有限数或解析 +失败(包括递归错误)安全降级。每个覆盖的真实事件须有非负整数 `frameCount`、恰好 `0..frameCount-1` 索引及保留全部 +原始类型字段的 JSON **object**。保持顺序和重复帧;空数组须有明确 count=0 行。 +只有已识别版本/schema 及有效逐事件覆盖才输出 `frames`、`frames_status=ordered`。 +未知版本、缺失 schema/行、无效 JSON 或覆盖不全降级为 `frames=null`、 +`frames_status=text_only`;不会切分/反转格式化文本伪造原始 frames。P0/overview 的 +数据集级 `structured_frames=false` 仍是保守校验能力,不表示定点来源载荷必然无 frames。 +本接口不提供树渲染或 GUI 验收。 + ## 查询模板 模板分为三个分类。用 `pt-snap capabilities --json` 一次查看完整清单(CLI 版本、全部模板契约和随包 skill),或用 `pt-snap query --list` 查看名称与描述。用 `--category` 过滤。 diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 2bc7dd5..13f6f17 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -80,8 +80,9 @@ pt-snap query --template-use event --slice 0 --json `--no-focus` 完全跳过 focus。回滚自身失败会保留并明确报告恢复路径,检查证据前不要删除。 不保证任意 I/O 故障、并发 writer 或崩溃下无条件原子恢复。 -数据集 focus 当前只支持有界 `event` 寻址,不会静默在一片上运行峰值/生命周期/聚合。 -详见[原生 manifest 合同](sharded-snapshotdb.md#已发布原生数据集p2)。 +数据集 focus 支持有界 `event` 寻址及共用跨片来源的定点 active block/调用栈归因。 +数据集全局 peak/list/leak 聚合仍明确失败,不按一片近似。详见 +[定点覆盖](querying.md#数据集定点事件归因)及[原生 manifest 合同](sharded-snapshotdb.md#已发布原生数据集p2)。 `SnapshotAnalyzer` 仍只负责分析,不新增 import/split 方法。 ### 可选:导出 msinsight 兼容数据集 @@ -102,7 +103,8 @@ workspace。可信 pickle 与完整加载的内存成本仍适用。 基础表是物理内联 v1 表,metadata import format 为 `1`。调用栈文本按**每个事件** 复制,不按不同栈只存一次,磁盘与暂存转换空间可能明显增加;原生去重不变。 -不从格式化文本虚构有序结构化 frames。数据集当前只支持有界 `event` 查询。 +不从格式化文本虚构有序结构化 frames。数据集可用有界 `event` 和定点 active +block/调用栈查询;全局聚合/报告事件选择仍不支持。 兼容发布采用 **no-replace**:外部/原版/用户缓存即使 `--force` 也不覆盖、不删除。 仅完整、已证明且身份相同的 pt-snap 缓存可复用(force 也复用);否则换新 diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index b08827b..03cf235 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -138,7 +138,8 @@ dictionary 的十进制整数 key 以文本存储,使用真实带设备后缀 设备间没有全局事件顺序。边界 ID 不能作为范围端点。P0 的 `DatasetValidation.query_execution=False` 描述校验而非执行。下述 P1 resolver -使 CLI/API 接受完整 manifest/目录;跨片执行仍留后续。 +使 CLI/API 接受完整 manifest/目录;通用跨片聚合仍留后续,定点来源解析见 +[查询指南](querying.md#数据集定点事件归因)。 ```python from pt_snap_cli.core.dataset_contract import QueryScope, validate_dataset @@ -182,12 +183,16 @@ with SnapshotAnalyzer(Path("/capture/snapshot.pkl.msinsight"), device_id=0) as a print(event["scope"], page["total_is_exact"]) ``` -数据集 focus 当前只支持 `event` 模板:真实 ID 路由至所在片,闭区间 `min_id/max_id` +数据集 focus 的 `event` 模板支持有界寻址:真实 ID 路由至所在片,闭区间 `min_id/max_id` 必须落在一个片内,`--slice`/`slice_index` 列出该片真实事件。负 ID 不能作为整份数据集 selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、数据集指纹和边界排除。 总数和分页针对该 scope,**不是**隐式截断的整份数据集。CLI `effective_params` 的调用者 -参数默认值另受 `scope` 约束。跨片区间、无界多片查询、聚合/生命周期模板明确失败,不会 -静默选首片/最新片。直接对单库 v1/v2 查询保持原有语义。 +参数默认值另受 `scope` 约束。定点 `active_blocks_at_event` 和 +`active_memory_callstack_at_event` 现于分组/排名前批量解析 alloc/free/完整栈来源, +原版无扩展产物也适用。生命周期身份、独立来源/frame 覆盖及版本化有序 frame reader +接口详见[定点归因](querying.md#数据集定点事件归因)。数据集全局聚合/peak/list/leak、 +跨片事件区间及无界多片 event 查询仍明确失败,不静默选首片/最新片。直接单库 v1/v2 +查询保留现有语义。 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 @@ -247,7 +252,8 @@ trace 列与单库 v2 一致(`callstackId`,非内联文本),保留 OOM a 最终化引用,保留 P1 打开前的别名/sidecar/持久 WAL 检查及有界借用 cache 所有权。 无需源 pickle 即可 focus、metadata、overview 和单片 `event` 寻址。原生数据集 metadata 为 available,外部兼容数据集为 unavailable(下述可识别兼容证明除外)。原生模式不隐含 -迁移、修复、跨片聚合/生命周期执行、兼容导出、salted hash 或 GUI 验收;P0 schema/manifest +迁移、修复、数据集全局聚合/peak/list/leak、兼容导出、salted hash 或 GUI 验收;定点 +来源解析通过上述共享查询路径可用。P0 schema/manifest 语义不变,仅兼容最终化 WAL 的只读传输修正见上文。 发布、force/focus 补偿见[导入指南](quickstart.md#可选导入原生分库数据集)。完整 manifest diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py new file mode 100644 index 0000000..d1e199b --- /dev/null +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -0,0 +1,263 @@ +"""Dataset point-event lifecycle queries using the shared batched source layer.""" + +from __future__ import annotations + +from typing import cast + +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, EventSource +from pt_snap_cli.core.models import QueryResult +from pt_snap_cli.core.stack_summary import summarize_stacks + +STATIC_LABEL = "[static] allocEventId=-1, freeEventId=-1" +PREEXISTING_LABEL = "[preexisting live] allocEventId=-1" +MISSING_LABEL = "[missing callstack]" + + +def event_attribution( + sources: DatasetSourceResolver, + template: str, + params: dict[str, object], + max_rows: int | None, + exact_total: bool, + slice_index: int | None, + semantics_version: int | None, +) -> QueryResult: + event_id = cast(int, params["event_id"]) + item = sources.target(event_id, slice_index) + path = sources.dataset.root / item.file + blocks = sources.read( + path, + f'SELECT * FROM "block_{sources.device.device_id}" WHERE size>=? ' + "AND (allocEventId=-1 OR allocEventId<=?) AND (freeEventId=-1 OR freeEventId>?) " + + ("AND allocEventId!=-1 " if not params["include_static"] else "") + + "ORDER BY id", + [params["min_size"], event_id, event_id], + ) + references = sources.events( + cast(int, block[field]) + for block in blocks + for field in ("allocEventId", "freeEventId") + if cast(int, block[field]) >= 0 + ) + allocation_events = allocation_stacks = free_events = ordered_blocks = dynamic_bytes = ( + captured_bytes + ) = 0 + dynamic_blocks = preexisting_blocks = 0 + for block in blocks: + sources.budget.remaining() + alloc, free = cast(int, block["allocEventId"]), cast(int, block["freeEventId"]) + category = ( + "dynamic_live_at_event" + if alloc >= 0 + else "static" if free == -1 else "preexisting_live_at_event" + ) + block["category"] = category + block["lifecycle_id"] = f"{sources.namespace}:" + ( + f"alloc:{alloc}" if alloc >= 0 else f"preexisting:{block['id']}" + ) + # The stored state is a replay/slice observation, not necessarily state at E. + block["state_scope"] = "slice_observation" + allocation, completion = references.get(alloc), references.get(free) + # Base schemas can be structurally valid yet have an invalid source action. + # Preserve bytes/category and report the missing proof instead of guessing. + allocation = ( + allocation if allocation is not None and allocation.event["action"] == 4 else None + ) + completion = ( + completion if completion is not None and completion.event["action"] == 6 else None + ) + block["allocation_source"] = allocation.to_dict() if allocation else None + block["free_source"] = completion.to_dict() if completion else None + block["allocation_source_status"] = ( + "unknown_preexisting" if alloc == -1 else "resolved" if allocation else "missing" + ) + block["free_source_status"] = ( + "live_or_unknown" if free == -1 else "resolved" if completion else "missing" + ) + if alloc >= 0: + dynamic_blocks += 1 + dynamic_bytes += cast(int, block["size"]) + allocation_events += allocation is not None + allocation_stacks += allocation is not None and allocation.stack_kind == "captured" + ordered_blocks += allocation is not None and allocation.frames is not None + if allocation is not None and allocation.stack_kind == "captured": + captured_bytes += cast(int, block["size"]) + else: + preexisting_blocks += 1 + free_events += completion is not None + coverage: dict[str, object] = { + "range_complete": True, + "event_position": "after", + "active_blocks": len(blocks), + "active_bytes": sum(cast(int, b["size"]) for b in blocks), + "dynamic_blocks": dynamic_blocks, + "dynamic_bytes": dynamic_bytes, + "allocation_events_resolved": allocation_events, + "allocation_stacks_captured": allocation_stacks, + "allocation_source_complete": allocation_events == dynamic_blocks, + "captured_dynamic_bytes": captured_bytes, + "unknown_preexisting_blocks": preexisting_blocks, + "free_references": sum(cast(int, b["freeEventId"]) >= 0 for b in blocks), + "free_events_resolved": free_events, + "ordered_frame_blocks": ordered_blocks, + "ordered_frames_complete": dynamic_blocks > 0 and ordered_blocks == dynamic_blocks, + "frame_degradation": ( + None + if dynamic_blocks > 0 and ordered_blocks == dynamic_blocks + else "text_only_or_uncovered" + ), + "source_queries": sources.query_count, + } + scope: dict[str, object] = { + "kind": "event", + "device_id": sources.device.device_id, + "slice_index": item.index, + "first_event_id": event_id, + "last_event_id": event_id, + "db_path": str(path), + "dataset_path": str(sources.dataset.root), + "fingerprint": sources.dataset.fingerprint, + "boundary_events_included": False, + "source_coverage": coverage, + } + if template == "active_blocks_at_event": + key = cast(str, params["order_by"]) + blocks.sort(key=lambda b: cast(int, b["id"])) + blocks.sort(key=lambda b: cast(int, b[key]), reverse=params["order_dir"] == "DESC") + all_rows = blocks + rank_full = False + limit = cast(int, params["limit"]) + offset = max(0, cast(int, params["offset"])) + selected = all_rows[offset : offset + limit] if limit >= 0 else all_rows[offset:] + else: + all_rows = _groups(sources, blocks, references) + top_n = cast(int, params["top_n"]) + dynamic = [row for row in all_rows if row["category"] == "dynamic_live_at_event"] + rank_full = top_n >= 0 and bool(dynamic) and len(dynamic) >= top_n + selected = [row for row in all_rows if row["category"] != "dynamic_live_at_event"] + ( + dynamic[:top_n] if top_n >= 0 else dynamic + ) + selected.sort(key=_rank) + denominator = sum(cast(int, row["size_bytes"]) for row in selected) + for row in selected: + row["percent_of_active_blocks"] = ( + round(cast(int, row["size_bytes"]) * 100.0 / denominator, 4) + if denominator + else None + ) + offset = 0 + has_more = len(selected) < len(all_rows) - offset or rank_full + if max_rows is not None and max_rows > 0: + has_more = has_more or len(selected) > max_rows + selected = selected[:max_rows] + if exact_total: + has_more = len(all_rows) > len(selected) + offset + sources.budget.remaining() + return QueryResult( + total=len(all_rows) if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=has_more, + truncated=has_more or offset > 0, + total_is_exact=exact_total or not has_more and offset == 0, + timeout_s=sources.budget.timeout_s, + scope=scope, + ) + + +def _rank(row: dict[str, object]) -> tuple[int, int, str, str]: + return ( + -cast(int, row["size_bytes"]), + -cast(int, row["block_count"]), + cast(str, row["callstack"]), + cast(str, row["source_stack_id"]), + ) + + +def _groups( + sources: DatasetSourceResolver, + blocks: list[dict[str, object]], + references: dict[int, EventSource], +) -> list[dict[str, object]]: + grouped: dict[str, dict[str, object]] = {} + for block in blocks: + sources.budget.remaining() + category = cast(str, block["category"]) + source = references.get(cast(int, block["allocEventId"])) + if block["allocation_source"] is None: + source = None + kind = source.stack_kind if source is not None else "missing" + if category != "dynamic_live_at_event": + kind = category + sourced = ( + source is not None + and category == "dynamic_live_at_event" + and (kind == "captured" or source.frames is not None) + ) + identity = ( + source.stack_id + if sourced and source is not None + else f"{sources.namespace}:category:{kind}" + ) + text = ( + source.event["callstack"] + if kind == "captured" and source is not None and source.text_kind == "captured" + else ( + STATIC_LABEL + if kind == "static" + else PREEXISTING_LABEL if kind == "preexisting_live_at_event" else MISSING_LABEL + ) + ) + group = grouped.setdefault( + identity, + { + "callstack": text, + "category": category, + "block_count": 0, + "size_bytes": 0, + "requested_bytes": 0, + "source_stack_id": identity, + "stack_kind": kind, + "text_kind": source.text_kind if sourced and source else kind, + "stack_id": identity, + "stack_event_id": source.event["id"] if sourced and source else None, + "frames": source.frames if sourced and source else None, + "frames_status": (source.frames_status if sourced and source else "text_only"), + }, + ) + group["block_count"] = cast(int, group["block_count"]) + 1 + group["size_bytes"] = cast(int, group["size_bytes"]) + cast(int, block["size"]) + group["requested_bytes"] = cast(int, group["requested_bytes"]) + cast( + int, block["requestedSize"] + ) + if source is not None and sourced: + if group["text_kind"] != "captured" and source.text_kind == "captured": + group["callstack"] = source.event["callstack"] + group["text_kind"] = "captured" + group["stack_event_id"] = source.event["id"] + elif group["text_kind"] != "captured" or source.text_kind == "captured": + group["stack_event_id"] = min( + cast(int, group["stack_event_id"]), cast(int, source.event["id"]) + ) + rows = list(grouped.values()) + for row in rows: + row["size_gib"] = round(cast(int, row["size_bytes"]) / 1073741824.0, 6) + row["requested_gib"] = round(cast(int, row["requested_bytes"]) / 1073741824.0, 6) + rows.sort(key=_rank) + return rows + + +def summarize_dataset_stacks(rows: list[dict[str, object]], budget: int) -> None: + """Shorten display only; retain the already proved cross-shard identity.""" + identities = [(row["stack_id"], row["stack_kind"]) for row in rows] + for row in rows: + # Raw frames may be captured even when the formatted TEXT is absent. + if not isinstance(row.get("callstack"), str): + row["stack_kind"] = "missing" + summarize_stacks(rows, budget, "v1") + for row, (identity, kind) in zip(rows, identities, strict=True): + row["stack_id"] = identity + row["stack_kind"] = kind diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index 541e529..004d727 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -38,6 +38,7 @@ class ResolvedDataset: root: Path validation: DatasetValidation fingerprint: str + ordered_frames_version: int | None = None @property def device_ids(self) -> list[int]: @@ -91,7 +92,14 @@ def to_dict(self) -> dict[str, object]: "text_callstacks": True, "structured_frames": False, "cross_slice_queries": False, - "supported_queries": ["event: real ID, single-slice range, or explicit slice"], + "supported_queries": [ + "event: real ID, single-slice range, or explicit slice", + "active_blocks_at_event: real point-event with batched alloc/free sources", + "active_memory_callstack_at_event: real point-event attribution before top_n", + ], + "cross_slice_sources": True, + "ordered_frames_reader_version": 1, + "structured_frames_scope": "validation_only; query source coverage is separate", "devices": [ { "device_id": device.device_id, @@ -162,6 +170,11 @@ def inspect(self, path: Path | str) -> ResolvedDataset | None: digest.update(_hash_file(member).encode("ascii")) if before != _hash_file(manifest_path): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") - return ResolvedDataset(root, validation, digest.hexdigest()) + extensions = raw.get("extensions") if isinstance(raw, dict) else None + frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None + version = frames.get("version") if isinstance(frames, dict) else None + return ResolvedDataset( + root, validation, digest.hexdigest(), version if type(version) is int else None + ) except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py new file mode 100644 index 0000000..162bb29 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -0,0 +1,310 @@ +"""Read-only, batched device/event source lookup over a validated dataset. + +No address joins, foreign-event insertion, text-to-frame reconstruction, or retained +source cache. The caller owns the Context LRU and the entire operation's deadline. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import sqlite3 +import time +from bisect import bisect_right +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from pt_snap_cli.context import DatabaseNotFoundError, SchemaVersionError +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_contract import SliceRecord, _unique_object +from pt_snap_cli.core.dataset_files import require_readonly_member +from pt_snap_cli.core.dataset_resolver import ResolvedDataset +from pt_snap_cli.core.errors import ( + DatabaseSchemaError, + InvalidParameterError, + QueryExecutionError, + QueryTimeoutError, +) +from pt_snap_cli.query.executor import QueryExecutionError as ExecutorError +from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorTimeout + +SOURCE_BATCH_SIZE = 256 + + +def _invalid_constant(value: str) -> object: + raise ValueError(f"Non-JSON numeric constant: {value}") + + +def _finite_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError("Non-finite frame number") + return result + + +def _bounded_depth(value: object) -> bool: + pending = [(value, 0)] + while pending: + current, depth = pending.pop() + if depth > 128: + return False + if isinstance(current, dict): + pending.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + pending.extend((item, depth + 1) for item in current) + return True + + +@dataclass(frozen=True) +class QueryBudget: + timeout_s: float | None + started: float + + def remaining(self) -> float | None: + if self.timeout_s is None: + return None + left = self.timeout_s - (time.monotonic() - self.started) + if left <= 0: + raise QueryTimeoutError(f"Query timed out after {self.timeout_s} seconds") + return left + + +@dataclass(frozen=True) +class EventSource: + event: dict[str, object] + slice_index: int + stack_id: str + stack_kind: str + frames: list[dict[str, object]] | None = None + frames_status: str = "text_only" + local_stack_id: int | None = None + + @property + def text_kind(self) -> str: + text = self.event.get("callstack") + return "captured" if isinstance(text, str) and text != "" else "missing" + + def to_dict(self) -> dict[str, object]: + return { + "event": self.event, + "slice_index": self.slice_index, + "local_stack_id": self.local_stack_id, + "stack_id": self.stack_id, + "stack_kind": self.stack_kind, + "text_kind": self.text_kind, + "frames": self.frames, + "frames_status": self.frames_status, + } + + +class DatasetSourceResolver: + """One operation's sources; at most one SQL batch per shard/256 distinct IDs. + + Schema/optional frame checks add a constant number of queries per visited + shard. Connections are sequential and bounded by the borrowed LRU, not blocks. + """ + + def __init__( + self, dataset: ResolvedDataset, device_id: int, cache: ContextCache, budget: QueryBudget + ) -> None: + self.dataset = dataset + self.device = dataset.device(device_id) + self.cache = cache + self.budget = budget + self.query_count = 0 + self._starts = [s.start_event_id for s in self.device.slices] + + @property + def namespace(self) -> str: + return f"dataset:{self.dataset.fingerprint}:device:{self.device.device_id}" + + def owner(self, event_id: int) -> SliceRecord | None: + if type(event_id) is not int or event_id < 0: + return None + index = bisect_right(self._starts, event_id) - 1 + if index < 0: + return None + item = self.device.slices[index] + return item if event_id <= item.end_event_id else None + + def target(self, event_id: int, slice_index: int | None = None) -> SliceRecord: + item = self.owner(event_id) + if item is None or (slice_index is not None and item.index != slice_index): + raise InvalidParameterError("Event ID must be a real ID within the selected scope.") + if event_id not in self.events([event_id]): + raise InvalidParameterError( + f"No real event at ID {event_id}; sparse gaps are not events." + ) + return item + + def read( + self, path: Path, sql: str, values: list[object] | None = None + ) -> list[dict[str, object]]: + self.budget.remaining() + require_readonly_member( + self.dataset.root, path, immutable=self.dataset.callstack_layout == "v1" + ) + try: + context = self.cache.get( + path, + generation=self.dataset.fingerprint, + immutable=self.dataset.callstack_layout == "v1", + ) + self.query_count += 1 + return QueryExecutor(context).execute(sql, values, timeout_s=self.budget.remaining()) + except ExecutorTimeout as exc: + raise QueryTimeoutError( + f"Query timed out after {self.budget.timeout_s} seconds" + ) from exc + except ExecutorError as exc: + raise QueryExecutionError(str(exc)) from exc + except (DatabaseNotFoundError, SchemaVersionError, sqlite3.DatabaseError) as exc: + raise DatabaseSchemaError(str(exc)) from exc + + def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: + grouped: dict[int, list[int]] = defaultdict(list) + for event in sorted(set(event_ids)): + self.budget.remaining() + item = self.owner(event) + if item is not None: + grouped[item.index].append(event) + result: dict[int, EventSource] = {} + for index, ids in grouped.items(): + item = self.device.slices[index] + path = self.dataset.root / item.file + trace = f'"trace_entry_{self.device.device_id}"' + for start in range(0, len(ids), SOURCE_BATCH_SIZE): + batch = ids[start : start + SOURCE_BATCH_SIZE] + placeholders = ",".join("?" for _ in batch) + sql = ( + f"SELECT t.*, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + if self.dataset.callstack_layout == "v2" + else f"SELECT t.* FROM {trace} t" + ) + rows = self.read( + path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) + ) + for row in rows: + event_id = cast(int, row["id"]) + text = row.get("callstack") + captured = isinstance(text, str) and text != "" + local_id = cast(int | None, row.pop("callstackId", None)) + # V2 IDs select text WITHIN the owning DB; they are provenance, + # not dataset grouping keys. Global interning equivalence uses + # the full captured text, never a shortened display prefix. + identity = ( + "text:sha256:" + + hashlib.sha256( + (text if isinstance(text, str) else "").encode("utf-8") + ).hexdigest() + ) + result[event_id] = EventSource( + row, + index, + ( + f"{self.namespace}:{identity}" + if captured + else f"{self.namespace}:category:missing" + ), + "captured" if captured else "missing", + local_stack_id=local_id, + ) + self._attach_frames(path, ids, result) + self.budget.remaining() + return result + + def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: + # This is a NEW, explicit read contract, not a guessed original-producer + # schema. Existing pt_snap_block_reference has formatted text only. + if self.dataset.ordered_frames_version != 1: + return + expected = { + "pt_snap_frame_coverage": [("eventId", "INTEGER", 1), ("frameCount", "INTEGER", 0)], + "pt_snap_frame": [ + ("eventId", "INTEGER", 1), + ("frameIndex", "INTEGER", 2), + ("frameJson", "TEXT", 0), + ], + } + for table, columns in expected.items(): + kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = self.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + return # Unknown/absent schemas never imply precision. + for start in range(0, len(ids), SOURCE_BATCH_SIZE): + batch = ids[start : start + SOURCE_BATCH_SIZE] + placeholders = ",".join("?" for _ in batch) + coverage = self.read( + path, + f"SELECT * FROM pt_snap_frame_coverage WHERE eventId IN ({placeholders})", + list(batch), + ) + frames = self.read( + path, + f"SELECT * FROM pt_snap_frame WHERE eventId IN ({placeholders}) ORDER BY eventId,frameIndex", + list(batch), + ) + by_event: dict[int, list[dict[str, object]]] = defaultdict(list) + for row in frames: + by_event[cast(int, row["eventId"])].append(row) + for row in coverage: + event = cast(int, row["eventId"]) + count = row["frameCount"] + entries = by_event[event] + if ( + event not in sources + or type(count) is not int + or count < 0 + or len(entries) != count + ): + continue + if any( + type(entry["frameIndex"]) is not int or not isinstance(entry["frameJson"], str) + for entry in entries + ) or [entry["frameIndex"] for entry in entries] != list(range(count)): + continue + try: + decoded = [ + json.loads( + cast(str, entry["frameJson"]), + object_pairs_hook=_unique_object, + parse_constant=_invalid_constant, + parse_float=_finite_float, + ) + for entry in entries + ] + if not all(isinstance(frame, dict) for frame in decoded) or not _bounded_depth( + decoded + ): + continue + # Canonical raw arrays retain order, duplicates and typed fields. + # Text formatting and shard-local IDs are only provenance here. + digest = hashlib.sha256( + json.dumps( + decoded, + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + ).hexdigest() + except (ValueError, TypeError, RecursionError, OverflowError): + continue + old = sources[event] + sources[event] = EventSource( + old.event, + old.slice_index, + f"{self.namespace}:frames:sha256:{digest}", + "captured", + decoded, + "ordered", + old.local_stack_id, + ) diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index 71030c3..e59f7ea 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -127,6 +127,7 @@ class PeakMemoryReport: percent_denominator: str = "included_bytes" active_bytes_at_event: int | None = None coverage_percent: float | None = None + source_coverage: dict[str, object] | None = None @dataclass(frozen=True) diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index cf03bb7..6c4c0a5 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -9,8 +9,10 @@ from pt_snap_cli.context import Context, DatabaseNotFoundError, SchemaVersionError from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_attribution import event_attribution, summarize_dataset_stacks from pt_snap_cli.core.dataset_contract import QueryScope from pt_snap_cli.core.dataset_resolver import DatasetResolver, ResolvedDataset +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget from pt_snap_cli.core.errors import ( DatabaseMissingError, DatabaseSchemaError, @@ -206,6 +208,9 @@ def execute_query( timeout_s: float | None = None, slice_index: int | None = None, ) -> QueryResult: + timeout_s = resolve_query_timeout(timeout_s) + started = time.monotonic() + budget = QueryBudget(timeout_s, started) resolved = self._focus_service.resolve_focus( explicit_db_path=db_path, explicit_device_id=device_id, @@ -215,10 +220,39 @@ def execute_query( raise FocusNotConfiguredError("No database path specified and no database configured.") dataset = DatasetResolver().inspect(resolved.db_path) + budget.remaining() scope: dict[str, object] | None = None query_params = params or {} + sources: DatasetSourceResolver | None = None if dataset is not None: selected = dataset.device(device_id if device_id is not None else resolved.device_id) + sources = DatasetSourceResolver( + dataset, selected.device_id, self._context_cache, budget + ) + if template in ("active_blocks_at_event", "active_memory_callstack_at_event"): + template_obj = get_query(template) + if template_obj is None: + raise TemplateNotFoundError(f"Template '{template}' not found") + try: + validated = template_obj.validate_params(query_params) + except (TypeError, ValueError) as exc: + raise TemplateRenderError(str(exc)) from exc + if slice_index is not None: + dataset.paths(QueryScope("slice", selected.device_id, slice_index)) + result = event_attribution( + sources, + template, + validated, + max_rows, + exact_total, + slice_index, + template_obj.semantics_version, + ) + stack_bytes = validated.get("stack_bytes") + if isinstance(stack_bytes, int) and stack_bytes >= 0: + summarize_dataset_stacks(result.rows, stack_bytes) + budget.remaining() + return result member, query_params, scope = self._dataset_query( dataset, selected.device_id, template, query_params, slice_index ) @@ -232,8 +266,6 @@ def execute_query( ctx = self._validated_context(resolved.db_path) target_device = self._resolve_device_id(ctx, resolved.device_id, device_id) executor = self._get_executor(ctx) - timeout_s = resolve_query_timeout(timeout_s) - started = time.monotonic() try: rows, has_more, _applied_limit = executor.execute_template_page( @@ -241,7 +273,7 @@ def execute_query( query_params, device_id=target_device, max_rows=max_rows, - timeout_s=timeout_s, + timeout_s=budget.remaining(), ) offset = _validated_offset(template, query_params) rank_cap = _finite_top_n(template, query_params) @@ -266,17 +298,55 @@ def execute_query( except ExecutorTemplateRenderError as exc: raise TemplateRenderError(str(exc)) from exc except ExecutorQueryTimeoutError as exc: - raise QueryTimeoutError(str(exc)) from exc + raise QueryTimeoutError(f"Query timed out after {timeout_s} seconds") from exc except ExecutorQueryExecutionError as exc: if get_query(template) is None: raise TemplateNotFoundError(f"Template '{template}' not found") from exc raise QueryExecutionError(str(exc)) from exc + if sources is not None and scope is not None: + event_sources = sources.events(row["id"] for row in rows) + # Preserve the original default nine-column event payload. Explicit + # stack summaries or a recognized frame reader add source details. + event_template = get_query(template) + detail_budget = ( + event_template.validate_params(query_params).get("stack_bytes", -1) + if event_template + else -1 + ) + detailed_rows = ( + rows if sources.dataset.ordered_frames_version == 1 or detail_budget >= 0 else [] + ) + for row in detailed_rows: + source = event_sources[row["id"]] + row.update( + { + "source_stack_id": source.stack_id, + "stack_id": source.stack_id, + "stack_kind": source.stack_kind, + "text_kind": source.text_kind, + "stack_event_id": row["id"] if source.stack_kind == "captured" else None, + "frames": source.frames, + "frames_status": source.frames_status, + } + ) + scope["source_coverage"] = { + "range_complete": True, + "event_position": "after", + "coverage_scope": "returned_events", + "events_resolved": len(event_sources), + "ordered_frame_events": sum(s.frames is not None for s in event_sources.values()), + "source_queries": sources.query_count, + } template_obj = get_query(template) if template_obj is not None and "stack_bytes" in template_obj.parameters: stack_bytes = template_obj.validate_params(query_params)["stack_bytes"] if isinstance(stack_bytes, int) and stack_bytes >= 0: - summarize_stacks(rows, stack_bytes, ctx.callstack_layout) + if sources is not None: + summarize_dataset_stacks(rows, stack_bytes) + else: + summarize_stacks(rows, stack_bytes, ctx.callstack_layout) + budget.remaining() return QueryResult( total=total, returned=returned, @@ -304,8 +374,8 @@ def _dataset_query( raise TemplateNotFoundError(f"Template '{template}' not found") if template != "event": raise QueryExecutionError( - "Dataset queries currently support only event addressing; " - "cross-slice aggregation/lifecycle queries are not supported." + "Dataset queries support event addressing and point-event active attribution; " + "dataset-global cross-slice aggregation/list/peak/leak queries are not supported." ) try: validated = template_obj.validate_params(params) diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index 7761eb7..c7e4583 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -4,7 +4,7 @@ from typing import Literal, cast from pt_snap_cli.core.focus_service import FocusService -from pt_snap_cli.core.models import PeakMemoryReport +from pt_snap_cli.core.models import PeakMemoryReport, QueryResult from pt_snap_cli.core.query_service import QueryService PeakMetric = Literal["active", "allocated", "reserved"] @@ -31,6 +31,38 @@ def close(self) -> None: """Release connections owned by this report service; allow later reuse.""" self._query_service.close() + def event_attribution( + self, + event_id: int, + db_path: Path | str | None = None, + device_id: int | None = None, + *, + include_static: bool = True, + limit: int = 20, + stack_bytes: int = -1, + start_dir: Path | None = None, + timeout_s: float | None = None, + ) -> QueryResult: + """Reuse core point-event attribution, also for externally selected peaks. + + Dataset-global peak selection is a separate contract; this method does + not silently choose a local shard peak or invent an event for empty data. + """ + return self._query_service.execute_query( + "active_memory_callstack_at_event", + params={ + "event_id": event_id, + "include_static": include_static, + "min_size": 0, + "top_n": limit, + "stack_bytes": stack_bytes, + }, + db_path=db_path, + device_id=device_id, + start_dir=start_dir, + timeout_s=timeout_s, + ) + def peak_memory_report( self, db_path: Path | str | None = None, @@ -81,11 +113,13 @@ def peak_memory_report( device_id=device_id, start_dir=start_dir, ) - callstack_result = self._query_service.execute_query( - "active_memory_callstack_at_event", - params=attribution_params, - db_path=db_path, - device_id=device_id, + callstack_result = self.event_attribution( + event_id, + db_path, + device_id, + include_static=include_static, + limit=limit, + stack_bytes=stack_bytes, start_dir=start_dir, ) @@ -116,4 +150,7 @@ def peak_memory_report( included_bytes=included_bytes, active_bytes_at_event=active_bytes, coverage_percent=coverage, + source_coverage=cast( + dict[str, object] | None, (callstack_result.scope or {}).get("source_coverage") + ), ) diff --git a/src/pt_snap_cli/query/templates/basic/event.yaml b/src/pt_snap_cli/query/templates/basic/event.yaml index 9b4b9e0..6f53326 100644 --- a/src/pt_snap_cli/query/templates/basic/event.yaml +++ b/src/pt_snap_cli/query/templates/basic/event.yaml @@ -149,6 +149,18 @@ queries: LIMIT {{ limit }} {% if offset > 0 %}OFFSET {{ offset }}{% endif %} output_schema: + - column: source_stack_id + type: str + interpretation_limits: ["Dataset only with stack_bytes>=0 or recognized ordered-frame declaration: full generation/device/canonical text identity for both layouts, or exact raw ordered-frame identity when covered; source owner/local IDs are provenance, not grouping keys; never merge by shortened text or bare local IDs."] + - column: text_kind + type: str + interpretation_limits: ["Dataset source detail only: captured or missing formatted text. Independent of captured ordered raw frames; text availability never changes raw-array identity."] + - column: frames + type: list + interpretation_limits: ["Dataset only with stack_bytes>=0 or recognized ordered-frame declaration: ordered raw JSON objects preserving duplicates, only for explicit recognized version/schema/per-event coverage; NULL is text-only/uncovered, not reconstructed from formatted text."] + - column: frames_status + type: str + interpretation_limits: ["Dataset only with stack_bytes>=0 or recognized ordered-frame declaration: ordered or text_only; scope.source_coverage covers returned_events separately from result truncation."] - column: id type: int - column: action diff --git a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml index baf0eac..8a9c8c7 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml @@ -230,10 +230,22 @@ queries: ) ORDER BY size_bytes DESC, block_count DESC, callstack ASC output_schema: + - column: source_stack_id + type: str + interpretation_limits: ["Dataset only, always present: full generation/device/canonical text identity for both layouts, or exact raw ordered-frame identity when covered; source owner/local IDs are provenance, not grouping keys; resolve sources before grouping/top_n."] + - column: text_kind + type: str + interpretation_limits: ["Dataset only: captured/missing formatted text or special category; independent of raw ordered-array identity. Representative text is preferred when available."] + - column: frames + type: list + interpretation_limits: ["Dataset only: ordered raw JSON objects preserving duplicates, only for explicit recognized version/schema/per-event coverage; NULL means text-only or uncovered, never reconstructed from text."] + - column: frames_status + type: str + interpretation_limits: ["Dataset only: ordered or text_only; scope.source_coverage describes source/frame coverage before ranking/paging independently of truncated."] - column: stack_id type: str interpretation_limits: - - "Opt-in stack_bytes >= 0 only. Database/device-scoped identity: v1 full-text SHA-256, v2 callstackId, or category identity. Missing, static, preexisting and captured literal labels stay distinct. Never merge by display text." + - "Standalone opt-in stack_bytes >= 0 only; datasets always expose canonical content/ordered-frame identity. Standalone database/device-scoped identity: v1 full-text SHA-256, v2 callstackId, or category identity. Missing, static, preexisting and captured literal labels stay distinct. Never merge by display text." - column: stack_kind type: str interpretation_limits: @@ -264,7 +276,7 @@ queries: interpretation_limits: - "Static and preexisting groups use synthetic labels rather than a captured allocation callstack. Dynamic rows use '[missing callstack]' for empty strings, NULL text or IDs, or missing joined allocation events or callstack records." - "All missing dynamic callstacks merge before top_n ranking; bytes, requested bytes, and block counts are retained. Missing attribution does not imply static or preexisting memory." - - "Nonempty callstacks are grouped by text in v1 and callstackId in v2. Whitespace-only text and a captured literal '[missing callstack]' are not missing and do not merge into the missing group. Display labels alone are not unique group identities." + - "Standalone nonempty callstacks are grouped by text in v1 and callstackId in v2. Datasets resolve local v2 IDs then group canonical full text across owners, or exact raw ordered-frame identity when explicitly covered. Whitespace-only text and a captured literal '[missing callstack]' are not missing and do not merge into the missing group. Display labels alone are not unique group identities." - column: category type: str units: text diff --git a/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml b/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml index 227f470..c6bfd94 100644 --- a/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml +++ b/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml @@ -75,7 +75,27 @@ queries: {% if order_dir|upper == 'ASC' %}ASC{% else %}DESC{% endif %}, id ASC LIMIT {{ limit|int }} {% if offset > 0 %}OFFSET {{ offset|int }}{% endif %} + interpretation_limits: + - "Dataset point-event queries resolve alloc/free sources in batches before paging; scope.source_coverage describes all filtered blocks and is separate from row truncation. Stored state is a slice observation, not necessarily state at the target event." output_schema: + - column: lifecycle_id + type: str + interpretation_limits: ["Dataset only: generation/device/original alloc-event or stable negative lifetime identity; never address identity."] + - column: state_scope + type: str + interpretation_limits: ["Dataset only: slice_observation; state is not reconstructed at the requested event."] + - column: allocation_source + type: object + interpretation_limits: ["Dataset only: resolved original event, owning slice, full stack identity and optional covered ordered frames; NULL for unknown/preexisting or missing proof."] + - column: free_source + type: object + interpretation_limits: ["Dataset only: completed-free source, never free_requested; NULL for live/unknown or missing proof."] + - column: allocation_source_status + type: str + interpretation_limits: ["Dataset only: resolved, unknown_preexisting or missing; missing dynamic sources retain bytes/category."] + - column: free_source_status + type: str + interpretation_limits: ["Dataset only: resolved, live_or_unknown or missing; no recorded free does not prove a leak."] - column: id type: int - column: address diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py new file mode 100644 index 0000000..49bde5e --- /dev/null +++ b/tests/core/test_dataset_attribution.py @@ -0,0 +1,700 @@ +"""Real SQLite contracts for point-event attribution; no original producer execution.""" + +import dataclasses +import json +import sqlite3 +import time +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import SOURCE_BATCH_SIZE, DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.core.report_service import ReportService +from tests.core.test_dataset_import import source_options +from tests.test_dataset_focus import hashes, make_dataset + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def case(tmp_path, devices=(0, 2)): + root = make_dataset(tmp_path / "no-extensions", devices=devices, slices=8) + blocks = [ + (0, 100, 90, 85, 0, 0, 10), + (1, 200, 60, 55, 1, 1, 14), + (3, 300, 40, 35, 1, 3, -1), + (4, 400, 30, 25, 1, 4, -1), + (5, 500, 25, 20, 1, 5, -1), + (6, 600, 20, 15, 1, 6, -1), + (7, 700, 15, 10, 1, 7, -1), + (11, 100, 80, 75, 1, 11, -1), + (-20, 800, 10, 5, 1, -1, 12), + (-21, 900, 5, 5, 1, -1, -1), + ] + for device in devices: + for index in range(8): + with ( + closing(sqlite3.connect(root / f"device_{device}/slice_{index:05d}.db")) as conn, + conn, + ): + conn.executemany(f"INSERT INTO block_{device} VALUES (?,?,?,?,?,?,?)", blocks) + for event in range(index * 2, index * 2 + 2): + text = { + 0: "shared:" + "长" * 100, + 1: "shared:" + "长" * 100, + 3: "third", + 4: "[missing callstack]", + 5: " \n", + 6: "", + 7: None, + 11: "reused", + }.get(event, f"free:{event}") + action = ( + 4 + if event in (0, 1, 3, 4, 5, 6, 7, 11) + else 6 if event in (10, 12, 14) else 5 if event == 2 else 7 + ) + active = sum( + b[2] + for b in blocks + if (b[5] == -1 or b[5] <= event) and (b[6] == -1 or b[6] > event) + ) + conn.execute( + f"UPDATE trace_entry_{device} SET action=?, active=?, allocated=?, reserved=1000, callstack=? WHERE id=?", + (action, active, active - (90 if 2 <= event < 10 else 0), text, event), + ) + return root + + +def standalone(root, tmp_path, device=0): + target = tmp_path / f"standalone-{device}.db" + with closing(sqlite3.connect(target)) as out, out: + out.executescript( + 'CREATE TABLE dictionary ("table" TEXT,"column" TEXT,"key" TEXT,"value" TEXT);' + f"CREATE TABLE trace_entry_{device} (id INTEGER PRIMARY KEY, action INTEGER,address INTEGER,size INTEGER,stream INTEGER,allocated INTEGER,active INTEGER,reserved INTEGER,callstack TEXT);" + f"CREATE TABLE block_{device} (id INTEGER PRIMARY KEY,address INTEGER,size INTEGER,requestedSize INTEGER,state INTEGER,allocEventId INTEGER,freeEventId INTEGER);" + ) + for index in range(8): + with closing( + sqlite3.connect( + (root / f"device_{device}/slice_{index:05d}.db").as_uri() + + "?mode=ro&immutable=1", + uri=True, + ) + ) as conn: + rows = conn.execute(f"SELECT * FROM trace_entry_{device} WHERE id>=0").fetchall() + out.executemany( + f"INSERT INTO trace_entry_{device} VALUES (?,?,?,?,?,?,?,?,?)", rows + ) + if index == 0: + out.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + conn.execute("SELECT * FROM dictionary").fetchall(), + ) + out.executemany( + f"INSERT INTO block_{device} VALUES (?,?,?,?,?,?,?)", + conn.execute(f"SELECT * FROM block_{device}").fetchall(), + ) + return target + + +def normalized_groups(rows): + return sorted( + ( + r["callstack"], + r["category"], + r["size_bytes"], + r["requested_bytes"], + r["block_count"], + r["percent_of_active_blocks"], + ) + for r in rows + ) + + +@pytest.mark.parametrize("event", [0, 2, 8, 10, 11, 12, 14, 15]) +def test_original_noextensions_matches_single_db_after_event_and_completed_free(tmp_path, event): + root = case(tmp_path) + single = standalone(root, tmp_path) + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + blocks = dataset.execute_query("active_blocks_at_event", {"event_id": event}) + local = reference.execute_query("active_blocks_at_event", {"event_id": event}) + assert {b["id"] for b in blocks["rows"]} == {b["id"] for b in local["rows"]} + counter = dataset.execute_query("event", {"id": event})["rows"][0]["active"] + assert sum(b["size"] for b in blocks["rows"]) == counter + params = {"event_id": event, "top_n": -1} + grouped = dataset.execute_query("active_memory_callstack_at_event", params) + assert normalized_groups(grouped["rows"]) == normalized_groups( + reference.execute_query("active_memory_callstack_at_event", params)["rows"] + ) + coverage = grouped["scope"]["source_coverage"] + assert coverage["active_bytes"] == counter + assert coverage["allocation_source_complete"] + assert not coverage["ordered_frames_complete"] + assert grouped["scope"]["boundary_events_included"] is False + assert ( + blocks["scope"]["source_coverage"]["free_events_resolved"] + == blocks["scope"]["source_coverage"]["free_references"] + ) + assert all(b["state_scope"] == "slice_observation" for b in blocks["rows"]) + if event == 2: + pending = next(b for b in blocks["rows"] if b["id"] == 0) + assert pending["free_source"]["event"]["id"] == 10 # Not free_requested=2. + assert pending["free_source"]["event"]["action"] == 6 + if event == 11: + reused = next(b for b in blocks["rows"] if b["address"] == 100) + assert reused["id"] == 11 and reused["allocation_source"]["event"]["id"] == 11 + assert before == hashes(root) + assert not list(root.rglob("*-wal")) + + +def test_source_before_top_n_bytes_denominator_specials_missing_literal_and_summary(tmp_path): + root = case(tmp_path) + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": 1, "stack_bytes": 5} + ) + assert [r["size_bytes"] for r in result["rows"]] == [150, 10, 5] + assert result["rows"][0]["percent_of_active_blocks"] == round(15000 / 165, 4) + assert result["rows"][0]["stack_event_id"] == 0 + assert result["rows"][0]["stack_truncated"] + assert result["has_more"] and result["truncated"] + assert result["scope"]["source_coverage"]["active_bytes"] == 295 + assert result["scope"]["source_coverage"]["allocation_source_complete"] + smaller = analyzer.execute_query( + "active_memory_callstack_at_event", + {"event_id": 8, "top_n": 1}, + max_rows=1, + exact_total=True, + ) + assert smaller["total"] == 7 and smaller["returned"] == 1 + assert ( + smaller["rows"][0]["percent_of_active_blocks"] + == result["rows"][0]["percent_of_active_blocks"] + ) + complete = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": -1} + ) + labels = [r for r in complete["rows"] if r["callstack"] == "[missing callstack]"] + assert len(labels) == 2 + assert {r["stack_kind"] for r in labels} == {"missing", "captured"} + assert len({r["stack_id"] for r in labels}) == 2 + assert next(r for r in complete["rows"] if r["stack_kind"] == "missing")["size_bytes"] == 35 + assert ( + next(r for r in complete["rows"] if r["callstack"] == " \n")["stack_kind"] + == "captured" + ) + exclude = analyzer.execute_query( + "active_memory_callstack_at_event", + {"event_id": 8, "include_static": False, "min_size": 40, "top_n": -1}, + exact_total=True, + ) + assert sum(r["size_bytes"] for r in exclude["rows"]) == 190 + assert sum(r["percent_of_active_blocks"] for r in exclude["rows"]) == pytest.approx( + 100, abs=0.0001 + ) + + +def test_devices_address_and_negative_lifecycle_identity_are_independent(tmp_path): + root = case(tmp_path) + with SnapshotAnalyzer(root) as analyzer: + one = analyzer.execute_query("active_blocks_at_event", {"event_id": 8}, device_id=0)["rows"] + two = analyzer.execute_query("active_blocks_at_event", {"event_id": 8}, device_id=2)["rows"] + assert not {b["lifecycle_id"] for b in one} & {b["lifecycle_id"] for b in two} + unknown = {b["category"]: b for b in one if b["allocEventId"] == -1} + assert set(unknown) == {"static", "preexisting_live_at_event"} + assert unknown["static"]["free_source_status"] == "live_or_unknown" + assert all(b["allocation_source_status"] == "unknown_preexisting" for b in unknown.values()) + old = next(b for b in one if b["id"] == 0) + new = next( + b + for b in analyzer.execute_query("active_blocks_at_event", {"event_id": 11})["rows"] + if b["id"] == 11 + ) + assert old["address"] == new["address"] and old["lifecycle_id"] != new["lifecycle_id"] + + +def test_missing_dynamic_action_is_not_static_or_row_truncation(tmp_path): + root = case(tmp_path) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET action=2 WHERE id=0") + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": -1} + ) + assert not result["truncated"] + assert not result["scope"]["source_coverage"]["allocation_source_complete"] + assert sum(r["size_bytes"] for r in result["rows"]) == 295 + missing = next(r for r in result["rows"] if r["stack_kind"] == "missing") + assert missing["category"] == "dynamic_live_at_event" and missing["size_bytes"] == 125 + + +@pytest.mark.parametrize("template", ["active_blocks_at_event", "active_memory_callstack_at_event"]) +@pytest.mark.parametrize("event,slice_index", [(-1, None), (16, None), (8, 0), (8, True), (8, 99)]) +def test_point_event_rejects_boundary_outside_or_wrong_scope( + tmp_path, template, event, slice_index +): + root = case(tmp_path) + with SnapshotAnalyzer(root) as analyzer, pytest.raises(InvalidParameterError): + analyzer.execute_query(template, {"event_id": event}, slice_index=slice_index) + + +def test_active_block_limit_offset_exact_and_zero_top_n_keep_coverage(tmp_path): + root = case(tmp_path) + with SnapshotAnalyzer(root) as analyzer: + page = analyzer.execute_query( + "active_blocks_at_event", + {"event_id": 8, "order_by": "id", "order_dir": "ASC", "limit": 2, "offset": 1}, + max_rows=1, + exact_total=True, + ) + assert page["total"] == 9 and page["returned"] == 1 and page["truncated"] + assert page["rows"][0]["id"] == -20 + assert page["scope"]["source_coverage"]["active_blocks"] == 9 + zero = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": 0}, exact_total=True + ) + assert {r["category"] for r in zero["rows"]} == {"static", "preexisting_live_at_event"} + assert sum(r["percent_of_active_blocks"] for r in zero["rows"]) == 100 + assert zero["has_more"] and zero["total"] == 7 + + +def add_frames(root, version=1, corrupt=None): + manifest = root / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["extensions"] = {"ptSnapOrderedFrames": {"version": version}} + manifest.write_text(json.dumps(raw)) + frame = {"filename": "raw.py", "name": "same", "line": 1, "extra": {"native": True}} + for device in (0, 2): + for index in range(8): + with ( + closing(sqlite3.connect(root / f"device_{device}/slice_{index:05d}.db")) as conn, + conn, + ): + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER); CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT, PRIMARY KEY(eventId,frameIndex));" + ) + for event in range(index * 2, index * 2 + 2): + conn.execute("INSERT INTO pt_snap_frame_coverage VALUES (?,3)", (event,)) + # order and repeats are significant. Events 0 and 1 share text + # but raw identity differs in the final frame. + frames = [frame, frame, {"name": f"event:{event}"}] + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,?,?)", + [(event, i, json.dumps(f)) for i, f in enumerate(frames)], + ) + if index == 0: + if corrupt == "gap": + conn.execute("DELETE FROM pt_snap_frame WHERE eventId=0 AND frameIndex=1") + elif corrupt == "json": + conn.execute("UPDATE pt_snap_frame SET frameJson='[]' WHERE eventId=0") + elif corrupt == "absent": + conn.execute("DROP TABLE pt_snap_frame") + return frame + + +def test_recognized_ordered_frames_retain_order_repeats_exact_identity_and_shared_sources(tmp_path): + root = case(tmp_path) + frame = add_frames(root) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + row = analyzer.execute_query("event", {"id": 0, "stack_bytes": 0})["rows"][0] + assert row["frames"] == [frame, frame, {"name": "event:0"}] + assert row["frames_status"] == "ordered" and row["stack_truncated"] + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": -1, "stack_bytes": 0} + ) + shared = [r for r in result["rows"] if r["stack_event_id"] in (0, 1)] + assert len(shared) == 2 and shared[0]["stack_id"] != shared[1]["stack_id"] + assert shared[0]["stack_id"] == row["stack_id"] + assert shared[0]["frames"][:2] == [frame, frame] + assert result["scope"]["source_coverage"]["ordered_frames_complete"] + assert ( + analyzer.get_database_overview()["dataset"]["structured_frames"] is False + ) # P0 validation, not scoped source coverage. + assert before == hashes(root) + + +@pytest.mark.parametrize( + "version,corrupt", [(999, None), (True, None), (1, "gap"), (1, "json"), (1, "absent")] +) +def test_unsupported_or_uncovered_frames_degrade_without_reconstructing_text( + tmp_path, version, corrupt +): + root = case(tmp_path) + add_frames(root, version, corrupt) + with SnapshotAnalyzer(root) as analyzer: + row = analyzer.execute_query("event", {"id": 0, "stack_bytes": 1000})["rows"][0] + assert row["frames"] is None and row["frames_status"] == "text_only" + assert row["callstack"].startswith("shared:") + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": -1} + ) + assert not result["scope"]["source_coverage"]["ordered_frames_complete"] + + +def test_native_sources_preserve_sparse_ids_local_stack_collision_and_borrowed_lifetime(tmp_path): + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + root = ImportService().import_snapshot(request).db_path + cache = ContextCache(maxsize=1) + try: + with SnapshotAnalyzer(root, context_cache=cache) as analyzer: + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 57, "top_n": -1} + ) + assert result["scope"]["first_event_id"] == 57 + assert result["scope"]["source_coverage"]["allocation_source_complete"] + blocks = analyzer.execute_query("active_blocks_at_event", {"event_id": 57}) + sources = [b["allocation_source"] for b in blocks["rows"] if b["allocation_source"]] + assert sources and all( + type(s["local_stack_id"]) is int and type(s["slice_index"]) is int for s in sources + ) + assert len(cache) == 1 + with pytest.raises(InvalidParameterError, match="sparse"): + analyzer.execute_query("active_blocks_at_event", {"event_id": 48}) + assert analyzer.execute_query("event", {"id": 48})["rows"] == [] + with pytest.raises(InvalidParameterError): + analyzer.execute_query("active_blocks_at_event", {"event_id": 58}) + held = next(iter(cache._entries.values()))[0] + assert len(cache) == 1 + with held.connect() as conn: + assert conn.execute("SELECT 1").fetchone()[0] == 1 + with pytest.raises(RuntimeError, match="closed"): + analyzer.execute_query("active_blocks_at_event", {"event_id": 57}) + finally: + cache.close() + + +@pytest.mark.parametrize("event", [7, 17, 27, 37, 47, 57, 67, 77, 87]) +def test_native_same_event_active_sets_bytes_and_attribution_match_standalone(tmp_path, event): + request = dataclasses.replace(source_options(tmp_path), set_focus=False) + service = ImportService() + root = service.import_snapshot(request).db_path + single = service.import_snapshot(dataclasses.replace(request, events_per_slice=None)).db_path + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + params = {"event_id": event} + blocks = dataset.execute_query("active_blocks_at_event", params)["rows"] + original = reference.execute_query("active_blocks_at_event", params)["rows"] + # Unknown negative lifetimes are stable WITHIN a dataset, not globally + # comparable to a separately imported standalone DB's negative tokens. + from collections import Counter + + assert { + (r["id"], r["size"], r["allocEventId"], r["freeEventId"]) + for r in blocks + if r["allocEventId"] >= 0 + } == { + (r["id"], r["size"], r["allocEventId"], r["freeEventId"]) + for r in original + if r["allocEventId"] >= 0 + } + assert Counter( + (r["address"], r["size"], r["requestedSize"], r["allocEventId"], r["freeEventId"]) + for r in blocks + if r["allocEventId"] == -1 + ) == Counter( + (r["address"], r["size"], r["requestedSize"], r["allocEventId"], r["freeEventId"]) + for r in original + if r["allocEventId"] == -1 + ) + assert ( + sum(r["size"] for r in blocks) + == dataset.execute_query("event", {"id": event})["rows"][0]["active"] + ) + params["top_n"] = -1 + assert normalized_groups( + dataset.execute_query("active_memory_callstack_at_event", params)["rows"] + ) == normalized_groups( + reference.execute_query("active_memory_callstack_at_event", params)["rows"] + ) + + +@pytest.mark.parametrize("ordered", [False, True]) +def test_cross_owner_same_full_stack_merges_before_top_n_like_standalone(tmp_path, ordered): + from pt_snap_cli.core.models import ImportOptions + from tests.snapshot.test_sharded_replay import block, event, segment, write_source + + data = { + "segments": [ + segment(0, 64, [block(1000, 16, 16), block(1016, 16, 16), block(1032, 24, 24)]) + ], + "device_traces": [ + [ + event("alloc", addr=1000), + event("alloc", addr=1016), + event("alloc", addr=1032, size=24), + ] + ], + } + data["device_traces"][0][2]["frames"] = [{"filename": "other.py", "line": 8, "name": "other"}] + source = write_source(tmp_path, data) + service = ImportService() + request = ImportOptions(source, tmp_path / "outputs", set_focus=False, events_per_slice=1) + root = service.import_snapshot(request).db_path + single = service.import_snapshot(dataclasses.replace(request, events_per_slice=None)).db_path + if ordered: + raw = json.loads((root / "manifest.json").read_text()) + raw["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + import hashlib + + for i, item in enumerate(raw["devices"]["0"]["slices"]): + path = root / item["file"] + with closing(sqlite3.connect(path)) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER); CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT, PRIMARY KEY(eventId,frameIndex));" + ) + conn.execute("INSERT INTO pt_snap_frame_coverage VALUES (?,1)", (i,)) + conn.execute( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", + (i, json.dumps(data["device_traces"][0][i]["frames"][0])), + ) + item["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + (root / "manifest.json").write_text(json.dumps(raw)) + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + params = {"event_id": 2, "top_n": 1} + result = dataset.execute_query("active_memory_callstack_at_event", params) + expected = reference.execute_query("active_memory_callstack_at_event", params) + assert normalized_groups(result["rows"]) == normalized_groups(expected["rows"]) + assert result["rows"][0]["block_count"] == 2 and result["rows"][0]["size_bytes"] == 32 + assert result["rows"][0]["percent_of_active_blocks"] == 100 + assert before == hashes(root) + + +@pytest.mark.parametrize("missing", [None, ""]) +def test_ordered_identity_ignores_formatted_text_availability_across_owners(tmp_path, missing): + root = case(tmp_path) + add_frames(root) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=0", (missing,)) + with closing(sqlite3.connect(root / "device_0/slice_00001.db")) as conn, conn: + conn.execute( + "UPDATE pt_snap_frame SET frameJson=? WHERE eventId=3 AND frameIndex=2", + (json.dumps({"name": "event:0"}),), + ) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + blocks = analyzer.execute_query("active_blocks_at_event", {"event_id": 8})["rows"] + first = next(b["allocation_source"] for b in blocks if b["id"] == 0) + other = next(b["allocation_source"] for b in blocks if b["id"] == 3) + assert first["frames"] == other["frames"] + assert first["stack_id"] == other["stack_id"] + result = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 8, "top_n": 1, "stack_bytes": 0} + ) + group = result["rows"][0] + assert (group["size_bytes"], group["block_count"]) == (130, 2) + assert group["stack_event_id"] == 3 + assert group["text_kind"] == "captured" and first["text_kind"] == "missing" + assert first["event"]["callstack"] == missing + row = analyzer.execute_query("event", {"id": 0, "stack_bytes": 0})["rows"][0] + assert row["frames"] == first["frames"] and row["text_kind"] == "missing" + json.dumps(row, allow_nan=False) + assert before == hashes(root) + + +@pytest.mark.parametrize( + "raw", + ['{"line":1e309}', '{"nested":' + "[" * 1200 + "0" + "]" * 1200 + "}"], + ids=["nonfinite", "too_deep"], +) +def test_nonfinite_and_deep_optional_frames_degrade_readonly(tmp_path, raw): + root = case(tmp_path) + add_frames(root) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute( + "UPDATE pt_snap_frame SET frameJson=? WHERE eventId=0 AND frameIndex=0", (raw,) + ) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + row = analyzer.execute_query("event", {"id": 0})["rows"][0] + assert row["frames"] is None and row["frames_status"] == "text_only" + json.dumps(row, allow_nan=False) + assert before == hashes(root) + + +def test_native_bare_local_stack_ids_do_not_alias_foreign_sources(tmp_path): + root = ( + ImportService() + .import_snapshot(dataclasses.replace(source_options(tmp_path), set_focus=False)) + .db_path + ) + raw = json.loads((root / "manifest.json").read_text()) + for event in (17, 67): + item = next( + s + for s in raw["devices"]["0"]["slices"] + if s["startEventId"] <= event <= s["endEventId"] + ) + path = root / item["file"] + with closing(sqlite3.connect(path)) as conn, conn: + local = conn.execute( + "SELECT callstackId FROM trace_entry_0 WHERE id=?", (event,) + ).fetchone()[0] + conn.execute("UPDATE callstack SET id=42 WHERE id=?", (local,)) + conn.execute("UPDATE trace_entry_0 SET callstackId=42 WHERE callstackId=?", (local,)) + import hashlib + + item["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + # Different complete texts with the same local ID must NOT alias. + for item in raw["devices"]["0"]["slices"]: + path = root / item["file"] + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute( + "UPDATE callstack SET callstack='different complete stack' WHERE id IN (SELECT callstackId FROM trace_entry_0 WHERE id=67)" + ) + conn.execute( + "UPDATE pt_snap_block_reference SET allocCallstack='different complete stack' WHERE blockId=67" + ) + item["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + (root / "manifest.json").write_text(json.dumps(raw)) + cache = ContextCache(maxsize=1) + try: + resolver = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(None, time.monotonic()) + ) + sources = resolver.events([17, 67]) + assert len(sources) == 2 + assert all(s.local_stack_id == 42 for s in sources.values()) + assert sources[17].slice_index != sources[67].slice_index + assert sources[17].event["callstack"] != sources[67].event["callstack"] + assert sources[17].stack_id != sources[67].stack_id + assert {s.event["id"] for s in sources.values()} == {17, 67} + finally: + cache.close() + + +def test_high_block_count_batches_distinct_sources_and_bounds_cache_queries(tmp_path): + root = make_dataset(tmp_path / "large", devices=(0,), slices=300) + with closing(sqlite3.connect(root / "device_0/slice_00299.db")) as conn, conn: + conn.executemany( + "INSERT INTO block_0 VALUES (?,100,8,8,1,?,-1)", + [(event, event) for event in range(599)], + ) + dataset = DatasetResolver().inspect(root) + cache = ContextCache(maxsize=2) + try: + sources = DatasetSourceResolver(dataset, 0, cache, QueryBudget(None, time.monotonic())) + result = sources.events([*range(599), *range(599)]) + assert len(result) == 599 + assert sources.query_count == 300 # Per owning shard, not each block. + assert len(cache) <= 2 + with SnapshotAnalyzer(root, context_cache=cache) as analyzer: + high = analyzer.execute_query( + "active_memory_callstack_at_event", {"event_id": 599, "top_n": -1} + ) + assert sum(r["block_count"] for r in high["rows"]) == 599 + assert high["scope"]["source_coverage"]["source_queries"] == 302 + assert len(cache) <= 2 + # A single large shard specifically crosses the parameter batch bound. + single = make_dataset(tmp_path / "wide", devices=(0,), slices=1) + path = single / "device_0/slice_00000.db" + count = SOURCE_BATCH_SIZE * 3 + 7 + with closing(sqlite3.connect(path)) as conn, conn: + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,100,8,0,8,8,16,'wide')", + [(i,) for i in range(2, count)], + ) + raw = json.loads((single / "manifest.json").read_text()) + raw["eventsPerSlice"] = count + raw["devices"]["0"]["eventCount"] = count + raw["devices"]["0"]["slices"][0]["endEventId"] = count - 1 + (single / "manifest.json").write_text(json.dumps(raw)) + resolver = DatasetSourceResolver( + DatasetResolver().inspect(single), 0, cache, QueryBudget(None, time.monotonic()) + ) + assert len(resolver.events([*range(count), *range(count)])) == count + assert resolver.query_count == 4 + finally: + cache.close() + + +def test_global_budget_counts_source_work_and_clears_sqlite_progress_handler(tmp_path, monkeypatch): + root = case(tmp_path) + cache = ContextCache(maxsize=1) + service = QueryService(context_cache=cache) + try: + original = DatasetSourceResolver.read + values = [] + + def counted(self, path, sql, params=None): + values.append(self.budget.remaining()) + return original(self, path, sql, params) + + monkeypatch.setattr(DatasetSourceResolver, "read", counted) + service.execute_query( + "active_memory_callstack_at_event", + {"event_id": 8}, + root, + timeout_s=10, + exact_total=True, + ) + assert len(values) > 3 and values == sorted(values, reverse=True) + expired = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(0.001, time.monotonic() - 1) + ) + with pytest.raises(QueryTimeoutError): + expired.events([0, 1]) + sources = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(0.0001, time.monotonic()) + ) + path = root / "device_0/slice_00000.db" + with pytest.raises(QueryTimeoutError): + sources.read( + path, + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<10000000) SELECT SUM(x) FROM n", + ) + sources = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(None, time.monotonic()) + ) + assert sources.read(path, "SELECT 1 AS okay") == [{"okay": 1}] + service.close() + assert len(cache) == 1 # Borrowed ownership, even following cancellation. + finally: + service.close() + cache.close() + + +def test_report_uses_same_point_event_core_and_global_aggregates_stay_explicit(tmp_path): + root = case(tmp_path) + report = ReportService() + try: + attribution = report.event_attribution(8, root, limit=-1) + with SnapshotAnalyzer(root) as analyzer: + assert dataclasses.asdict(attribution) == analyzer.execute_query( + "active_memory_callstack_at_event", + { + "event_id": 8, + "include_static": True, + "min_size": 0, + "top_n": -1, + "stack_bytes": -1, + }, + ) + for template in ( + "memory_peak", + "allocator_gap", + "leak_detection", + "block", + "callstack_analysis", + ): + with pytest.raises(QueryExecutionError, match="dataset-global"): + analyzer.execute_query(template) + with pytest.raises(QueryExecutionError, match="dataset-global"): + report.peak_memory_report(root) + finally: + report.close() diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index 852b764..2fbfd61 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -918,6 +918,59 @@ def test_compatible_import_focus_metadata_query_contract(tmp_path: Path, monkeyp assert not hasattr(analyzer, "import_snapshot") +@pytest.mark.parametrize( + "template", ["event", "active_blocks_at_event", "active_memory_callstack_at_event"] +) +@pytest.mark.parametrize("frames", [False, True]) +def test_dataset_point_event_sources_cli_api_report_contract( + tmp_path: Path, template: str, frames: bool +) -> None: + from pt_snap_cli.core.report_service import ReportService + from tests.core.test_dataset_attribution import add_frames, case + + root = case(tmp_path) + if frames: + add_frames(root) + params = {"id": 0, "stack_bytes": 0} if template == "event" else {"event_id": 8} + if template == "active_memory_callstack_at_event": + params.update({"top_n": 1, "stack_bytes": 0}) + cli = _json_stdout( + runner.invoke( + app, + [ + "query", + str(root), + "--device", + "2", + "--template-use", + template, + "--params", + json.dumps(params), + "-n", + "1", + "--exact-total", + "--json", + ], + ) + ) + with SnapshotAnalyzer(root, device_id=2) as analyzer: + api = analyzer.execute_query(template, params, max_rows=1, exact_total=True) + assert {key: cli[key] for key in api} == api + assert cli["scope"]["source_coverage"]["range_complete"] + if template != "event": + assert ( + cli["truncated"] and cli["scope"]["source_coverage"]["allocation_source_complete"] + ) + if template == "active_memory_callstack_at_event": + report = ReportService() + try: + shared = report.event_attribution(8, root, 2, limit=1, stack_bytes=0) + assert shared.rows[0] == api["rows"][0] + assert shared.scope == api["scope"] + finally: + report.close() + + def test_overview_error_contract_maps_four_domain_failures(tmp_path: Path) -> None: missing = runner.invoke(app, ["overview", str(tmp_path / "missing.db"), "--json"]) missing_error = json.loads(missing.stderr)["error"]