From a0f9738fedaa31b7c3929f5884c48df74616a75a Mon Sep 17 00:00:00 2001 From: codeasier Date: Tue, 6 Oct 2026 11:51:46 +0800 Subject: [PATCH] fix: implement dataset-global query semantics (#204) --- README.md | 8 +- README_zh.md | 6 +- docs/en/querying.md | 97 +++- docs/en/sharded-snapshotdb.md | 35 +- docs/zh/querying.md | 80 ++- docs/zh/sharded-snapshotdb.md | 26 +- src/pt_snap_cli/cli.py | 1 + src/pt_snap_cli/cli_reports.py | 15 + src/pt_snap_cli/core/dataset_attribution.py | 2 +- src/pt_snap_cli/core/dataset_global.py | 484 +++++++++++++++ src/pt_snap_cli/core/dataset_resolver.py | 9 +- src/pt_snap_cli/core/dataset_sources.py | 39 +- src/pt_snap_cli/core/dataset_support.py | 28 + src/pt_snap_cli/core/models.py | 4 + src/pt_snap_cli/core/query_service.py | 200 ++----- src/pt_snap_cli/core/report_service.py | 21 +- src/pt_snap_cli/query/executor.py | 34 +- src/pt_snap_cli/query/registry.py | 23 +- .../query/templates/basic/block.yaml | 57 ++ .../templates/business/leak_detection.yaml | 75 +++ .../templates/statistical/allocator_gap.yaml | 5 +- .../statistical/callstack_analysis.yaml | 71 ++- .../templates/statistical/memory_peak.yaml | 5 +- tests/core/test_dataset_attribution.py | 9 +- tests/core/test_dataset_global.py | 549 ++++++++++++++++++ tests/query/test_callstack_schema_compat.py | 9 +- tests/query/test_registry.py | 19 +- tests/query/test_template_semantics.py | 2 +- tests/test_contract_cli_api.py | 53 +- tests/test_dataset_focus.py | 5 +- tests/test_fragmentation_query_scope.py | 13 +- tests/test_native_dataset.py | 9 +- 32 files changed, 1727 insertions(+), 266 deletions(-) create mode 100644 src/pt_snap_cli/core/dataset_global.py create mode 100644 src/pt_snap_cli/core/dataset_support.py create mode 100644 tests/core/test_dataset_global.py diff --git a/README.md b/README.md index 13e038b..92fc058 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,11 @@ rollback fails (force is not a single crash-atomic swap). See the [native import guide](docs/en/quickstart.md#optional-import-a-native-sharded-dataset). Dataset `event` addressing and point-event `active_blocks_at_event` / `active_memory_callstack_at_event` share batched cross-shard alloc/free sources, -including original artifacts without extensions. Dataset-global peak/list/leak -queries and `report peak-memory` remain explicitly unsupported. See -[dataset attribution and coverage](docs/en/querying.md#dataset-point-event-attribution). +including original artifacts without extensions. Built-in global peaks, event pages, +canonical stack statistics, deduplicated lifecycles, terminal leak candidates and +`report peak-memory` share bounded core semantics; custom SQL/overrides remain unsupported +on datasets. See the [global support matrix](docs/en/querying.md#dataset-global-built-in-support-p3) +and [dataset attribution and coverage](docs/en/querying.md#dataset-point-event-attribution). Explicit `pt-snap import snapshot.pkl --format msinsight --json` produces `snapshot.pkl.msinsight/` for the fixed msinsight revision's **same original diff --git a/README_zh.md b/README_zh.md index 7eaf3bd..aa26aec 100644 --- a/README_zh.md +++ b/README_zh.md @@ -58,8 +58,10 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices (force 不是单次崩溃原子 swap)。详见 [原生分库导入](docs/zh/quickstart.md#可选导入原生分库数据集)。数据集 `event` 寻址及定点 `active_blocks_at_event` / `active_memory_callstack_at_event` 共用跨片 alloc/free 批量来源, -原版无扩展产物也适用。数据集全局 peak/list/leak 及 `report peak-memory` 仍明确不支持。 -详见[数据集定点归因与覆盖](docs/zh/querying.md#数据集定点事件归因)。 +原版无扩展产物也适用。内建全局峰值、事件分页、canonical 栈统计、去重生命周期、 +终片泄漏候选及 `report peak-memory` 共用有界 core 语义;数据集仍不支持自定义 SQL/override。 +详见[全局支持矩阵](docs/zh/querying.md#数据集全局内建支持p3)及 +[数据集定点归因与覆盖](docs/zh/querying.md#数据集定点事件归因)。 显式 `pt-snap import snapshot.pkl --format msinsight --json` 生成 `snapshot.pkl.msinsight/`,对齐固定 msinsight 版本的**同一原始 pickle + 邻接缓存**入口。 diff --git a/docs/en/querying.md b/docs/en/querying.md index 7dc54be..d466b13 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -24,7 +24,7 @@ pt-snap query [DB_PATH] [--template-use ] [--params ] \ | `--category` | Filter templates by category: `basic`, `statistical`, `business` | | `--template-info` | Show template details (parameters, output schema, and field semantics) | | `-n` | Maximum displayed rows; zero or a negative value means unlimited. Separate from `--timeout`. | -| `--exact-total` | Run a `COUNT` of the matching set. Default `total` is the returned-row count. | +| `--exact-total` | Count the complete matching set: standalone SQL COUNT or dataset post-merge count. Default `total` is returned rows. | | `--timeout` | Wall-clock budget for one `query` / `QueryService` call (page query and optional `--exact-total` COUNT share it) via a SQLite progress handler. `<= 0` disables. Default: `PT_SNAP_QUERY_TIMEOUT` or unlimited. The environment variable applies to every template query, including `report peak-memory`. | | `--json` | Emit machine-readable JSON (execute, `--list`, and `--template-info`) | @@ -78,6 +78,76 @@ group identity. Missing/static/preexisting groups have no captured allocation stack to retrieve; omit `stack_bytes` on the original query to see their full display labels. Results are read-only and summaries do not change the database. +## Dataset-global built-in support (P3) + +Complete native-v2 and compatibility-v1 datasets share the following **one-device** +query support matrix, also exposed as each catalog/template-info `dataset_support`. +Device selection stays explicit → focused → first discovered; peaks are never summed +across devices and there is no shared cross-device event order. + +| Template | Dataset semantics before the final output window | +| --- | --- | +| `event`, `allocation` | Filter all selected real events, globally sort with the declared ID tie-break, then offset/limit/max_rows; negative boundaries excluded | +| `memory_peak` | Independent allocated/active/reserved maxima over real events; earliest real ID wins ties; `start_id/end_id` may cross shards | +| `allocator_gap` | The same independently selected global events, with each gap computed from counters **at that event** | +| `active_blocks_at_event`, `active_memory_callstack_at_event` | Route a real event to its owner and resolve full cross-shard sources before grouping/top_n (details below) | +| `preexisting_live` | Owner-event preexisting occupancy, not a leak count | +| `block` | Merge proved allocation-event lifetimes and stable negative tokens once, then apply filters/order/page; latest containing-shard state is an observation, not state at arbitrary E | +| `freed_block_lifetime` | Deduplicated captured allocations with a proved `free_completed` source; event-ID distance buckets, not time | +| `leak_detection` | Captured dynamic lifetimes still present without completion in the **terminal dataset shard**, not union of per-shard unfreed lists; survival is only a candidate, not a confirmed leak; `--slice` is rejected | +| `callstack_analysis` | All real stack-bearing trace actions, including free/segment/workspace, merged by full canonical source identity before global thresholds/ranking; weighted `avg_size=total_size/alloc_count` | +| Runtime overrides/custom templates/arbitrary SQL | Explicitly unsupported on datasets, even when named after a built-in; pass an explicit standalone member to use local SQL | + +`memory_peak` and `allocator_gap` semantics version **2** exclude negative synthetic +rows in **standalone** SQL too. `callstack_analysis` version **2** retains the legacy +`alloc_count` column name but does **not** mean allocation count: v1 counts all real +non-NULL-text rows; v2 counts all real non-NULL stack references, retaining absent/NULL +joined text as one distinct missing group. Empty text counts as its own group, separate +from missing and captured literal labels. Both standalone layouts now group full text, +not local v2 IDs. Datasets also accept explicitly covered ordered +raw arrays as stack evidence; missing text does not change raw-array identity. Size +sums include nonallocation actions and are not instantaneous live occupancy. + +The shared YAML output schemas declare every optional dataset-only row column; +standalone row shapes stay unchanged. `block` and dataset `leak_detection` expose +`lifecycle_id`, proof/source statuses, source payloads, latest observation slice/state +scope, category and terminal survival; leak rows additionally retain `requestedSize`, +`state` and `freeEventId`. These observations and proofs are not leak confirmation. +Dataset statistics expose `source_stack_id`, `stack_kind`, `text_kind`, +`stack_event_id` and `frames_status`. The representative real event prefers nonempty +captured text when available but is not the canonical group identity and need not be +an allocation action; otherwise it may have empty/NULL text. `text_kind` is provenance +only and never changes raw-array identity. Retrieve actual text/arrays using `event` +on that same dataset/device; `frames_status` is not frame reconstruction. + +```bash +pt-snap query '' --device 0 --template-use memory_peak --params '{"start_id":100,"end_id":700}' --json +pt-snap query '' --device 0 --template-use event --params '{"min_id":100,"max_id":700,"limit":20,"offset":20}' --exact-total --json +pt-snap query '' --device 0 --template-use leak_detection -n 20 --exact-total --json +pt-snap report peak-memory '' --device 0 --metric reserved --start-id 100 --end-id 700 --timeout 10 --json +``` + +Use actual bounds. Ranges may include sparse gaps; they never create an event. +An empty real range has NULL peak values/IDs. Outside/negative/reversed range +endpoints fail; `event(id=gap)` may return no rows, while active-at-gap fails. +Global filters/dedup/source identity/thresholds/sort precede the final window. +`exact_total` ignores row caps; default total is returned rows with honest exactness +flags. Result `has_more/truncated` describes the window, not source/range completeness. +`scope` records requested bounds, actual real-event coverage where applicable, +involved `slice_indices`, device, fingerprint and boundary exclusion. Source coverage +is separate. Unproved historical lifecycle identities are shard-local, explicitly +marked, never guessed by address/local stack ID and never asserted as proved leaks. + +Dataset reads detach batches before foreign-owner lookups, using the borrowed bounded +Context LRU. One diminishing deadline includes all shards, sources, grouping and exact +totals; reports explicitly share **one** budget across their three queries. Cumulative +fetched/output work is limited to **100000 rows / 64 MiB serialized values**, including +repeated source reads. Exceeding either fails the entire operation, never an alleged +complete partial result. Materialization/sort are bounded by this work ceiling, **not** +by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual +cells have overhead. Manifest hashing/validation remains proportional to artifact size +and phase-checked, not hard-preempted. No temporary merge DB, repair or artifact write. + ## Dataset point-event attribution A complete compatibility-v1 or native-v2 directory/manifest supports `event` @@ -136,9 +206,9 @@ filesystem I/O/validation is checked at phase boundaries, not hard-preempted; no OS-level time guarantee is claimed. SQL uses cleared-on-exit progress handlers. The Context LRU remains bounded (default four) and borrowed caches remain caller-owned. No merged temporary database is created. `ReportService.event_attribution(E, ...)` -reuses this same query path for a supplied event. Dataset-global peak selection, -`report peak-memory`, cross-slice event ranges, arbitrary SQL/global list/leak/group -merging remain explicitly unsupported until their own contracts are implemented. +reuses this same query path for a supplied event. Dataset-global queries and reports +follow the explicit support matrix above; arbitrary SQL/custom overrides never inherit +those merge semantics merely by using a built-in name. ### Optional ordered-frame reader contract @@ -358,7 +428,7 @@ This is useful because `reserved` may peak at a different event from `active` or For a higher-level summary, use the report command: ```bash -pt-snap report peak-memory [db_path] [--device ] [--metric active|allocated|reserved] [--include-static|--exclude-static] [--limit ] [--json] +pt-snap report peak-memory [db_path] [--device ] [--metric active|allocated|reserved] [--include-static|--exclude-static] [--limit ] [--start-id ] [--end-id ] [--timeout ] [--json] ``` Examples: @@ -380,7 +450,12 @@ The report command combines: - `allocator_gap` - `active_memory_callstack_at_event` -and prints either a human-readable summary or JSON. +and prints either a human-readable summary or JSON. `--start-id/--end-id` restrict +peak selection; attribution still resolves the complete active set at that selected +event, including earlier allocations. `--timeout` (or `PT_SNAP_QUERY_TIMEOUT`) is +one diminishing **report-wide** deadline, not three reset per-query timeouts. +JSON adds peak `scope`, separate attribution `source_coverage`, `timeout_s`, and +`budget_scope="report_composition"`. The existing `callstack_groups` and `percent_of_active_blocks` values are preserved. JSON also exposes attribution `has_more`, `truncated`, `total_is_exact`, and @@ -417,8 +492,10 @@ An empty trace returns no groups, zero included bytes, and unknown coverage. `-n` still caps `rows` and `returned`. By default `total` equals `returned` (this page's row count) and `total_is_exact` is true only when the page is the complete matching set (`has_more` is false and `offset` is 0). -`--exact-total` runs a `COUNT` of the matching set (ignoring `limit` / -`offset` / `top_n`) and sets `total_is_exact` to true. When a finite +`--exact-total` counts the matching set (ignoring `limit` / `offset` / `top_n`) +and sets `total_is_exact` to true. Standalone queries use SQL `COUNT`; dataset +built-ins count the fully merged/deduplicated matching set before its global window, +not capped shard-local rows. On the standalone path, when a finite trailing `LIMIT` is in effect, the executor fetches one extra row to set `has_more` without that count. A finite inner `top_n` (for example inside a CTE) is not a trailing `LIMIT`: when that ranked window is full, @@ -434,7 +511,9 @@ change the row cap. A non-numeric `PT_SNAP_QUERY_TIMEOUT` is `INVALID_PARAMETER`, not `QUERY_FAILED`. `effective_params` is the validated parameter set after defaults and `choices` -normalization. When `-n` is set, `limit` is the trailing SQL LIMIT after the +normalization. Dataset `limit` reports the final global output-window cap (with the +same min-of-positive-caps rule), not a per-shard SQL LIMIT; global filters and merges +are already complete. On the standalone path, when `-n` is set, `limit` is the trailing SQL LIMIT after the same merge the executor applies: `min` of a declared template `limit` and `-n`, or `-n` appended when the rendered SQL has no trailing LIMIT. Inner caps such as `top_n` stay their own parameters; they do not rewrite diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 9397bec..b1c3f1d 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -5,9 +5,11 @@ ## Scope and versions This defines artifact contracts, shared validators and an **explicit** compatible -CLI exporter below, not a cross-slice query engine. Existing `import` defaults to one +CLI exporter below. The [P3 built-in query matrix](querying.md#dataset-global-built-in-support-p3) +defines supported cross-slice execution separately. Existing `import` defaults to one native v2 DB; `split` still produces replayable pickle/JSON. Existing single-DB -v1/v2 single-DB analysis is unchanged. The P1 reader below additionally accepts a +v1/v2 analysis retains its entry points; P3 real-event peak/gap and all-action full-text +stack-statistics semantics are explicitly versioned in the querying guide. The P1 reader below additionally accepts a complete compatibility-v1 manifest or directory as focus. The `complete` gate applies only to datasets, not to legacy single-DB analysis. @@ -170,8 +172,8 @@ which P0 does not implement. Text stacks cannot substitute for these guarantees. There is no global event order across devices. Boundary IDs cannot be range endpoints. P0 `DatasetValidation.query_execution=False` describes validation, not execution. P1 accepts complete manifests/directories through the resolver -below; general cross-slice aggregation remains deferred; point-event source -resolution is described in the [querying guide](querying.md#dataset-point-event-attribution). +below; the explicit [P3 support matrix](querying.md#dataset-global-built-in-support-p3) +and [point-event sources](querying.md#dataset-point-event-attribution) define execution. ```python from pt_snap_cli.core.dataset_contract import QueryScope, validate_dataset @@ -198,8 +200,9 @@ pt-snap query --template-use event --slice 1 --json `overview.dataset` reports format, manifest version/status, content fingerprint, real/boundary counts, per-device slice paths/ranges and capabilities. Event bounds exclude negative synthetic rows (also for standalone DB overview). Text stacks -are available, but not necessarily populated; structured frames and cross-slice -queries are **unavailable**, including when unknown extensions claim them. +are available, but not necessarily populated; P0 structured-frame validation remains +**unavailable** despite unknown extension claims. Cross-slice built-ins follow the P3 +matrix; recognized ordered-frame query coverage is separate from P0 validation. External dataset-wide import metadata is unavailable; per-slice metadata alone is not a source attestation. The compatible exporter below supplies a recognized, whole-artifact `ptSnap` attestation, making its validated metadata available. @@ -220,8 +223,8 @@ with SnapshotAnalyzer(Path("/capture/snapshot.pkl.msinsight"), device_id=0) as a ``` The `event` template supports bounded addressing through dataset focus: a real ID -routes to its containing slice; an inclusive `min_id/max_id` range must fit one -slice; `--slice`/`slice_index` lists that slice's real events. Negative IDs are not +routes to its containing slice; inclusive `min_id/max_id` ranges may cross shards +with global filtering/sorting/paging; `--slice`/`slice_index` restricts real events. Negative IDs are not whole-dataset selectors. Output `scope` identifies the actual DB/device/slice, restricted real interval, dataset fingerprint and boundary exclusion. Totals and pagination apply to this scope, **not** to an implicitly truncated full dataset. @@ -231,10 +234,10 @@ by `scope`. Point-event `active_blocks_at_event` and bounded batches before grouping/ranking, including original no-extension artifacts. See [point-event attribution](querying.md#dataset-point-event-attribution) for lifecycle identity, separate source/frame coverage, and the optional versioned -ordered-frame reader interface. Dataset-global aggregates, peak/list/leak queries, -cross-slice event ranges and unbounded multi-slice event queries still fail -explicitly; they never silently choose the first/latest slice. Direct single-DB -v1/v2 queries retain their existing semantics. +ordered-frame reader interface. Dataset-global built-ins use the explicit P3 matrix +and cumulative work ceilings; arbitrary SQL/custom overrides fail explicitly, never +silently choosing the first/latest slice. Single-DB real-event peak/gap and all-action +full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` @@ -262,7 +265,7 @@ This is read-only validation, not a filesystem sandbox or a producer lock. Native-v2 private staging has **no published manifest**, so it is not a dataset; individual DBs remain readable via the single-DB entry. Published native format and compatible export are separate explicit protocols below. Cross-slice lifecycle -execution remains separate work; safe original-WAL reading is not GUI acceptance. +execution follows the P3 matrix; safe original-WAL reading is not GUI acceptance. ## Published Native Dataset (P2) @@ -309,8 +312,8 @@ alias/sidecar/persistent-WAL checks and bounded borrowed-cache ownership. Focus, metadata, overview and single-slice `event` addressing work without the source pickle. Native metadata is available dataset-wide; external compatibility metadata remains unavailable unless the recognized compatible attestation below is present. -Native mode implies no migration, repair, dataset-global aggregation/peak/list/leak, -compatible export, salted hash or GUI acceptance. Point-event source resolution is +Native mode implies no migration, repair, compatible export, salted hash or GUI +acceptance. Dataset-global built-ins follow the P3 query matrix. Point-event source resolution is available through the shared query path described above. P0 schema/manifest semantics are unchanged; the compatibility-only finalized-WAL transport is described above. @@ -386,7 +389,7 @@ JOIN gaps. Unsupported OOM/unknown actions, nonzero/sparse IDs and empty/static- selections fail; native retains its wider contract. All selected members finalize, close and validate before complete publication. `pt_snap_block_reference` v1 is an isolated text extension, not ordered frames or a claim of cross-source queries. -Those queries and global aggregation remain separate work. Text cannot recover +Those queries and global aggregation follow the P3 query matrix. Text cannot recover original ordered frame arrays; original workspace raw-frame loss and string-action OOM coercion are known upstream limitations, not compatible features to imitate. diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 62c4a54..8259c90 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -24,7 +24,7 @@ pt-snap query [DB_PATH] [--template-use ] [--params ] \ | `--category` | 按分类过滤模板:`basic`、`statistical`、`business` | | `--template-info` | 显示模板详情(参数、输出 schema 和字段语义) | | `-n` | 最大显示行数;零或负数表示不限制。与 `--timeout` 相互独立。 | -| `--exact-total` | 对匹配集合做 `COUNT`。默认 `total` 等于已返回行数。 | +| `--exact-total` | 计数完整匹配集合:单库 SQL COUNT 或数据集合并后的总数。默认 `total` 为返回行数。 | | `--timeout` | 一次 `query` / `QueryService` 调用的墙钟预算(页面查询与可选 `--exact-total` COUNT 共用),经 SQLite progress handler 生效。`<= 0` 表示关闭。默认:`PT_SNAP_QUERY_TIMEOUT` 或不限制。该环境变量作用于所有模板查询,包括 `report peak-memory`。 | | `--json` | 输出机器可读 JSON(执行、`--list` 与 `--template-info`) | @@ -71,6 +71,63 @@ pt-snap query '' --device 0 --template-use event --params '{"id":123}' 分组没有可取回的采集分配栈;在原查询中省略 `stack_bytes` 可查看完整显示标签。 查询保持只读,摘要不会改动数据库。 +## 数据集全局内建支持(P3) + +完整 native-v2 与 compatibility-v1 数据集共用以下**单设备**支持矩阵;catalog 与 +模板详情的 `dataset_support` 同步返回。设备优先级仍为显式 → focused → 首个发现; +不累加跨设备峰值,设备间没有共同全局事件顺序。 + +| 模板 | 最终输出窗口之前的数据集语义 | +| --- | --- | +| `event`, `allocation` | 筛选选定范围全部真实事件,按声明的 ID tie-break 全局排序,最后 offset/limit/max_rows;排除负边界 | +| `memory_peak` | allocated/active/reserved 独立真实峰值;同值选最早真实 ID;`start_id/end_id` 可跨片 | +| `allocator_gap` | 使用同样独立选择的全局事件,每个 gap 由**该事件**计数器计算 | +| `active_blocks_at_event`, `active_memory_callstack_at_event` | 路由实际事件到所有者片,分组/top_n 前解析完整跨片来源(见下文) | +| `preexisting_live` | 所有者事件的 preexisting 占用,不是泄漏数 | +| `block` | 已证明的分配事件生命周期及稳定负 token 去重后筛选/排序/分页;最新出现片 state 是观察,不是任意 E 的状态 | +| `freed_block_lifetime` | 已采集分配与已证明 `free_completed` 来源的去重生命周期;event-ID 距离桶,不是时间 | +| `leak_detection` | **数据集终片**中仍存在且无完成释放的已采集动态生命周期,不拼接逐片未释放列表;存活仅为候选,不证明泄漏;拒绝 `--slice` | +| `callstack_analysis` | 全部带栈真实 trace action(含 free/segment/workspace),完整 canonical 来源身份合并后再做全局阈值/排名;加权 `avg_size=total_size/alloc_count` | +| runtime override/自定义模板/任意 SQL | 数据集明确不支持,即使命名为内建模板;需显式传单库 member 使用局部 SQL | + +`memory_peak` 与 `allocator_gap` 语义版本 **2** 在**单库** SQL 中也排除负合成行。 +`callstack_analysis` 版本 **2** 保留旧 `alloc_count` 字段名,但**不是分配次数**: +v1 统计非 NULL 文本的全部真实行;v2 统计非 NULL 栈引用的全部真实行,缺失/NULL +joined 文本保留为独立 missing 组。空文本单独计数组,与 missing 及真实字面量标签分开; +两种单库布局都按完整文本分组而非局部 v2 ID。数据集还接受明确覆盖的有序原始数组 +作为栈证据;缺失文本不改变原始数组 +身份。size 总和含非分配 action,不是瞬时活跃占用。 + +共享 YAML 输出 schema 声明全部可选的仅数据集行字段;单库行形状不变。`block` 与 +数据集 `leak_detection` 提供 `lifecycle_id`、身份/来源证明状态、来源载荷、最新观察片/ +state scope、category 与终片存活;leak 行另保留 `requestedSize`、`state`、`freeEventId`。 +这些观察与证明不等于确认泄漏。数据集统计提供 `source_stack_id`、`stack_kind`、 +`text_kind`、`stack_event_id`、`frames_status`。真实代表事件优先选可用的非空采集文本, +但不是 canonical 分组身份,也不一定是 allocation action;无采集文本时可指向空/NULL +文本。`text_kind` 仅为 provenance,不改变原始数组身份。在同一数据集/设备用 `event` +取回真实文本/数组;`frames_status` 不表示从文本重建 frame。 + +```bash +pt-snap query '' --device 0 --template-use memory_peak --params '{"start_id":100,"end_id":700}' --json +pt-snap query '' --device 0 --template-use event --params '{"min_id":100,"max_id":700,"limit":20,"offset":20}' --exact-total --json +pt-snap query '' --device 0 --template-use leak_detection -n 20 --exact-total --json +pt-snap report peak-memory '' --device 0 --metric reserved --start-id 100 --end-id 700 --timeout 10 --json +``` + +请使用实际范围。区间可包含稀疏缺口,不虚构事件;无真实事件的范围返回 NULL 峰值/ID。 +越界/负/反向端点失败;`event(id=gap)` 可为空,active-at-gap 失败。全局筛选、去重、来源 +身份、阈值、排序都在最终窗口之前。`exact_total` 忽略行数上限;默认 total 为返回行数, +exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范围完整性。`scope` 返回 +请求区间、适用时实际真实事件覆盖、涉及 `slice_indices`、设备、指纹及边界排除;来源覆盖 +独立。未证明历史身份按片局部单独表达,不按地址/局部栈 ID 猜合并,不断言为已证明泄漏。 + +数据集读取先分离批次再查外片来源,使用借用的有界 Context LRU。所有片、来源、分组、 +精确总数共用递减 deadline;报告三条查询明确共用**一个**预算。累计取回/输出工作上限为 +**100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 +结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ +SQLite、一个批次及单个 cell 有额外开销。manifest hash/校验仍与产物大小成正比且仅在 +阶段边界检查,不硬抢占。不生成合并临时 DB、不修复或写产物。 + ## 数据集定点事件归因 完整兼容 v1 或原生 v2 目录/manifest 支持 `event` 寻址,以及接受真实 `event_id` 的 @@ -116,8 +173,8 @@ max_rows 之前**所含字节,不是数据集 active 计数器。精确总数 设置耗时计入预算,但同步文件 I/O/校验只在阶段边界检查,不硬抢占,不保证 OS 级 时间上限。SQL progress handler 退出即清除。Context LRU 有界(默认四个),借用 cache 仍由调用者所有。不生成合并临时数据库。`ReportService.event_attribution(E, ...)` -对给定事件复用同一路径。数据集全局峰值选择、`report peak-memory`、跨片事件范围、 -任意 SQL/全局 list/leak/group 合并仍明确不支持,等待各自合同实现。 +对给定事件复用同一路径。数据集全局查询及报告遵循上文明确支持矩阵;任意 SQL/ +自定义 override 不会因使用内建名字就继承全局合并语义。 ### 可选有序 frame reader 合同 @@ -316,7 +373,7 @@ pt-snap query --template-use allocator_gap 如果需要更高层的摘要,可以使用: ```bash -pt-snap report peak-memory [db_path] [--device ] [--metric active|allocated|reserved] [--include-static|--exclude-static] [--limit ] [--json] +pt-snap report peak-memory [db_path] [--device ] [--metric active|allocated|reserved] [--include-static|--exclude-static] [--limit ] [--start-id ] [--end-id ] [--timeout ] [--json] ``` 示例: @@ -338,7 +395,10 @@ pt-snap report peak-memory /path/to/snapshot.db --json - `allocator_gap` - `active_memory_callstack_at_event` -并输出人类可读摘要或 JSON。 +并输出人类可读摘要或 JSON。`--start-id/--end-id` 限制峰值选择;归因仍解析所选事件 +完整活跃集合,包含更早分配。`--timeout`(或 `PT_SNAP_QUERY_TIMEOUT`)是递减的 +**报告整体** deadline,不是重置三次的逐查询超时。JSON 新增峰值 `scope`、独立归因 +`source_coverage`、`timeout_s` 与 `budget_scope="report_composition"`。 原有 `callstack_groups` 和 `percent_of_active_blocks` 数值保持不变。JSON 还会 返回归因的 `has_more`、`truncated`、`total_is_exact` 和 `effective_params` @@ -371,8 +431,9 @@ pt-snap report peak-memory /path/to/snapshot.db --json `-n` 仍然限制 `rows` 与 `returned`。默认 `total` 等于 `returned`(本页行数); 仅当本页就是完整匹配集合(`has_more` 为 false 且 `offset` 为 0)时 -`total_is_exact` 为 true。`--exact-total` 会对匹配集合做 `COUNT`(忽略 -`limit` / `offset` / `top_n`)并把 `total_is_exact` 设为 true。存在有限 +`total_is_exact` 为 true。`--exact-total` 计数匹配集合(忽略 `limit` / `offset` / +`top_n`)并设置 exactness;单库使用 SQL `COUNT`,数据集内建查询计数完整合并/去重后、 +全局窗口前的匹配集合,不计数已截断的局部片行。单库路径存在有限 尾部 `LIMIT` 时,执行器会多取一行来设置 `has_more`,不必先做计数。CTE 内部的有限 `top_n` 不是尾部 `LIMIT`:排名窗口已满时会置 `has_more` / `truncated`,避免把默认页报成完整集合;精确 `COUNT` 若等于 `returned` @@ -383,8 +444,9 @@ QueryService 调用的共享预算(页面查询与可选 COUNT 共用),作 模板查询(含 `report peak-memory`),不改变行数上限。非数字的 `PT_SNAP_QUERY_TIMEOUT` 归为 `INVALID_PARAMETER`,不是 `QUERY_FAILED`。 -`effective_params` 是应用默认值并按 `choices` 规范化后的参数。带 `-n` 时, -`limit` 是与执行器相同的**尾部** SQL `LIMIT`:已声明的模板 `limit` 与 `-n` +`effective_params` 是应用默认值并按 `choices` 规范化后的参数。数据集 `limit` 表示 +最终全局输出窗口上限(相同的正上限取小规则),不是逐片 SQL LIMIT;筛选/合并已经完成。 +单库路径带 `-n` 时,`limit` 是与执行器相同的**尾部** SQL `LIMIT`:已声明的模板 `limit` 与 `-n` 取较小值;渲染后的 SQL 没有尾部 `LIMIT` 时追加 `-n`。CTE 内部的 `top_n` 仍是独立参数,不会改写 `limit`。 `--template-info --json` 包含 `semantics_version`、`interpretation_limits` diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 03cf235..83dc9b5 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -4,9 +4,10 @@ ## 范围与版本 -本文定义产物协议、共享校验模型及下述**显式**兼容 CLI 导出器,不实现跨片查询引擎。 +本文定义产物协议、共享校验模型及下述**显式**兼容 CLI 导出器;跨片执行由 +[P3 内建支持矩阵](querying.md#数据集全局内建支持p3) 单独定义。 `import` 默认仍生成一个原生 v2 DB;`split` 仍输出可回放 pickle/JSON。现有单库 -v1/v2 单库分析不变。下述 P1 reader 新增将 complete 兼容 v1 manifest 或目录作为 +v1/v2 分析入口保留;P3 真实事件 peak/gap 与全部 action 完整文本栈统计语义在查询指南明确版本化。下述 P1 reader 新增将 complete 兼容 v1 manifest 或目录作为 focus;`complete` 门槛只约束数据集,不约束旧单库分析。 `pt_snap_cli.core.dataset_contract` 提供 `parse_manifest`、`validate_event_ids`、 @@ -138,8 +139,8 @@ dictionary 的十进制整数 key 以文本存储,使用真实带设备后缀 设备间没有全局事件顺序。边界 ID 不能作为范围端点。P0 的 `DatasetValidation.query_execution=False` 描述校验而非执行。下述 P1 resolver -使 CLI/API 接受完整 manifest/目录;通用跨片聚合仍留后续,定点来源解析见 -[查询指南](querying.md#数据集定点事件归因)。 +使 CLI/API 接受完整 manifest/目录;执行能力见 [P3 支持矩阵](querying.md#数据集全局内建支持p3) +及[定点来源](querying.md#数据集定点事件归因)。 ```python from pt_snap_cli.core.dataset_contract import QueryScope, validate_dataset @@ -165,7 +166,8 @@ pt-snap query --template-use event --slice 1 --json `overview.dataset` 输出格式、manifest 版本/状态、内容指纹、真实/边界事件数、逐设备 分片路径/范围及能力。事件范围排除负 ID 合成行(单库 overview 也如此)。支持文本调用栈, -不保证每行有栈;结构化 frames 与跨片查询**不可用**,未知扩展的能力声明不会提升它们。 +不保证每行有栈;P0 结构化 frame 校验仍**不可用**,未知声明不会提升。跨片内建查询 +遵循 P3 矩阵;已识别有序 frame 的查询覆盖与 P0 校验能力独立。 外部数据集级导入 metadata 为 unavailable;逐片 metadata 不足以证明整体来源。 下述兼容导出器提供可识别的整份 `ptSnap` 证明,其完整校验后的 metadata 为 available。 @@ -184,15 +186,15 @@ with SnapshotAnalyzer(Path("/capture/snapshot.pkl.msinsight"), device_id=0) as a ``` 数据集 focus 的 `event` 模板支持有界寻址:真实 ID 路由至所在片,闭区间 `min_id/max_id` -必须落在一个片内,`--slice`/`slice_index` 列出该片真实事件。负 ID 不能作为整份数据集 +可跨片并全局筛选/排序/分页,`--slice`/`slice_index` 限制为该片真实事件。负 ID 不能作为整份数据集 selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、数据集指纹和边界排除。 总数和分页针对该 scope,**不是**隐式截断的整份数据集。CLI `effective_params` 的调用者 参数默认值另受 `scope` 约束。定点 `active_blocks_at_event` 和 `active_memory_callstack_at_event` 现于分组/排名前批量解析 alloc/free/完整栈来源, 原版无扩展产物也适用。生命周期身份、独立来源/frame 覆盖及版本化有序 frame reader -接口详见[定点归因](querying.md#数据集定点事件归因)。数据集全局聚合/peak/list/leak、 -跨片事件区间及无界多片 event 查询仍明确失败,不静默选首片/最新片。直接单库 v1/v2 -查询保留现有语义。 +接口详见[定点归因](querying.md#数据集定点事件归因)。数据集全局内建查询遵循 P3 矩阵 +及累计工作上限;任意 SQL/自定义 override 明确失败,不静默选首片/最新片。单库真实 +事件 peak/gap 与全部 action 完整文本栈统计版本在相同合同中明确说明。 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 @@ -212,7 +214,7 @@ building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写 不是文件系统沙箱或生产端锁。 native-v2 私有 staging **没有已发布 manifest**,不能作为数据集;单个 DB 仍由单库 -入口读取。已发布原生格式与兼容导出是下述独立显式协议。跨片生命周期执行仍属独立工作, +入口读取。已发布原生格式与兼容导出是下述独立显式协议。跨片生命周期执行遵循 P3 矩阵, 安全读取原版 WAL 不等于 GUI 验收。 ## 已发布原生数据集(P2) @@ -252,7 +254,7 @@ trace 列与单库 v2 一致(`callstackId`,非内联文本),保留 OOM a 最终化引用,保留 P1 打开前的别名/sidecar/持久 WAL 检查及有界借用 cache 所有权。 无需源 pickle 即可 focus、metadata、overview 和单片 `event` 寻址。原生数据集 metadata 为 available,外部兼容数据集为 unavailable(下述可识别兼容证明除外)。原生模式不隐含 -迁移、修复、数据集全局聚合/peak/list/leak、兼容导出、salted hash 或 GUI 验收;定点 +迁移、修复、兼容导出、salted hash 或 GUI 验收;数据集全局内建查询遵循 P3 矩阵。定点 来源解析通过上述共享查询路径可用。P0 schema/manifest 语义不变,仅兼容最终化 WAL 的只读传输修正见上文。 @@ -311,7 +313,7 @@ native-v2 输出仍独立且不变。 外片真实事件掩盖片内 JOIN 缺口。OOM/未知 action、非零/稀疏 ID、空/仅静态选择失败; 原生保留更广合同。所有选中成员最终化、关闭、校验后才发布 complete。 `pt_snap_block_reference` v1 是隔离文本扩展,不是有序 frames 或跨来源查询承诺; -跨来源查询/全局聚合仍属后续独立工作。文本不能恢复原始有序 frame 数组;原版 workspace +跨来源查询/全局聚合遵循 P3 查询矩阵。文本不能恢复原始有序 frame 数组;原版 workspace raw-frame 丢失与字符串 OOM action 强制转换属于已知局限,不应仿制为兼容能力。 `cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`,salt **前置**。 diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 2d23442..cfa20e1 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -665,6 +665,7 @@ def query_database( typer.echo(f"Description: {info.description}") typer.echo(f"Category: {info.category}") typer.echo(f"Devices: {info.devices}") + typer.echo(f"Dataset Support: {json.dumps(info.dataset_support, sort_keys=True)}") typer.echo( "Semantics Version: " f"{info.semantics_version if info.semantics_version is not None else 'none'}" diff --git a/src/pt_snap_cli/cli_reports.py b/src/pt_snap_cli/cli_reports.py index b9abd25..f20831f 100644 --- a/src/pt_snap_cli/cli_reports.py +++ b/src/pt_snap_cli/cli_reports.py @@ -29,6 +29,7 @@ FocusNotConfiguredError, FocusService, InvalidDeviceError, + InvalidParameterError, QueryExecutionError, ReportService, TemplateNotFoundError, @@ -71,6 +72,16 @@ def report_peak_memory( help="Opt-in UTF-8 byte budget per callstack (0: identity only; negative: full text). Not a response byte cap or token budget.", ), ] = -1, + start_id: Annotated[ + int | None, typer.Option("--start-id", help="First real event ID in peak range") + ] = None, + end_id: Annotated[ + int | None, typer.Option("--end-id", help="Last real event ID in peak range") + ] = None, + timeout: Annotated[ + float | None, + typer.Option("--timeout", help="One report-wide deadline in seconds; <=0 disables"), + ] = None, json_output: Annotated[bool, _json_flag()] = False, ) -> None: """Generate a peak memory attribution report.""" @@ -84,6 +95,9 @@ def report_peak_memory( include_static=include_static, limit=limit, stack_bytes=stack_bytes, + start_id=start_id, + end_id=end_id, + timeout_s=timeout, ) except ValueError as e: _error(str(e), code=INVALID_PARAMETER, hint="Use --metric active, allocated, or reserved.") @@ -120,6 +134,7 @@ def report_peak_memory( except ( TemplateNotFoundError, TemplateRenderError, + InvalidParameterError, QueryExecutionError, DatabaseSchemaError, ) as e: diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index d1e199b..0fcc5a0 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -152,7 +152,7 @@ def event_attribution( selected = selected[:max_rows] if exact_total: has_more = len(all_rows) > len(selected) + offset - sources.budget.remaining() + sources.budget.consume(selected) return QueryResult( total=len(all_rows) if exact_total else len(selected), returned=len(selected), diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py new file mode 100644 index 0000000..cbc73f6 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_global.py @@ -0,0 +1,484 @@ +"""Explicit built-in global algorithms; no local caps or temporary merge database. + +Fetched work is cumulatively bounded by QueryBudget, including sources and final +output. Sorting/materialization is O(bounded work), NOT O(max_rows). A budget +failure rejects the operation rather than returning an alleged global answer. +""" + +from __future__ import annotations + +import hashlib +from typing import cast + +from pt_snap_cli.core.dataset_contract import QueryScope, SliceRecord +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError +from pt_snap_cli.core.models import QueryResult + +METRICS = ("allocated", "active", "reserved") + + +def _range( + sources: DatasetSourceResolver, params: dict[str, object], slice_index: int | None +) -> tuple[int, int, list[SliceRecord]]: + device = sources.device + if slice_index is not None: + sources.dataset.paths(QueryScope("slice", device.device_id, slice_index)) + members = [device.slices[slice_index]] + else: + members = list(device.slices) + low, high = members[0].start_event_id, members[-1].end_event_id + start = params.get("start_id", params.get("min_id")) + end = params.get("end_id", params.get("max_id")) + event = params.get("id") + # `id` on block means block ID, so its caller does not pass it here. + for endpoint in (start, end, event): + if endpoint is not None and (type(endpoint) is not int or not low <= endpoint <= high): + raise InvalidParameterError("Event range must lie within the selected real scope.") + low = cast(int, start) if start is not None else low + high = cast(int, end) if end is not None else high + if event is not None: + low, high = max(low, cast(int, event)), min(high, cast(int, event)) + # Contradictory filters remain an empty matching set, not a made-up event. + elif low > high: + raise InvalidParameterError("Event range start must not exceed end.") + return low, high, [s for s in members if s.end_event_id >= low and s.start_event_id <= high] + + +def _scope( + sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord] +) -> dict[str, object]: + return { + "kind": "event_range", + "device_id": sources.device.device_id, + "slice_index": items[0].index if len(items) == 1 else None, + "slice_indices": [s.index for s in items], + "first_event_id": low, + "last_event_id": high, + "requested_range": {"first_event_id": low, "last_event_id": high}, + "dataset_path": str(sources.dataset.root), + "fingerprint": sources.dataset.fingerprint, + "boundary_events_included": False, + "range_complete": True, + "work_budget": { + "max_rows": sources.budget.max_work_rows, + "max_bytes": sources.budget.max_work_bytes, + "unit": "cumulative_fetched_and_output_serialized_values; not_RSS", + }, + } + + +def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord]): + for item in items: + rows = sources.read( + sources.dataset.root / item.file, + f'SELECT * FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + [low, high], + ) + yield item, rows + + +def _peaks(sources, low, high, items, template, scope): + chosen: dict[str, dict[str, object]] = {} + count, first, last = 0, None, None + for _item, rows in _traces(sources, low, high, items): + for row in rows: + sources.budget.remaining() + event = cast(int, row["id"]) + count += 1 + first = event if first is None else min(first, event) + last = event if last is None else max(last, event) + for metric in METRICS: + if metric not in chosen or (cast(int, row[metric]), -event) > ( + cast(int, chosen[metric][metric]), + -cast(int, chosen[metric]["id"]), + ): + chosen[metric] = row + result: dict[str, object] = {} + for metric in METRICS: + row = chosen.get(metric, {}) + result[f"peak_{metric}"] = row.get(metric) + result[f"peak_{metric}_event_id"] = row.get("id") + if template == "allocator_gap": + for other in METRICS: + if other != metric: + result[f"{other}_at_{metric}_peak"] = row.get(other) + for other in ("active", "allocated"): + result[f"reserved_{other}_gap_at_{metric}_peak"] = ( + cast(int, row["reserved"]) - cast(int, row[other]) if row else None + ) + if template == "allocator_gap": + ids = [result[f"peak_{m}_event_id"] for m in METRICS] + result["all_peaks_same_event"] = int(None not in ids and len(set(ids)) == 1) + for a, b in (("allocated", "active"), ("active", "reserved"), ("allocated", "reserved")): + result[f"{a}_{b}_same_event"] = int( + result[f"peak_{a}_event_id"] is not None + and result[f"peak_{a}_event_id"] == result[f"peak_{b}_event_id"] + ) + scope["actual_range"] = { + "first_event_id": first, + "last_event_id": last, + "real_event_count": count, + } + scope["peak_owner_slices"] = { + m: sources.owner(cast(int, r["id"])).index for m, r in chosen.items() + } + return [result] + + +def _matches(row: dict[str, object], params: dict[str, object], fields: tuple[str, ...]) -> bool: + for field in fields: + value = params.get(field) + if value is not None and row[field] != value: + return False + for prefix, predicate in (("min_", lambda a, b: a >= b), ("max_", lambda a, b: a <= b)): + value = params.get(prefix + field) + if value is not None and (row[field] is None or not predicate(row[field], value)): + return False + return True + + +def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: + # SQLite NULL placement and declared primary/id direction, plus full identity + # tie-breaks for unusual unproved historical rows with the same local ID. + rows.sort(key=lambda r: cast(str, r.get("lifecycle_id", "")), reverse=descending) + rows.sort(key=lambda r: cast(int, r["id"]), reverse=descending) + rows.sort(key=lambda r: (r[key] is not None, r[key]), reverse=descending) + + +def _events(sources, low, high, items, params, template, scope): + rows = [] + first, last, count = None, None, 0 + for _item, batch in _traces(sources, low, high, items): + for row in batch: + sources.budget.remaining() + event = cast(int, row["id"]) + first = event if first is None else min(first, event) + last = event if last is None else max(last, event) + count += 1 + fields = ("allocated", "active", "reserved") + if template == "event": + fields += ("id", "action", "address", "size", "stream") + if _matches(row, params, fields): + rows.append(row) + scope["actual_range"] = { + "first_event_id": first, + "last_event_id": last, + "real_event_count": count, + } + if template == "event": + # Only returned events need formatted text; resolve separately after the + # global window below. `order_by=callstack` requires text BEFORE sorting. + if params["order_by"] == "callstack": + refs = sources.events(cast(int, r["id"]) for r in rows) + for row in rows: + row["callstack"] = refs[cast(int, row["id"])].event.get("callstack") + _sort(rows, params["order_by"], params["order_dir"] == "DESC") + return rows + + +def _lifecycles(sources, items, scope): + merged: dict[str, dict[str, object]] = {} + terminal = sources.device.slices[-1].index + resolved_alloc = resolved_free = unproved = 0 + for item in items: + blocks = sources.read( + sources.dataset.root / item.file, + f'SELECT * FROM "block_{sources.device.device_id}" ORDER BY id', + ) + refs = sources.events( + cast(int, row[field]) + for row in blocks + for field in ("allocEventId", "freeEventId") + if type(row[field]) is int and cast(int, row[field]) >= 0 + ) + for block in blocks: + sources.budget.remaining() + alloc, free = cast(int, block["allocEventId"]), block["freeEventId"] + allocation = refs.get(alloc) + allocation = allocation if allocation and allocation.event["action"] == 4 else None + completion = refs.get(cast(int, free)) + completion = completion if completion and completion.event["action"] == 6 else None + proved = allocation is not None + stable_negative = alloc == -1 and cast(int, block["id"]) < 0 + token = ( + f"alloc:{alloc}" + if proved + else ( + f"preexisting:{block['id']}" + if stable_negative + else f"unproved:slice:{item.index}:block:{block['id']}" + ) + ) + identity = f"{sources.namespace}:{token}" + old = merged.get(identity) + if old is not None: + # A shared token with incompatible invariant observations is not + # evidence to merge by address or guess which allocation wins. + if any( + old[f] != block[f] for f in ("address", "size", "requestedSize", "allocEventId") + ): + raise QueryExecutionError( + "Conflicting observations for a proved lifecycle identity." + ) + if old["free_source"] and completion and old["freeEventId"] != free: + raise QueryExecutionError( + "Conflicting free-completion events for one lifecycle." + ) + if old["free_source"] and not completion: + free = old["freeEventId"] + completion_dict = old["free_source"] + else: + completion_dict = completion.to_dict() if completion else None + else: + completion_dict = completion.to_dict() if completion else None + block.update( + { + "lifecycle_id": identity, + "identity_status": ( + "proved_allocation_event" + if proved + else "stable_negative_token" if stable_negative else "unproved_shard_local" + ), + "allocation_source": allocation.to_dict() if allocation else None, + "allocation_source_status": ( + "resolved" + if proved + else "unknown_preexisting" if alloc == -1 else "missing" + ), + "free_source": completion_dict, + "free_source_status": ( + "resolved" + if completion_dict + else "live_or_unknown" if free is None or cast(int, free) < 0 else "missing" + ), + "freeEventId": free, + "observation_slice_index": item.index, + "state_scope": "latest_slice_observation", + "terminal_survivor": item.index == terminal + and block["state"] in (0, 1) + and (free is None or cast(int, free) < 0), + "category": ( + "dynamic" if alloc >= 0 else "static" if free == -1 else "preexisting" + ), + } + ) + merged[identity] = block + rows = list(merged.values()) + for row in rows: + resolved_alloc += row["allocation_source"] is not None + resolved_free += row["free_source"] is not None + unproved += row["identity_status"] == "unproved_shard_local" + scope["source_coverage"] = { + "coverage_scope": "all_deduplicated_lifecycles_before_filters_and_page", + "range_complete": True, + "lifecycles": len(rows), + "allocation_events_resolved": resolved_alloc, + "free_events_resolved": resolved_free, + "unproved_identities": unproved, + "allocation_source_complete": all( + r["allocation_source"] is not None for r in rows if cast(int, r["allocEventId"]) >= 0 + ), + "source_queries": sources.query_count, + } + return rows + + +def _stacks(sources, low, high, items, params, scope): + grouped = {} + events = ordered = 0 + for _item, batch in _traces(sources, low, high, items): + refs = sources.events(cast(int, r["id"]) for r in batch) + for source in refs.values(): + sources.budget.remaining() + text = source.event.get("callstack") + # Historical alloc_count is ALL stack-bearing real trace actions, + # including free/segment/workspace. Never claim allocation-only. + referenced = ( + sources.dataset.callstack_layout == "v2" and source.local_stack_id is not None + ) + if text is None and source.frames is None and not referenced: + continue + identity = source.stack_id + # Point attribution normalizes empty text into missing; statistics + # historically count the non-NULL empty string as its own group. + if text == "" and source.frames is None: + identity = f"{sources.namespace}:text:sha256:{hashlib.sha256(b'').hexdigest()}" + row = grouped.setdefault( + identity, + { + "callstack": text if text is not None else "[missing callstack]", + "stack_kind": source.stack_kind, + "text_kind": source.text_kind, + "alloc_count": 0, + "total_size": 0, + "max_size": 0, + "source_stack_id": identity, + "stack_event_id": source.event["id"], + "frames_status": source.frames_status, + }, + ) + row["alloc_count"] += 1 + row["total_size"] += source.event["size"] + row["max_size"] = max(row["max_size"], source.event["size"]) + if row["text_kind"] != "captured" and source.text_kind == "captured": + row["callstack"], row["stack_event_id"] = text, source.event["id"] + row["text_kind"] = "captured" + events += 1 + ordered += source.frames is not None + rows = [ + r + for r in grouped.values() + if r["alloc_count"] >= params["min_count"] and r["total_size"] >= params["min_size"] + ] + for row in rows: + row["avg_size"] = row["total_size"] / row["alloc_count"] + rows.sort( + key=lambda r: ( + -r["total_size"], + r["callstack"] or "", + r["stack_kind"] == "missing", + r["source_stack_id"], + ) + ) + scope["source_coverage"] = { + "range_complete": True, + "coverage_scope": "all_real_stack_bearing_actions_before_global_thresholds", + "stack_events": events, + "ordered_frame_events": ordered, + "source_queries": sources.query_count, + } + return rows + + +def global_query( + sources: DatasetSourceResolver, + template: str, + params: dict[str, object], + max_rows: int | None, + exact_total: bool, + slice_index: int | None, + semantics_version: int | None, +) -> QueryResult: + range_params = ( + params if template in ("event", "allocation", "memory_peak", "allocator_gap") else {} + ) + low, high, items = _range(sources, range_params, slice_index) + scope = _scope(sources, low, high, items) + if template in ("memory_peak", "allocator_gap"): + rows = _peaks(sources, low, high, items, template, scope) + elif template in ("event", "allocation"): + rows = _events(sources, low, high, items, params, template, scope) + elif template == "callstack_analysis": + rows = _stacks(sources, low, high, items, params, scope) + elif template == "preexisting_live": + event = cast(int, params["event_id"]) + item = sources.target(event, slice_index) + blocks = sources.read( + sources.dataset.root / item.file, + f'SELECT COUNT(*) AS block_count, COALESCE(SUM(size),0) AS size_bytes FROM "block_{sources.device.device_id}" WHERE allocEventId=-1 AND (freeEventId IS NULL OR freeEventId> ? OR (freeEventId<0 AND freeEventId!=-1))', + [event], + ) + rows = blocks + scope = _scope(sources, event, event, [item]) + else: + if template == "leak_detection" and slice_index is not None: + raise QueryExecutionError( + "Dataset leak_detection requires terminal dataset scope, not --slice; select an explicit member for local observations." + ) + rows = _lifecycles(sources, items, scope) + if template == "block": + rows = [ + r + for r in rows + if _matches( + r, + params, + ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"), + ) + ] + _sort(rows, cast(str, params["order_by"]), params["order_dir"] == "DESC") + elif template == "leak_detection": + rows = [ + r + for r in rows + if r["terminal_survivor"] + and r["allocation_source"] is not None + and cast(int, r["size"]) >= cast(int, params["min_size"]) + ] + _sort(rows, "size", True) + elif template == "freed_block_lifetime": + buckets = {} + for row in rows: + if row["allocation_source"] is None or row["free_source"] is None: + continue + distance = cast(int, row["freeEventId"]) - cast(int, row["allocEventId"]) + if distance < 0: + continue + index = next( + (i for i, cap in enumerate((1000, 5000, 20000, 100000)) if distance < cap), 4 + ) + bucket = buckets.setdefault( + index, + { + "lifetime_events": ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[index], + "block_count": 0, + "size_bytes": 0, + }, + ) + bucket["block_count"] += 1 + bucket["size_bytes"] += row["size"] + rows = [buckets[i] for i in sorted(buckets)] + else: + raise QueryExecutionError("No dataset-global merge implementation for this template.") + total = len(rows) + offset = max(0, cast(int, params.get("offset", 0))) + limit = cast(int, params.get("limit", -1)) + if max_rows is not None and max_rows > 0: + limit = min(limit, max_rows) if limit >= 0 else max_rows + selected = rows[offset : offset + limit] if limit >= 0 else rows[offset:] + if template == "event": + refs = sources.events(cast(int, r["id"]) for r in selected) + detailed = ( + sources.dataset.ordered_frames_version == 1 or cast(int, params["stack_bytes"]) >= 0 + ) + selected = [dict(refs[cast(int, r["id"])].event) for r in selected] + if detailed: + for row in selected: + source = refs[cast(int, row["id"])] + row.update( + { + "source_stack_id": source.stack_id, + "stack_id": source.stack_id, + "stack_kind": source.stack_kind, + "text_kind": source.text_kind, + "stack_event_id": row["id"] if source.stack_kind == "captured" else None, + "frames": source.frames, + "frames_status": source.frames_status, + } + ) + scope["source_coverage"] = { + "range_complete": True, + "event_position": "after", + "coverage_scope": "returned_events", + "events_resolved": len(refs), + "ordered_frame_events": sum(s.frames is not None for s in refs.values()), + "source_queries": sources.query_count, + } + elif template == "allocation": + selected = [{k: r[k] for k in ("id", *METRICS)} for r in selected] + sources.budget.consume(selected) + more = total > offset + len(selected) + return QueryResult( + total=total if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=more, + truncated=more or offset > 0, + total_is_exact=exact_total or not more and offset == 0, + timeout_s=sources.budget.timeout_s, + scope=scope, + ) diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index 004d727..94126ec 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -23,6 +23,7 @@ ) from pt_snap_cli.core.dataset_files import hash_file as _hash_file from pt_snap_cli.core.dataset_files import require_readonly_member as _require_readonly_member +from pt_snap_cli.core.dataset_support import DATASET_SUPPORT from pt_snap_cli.core.errors import DatabaseSchemaError, InvalidDeviceError, InvalidParameterError from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, @@ -91,12 +92,8 @@ def to_dict(self) -> dict[str, object]: "boundary_event_count": self.validation.boundary_event_count, "text_callstacks": True, "structured_frames": False, - "cross_slice_queries": False, - "supported_queries": [ - "event: real ID, single-slice range, or explicit slice", - "active_blocks_at_event: real point-event with batched alloc/free sources", - "active_memory_callstack_at_event: real point-event attribution before top_n", - ], + "cross_slice_queries": True, + "supported_queries": dict(DATASET_SUPPORT), "cross_slice_sources": True, "ordered_frames_reader_version": 1, "structured_frames_scope": "validation_only; query source coverage is separate", diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 162bb29..7a8e3f0 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -60,10 +60,32 @@ def _bounded_depth(value: object) -> bool: return True -@dataclass(frozen=True) +@dataclass class QueryBudget: timeout_s: float | None started: float + max_work_rows: int = 100_000 + max_work_bytes: int = 64 * 1024 * 1024 + work_rows: int = 0 + work_bytes: int = 0 + + def consume(self, rows: list[dict[str, object]]) -> None: + """Bound cumulative fetched rows/serialized values, not merely output. + + Includes repeated source reads. Exceeding either ceiling fails the whole + operation, never returning a silently partial global conclusion. This is + not an RSS ceiling: SQLite and one fetched batch have their own overhead. + """ + self.remaining() + self.work_rows += len(rows) + self.work_bytes += sum( + len(json.dumps(row, ensure_ascii=True, default=str).encode("utf-8")) for row in rows + ) + if self.work_rows > self.max_work_rows or self.work_bytes > self.max_work_bytes: + raise QueryExecutionError( + "Dataset query work budget exceeded (100000 fetched rows / 64 MiB serialized " + "values by default); no partial global result. Narrow the scope or select a member." + ) def remaining(self) -> float | None: if self.timeout_s is None: @@ -142,6 +164,17 @@ def target(self, event_id: int, slice_index: int | None = None) -> SliceRecord: ) return item + def _consume(self, rows: list[dict[str, object]]) -> None: + # The executor intentionally knows only its own errors. Translate the + # core callback's timeout before its generic exception boundary, so + # cancellation keeps the domain kind and CLI QUERY_TIMEOUT code. + try: + self.budget.consume(rows) + except QueryTimeoutError as exc: + raise ExecutorTimeout(str(exc)) from exc + except QueryExecutionError as exc: + raise ExecutorError(str(exc)) from exc + def read( self, path: Path, sql: str, values: list[object] | None = None ) -> list[dict[str, object]]: @@ -156,7 +189,9 @@ def read( immutable=self.dataset.callstack_layout == "v1", ) self.query_count += 1 - return QueryExecutor(context).execute(sql, values, timeout_s=self.budget.remaining()) + return QueryExecutor(context).execute( + sql, values, timeout_s=self.budget.remaining(), consume=self._consume + ) except ExecutorTimeout as exc: raise QueryTimeoutError( f"Query timed out after {self.budget.timeout_s} seconds" diff --git a/src/pt_snap_cli/core/dataset_support.py b/src/pt_snap_cli/core/dataset_support.py new file mode 100644 index 0000000..6d268dc --- /dev/null +++ b/src/pt_snap_cli/core/dataset_support.py @@ -0,0 +1,28 @@ +"""One shared, explicit per-template dataset execution support matrix.""" + +DATASET_SUPPORT: dict[str, str] = { + "event": "real_events_global_filter_sort_page", + "allocation": "real_events_global_filter_sort_page", + "memory_peak": "independent_real_counter_peaks_earliest_id", + "allocator_gap": "global_peaks_same_event_counters", + "active_blocks_at_event": "owner_event_active_set_cross_slice_sources", + "active_memory_callstack_at_event": "owner_event_global_source_groups_before_top_n", + "preexisting_live": "owner_event_preexisting_occupancy", + "block": "proved_lifecycles_dedup_latest_observation", + "freed_block_lifetime": "proved_completed_lifecycles_once", + "leak_detection": "terminal_dataset_dynamic_survivors_not_confirmed_leaks", + "callstack_analysis": "all_real_stack_bearing_actions_global_identity_weighted_stats", +} + + +def dataset_support(name: str, packaged: bool) -> dict[str, object]: + semantics = DATASET_SUPPORT.get(name) if packaged else None + return { + "supported": semantics is not None, + "semantics": semantics, + "device_scope": "one_selected_device; never sum device peaks", + "custom_sql": "unsupported; select an explicit standalone member", + "work_limit_rows": 100_000, + "work_limit_bytes": 64 * 1024 * 1024, + "budget_scope": "one_query_or_explicit_report_composition", + } diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index e59f7ea..e994284 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -94,6 +94,7 @@ class TemplateInfo: output_schema: list[dict[str, object]] | None semantics_version: int | None = None interpretation_limits: list[str] = field(default_factory=list) + dataset_support: dict[str, object] = field(default_factory=dict) @dataclass(frozen=True) @@ -128,6 +129,9 @@ class PeakMemoryReport: active_bytes_at_event: int | None = None coverage_percent: float | None = None source_coverage: dict[str, object] | None = None + scope: dict[str, object] | None = None + timeout_s: float | None = None + budget_scope: str = "report_composition" @dataclass(frozen=True) diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 6c4c0a5..138e342 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -11,8 +11,10 @@ from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_attribution import event_attribution, summarize_dataset_stacks from pt_snap_cli.core.dataset_contract import QueryScope -from pt_snap_cli.core.dataset_resolver import DatasetResolver, ResolvedDataset +from pt_snap_cli.core.dataset_global import global_query +from pt_snap_cli.core.dataset_resolver import DatasetResolver from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.dataset_support import dataset_support from pt_snap_cli.core.errors import ( DatabaseMissingError, DatabaseSchemaError, @@ -36,6 +38,7 @@ discover_categories, get_query, get_template_info, + is_packaged_query, list_by_category_with_details, list_queries_with_details, ) @@ -160,6 +163,11 @@ def get_template_info(self, name: str) -> TemplateInfo: for param_name, param_details in info["parameters"].items() }, output_schema=copy.deepcopy(info["output_schema"]), + dataset_support=dataset_support( + name, + is_packaged_query(name) + and not (self._executor is not None and self._executor.has_runtime_template(name)), + ), semantics_version=info.get("semantics_version"), interpretation_limits=( [str(item) for item in info["interpretation_limits"]] @@ -193,6 +201,7 @@ def template_info_to_dict(info: TemplateInfo) -> dict[str, Any]: "output_schema": info.output_schema, "semantics_version": info.semantics_version, "interpretation_limits": list(info.interpretation_limits), + "dataset_support": dict(info.dataset_support), } def execute_query( @@ -207,10 +216,15 @@ def execute_query( exact_total: bool = False, timeout_s: float | None = None, slice_index: int | None = None, + _budget: QueryBudget | None = None, ) -> QueryResult: - timeout_s = resolve_query_timeout(timeout_s) - started = time.monotonic() - budget = QueryBudget(timeout_s, started) + budget = ( + _budget + if _budget is not None + else QueryBudget(resolve_query_timeout(timeout_s), time.monotonic()) + ) + timeout_s, started = budget.timeout_s, budget.started + budget.remaining() resolved = self._focus_service.resolve_focus( explicit_db_path=db_path, explicit_device_id=device_id, @@ -229,37 +243,46 @@ def execute_query( sources = DatasetSourceResolver( dataset, selected.device_id, self._context_cache, budget ) - if template in ("active_blocks_at_event", "active_memory_callstack_at_event"): - template_obj = get_query(template) - if template_obj is None: - raise TemplateNotFoundError(f"Template '{template}' not found") - try: - validated = template_obj.validate_params(query_params) - except (TypeError, ValueError) as exc: - raise TemplateRenderError(str(exc)) from exc - if slice_index is not None: - dataset.paths(QueryScope("slice", selected.device_id, slice_index)) - result = event_attribution( - sources, - template, - validated, - max_rows, - exact_total, - slice_index, - template_obj.semantics_version, + template_obj = get_query(template) + if template_obj is None: + raise TemplateNotFoundError(f"Template '{template}' not found") + support = dataset_support(template, is_packaged_query(template)) + overridden = self._executor is not None and self._executor.has_runtime_template( + template + ) + if not support["supported"] or overridden: + raise QueryExecutionError( + "Unsupported dataset-global template/custom SQL or runtime override; " + "select an explicit standalone member. A built-in name is not a merge contract." ) - stack_bytes = validated.get("stack_bytes") - if isinstance(stack_bytes, int) and stack_bytes >= 0: - summarize_dataset_stacks(result.rows, stack_bytes) - budget.remaining() - return result - member, query_params, scope = self._dataset_query( - dataset, selected.device_id, template, query_params, slice_index + for name in ("id", "min_id", "max_id", "start_id", "end_id", "event_id"): + if isinstance(query_params.get(name), bool): + raise InvalidParameterError("Event/block endpoints must not be booleans.") + try: + validated = template_obj.validate_params(query_params) + except (TypeError, ValueError) as exc: + raise TemplateRenderError(str(exc)) from exc + if slice_index is not None: + dataset.paths(QueryScope("slice", selected.device_id, slice_index)) + implementation = ( + event_attribution + if template in ("active_blocks_at_event", "active_memory_callstack_at_event") + else global_query ) - ctx = self._validated_context( - member, generation=dataset.fingerprint, immutable=dataset.callstack_layout == "v1" + result = implementation( + sources, + template, + validated, + max_rows, + exact_total, + slice_index, + template_obj.semantics_version, ) - target_device = selected.device_id + stack_bytes = validated.get("stack_bytes") + if isinstance(stack_bytes, int) and stack_bytes >= 0: + summarize_dataset_stacks(result.rows, stack_bytes) + budget.remaining() + return result else: if slice_index is not None: raise InvalidParameterError("slice_index requires a manifest dataset.") @@ -304,40 +327,6 @@ def execute_query( raise TemplateNotFoundError(f"Template '{template}' not found") from exc raise QueryExecutionError(str(exc)) from exc - if sources is not None and scope is not None: - event_sources = sources.events(row["id"] for row in rows) - # Preserve the original default nine-column event payload. Explicit - # stack summaries or a recognized frame reader add source details. - event_template = get_query(template) - detail_budget = ( - event_template.validate_params(query_params).get("stack_bytes", -1) - if event_template - else -1 - ) - detailed_rows = ( - rows if sources.dataset.ordered_frames_version == 1 or detail_budget >= 0 else [] - ) - for row in detailed_rows: - source = event_sources[row["id"]] - row.update( - { - "source_stack_id": source.stack_id, - "stack_id": source.stack_id, - "stack_kind": source.stack_kind, - "text_kind": source.text_kind, - "stack_event_id": row["id"] if source.stack_kind == "captured" else None, - "frames": source.frames, - "frames_status": source.frames_status, - } - ) - scope["source_coverage"] = { - "range_complete": True, - "event_position": "after", - "coverage_scope": "returned_events", - "events_resolved": len(event_sources), - "ordered_frame_events": sum(s.frames is not None for s in event_sources.values()), - "source_queries": sources.query_count, - } template_obj = get_query(template) if template_obj is not None and "stack_bytes" in template_obj.parameters: stack_bytes = template_obj.validate_params(query_params)["stack_bytes"] @@ -361,85 +350,6 @@ def execute_query( scope=scope, ) - def _dataset_query( - self, - dataset: ResolvedDataset, - device_id: int, - template: str, - params: dict[str, Any], - slice_index: int | None, - ) -> tuple[Path, dict[str, Any], dict[str, object]]: - template_obj = get_query(template) - if template_obj is None: - raise TemplateNotFoundError(f"Template '{template}' not found") - if template != "event": - raise QueryExecutionError( - "Dataset queries support event addressing and point-event active attribution; " - "dataset-global cross-slice aggregation/list/peak/leak queries are not supported." - ) - try: - validated = template_obj.validate_params(params) - except (TypeError, ValueError) as exc: - raise TemplateRenderError(str(exc)) from exc - device = dataset.device(device_id) - if slice_index is not None: - paths = dataset.paths(QueryScope("slice", device_id, slice_index)) - item = device.slices[slice_index] - low, high = item.start_event_id, item.end_event_id - else: - low, high = device.slices[0].start_event_id, device.slices[-1].end_event_id - event_id = validated.get("id") - start = validated.get("min_id") - end = validated.get("max_id") - if event_id is not None: - if ( - not isinstance(event_id, int) - or isinstance(event_id, bool) - or not low <= event_id <= high - ): - raise InvalidParameterError("Event ID must be a real ID within the selected scope.") - low = high = event_id - else: - for value in (start, end): - if value is not None and ( - not isinstance(value, int) - or isinstance(value, bool) - or not low <= value <= high - ): - raise InvalidParameterError( - "Event range must lie within the selected real scope." - ) - low = start if isinstance(start, int) else low - high = end if isinstance(end, int) else high - paths = dataset.paths( - QueryScope("event_range", device_id, start_event_id=low, end_event_id=high) - ) - if len(paths) != 1: - raise QueryExecutionError( - "Event range spans slices; cross-slice execution is not supported. " - "Select a real id, a single-slice range, or --slice." - ) - item = next(item for item in device.slices if dataset.root / item.file == paths[0]) - return ( - paths[0], - { - **params, - "min_id": max(low, start) if isinstance(start, int) else low, - "max_id": min(high, end) if isinstance(end, int) else high, - }, - { - "kind": "event_range", - "device_id": device_id, - "slice_index": item.index, - "first_event_id": low, - "last_event_id": high, - "db_path": str(paths[0]), - "dataset_path": str(dataset.root), - "fingerprint": dataset.fingerprint, - "boundary_events_included": False, - }, - ) - def _get_executor(self, ctx: Context) -> QueryExecutor: """Reuse the executor while its cached database context remains valid. diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index c7e4583..69a3090 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -1,11 +1,13 @@ from __future__ import annotations +import time from pathlib import Path from typing import Literal, cast +from pt_snap_cli.core.dataset_sources import QueryBudget from pt_snap_cli.core.focus_service import FocusService from pt_snap_cli.core.models import PeakMemoryReport, QueryResult -from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.core.query_service import QueryService, resolve_query_timeout PeakMetric = Literal["active", "allocated", "reserved"] @@ -42,6 +44,7 @@ def event_attribution( stack_bytes: int = -1, start_dir: Path | None = None, timeout_s: float | None = None, + _budget: QueryBudget | None = None, ) -> QueryResult: """Reuse core point-event attribution, also for externally selected peaks. @@ -61,6 +64,7 @@ def event_attribution( device_id=device_id, start_dir=start_dir, timeout_s=timeout_s, + _budget=_budget, ) def peak_memory_report( @@ -72,14 +76,22 @@ def peak_memory_report( limit: int = 20, start_dir: Path | None = None, stack_bytes: int = -1, + *, + start_id: int | None = None, + end_id: int | None = None, + timeout_s: float | None = None, ) -> PeakMemoryReport: if metric not in _EVENT_ID_BY_METRIC: raise ValueError( f"Invalid metric '{metric}'. Must be one of: active, allocated, reserved" ) + budget = QueryBudget(resolve_query_timeout(timeout_s), time.monotonic()) + range_params = {"start_id": start_id, "end_id": end_id} peak_result = self._query_service.execute_query( "memory_peak", + params=range_params, + _budget=budget, db_path=db_path, device_id=device_id, start_dir=start_dir, @@ -105,10 +117,14 @@ def peak_memory_report( callstack_groups=[], total_is_exact=True, effective_params=attribution_params, + scope=peak_result.scope, + timeout_s=budget.timeout_s, ) gap_result = self._query_service.execute_query( "allocator_gap", + params=range_params, + _budget=budget, db_path=db_path, device_id=device_id, start_dir=start_dir, @@ -121,6 +137,7 @@ def peak_memory_report( limit=limit, stack_bytes=stack_bytes, start_dir=start_dir, + _budget=budget, ) gap = gap_result.rows[0] if gap_result.rows else None @@ -150,6 +167,8 @@ def peak_memory_report( included_bytes=included_bytes, active_bytes_at_event=active_bytes, coverage_percent=coverage, + scope=peak_result.scope, + timeout_s=budget.timeout_s, source_coverage=cast( dict[str, object] | None, (callstack_result.scope or {}).get("source_coverage") ), diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 7537bc0..1704c3d 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -5,7 +5,7 @@ import re import sqlite3 import time -from collections.abc import Generator, Mapping +from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from pathlib import Path from typing import Any @@ -326,6 +326,8 @@ def execute( sql: str, params: list[Any] | None = None, timeout_s: float | None = None, + *, + consume: Callable[[list[dict[str, Any]]], None] | None = None, ) -> list[dict[str, Any]]: """Execute raw SQL query. @@ -345,12 +347,26 @@ def execute( with self._context.connect() as conn: with _sqlite_timeout(conn, timeout_s): cursor = conn.cursor() - if params: - cursor.execute(sql, params) - else: - cursor.execute(sql) - columns = [desc[0] for desc in cursor.description] - return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + try: + if params: + cursor.execute(sql, params) + else: + cursor.execute(sql) + columns = [desc[0] for desc in cursor.description] + if consume is None: + # Preserve standalone behavior and its established + # cursor contract. Dataset reads explicitly opt in. + return [ + dict(zip(columns, row, strict=True)) for row in cursor.fetchall() + ] + result = [] + while batch := cursor.fetchmany(256): + rows = [dict(zip(columns, row, strict=True)) for row in batch] + consume(rows) + result.extend(rows) + return result + finally: + cursor.close() except sqlite3.OperationalError as e: if timeout_s is not None and timeout_s > 0 and _is_sqlite_interrupt(e): raise self._timeout_error(timeout_s) from e @@ -570,6 +586,10 @@ def register_template(self, template: QueryTemplate, config_name: str = "default self._configs[config_name] = QueryConfig() self._configs[config_name].queries[template.name] = template + def has_runtime_template(self, name: str) -> bool: + """A local override cannot inherit a packaged dataset merge contract.""" + return any(name in config.queries for config in self._configs.values()) + def list_templates(self) -> list[str]: """List all available template names. diff --git a/src/pt_snap_cli/query/registry.py b/src/pt_snap_cli/query/registry.py index 4fcbdf3..08e9208 100644 --- a/src/pt_snap_cli/query/registry.py +++ b/src/pt_snap_cli/query/registry.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import warnings from collections.abc import Callable from pathlib import Path @@ -331,9 +332,29 @@ def _load_yaml_templates(template_dir: Path | str | None = None) -> None: warnings.warn(f"Failed to load query template from {yaml_file}: {e}", stacklevel=2) +_packaged: dict[str, tuple[QueryTemplate, QueryTemplate]] = {} + + +def is_packaged_query(name: str) -> bool: + """Only unchanged, actually loaded packaged objects have global semantics. + + A runtime override, even with a built-in name or identical SQL, is custom. + Retain a deep copy too, since QueryTemplate is mutable. + """ + current = get_query(name) + original = _packaged.get(name) + return original is not None and current is original[0] and current == original[1] + + def _load_all_templates() -> None: - """Load all templates from YAML files.""" + """Load all templates from YAML files and record their actual identities.""" _load_yaml_templates() + _packaged.clear() + _packaged.update( + (name, (template, copy.deepcopy(template))) + for name in list_queries() + if (template := get_query(name)) is not None + ) try: diff --git a/src/pt_snap_cli/query/templates/basic/block.yaml b/src/pt_snap_cli/query/templates/basic/block.yaml index e18f1af..195ed74 100644 --- a/src/pt_snap_cli/query/templates/basic/block.yaml +++ b/src/pt_snap_cli/query/templates/basic/block.yaml @@ -120,3 +120,60 @@ queries: type: int - column: freeEventId type: int + - column: lifecycle_id + type: str + units: text + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: generation/device/proved allocation-event or stable negative lifetime token; unproved histories use a separate shard-local identity. Never address or local stack identity."] + - column: identity_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: proved_allocation_event, stable_negative_token, or unproved_shard_local. Unknown histories are not guessed into a global lifetime."] + - column: allocation_source + type: object + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: resolved alloc-action event, owning slice/local stack provenance, canonical full source identity and optionally covered ordered raw frames; NULL for preexisting or missing proof."] + - column: allocation_source_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: resolved, unknown_preexisting, or missing. Missing proof does not discard dynamic bytes or invent an allocation event."] + - column: free_source + type: object + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: proved free_completed event with owning-source provenance, never free_requested; NULL for live/unknown or missing completion proof."] + - column: free_source_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: resolved, live_or_unknown, or missing. No recorded completion alone does not prove a leak."] + - column: observation_slice_index + type: int + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: latest selected shard containing this lifetime observation, not necessarily the terminal dataset shard."] + - column: state_scope + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: latest_slice_observation. Stored state is not reconstructed at an arbitrary event or at dataset end when the lifetime is absent from the terminal shard."] + - column: terminal_survivor + type: bool + units: flag + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: present in the terminal dataset shard with active/pending-free state and no recorded completion. Survival is not a confirmed leak or proof of allocation history."] + - column: category + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: dynamic, static, or preexisting according to captured lifecycle IDs. Identity/source proof and terminal survival are separate fields."] diff --git a/src/pt_snap_cli/query/templates/business/leak_detection.yaml b/src/pt_snap_cli/query/templates/business/leak_detection.yaml index f9e3004..374b1bf 100644 --- a/src/pt_snap_cli/query/templates/business/leak_detection.yaml +++ b/src/pt_snap_cli/query/templates/business/leak_detection.yaml @@ -51,3 +51,78 @@ queries: interpretation_limits: - "Points to the captured allocation event. This template keeps only allocEventId >= 0, so the uncaptured sentinel -1 is excluded." - "Event IDs are ordering markers, not timestamps." + - column: requestedSize + type: int + units: bytes + metric_semantics: instantaneous_occupancy + scope: dynamic + interpretation_limits: ["Dataset only, optional: user-requested bytes of the terminal candidate's observed block, distinct from its rounded size."] + - column: state + type: int + metric_semantics: classification + scope: dynamic + interpretation_limits: ["Dataset only, optional: terminal-shard observation, active_allocated=1 or active_pending_free=0 for these candidates; not a confirmed leak classification."] + - column: freeEventId + type: int + units: event_id + metric_semantics: ordering_marker + scope: captured_range + sentinel: -1 + interpretation_limits: ["Dataset only, optional: recorded completion ID; these terminal candidates have no recorded free_completed event. -1 is live or unknown, not proof of a leak."] + - column: lifecycle_id + type: str + units: text + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: generation/device/proved allocation-event identity, deduplicated before the final window; never address or local stack identity."] + - column: identity_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: proved_allocation_event for these candidates. Static, preexisting and unproved shard-local histories are excluded."] + - column: allocation_source + type: object + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: resolved alloc-action event, owning slice/local stack provenance, canonical full source identity and optionally covered ordered raw frames."] + - column: allocation_source_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: resolved for these captured dynamic candidates; missing or preexisting allocation proof is not asserted as a leak candidate."] + - column: free_source + type: object + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: completion-source payload; NULL for these candidates without a recorded free_completed event. A free_requested action is not a completed free."] + - column: free_source_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: live_or_unknown for these candidates; absence of completion is not a confirmed leak."] + - column: observation_slice_index + type: int + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: terminal dataset shard containing the candidate observation; earlier-shard locally unfreed observations are not unioned into this result."] + - column: state_scope + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: latest_slice_observation, which is the terminal shard for these candidates. State is an observation, not a diagnosis of leaking."] + - column: terminal_survivor + type: bool + units: flag + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: true for these present, active/pending-free terminal lifetimes without recorded completion; survival is not a confirmed leak."] + - column: category + type: str + units: text + metric_semantics: classification + scope: dynamic + interpretation_limits: ["Dataset only, optional: dynamic for these candidates; static/preexisting categories are excluded independently of output caps."] diff --git a/src/pt_snap_cli/query/templates/statistical/allocator_gap.yaml b/src/pt_snap_cli/query/templates/statistical/allocator_gap.yaml index a662b8e..7bc0cdf 100644 --- a/src/pt_snap_cli/query/templates/statistical/allocator_gap.yaml +++ b/src/pt_snap_cli/query/templates/statistical/allocator_gap.yaml @@ -4,8 +4,9 @@ queries: description: Compare allocated, active, and reserved peak events and same-event gaps devices: - all - semantics_version: 1 + semantics_version: 2 interpretation_limits: + - "Only real trace events (id >= 0) select peaks and same-event counters, matching memory_peak semantics version 2." - "Gaps are computed from counters at the same peak event. A reserved-active gap is not itself fragmentation." - "Allocated, active, and reserved may peak at different events; use the same-event flags before comparing across metrics." - "Event IDs are ordering markers, not elapsed time." @@ -24,7 +25,7 @@ queries: WITH filtered AS ( SELECT id, allocated, active, reserved FROM {{ device_trace_table }} - WHERE 1=1 + WHERE id >= 0 {% if start_id is not none %}AND id >= {{ start_id|int }}{% endif %} {% if end_id is not none %}AND id <= {{ end_id|int }}{% endif %} ), peaks AS ( diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index 7cdb3a8..67512a5 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -1,7 +1,12 @@ version: "1.0" queries: callstack_analysis: - description: Analyze memory allocation by callstack + description: Analyze all real stack-bearing trace actions by full callstack identity + semantics_version: 2 + interpretation_limits: + - "alloc_count is a retained legacy field name: it counts ALL real stack-bearing trace actions, not allocation-only events. v1 requires non-NULL text; v2 requires a non-NULL stack reference and retains absent/NULL joined text as a separate missing group. Empty text counts. Negative boundaries are excluded." + - "Statistics group full captured text in both standalone layouts; dataset grouping uses canonical text or explicitly covered ordered raw arrays before thresholds and ranking." + - "avg_size is total_size/alloc_count, not an average of shard averages; size sums include free/segment/workspace actions and are not live occupancy." devices: - all - "0" @@ -11,7 +16,7 @@ queries: type: int default: 5 required: false - description: Minimum allocation count to include + description: Minimum real stack-bearing action count (legacy alloc_count) to include min_size: type: int default: 0 @@ -31,7 +36,7 @@ queries: AVG(size) as avg_size, MAX(size) as max_size FROM {{ device_trace_table }} - WHERE callstack IS NOT NULL + WHERE id >= 0 AND callstack IS NOT NULL GROUP BY callstack HAVING alloc_count >= {{ min_count }} AND total_size >= {{ min_size }} @@ -40,25 +45,17 @@ queries: v2: | SELECT COALESCE(c.callstack, '[missing callstack]') AS callstack, - grouped.alloc_count AS alloc_count, - grouped.total_size AS total_size, - grouped.avg_size AS avg_size, - grouped.max_size AS max_size - FROM ( - SELECT - callstackId, - COUNT(*) as alloc_count, - SUM(size) as total_size, - AVG(size) as avg_size, - MAX(size) as max_size - FROM {{ device_trace_table }} - WHERE callstackId IS NOT NULL - GROUP BY callstackId - HAVING alloc_count >= {{ min_count }} - AND total_size >= {{ min_size }} - ) AS grouped - LEFT JOIN callstack c ON c.id = grouped.callstackId - ORDER BY total_size DESC, callstack ASC + COUNT(*) AS alloc_count, + SUM(t.size) AS total_size, + AVG(t.size) AS avg_size, + MAX(t.size) AS max_size + FROM {{ device_trace_table }} t + LEFT JOIN callstack c ON c.id = t.callstackId + WHERE t.id >= 0 AND t.callstackId IS NOT NULL + GROUP BY c.callstack + HAVING alloc_count >= {{ min_count }} + AND total_size >= {{ min_size }} + ORDER BY total_size DESC, callstack ASC, c.callstack IS NULL ASC LIMIT {{ limit|int }} output_schema: - column: callstack @@ -71,3 +68,33 @@ queries: type: float - column: max_size type: int + - column: source_stack_id + type: str + units: text + metric_semantics: identifier + scope: captured_range + interpretation_limits: ["Dataset only, optional: generation/device/canonical full-text or explicitly covered ordered raw-array identity, before global thresholds/ranking. Empty non-NULL text is a separate statistical group from a missing reference. Never group by local IDs, display labels or text_kind."] + - column: stack_kind + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: shared source availability, captured or missing; explicitly covered raw arrays (including empty arrays) are captured even without formatted text. Empty non-NULL text still counts statistically but does not prove captured attribution."] + - column: text_kind + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: captured for nonempty formatted text, otherwise missing. Provenance only: independent of raw-array identity/availability and never a grouping key."] + - column: stack_event_id + type: int + units: event_id + metric_semantics: ordering_marker + scope: captured_range + interpretation_limits: ["Dataset only, optional: representative real source event for retrieval on the same generation/device; prefers nonempty captured text when available, otherwise may point to empty/NULL text. Not the group identity, not necessarily an allocation action, and not a timestamp."] + - column: frames_status + type: str + units: text + metric_semantics: classification + scope: captured_range + interpretation_limits: ["Dataset only, optional: ordered only for the recognized version/schema and valid per-event raw-array coverage, otherwise text_only. Retrieve actual arrays with event(id=stack_event_id); statistics do not reconstruct or emit frame arrays."] diff --git a/src/pt_snap_cli/query/templates/statistical/memory_peak.yaml b/src/pt_snap_cli/query/templates/statistical/memory_peak.yaml index 5d576ea..3e4a2ec 100644 --- a/src/pt_snap_cli/query/templates/statistical/memory_peak.yaml +++ b/src/pt_snap_cli/query/templates/statistical/memory_peak.yaml @@ -4,8 +4,9 @@ queries: description: Get peak memory metrics from trace events with corresponding event IDs devices: - all - semantics_version: 1 + semantics_version: 2 interpretation_limits: + - "Only real trace events (id >= 0) participate in standalone databases and datasets; synthetic boundary occupancy cannot be a peak." - "Each peak is the maximum of that counter in the filtered event range; ties resolve to the earliest event ID." - "Peaks of allocated, active, and reserved may occur at different events; do not subtract them as if they were simultaneous." - "Event IDs are ordering markers, not elapsed time." @@ -30,7 +31,7 @@ queries: WITH filtered AS ( SELECT id, allocated, active, reserved FROM {{ device_trace_table }} - WHERE 1=1 + WHERE id >= 0 {% if start_id is not none %}AND id >= {{ start_id|int }}{% endif %} {% if end_id is not none %}AND id <= {{ end_id|int }}{% endif %} ), peaks AS ( diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py index 49bde5e..e797a9f 100644 --- a/tests/core/test_dataset_attribution.py +++ b/tests/core/test_dataset_attribution.py @@ -13,7 +13,7 @@ from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_resolver import DatasetResolver from pt_snap_cli.core.dataset_sources import SOURCE_BATCH_SIZE, DatasetSourceResolver, QueryBudget -from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.errors import InvalidParameterError, QueryTimeoutError from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService @@ -692,9 +692,8 @@ def test_report_uses_same_point_event_core_and_global_aggregates_stay_explicit(t "block", "callstack_analysis", ): - with pytest.raises(QueryExecutionError, match="dataset-global"): - analyzer.execute_query(template) - with pytest.raises(QueryExecutionError, match="dataset-global"): - report.peak_memory_report(root) + assert analyzer.execute_query(template)["scope"]["range_complete"] + peak = report.peak_memory_report(root) + assert peak.event_id is not None and peak.source_coverage["allocation_source_complete"] finally: report.close() diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py new file mode 100644 index 0000000..82a0dc5 --- /dev/null +++ b/tests/core/test_dataset_global.py @@ -0,0 +1,549 @@ +"""Device-global real SQLite regressions; synthetic inputs, never an upstream producer.""" + +import copy +import dataclasses +import hashlib +import json +import sqlite3 +import time +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.dataset_support import DATASET_SUPPORT +from pt_snap_cli.core.errors import ( + InvalidParameterError, + QueryExecutionError, + QueryTimeoutError, + TemplateNotFoundError, +) +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.core.report_service import ReportService +from pt_snap_cli.query.registry import get_query, register_query +from tests.core.test_dataset_attribution import add_frames, case, standalone +from tests.core.test_dataset_import import source_options +from tests.test_dataset_focus import hashes, make_dataset + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def projected(rows, keys): + return [{k: row[k] for k in keys} for row in rows] + + +@pytest.mark.parametrize( + "template", + [ + "memory_peak", + "allocator_gap", + "allocation", + "event", + "block", + "leak_detection", + "freed_block_lifetime", + "callstack_analysis", + "preexisting_live", + ], +) +def test_inline_global_matches_real_standalone_without_extensions(tmp_path, template): + root = case(tmp_path) + single = standalone(root, tmp_path) + before = hashes(root) + params = ( + {"min_count": 1} + if template == "callstack_analysis" + else {"event_id": 8} if template == "preexisting_live" else {} + ) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + actual = dataset.execute_query(template, params, exact_total=True) + expected = reference.execute_query(template, params, exact_total=True) + keys = list(expected["rows"][0]) if expected["rows"] else [] + assert projected(actual["rows"], keys) == expected["rows"] + assert actual["total"] == expected["total"] and actual["total_is_exact"] + assert actual["scope"]["range_complete"] + assert actual["scope"]["boundary_events_included"] is False + assert hashes(root) == before + + +@pytest.mark.parametrize( + "template", + [ + "memory_peak", + "allocator_gap", + "allocation", + "event", + "block", + "leak_detection", + "freed_block_lifetime", + "callstack_analysis", + ], +) +def test_native_sparse_multiple_devices_matches_standalone(tmp_path, template): + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + service = ImportService() + root = service.import_snapshot(request).db_path + single = service.import_snapshot(dataclasses.replace(request, events_per_slice=None)).db_path + before = hashes(root) + params = ( + {"min_count": 1} + if template == "callstack_analysis" + else {"min_id": 7} if template in ("event", "allocation") else {} + ) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in (0, 1): + actual = dataset.execute_query(template, params, device_id=device, exact_total=True) + expected = reference.execute_query(template, params, device_id=device, exact_total=True) + keys = list(expected["rows"][0]) if expected["rows"] else [] + # Negative IDs are dataset-local tokens, not comparable across imports. + if template == "block": + keys.remove("id") + # Stored state is an observation, not a lifecycle invariant. + # The standalone dump and latest containing slice can observe + # different pending-free states; neither is state at arbitrary E. + keys.remove("state") + assert all(r["state_scope"] == "latest_slice_observation" for r in actual["rows"]) + + def norm(rs, fields=tuple(keys)): + return sorted(tuple(str(r[k]) for k in fields) for r in rs) + + assert norm(actual["rows"]) == norm(expected["rows"]) + else: + assert projected(actual["rows"], keys) == expected["rows"] + assert actual["total"] == expected["total"] + assert hashes(root) == before + + +def test_peaks_ties_last_shard_boundary_and_same_event_report(tmp_path): + root = case(tmp_path) + single = standalone(root, tmp_path) + for index in range(8): + with closing(sqlite3.connect(root / f"device_0/slice_{index:05d}.db")) as conn, conn: + conn.execute( + "UPDATE trace_entry_0 SET allocated=99999,active=99999,reserved=99999 WHERE id<0" + ) + conn.execute("UPDATE trace_entry_0 SET reserved=5000 WHERE id IN (4,15)") + conn.execute("UPDATE trace_entry_0 SET allocated=3000 WHERE id=15") + with closing(sqlite3.connect(single)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET reserved=5000 WHERE id IN (4,15)") + conn.execute("UPDATE trace_entry_0 SET allocated=3000 WHERE id=15") + conn.execute("INSERT INTO trace_entry_0 VALUES (-1,2,0,0,0,99999,99999,99999,'fake')") + report = ReportService() + try: + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + peak = dataset.execute_query("memory_peak")["rows"][0] + assert peak == reference.execute_query("memory_peak")["rows"][0] + assert peak["peak_allocated_event_id"] == 15 and peak["peak_reserved_event_id"] == 4 + assert ( + dataset.execute_query("allocator_gap")["rows"] + == reference.execute_query("allocator_gap")["rows"] + ) + for metric in ("allocated", "active", "reserved"): + result = report.peak_memory_report(root, metric=metric, limit=-1, timeout_s=10) + original = report.peak_memory_report(single, metric=metric, limit=-1, timeout_s=10) + assert result.event_id == original.event_id + assert result.active_bytes_at_event == original.active_bytes_at_event + assert result.included_bytes == original.included_bytes + assert result.coverage_percent == original.coverage_percent + assert result.allocator_gap == original.allocator_gap + assert result.source_coverage["allocation_source_complete"] + assert result.budget_scope == "report_composition" + finally: + report.close() + + +def test_global_stack_winner_threshold_weighted_average_and_all_actions(tmp_path): + root = make_dataset(tmp_path / "ranking", devices=(0,), slices=3) + for index in range(3): + with closing(sqlite3.connect(root / f"device_0/slice_{index:05d}.db")) as conn, conn: + conn.execute( + "UPDATE trace_entry_0 SET callstack=?,size=11 WHERE id=?", + (f"local:{index}", index * 2), + ) + conn.execute( + "UPDATE trace_entry_0 SET callstack='global',size=?,action=6 WHERE id=?", + (10 + index, index * 2 + 1), + ) + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query( + "callstack_analysis", {"min_count": 2, "min_size": 30, "limit": 1}, exact_total=True + ) + assert result["total"] == 1 and not result["truncated"] + row = result["rows"][0] + assert ( + row["callstack"], + row["alloc_count"], + row["total_size"], + row["avg_size"], + row["max_size"], + ) == ("global", 3, 33, 11, 12) + assert result["semantics_version"] == 2 # counts free_completed too, not allocations + assert ( + analyzer.execute_query("callstack_analysis", {"min_count": 1, "limit": 1})["rows"][0][ + "callstack" + ] + == "global" + ) + + +def test_lifecycles_terminal_survival_completed_free_reuse_and_unknown(tmp_path): + root = case(tmp_path) + # An allocation present in early shards but absent at the terminal shard is + # NOT a terminal candidate, even if its earlier observation says unfreed. + with closing(sqlite3.connect(root / "device_0/slice_00007.db")) as conn, conn: + conn.execute("DELETE FROM block_0 WHERE id=3") + with SnapshotAnalyzer(root) as analyzer: + all_rows = analyzer.execute_query("block", exact_total=True) + assert all_rows["total"] == 10 # not 8 * 10 + assert len({r["lifecycle_id"] for r in all_rows["rows"]}) == 10 + same_address = [r for r in all_rows["rows"] if r["address"] == 100] + assert ( + len(same_address) == 2 + and same_address[0]["lifecycle_id"] != same_address[1]["lifecycle_id"] + ) + candidates = analyzer.execute_query("leak_detection")["rows"] + assert {r["id"] for r in candidates} == {4, 5, 6, 7, 11} + assert all(r["terminal_survivor"] for r in candidates) + freed = analyzer.execute_query("freed_block_lifetime")["rows"] + assert sum(r["block_count"] for r in freed) == 2 + old = next(r for r in all_rows["rows"] if r["id"] == 0) + assert ( + old["free_source"]["event"]["id"] == 10 and old["free_source"]["event"]["action"] == 6 + ) + assert all( + r["identity_status"] == "stable_negative_token" for r in all_rows["rows"] if r["id"] < 0 + ) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET action=2 WHERE id=0") + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query("block") + assert result["scope"]["source_coverage"]["unproved_identities"] == 8 + assert not result["scope"]["source_coverage"]["allocation_source_complete"] + assert all(r["id"] != 0 for r in analyzer.execute_query("leak_detection")["rows"]) + + +def test_global_page_filters_ties_exact_total_source_range_and_empty(tmp_path): + root = case(tmp_path) + single = standalone(root, tmp_path) + params = { + "min_id": 1, + "max_id": 14, + "min_active": 50, + "order_by": "active", + "order_dir": "DESC", + "offset": 2, + "limit": 3, + } + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + actual = dataset.execute_query("event", params, max_rows=2, exact_total=True) + expected = reference.execute_query("event", params, max_rows=2, exact_total=True) + assert actual["rows"] == expected["rows"] and actual["total"] == expected["total"] + assert len(actual["rows"][0]) == 9 + assert actual["has_more"] and actual["truncated"] + assert actual["scope"]["slice_indices"] == list(range(8)) + assert actual["scope"]["source_coverage"]["events_resolved"] == 2 + assert actual["scope"]["range_complete"] + empty = dataset.execute_query("event", {"action": 99}, exact_total=True) + assert empty["total"] == 0 and not empty["truncated"] + for endpoints in ({"min_id": -1}, {"max_id": 99}, {"min_id": 3, "max_id": 2}, {"id": True}): + with pytest.raises(InvalidParameterError): + dataset.execute_query("event", endpoints) + request = dataclasses.replace(source_options(tmp_path), set_focus=False) + native = ImportService().import_snapshot(request).db_path + with SnapshotAnalyzer(native) as analyzer: + gap = analyzer.execute_query("memory_peak", {"start_id": 18, "end_id": 26}) + assert all(v is None for v in gap["rows"][0].values()) + assert gap["scope"]["actual_range"]["real_event_count"] == 0 + assert analyzer.execute_query("event", {"min_id": 18, "max_id": 26})["rows"] == [] + + +def test_runtime_overrides_custom_sql_and_capabilities_are_not_builtin_semantics(tmp_path): + root = case(tmp_path) + original = get_query("memory_peak") + custom = copy.deepcopy(original) + custom.query = "SELECT 999 AS fake" + register_query(custom) + try: + with SnapshotAnalyzer(root) as analyzer: + assert ( + analyzer.get_template_info("memory_peak")["dataset_support"]["supported"] is False + ) + with pytest.raises(QueryExecutionError, match="runtime override"): + analyzer.execute_query("memory_peak") + with pytest.raises(TemplateNotFoundError): + analyzer.execute_query("SELECT * FROM block_0") + finally: + register_query(original) + service = QueryService() + try: + member = root / "device_0/slice_00000.db" + executor = service._get_executor(service._validated_context(member)) + executor.register_template(custom) + assert service.get_template_info("memory_peak").dataset_support["supported"] is False + with pytest.raises(QueryExecutionError, match="runtime override"): + service.execute_query("memory_peak", db_path=root) + finally: + service.close() + with SnapshotAnalyzer(root) as analyzer: + contracts = analyzer.list_capabilities()["templates"] + assert {c["name"] for c in contracts if c["dataset_support"]["supported"]} == set( + DATASET_SUPPORT + ) + + +def test_work_limits_detached_batches_borrowed_lru_timeout_cleanup_and_report_deadline( + tmp_path, monkeypatch +): + root = case(tmp_path) + cache = ContextCache(maxsize=1) + service = QueryService(context_cache=cache) + report = ReportService() + try: + budget = QueryBudget(None, time.monotonic(), max_work_rows=4) + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query("block", db_path=root, _budget=budget) + # The entire real range consumes 16 rows before final max_rows=1; + # its source lookup must use the SAME cumulative budget, not reset. + budget = QueryBudget(None, time.monotonic(), max_work_rows=16) + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query( + "event", db_path=root, max_rows=1, exact_total=True, _budget=budget + ) + budget = QueryBudget(None, time.monotonic(), max_work_bytes=1) + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query("event", db_path=root, _budget=budget) + assert service.execute_query("block", db_path=root).rows and len(cache) == 1 + held = next(iter(cache._entries.values()))[0] + with held.connect() as conn: + assert conn.execute("SELECT 1").fetchone()[0] == 1 + remaining, budgets = [], [] + original = DatasetSourceResolver.read + + def counted(self, path, sql, values=None): + remaining.append(self.budget.remaining()) + budgets.append(self.budget) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", counted) + report.peak_memory_report(root, timeout_s=10) + assert len({id(b) for b in budgets}) == 1 + assert remaining == sorted(remaining, reverse=True) + remaining.clear() + expired = QueryBudget(0.001, time.monotonic() - 1) + with pytest.raises(QueryTimeoutError): + service.execute_query("event", db_path=root, _budget=expired) + source = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(0.001, time.monotonic()) + ) + with pytest.raises(QueryTimeoutError): + source.read( + root / "device_0/slice_00000.db", + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<10000000) SELECT SUM(x) FROM n", + ) + source = DatasetSourceResolver( + DatasetResolver().inspect(root), 0, cache, QueryBudget(None, time.monotonic()) + ) + with pytest.raises(QueryExecutionError): + source.read(root / "device_0/slice_00000.db", "SELECT * FROM absent_table") + assert source.read(root / "device_0/slice_00000.db", "SELECT 1 AS okay") == [{"okay": 1}] + service.close() + assert len(cache) == 1 + finally: + service.close() + report.close() + cache.close() + + +def test_callback_deadline_keeps_domain_timeout_and_cli_json_code(tmp_path, monkeypatch): + root = case(tmp_path) + original = QueryBudget.consume + + def expire(self, rows): + self.started -= 100 + return original(self, rows) + + cache = ContextCache(maxsize=1) + try: + with monkeypatch.context() as patch: + patch.setattr(QueryBudget, "consume", expire) + with SnapshotAnalyzer(root, context_cache=cache) as analyzer: + with pytest.raises(QueryTimeoutError): + analyzer.execute_query("memory_peak", timeout_s=10) + cli = CliRunner().invoke( + app, + ["query", str(root), "--template-use", "memory_peak", "--timeout", "10", "--json"], + ) + assert cli.exit_code != 0 and not cli.stdout + assert json.loads(cli.stderr)["error"]["code"] == "QUERY_TIMEOUT" + with SnapshotAnalyzer(root, context_cache=cache) as analyzer: + assert analyzer.execute_query("memory_peak")["rows"] + assert len(cache) == 1 + finally: + cache.close() + + +def test_ordered_arrays_control_global_stack_identity_not_local_ids_or_display(tmp_path): + root = case(tmp_path) + add_frames(root) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query("callstack_analysis", {"min_count": 1})["rows"] + assert len(rows) == 16 # every explicit raw array differs, including repeated text + assert len({r["source_stack_id"] for r in rows}) == 16 + assert all(r["frames_status"] == "ordered" for r in rows) + + +def test_raw_stats_identity_prefers_captured_text_over_null_and_empty(tmp_path): + root = case(tmp_path) + add_frames(root) + for index, changes in [(0, [(0, None), (1, "")]), (1, [(3, "third")])]: + with closing(sqlite3.connect(root / f"device_0/slice_{index:05d}.db")) as conn, conn: + for event, text in changes: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", (text, event)) + conn.execute( + "UPDATE pt_snap_frame SET frameJson=? WHERE eventId=? AND frameIndex=2", + (json.dumps({"name": "event:0"}), event), + ) + with SnapshotAnalyzer(root) as analyzer: + identity = analyzer.execute_query("event", {"id": 0})["rows"][0]["source_stack_id"] + row = next( + r + for r in analyzer.execute_query("callstack_analysis", {"min_count": 1})["rows"] + if r["source_stack_id"] == identity + ) + assert row["alloc_count"] == 3 and row["callstack"] == "third" + assert row["stack_event_id"] == 3 and row["text_kind"] == "captured" + assert analyzer.execute_query("event", {"id": 1})["rows"][0]["text_kind"] == "missing" + + +@pytest.mark.parametrize("layout", ["inline", "native"]) +@pytest.mark.parametrize("detailed", [False, True]) +def test_all_dataset_row_keys_are_declared_by_shared_template_schema(tmp_path, layout, detailed): + if layout == "inline": + root = case(tmp_path) + event = 15 + if detailed: + add_frames(root) + else: + root = ( + ImportService() + .import_snapshot(dataclasses.replace(source_options(tmp_path), set_focus=False)) + .db_path + ) + event = 87 + if detailed: + manifest = root / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + for device, data in raw["devices"].items(): + for item in data["slices"]: + path = root / item["file"] + with closing(sqlite3.connect(path)) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT, PRIMARY KEY(eventId,frameIndex));" + ) + ids = [ + r[0] + for r in conn.execute( + f"SELECT id FROM trace_entry_{device} WHERE id>=0" + ) + ] + conn.executemany( + "INSERT INTO pt_snap_frame_coverage VALUES (?,1)", [(i,) for i in ids] + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", + [(i, '{"name":"same","line":1}') for i in ids], + ) + item["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + manifest.write_text(json.dumps(raw)) + params = { + "event": {"stack_bytes": 0 if detailed else -1}, + "callstack_analysis": {"min_count": 1}, + "active_blocks_at_event": {"event_id": event}, + "active_memory_callstack_at_event": { + "event_id": event, + "top_n": -1, + "stack_bytes": 0 if detailed else -1, + }, + "preexisting_live": {"event_id": event}, + } + with SnapshotAnalyzer(root) as analyzer: + for template in DATASET_SUPPORT: + rows = analyzer.execute_query(template, params.get(template), max_rows=1)["rows"] + info = analyzer.get_template_info(template) + columns = {field["column"] for field in info["output_schema"]} + assert all(set(row) <= columns for row in rows), ( + template, + [set(row) - columns for row in rows], + ) + cli = CliRunner().invoke(app, ["query", "--template-info", template, "--json"]) + assert cli.exit_code == 0, cli.output + assert json.loads(cli.stdout)["output_schema"] == info["output_schema"] + if template in ("block", "leak_detection", "callstack_analysis"): + dataset_fields = [ + field + for field in info["output_schema"] + if any( + "Dataset only" in text for text in field.get("interpretation_limits", []) + ) + ] + assert dataset_fields and all( + field.get("metric_semantics") and field.get("scope") for field in dataset_fields + ) + + +def test_cli_api_report_global_parity_and_range_options(tmp_path): + root = case(tmp_path) + runner = CliRunner() + with SnapshotAnalyzer(root) as analyzer: + api = analyzer.execute_query("memory_peak", {"start_id": 4, "end_id": 15}) + cli = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + "memory_peak", + "--params", + '{"start_id":4,"end_id":15}', + "--json", + ], + ) + assert cli.exit_code == 0, cli.output + payload = json.loads(cli.stdout) + for key in api: + assert payload[key] == api[key] + cli = runner.invoke( + app, + [ + "report", + "peak-memory", + str(root), + "--start-id", + "4", + "--end-id", + "15", + "--timeout", + "10", + "--json", + ], + ) + assert cli.exit_code == 0, cli.output + result = json.loads(cli.stdout) + assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15} + assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10 diff --git a/tests/query/test_callstack_schema_compat.py b/tests/query/test_callstack_schema_compat.py index 2cbfa53..9363e7a 100644 --- a/tests/query/test_callstack_schema_compat.py +++ b/tests/query/test_callstack_schema_compat.py @@ -368,16 +368,17 @@ def test_v1_callstack_analysis_pagination_and_total(v1_db: Path) -> None: assert [row["callstack"] for row in result.rows] == ["after.py:40", "free.py:30"] -def test_v2_callstack_analysis_keeps_id_aggregation(v2_db: Path) -> None: +def test_v2_callstack_analysis_v2_groups_full_text_and_real_actions(v2_db: Path) -> None: context = Context(v2_db) sql = QueryExecutor(context).render( get_query("callstack_analysis"), {"min_count": 1, "min_size": 0, "limit": -1}, device_id=0, ) - normalized = sql.replace("GROUP BY callstackId", "") - assert "GROUP BY callstackId" in sql - assert "GROUP BY callstack" not in normalized + assert "GROUP BY c.callstack" in sql + assert "t.id >= 0" in sql + assert "GROUP BY callstackId" not in sql + assert get_query("callstack_analysis").semantics_version == 2 context.close() diff --git a/tests/query/test_registry.py b/tests/query/test_registry.py index 23c7646..3a5994b 100644 --- a/tests/query/test_registry.py +++ b/tests/query/test_registry.py @@ -230,7 +230,8 @@ def test_block_template_schema_includes_every_selected_column(self): info = get_template_info("block") assert info is not None - assert [column["column"] for column in info["output_schema"]] == [ + columns = info["output_schema"] + assert [column["column"] for column in columns[:7]] == [ "id", "address", "size", @@ -239,6 +240,22 @@ def test_block_template_schema_includes_every_selected_column(self): "allocEventId", "freeEventId", ] + assert [column["column"] for column in columns[7:]] == [ + "lifecycle_id", + "identity_status", + "allocation_source", + "allocation_source_status", + "free_source", + "free_source_status", + "observation_slice_index", + "state_scope", + "terminal_survivor", + "category", + ] + assert all( + "Dataset only, optional" in " ".join(column["interpretation_limits"]) + for column in columns[7:] + ) def test_packaged_sql_fragment_parameters_declare_choices(self): """Regression for issue #120. diff --git a/tests/query/test_template_semantics.py b/tests/query/test_template_semantics.py index ed4931a..b25c368 100644 --- a/tests/query/test_template_semantics.py +++ b/tests/query/test_template_semantics.py @@ -61,7 +61,7 @@ def test_annotated_templates_share_a_json_serializable_contract() -> None: template = get_query(name) assert template is not None assert template.semantics_version == ( - 2 if name == "active_memory_callstack_at_event" else 1 + 2 if name in ("active_memory_callstack_at_event", "memory_peak", "allocator_gap") else 1 ) assert template.interpretation_limits info = get_template_info(name) diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index 2fbfd61..0cf9cfd 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -83,6 +83,47 @@ def test_stack_summary_cli_api_and_catalog_contract(contract_db: Path, template: assert entry["parameters"] == info["parameters"] +@pytest.mark.parametrize( + "template,params", + [ + ("memory_peak", {}), + ("allocator_gap", {"start_id": 1, "end_id": 4}), + ("event", {"min_id": 1, "max_id": 4, "limit": 2, "offset": 1}), + ("allocation", {"min_id": 1, "max_id": 4, "order_by": "active"}), + ("callstack_analysis", {"min_count": 1, "limit": 1}), + ("block", {}), + ("leak_detection", {}), + ], +) +def test_dataset_global_cli_api_scope_catalog_and_total_contract(tmp_path, template, params): + root = make_dataset(tmp_path / "global-contract", devices=(0,)) + with SnapshotAnalyzer(root) as analyzer: + api = analyzer.execute_query(template, params, exact_total=True) + cli = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + template, + "--params", + json.dumps(params), + "--exact-total", + "--json", + ], + ) + assert cli.exit_code == 0, cli.output + payload = json.loads(cli.stdout) + for key in api: + assert payload[key] == api[key] + info = analyzer.get_template_info(template) + assert info["dataset_support"]["supported"] + catalog = runner.invoke(app, ["capabilities", "--json"]) + entry = next(c for c in json.loads(catalog.stdout)["templates"] if c["name"] == template) + assert entry["dataset_support"] == info["dataset_support"] + assert api["scope"]["boundary_events_included"] is False + + def create_contract_db(db_path: Path) -> Path: """Create a tiny snapshot database shared by CLI and API contract paths.""" conn = sqlite3.connect(str(db_path)) @@ -237,6 +278,8 @@ def _normalize_cli_template_info(output: str) -> dict[str, object]: info["category"] = stripped.removeprefix("Category: ") elif stripped.startswith("Devices: "): info["devices"] = stripped.removeprefix("Devices: ") + elif stripped.startswith("Dataset Support: "): + info["dataset_support"] = json.loads(stripped.removeprefix("Dataset Support: ")) elif stripped.startswith("Semantics Version: "): text = stripped.removeprefix("Semantics Version: ") info["semantics_version"] = None if text == "none" else int(text) @@ -412,7 +455,7 @@ def test_template_semantics_contract_matches_cli_and_api(contract_db: Path) -> N assert cli_info == api_info json.dumps(api_info) assert api_info["semantics_version"] == ( - 2 if name == "active_memory_callstack_at_event" else 1 + 2 if name in ("active_memory_callstack_at_event", "memory_peak", "allocator_gap") else 1 ) assert api_info["interpretation_limits"] @@ -834,11 +877,13 @@ def test_dataset_focus_overview_query_contract(tmp_path: Path, use_manifest: boo ) assert error.exit_code == 1 assert json.loads(error.stderr)["error"]["code"] == "INVALID_PARAMETER" - failure = runner.invoke( + global_peak = runner.invoke( app, ["query", str(path), "--template-use", "memory_peak", "--json"] ) - assert failure.exit_code == 1 - assert "cross-slice" in json.loads(failure.stderr)["error"]["message"] + assert global_peak.exit_code == 0, global_peak.output + peak_payload = json.loads(global_peak.stdout) + assert peak_payload["scope"]["slice_indices"] == [0, 1, 2] + assert peak_payload["rows"][0]["peak_active"] == 8 # never synthetic 999 manifest = root / "manifest.json" data = json.loads(manifest.read_text()) data["status"] = "building" diff --git a/tests/test_dataset_focus.py b/tests/test_dataset_focus.py index a066ecd..8b525f4 100644 --- a/tests/test_dataset_focus.py +++ b/tests/test_dataset_focus.py @@ -132,7 +132,7 @@ def test_dataset_focus_readonly_and_complete_addressing(tmp_path, manifest_path) assert overview["dataset"]["format"] == "compatibility-v1" assert overview["dataset"]["structured_frames"] is False assert overview["dataset"]["text_callstacks"] is True - assert overview["dataset"]["cross_slice_queries"] is False + assert overview["dataset"]["cross_slice_queries"] is True assert overview["dataset"]["boundary_event_count"] == 6 assert analyzer.get_database_metadata()["status"] == "unavailable" for device in (0, 2): @@ -179,8 +179,6 @@ def test_precedence_device_and_completion(tmp_path, monkeypatch): @pytest.mark.parametrize( "template,params,slice_index,error", [ - ("event", {}, None, QueryExecutionError), - ("event", {"min_id": 1, "max_id": 2}, None, QueryExecutionError), ("event", {"id": -1}, None, InvalidParameterError), ("event", {"id": 6}, None, InvalidParameterError), ("event", {"id": 4}, 0, InvalidParameterError), @@ -189,7 +187,6 @@ def test_precedence_device_and_completion(tmp_path, monkeypatch): ("event", {}, 99, InvalidParameterError), ("event", {}, True, InvalidParameterError), ("event", {"unknown": 0}, 0, TemplateRenderError), - ("memory_peak", {}, None, QueryExecutionError), ("leak_detection", {}, 0, QueryExecutionError), ("missing_template", {}, None, TemplateNotFoundError), ], diff --git a/tests/test_fragmentation_query_scope.py b/tests/test_fragmentation_query_scope.py index 6fb6b61..2cab6ce 100644 --- a/tests/test_fragmentation_query_scope.py +++ b/tests/test_fragmentation_query_scope.py @@ -126,8 +126,13 @@ def test_explicit_full_trace_comparison_uses_its_own_peak(scope_db): assert sum(row["size_bytes"] for row in report["callstack_groups"]) == peak["peak_active"] -def test_full_trace_synthetic_peak_requires_stopping_comparison(scope_db): +def test_full_trace_real_peak_excludes_synthetic_boundary_and_drives_report(scope_db): section = SKILL_PATH.read_text().split("#### Separate full-trace comparison", 1)[1] - peak = run_skill_command(skill_commands(section)[0], scope_db)["rows"][0] - assert peak["peak_active_event_id"] == -2 - assert peak["peak_active"] == 1000 + commands = skill_commands(section.split("### 6.", 1)[0]) + peak = run_skill_command(commands[0], scope_db)["rows"][0] + assert peak["peak_active_event_id"] == 25 + assert peak["peak_active"] == 90 # never the synthetic id=-2/value=1000 + report = run_skill_command(commands[1], scope_db) + assert report["event_id"] == 25 + assert report["active_bytes_at_event"] == 90 + assert report["included_bytes"] == 90 and report["coverage_percent"] == 100 diff --git a/tests/test_native_dataset.py b/tests/test_native_dataset.py index bd341ef..07672f6 100644 --- a/tests/test_native_dataset.py +++ b/tests/test_native_dataset.py @@ -10,7 +10,7 @@ from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_contract import DatasetContractError, QueryScope, parse_manifest from pt_snap_cli.core.dataset_resolver import DatasetResolver, _hash_file -from pt_snap_cli.core.errors import DatabaseSchemaError, InvalidParameterError, QueryExecutionError +from pt_snap_cli.core.errors import DatabaseSchemaError, InvalidParameterError from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.native_dataset_contract import parse_native_manifest from tests.core.test_dataset_import import hashes, source_options @@ -56,8 +56,11 @@ def test_native_sparse_ids_positions_device_scope_borrowed_cache_and_readonly(tm assert result["scope"]["slice_index"] == position // 2 assert len(cache) <= 2 assert len(analyzer.execute_query("event", {}, device_id=1, slice_index=0)["rows"]) == 2 - with pytest.raises(QueryExecutionError, match="cross-slice"): - analyzer.execute_query("memory_peak") + peak = analyzer.execute_query("memory_peak") + assert peak["scope"]["slice_indices"] == [0, 1, 2, 3, 4] + assert peak["scope"]["boundary_events_included"] is False + assert peak["rows"][0]["peak_active_event_id"] in range(7, 88, 10) + assert len(cache) <= 2 with pytest.raises(InvalidParameterError): analyzer.execute_query("event", {"id": -1}) held = cache.get(root / "device_1/slice_00000.db")