diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 706afb4..1000c8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,6 +73,23 @@ jobs: - name: basedPyright check run: python -m basedpyright --pythonpath "$(python -c 'import sys; print(sys.executable)')" + browser: + name: Frame-tree browser regression + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install browser test dependencies + run: | + pip install -e ".[dev]" + python -m playwright install --with-deps chromium + - name: Run required Chromium interaction tests + env: + PT_SNAP_REQUIRE_BROWSER: "1" + run: pytest tests/test_fixture_provenance.py tests/test_memory_tree_browser.py + test: name: Test (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index bbcae61..462a25d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,17 @@ python -m basedpyright --pythonpath "$(python -c 'import sys; print(sys.executab git diff --check ``` +HTML interaction regressions run in a separate Chromium CI job. To run locally: + +```bash +python -m playwright install chromium +PT_SNAP_REQUIRE_BROWSER=1 pytest tests/test_memory_tree_browser.py +``` + +Without an installed browser, the ordinary suite skips these browser tests. The +required-browser flag makes missing browser support fail. `PT_SNAP_TEST_CHROMIUM` +may point to an existing Chromium executable instead of downloading one. + - Type checking covers all `src/pt_snap_cli`, including `snapshot/`. Zero errors is the gate; warnings are informational. Preserve the file-level Any/Unknown error gates in `api.py`, `core/models.py`, and `snapshot/representation.py`; diff --git a/README.md b/README.md index afacb19..8ab43cf 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ accept explicitly selected complete validated datasets, not arbitrary directorie | `pt-snap overview [database.db]` | Read-only device list, per-device event-id bounds, and import-metadata status | | `pt-snap query` | Run memory analysis queries | | `pt-snap report` | Generate higher-level memory analysis reports | +| `pt-snap report memory-tree --event-id ` | Frame-level live-memory breakdown; `--format html` emits an interactive flamegraph | | `pt-snap config` | Manage global configuration | | `pt-snap skill` | List and install bundled agent skills | @@ -109,6 +110,14 @@ Peak reports expose attribution completeness and same-event active-byte coverage group percentages use included bytes after filtering and ranking. See the [report guide](docs/en/querying.md#report-command) before interpreting a capped breakdown. +Standalone imports preserve ordered structured callstack frames. Use `event_frames` to +inspect them, or `report memory-tree --event-id 100 --format html > memory-tree.html` +to explore event-time occupancy by caller path. See +[frame memory breakdown](docs/en/querying.md#frame-memory-breakdown); older +text-only databases require re-importing the original snapshot. +`report memory-tree` also accepts `--json`. The HTML view has full-tree search and +keyboard-accessible child lists, so tiny and zero-byte frames remain reachable. + ## Agent Skills For long diagnostic stacks, opt into `stack_bytes` on `event` or diff --git a/README_zh.md b/README_zh.md index 1681c1e..9da42a1 100644 --- a/README_zh.md +++ b/README_zh.md @@ -87,6 +87,7 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices | `pt-snap overview [database.db]` | 只读输出设备列表、各设备 event id 边界和导入 metadata 状态 | | `pt-snap query` | 运行内存分析查询 | | `pt-snap report` | 生成高层内存分析报告 | +| `pt-snap report memory-tree --event-id ` | 按帧分层拆解存活内存;`--format html` 输出交互式火焰图 | | `pt-snap config` | 管理全局配置 | | `pt-snap skill` | 列出并安装随包 agent skill | @@ -96,6 +97,13 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices included bytes 为分母。解读有上限的分组结果前,请参阅 [报告指南](docs/zh/querying.md#report-命令)。 +单库导入时保留结构化调用栈帧及原始顺序。用 `event_frames` 查看帧,或执行 +`report memory-tree --event-id 100 --format html > memory-tree.html` +按调用路径查看事件时刻的占用。详见[调用栈帧内存拆解](docs/zh/querying.md#调用栈帧内存拆解); +旧文本数据库需要从原始快照重新导入。 +`report memory-tree` 同样支持 `--json`。HTML 支持全树搜索和可键盘操作的子节点列表, +极小占用及零字节帧也能进入查看。 + ## Agent Skills 诊断长调用栈时,可在 `event` 或 `active_memory_callstack_at_event` 上显式设置 diff --git a/docs/en/database.md b/docs/en/database.md index 234d0bc..212ff13 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -331,15 +331,65 @@ CREATE TABLE callstack ( | Property | Guarantee | |----------|-----------| -| Uniqueness | One row per distinct callstack text, so grouping by `callstackId` equals grouping by text | +| Uniqueness | One row per distinct ordered structured frame array; display text alone is not a lossless identity | | Scope | Shared by every device in the database; the table has no device suffix | | Coverage | Every event produced by import resolves to a row; `callstackId` is `NULL` only in externally generated databases | -Current `pt-snap import` writes this v2 layout (`import_format_version = 2`). +Current standalone `pt-snap import` retains this v2 text layout and adds structured frame +tables (`import_format_version = 3`). Format 2 databases have the same text +layout but no original frame records. Databases produced before callstack deduplication store inline `callstack` TEXT on each `trace_entry_` table and have no shared `callstack` table (`import_format_version = 1` when metadata is present). +### Structured frames and ordered arrays (format 3) + +`frame` stores each distinct original `(filename, line, name)` tuple once. +`callstack_frame` associates a stack with an ordered array of frame IDs: + +```sql +CREATE TABLE frame ( + id INTEGER PRIMARY KEY, + filename TEXT NOT NULL, + line INTEGER NOT NULL, + name TEXT NOT NULL +); +CREATE TABLE callstack_frame ( + callstackId INTEGER NOT NULL, + position INTEGER NOT NULL, + frameId INTEGER NOT NULL +); +CREATE UNIQUE INDEX idx_callstack_frame_position + ON callstack_frame (callstackId, position); +``` + +These tables are shared across devices. An event refers to the array through +`trace_entry_.callstackId`; read it in `position ASC` order to reproduce +the original snapshot array. Position 0 is the innermost frame. The frame tree +reverses traversal to start with the outermost caller. Recursive occurrences +retain distinct positions even when they share a frame ID. Empty arrays have +no association rows and a manifest frameCount of zero. Structured fields preserve colons, whitespace, Unicode +and embedded newlines without parsing display strings. + +Use `event_frames` to read an event's ordered frames and +`active_memory_frame_tree_at_event` to obtain parent/child occupancy. +See [Frame memory breakdown](querying.md#frame-memory-breakdown). +Text-only databases remain readable by existing queries, but these new +queries require re-importing the original snapshot. Import detects the changed +format and rebuilds the cache; read-only analysis never migrates a database. + +### Standalone frame capability boundary + +Format 3 applies only to standalone imports. Native sharded imports remain format 2 +and compatible exports remain format 1; neither claims the standalone frame-tree +capability. `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +frameCount INTEGER NOT NULL)` records the expected length of every captured stack, +including zero. Frame fields must be text filename/name and an integer line. +`event_frames` returns no rows for absent, known-empty or corrupt arrays, never a +partial reconstruction. Trees keep known-empty captured stacks separate from +missing/corrupt evidence. Dropped tail/all links, noninteger positions and invalid +field types fail validation. Read-only queries never repair stored evidence. + ### Callstack layout compatibility `pt-snap` reads both layouts without rewriting the file: @@ -351,7 +401,8 @@ on each `trace_entry_` table and have no shared `callstack` table `event`, `callstack_analysis`, and `active_memory_callstack_at_event` keep one template name, parameter set, and output schema. The query engine selects v1 or -v2 SQL from the detected layout. Other templates do not depend on this split. +v2 SQL from the detected layout. Structured-frame queries additionally require +the format 3 tables; their frame evidence is not reconstructed from legacy text. Detection is read-only: `focus`, `query`, reports, and the Python API all open the database with SQLite `mode=ro`. Layout is not stored in diff --git a/docs/en/querying.md b/docs/en/querying.md index 57d9383..47091a3 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -674,6 +674,80 @@ explanations; look up `semantics_version` and interpretation limits with rows. `SnapshotAnalyzer.execute_query()` results also include `template` and `semantics_version` so that lookup stays tied to the rows. +## Frame Memory Breakdown + +After a standalone import with this version, inspect the original ordered frame array or +decompose live memory after an event: + +```bash +pt-snap query --template-use event_frames --params '{"event_id":100}' --json +pt-snap query --template-use active_memory_frame_tree_at_event --params '{"event_id":100}' --json +pt-snap report memory-tree --event-id 100 +pt-snap report memory-tree --event-id 100 --json +pt-snap report memory-tree --event-id 100 --format html > memory-tree.html +``` + +The HTML file is self-contained: open it locally, click a frame to zoom, reset +to show the whole tree, or search filenames/functions. Width measures live +block bytes, not CPU time or allocation traffic. No static server is needed. +`--device` selects the device; `--exclude-static` omits static/preexisting +memory; `--min-size` filters individual blocks. `--event-id` is required. +The report requests a complete tree without a ranked or row cap. + +| Field | Meaning | +| --- | --- | +| `node_id`, `parent_id` | Caller-path identity and direct parent; `root` is the total | +| `depth`, `frame_id` | Caller-first level and structured frame identity | +| `size_bytes`, `requested_bytes`, `block_count` | Inclusive occupancy/count for this node and descendants | +| `self_bytes`, `self_requested_bytes`, `self_block_count` | Allocations whose recorded stack ends at this node | +| `percent_of_total` | 0–100 byte share of root occupancy after block/category filters, before output row caps | + +For each node, inclusive occupancy equals self occupancy plus the inclusive +occupancy of its direct children. Sum `self_bytes` across all nodes, or sum +inclusive bytes at one complete cut through the tree; summing inclusive bytes +across levels counts allocations repeatedly. An internal node can have self +bytes when a recorded stack ends there. Recursion and identical frames under +different callers remain separate paths. + +Only blocks with `allocEventId <= event_id` and a later or absent +`freeEventId` are attributed dynamically. `freeEventId` is `free_completed`, +so pending-free memory is still included. Static, preexisting, and missing or +damaged stack links remain explicit synthetic leaves. This is active-block +occupancy and does not decompose reserved bytes. Event IDs mark ordering, not +elapsed time. `min_size` and `include_static` change the root denominator. +If a direct query uses `-n`/`max_rows`, inspect its completeness flags: a +truncated set of node rows is not a complete tree. Old text-only databases +must be re-imported; the analyzer does not reconstruct frames from text. + +The HTML chart keeps byte-proportional widths. Tiny and zero-byte frames are +still reachable through the direct-child list or the search results for the +entire tree, even when zoomed into another branch. Search matches filename, +function or node ID; each result shows its full caller path, inclusive bytes and +node ID to distinguish identical labels. Lists include every match and support +Tab then Enter/Space. Parent frame moves up one level; Reset zoom returns to the +root without clearing the search. Clear the search field to remove the results. + +Frame-tree templates and `report memory-tree` support standalone format-3 databases +only. Dataset roots are rejected explicitly; the analyzer never selects a shard +implicitly. Direct native shards have no standalone frame tables: re-import the +original snapshot as a standalone database to use these templates. Existing +native `pt_snap_frame`/coverage readers remain unchanged. `--timeout` uses the +shared query deadline. Text and HTML renderers reject incomplete or nonconserving +trees. Known-empty stacks have an `empty_callstack` synthetic leaf; +missing/corrupt stacks retain the `missing` leaf. `event_frames` returns no rows +for missing events/stacks, known-empty stacks or invalid frame arrays. + +The Python API uses the same template and contracts: + +```python +from pathlib import Path +from pt_snap_cli import SnapshotAnalyzer + +with SnapshotAnalyzer(db_path=Path("snapshot.pkl.db"), device_id=0) as analyzer: + frames = analyzer.execute_query("event_frames", params={"event_id": 100}) + tree = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 100}) +``` + ## Template Architecture Query templates are defined in YAML format with: diff --git a/docs/zh/database.md b/docs/zh/database.md index 4022e4b..e542e7d 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -316,15 +316,57 @@ CREATE TABLE callstack ( | 属性 | 保证 | |------|------| -| 唯一性 | 每条不同的调用栈文本只有一行,因此按 `callstackId` 分组等价于按文本分组 | +| 唯一性 | 每组不同的有序结构化帧数组只有一行;显示文本本身不能无损标识调用栈 | | 作用域 | 数据库内所有设备共享,表名不带设备后缀 | | 覆盖度 | 导入产生的每个事件都能解析到一行;只有外部生成的数据库才可能出现 `callstackId` 为 `NULL` | -当前 `pt-snap import` 写入的是这种 v2 布局(`import_format_version = 2`)。 +当前 `pt-snap import` 保留这种 v2 文本布局,同时增加结构化帧表 +(`import_format_version = 3`)。格式 2 数据库的文本布局相同,但没有原始帧记录。 调用栈去重之前生成的数据库在每张 `trace_entry_` 表中内联存储 `callstack` TEXT,没有共享 `callstack` 表(有 metadata 时为 `import_format_version = 1`)。 +### 结构化帧与有序数组(格式 3) + +`frame` 对原始 `(filename, line, name)` 三元组去重;`callstack_frame` +把调用栈关联到有序的帧 ID 数组: + +```sql +CREATE TABLE frame ( + id INTEGER PRIMARY KEY, + filename TEXT NOT NULL, + line INTEGER NOT NULL, + name TEXT NOT NULL +); +CREATE TABLE callstack_frame ( + callstackId INTEGER NOT NULL, + position INTEGER NOT NULL, + frameId INTEGER NOT NULL +); +CREATE UNIQUE INDEX idx_callstack_frame_position + ON callstack_frame (callstackId, position); +``` + +两张表在设备之间共享。事件通过 `trace_entry_.callstackId` 关联数组, +按 `position ASC` 读取即可还原快照原始顺序;位置 0 是最内层帧。帧树分析反向 +遍历,从最外层调用者开始。递归中的同一帧保留各自位置,即使帧 ID 相同。 +空数组没有关联行。结构化字段直接保留冒号、空白、中文和内嵌换行,无需拆解显示文本。 + +使用 `event_frames` 读取事件有序帧,使用 `active_memory_frame_tree_at_event` +读取各父子节点的占用。详见[调用栈帧内存拆解](querying.md#调用栈帧内存拆解)。 +旧文本数据库仍可用于原有查询,新查询需要从原始快照重新导入。导入器检测格式变化 +后重建缓存;只读分析不会迁移数据库。 + +### 单库帧能力边界 + +格式 3 仅用于单库导入。原生分片导入保持格式 2,兼容导出保持格式 1; +它们不会声称支持单库帧树。新增 `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +frameCount INTEGER NOT NULL)` 保存每个采集调用栈的预期长度,包括零。 +filename/name 必须是文本,line 必须是整数。`event_frames` 对缺失、已知空数组 +或损坏数组返回零行,不输出部分重建的数组。树将已知空栈与缺失/损坏证据分开。 +删除末尾或全部关联、非整数位置以及无效字段类型均不能通过校验。 +只读查询不会修复已存储的证据。 + ### 调用栈布局兼容 `pt-snap` 可以读取两种布局,且不会改写数据库文件: @@ -335,8 +377,8 @@ CREATE TABLE callstack ( | v1(旧版) | `trace_entry_.callstack` TEXT,无 `callstackId`,无共享表 | 直接读该列 | `event`、`callstack_analysis` 和 `active_memory_callstack_at_event` 保持相同的 -模板名、参数和输出契约。查询引擎按识别到的布局选择 v1 或 v2 SQL。其他模板不依赖 -这次拆分。 +模板名、参数和输出契约。查询引擎按识别到的布局选择 v1 或 v2 SQL。 +结构化帧查询额外要求格式 3 的两张帧表,不会从旧显示文本猜测原始帧。 识别过程是只读的:`focus`、`query`、报告和 Python API 都以 SQLite `mode=ro` 打开数据库。布局不会写入 `.pt-snap/focus.json`。列冲突、设备布局不一致、 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 5f390b8..7975d55 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -570,6 +570,69 @@ CLI 和 Python API 的查询结果包含原始 SQLite 值。模板的 `output_sc `SnapshotAnalyzer.execute_query()` 的结果还包含 `template` 与 `semantics_version`, 便于把行与契约对应起来。 +## 调用栈帧内存拆解 + +使用当前版本导入后,可以读取原始有序帧数组,或拆解指定事件后的存活内存: + +```bash +pt-snap query --template-use event_frames --params '{"event_id":100}' --json +pt-snap query --template-use active_memory_frame_tree_at_event --params '{"event_id":100}' --json +pt-snap report memory-tree --event-id 100 +pt-snap report memory-tree --event-id 100 --json +pt-snap report memory-tree --event-id 100 --format html > memory-tree.html +``` + +HTML 文件可直接在本地打开:点击帧放大、重置查看整棵树、搜索文件或函数。 +宽度表示存活内存块的字节数,不是 CPU 时间或累计分配量,无需启动服务器。 +`--device` 选择设备,`--exclude-static` 排除静态和采集前已存在的内存, +`--min-size` 按单个块大小筛选;`--event-id` 必填。报告不做排名或行数截断。 + +| 字段 | 含义 | +| --- | --- | +| `node_id`、`parent_id` | 完整调用路径身份及直接父节点;`root` 为总占用 | +| `depth`、`frame_id` | 从调用者开始的层级和结构化帧身份 | +| `size_bytes`、`requested_bytes`、`block_count` | 当前节点及所有后代的占用和块数 | +| `self_bytes`、`self_requested_bytes`、`self_block_count` | 记录的分配栈终止于当前节点的自身占用和块数 | +| `percent_of_total` | 根节点总占用的 0–100 字节百分比;分母在块和类别筛选后、输出行截断前计算 | + +每个节点满足“总占用 = 自身占用 + 各直接子节点总占用”。可以对所有节点的 +`self_bytes` 求和,或对树的一组完整切面求和;不能跨层累加 `size_bytes`, +否则同一分配会重复计算。记录的调用栈可能止于中间节点,所以中间节点也可能有 +自身占用。递归帧以及不同调用者下的相同帧保留为不同路径。 + +动态内存只统计 `allocEventId <= event_id` 且 `freeEventId` 晚于该事件或未记录 +的块。`freeEventId` 对应 `free_completed`,因此待释放内存仍计入。 +静态内存、采集前已存在的内存、缺失或损坏调用栈分别作为显式虚拟叶节点。 +这表示 active 内存块占用,不是 reserved 内存拆解;事件 ID 是顺序标记,不是 +耗时。`min_size` 和 `include_static` 会改变根节点分母。直接查询若设置 +`-n`/`max_rows`,需检查完整性字段,截断结果不能当作完整树。 +旧文本数据库需要从原始快照重新导入,分析器不会从文本猜测原始帧。 + +HTML 图始终按字节比例绘制。即使帧过小或占用为零,也可以通过直属子节点列表或 +全树搜索结果进入;缩放到其他分支不会限制搜索范围。搜索支持文件名、函数名或 +节点 ID,每项显示完整调用路径、包含子节点的字节数和节点 ID,以区分同名帧。 +列表包含全部匹配项,支持 Tab 定位后按 Enter/空格进入。Parent frame 返回父节点, +Reset zoom 回到根节点且保留搜索;清空搜索框可清除搜索结果。 + +帧树模板和 `report memory-tree` 仅支持单库格式 3 数据库。dataset 根目录会 +明确拒绝,分析器不会隐式选取分片。直接访问原生分片也没有这些单库帧表; +使用新模板须将原始 snapshot 重新导入为单库。已有 `pt_snap_frame`/coverage +读取能力不变。`--timeout` 使用共享查询截止时间。文本和 HTML 渲染器拒绝 +不完整或不守恒的树。已知空栈使用 `empty_callstack` 合成叶节点,缺失/损坏 +调用栈使用 `missing` 叶节点。`event_frames` 对缺失事件/栈、已知空栈以及 +无效帧数组返回零行。 + +Python API 使用相同模板和契约: + +```python +from pathlib import Path +from pt_snap_cli import SnapshotAnalyzer + +with SnapshotAnalyzer(db_path=Path("snapshot.pkl.db"), device_id=0) as analyzer: + frames = analyzer.execute_query("event_frames", params={"event_id": 100}) + tree = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 100}) +``` + ## 模板架构 查询模板使用 YAML 格式定义,包含: diff --git a/pyproject.toml b/pyproject.toml index de01e4c..f97f49b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ Issues = "https://github.com/codeasier/pt-snap-cli/issues" dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", + "playwright>=1.40.0", "black>=23.0.0", "ruff>=0.0.280", "build>=1.0.0", diff --git a/src/pt_snap_cli/cli_reports.py b/src/pt_snap_cli/cli_reports.py index f20831f..9c16af6 100644 --- a/src/pt_snap_cli/cli_reports.py +++ b/src/pt_snap_cli/cli_reports.py @@ -36,10 +36,59 @@ TemplateRenderError, ) from pt_snap_cli.core.error_codes import DATABASE_NOT_FOUND, INVALID_PARAMETER +from pt_snap_cli.core.errors import PtSnapCoreError +from pt_snap_cli.memory_tree_render import render_memory_tree report_app = typer.Typer(help="Generate memory analysis reports") +@report_app.command("memory-tree") +def report_memory_tree( + event_id: Annotated[ + int, typer.Option("--event-id", help="Analyze live memory after this event") + ], + db_path: Annotated[ + Path | None, typer.Argument(help="Path to database (optional if configured)") + ] = None, + device: Annotated[ + int | None, typer.Option("--device", "-d", autocompletion=complete_device_ids) + ] = None, + include_static: Annotated[bool, typer.Option("--include-static/--exclude-static")] = True, + min_size: Annotated[int, typer.Option(help="Minimum individual block size in bytes")] = 0, + output_format: Annotated[ + Literal["text", "html"], + typer.Option("--format", help="Text breakdown or interactive HTML flamegraph on stdout"), + ] = "text", + timeout: Annotated[ + float | None, typer.Option("--timeout", help="Query deadline in seconds; <=0 disables") + ] = None, + json_output: Annotated[bool, _json_flag()] = False, +) -> None: + """Decompose event-time active memory by ordered allocation frames.""" + service = ReportService() + try: + if json_output and output_format != "text": + raise ValueError("--json cannot be combined with --format html.") + result = service.memory_tree_report( + event_id=event_id, + db_path=db_path, + device_id=device, + include_static=include_static, + min_size=min_size, + timeout_s=timeout, + ) + if json_output: + typer.echo(json.dumps({"event_id": event_id, **asdict(result)}, indent=2)) + else: + typer.echo(render_memory_tree(result, event_id, html=output_format == "html")) + except PtSnapCoreError as exc: + _error_from_exc(exc) + except ValueError as exc: + _error(str(exc), code=INVALID_PARAMETER) + finally: + service.close() + + @report_app.command("peak-memory") def report_peak_memory( db_path: Annotated[ diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index 140eca4..b7d915a 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -177,7 +177,7 @@ def _detect_callstack_layout( ) metadata_version = _import_format_version(cursor) - if metadata_version in (1, 2): + if metadata_version in (1, 2, 3): expected: CallstackLayout = "v1" if metadata_version == 1 else "v2" if layout is None: return None, ( diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index 725c4f0..e713803 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -27,9 +27,11 @@ METADATA_TABLE = "pt_snap_metadata" METADATA_SCHEMA_VERSION = 1 -# 2: trace_entry_ stores `callstackId` referencing the shared `callstack` -# table instead of an inlined `callstack` text column. -IMPORT_FORMAT_VERSION = 2 +# Standalone databases retain the v2 callstackId/text layout and add original- +# order structured frames in v3. Only standalone v2 caches require re-import. +IMPORT_FORMAT_VERSION = 3 +# Native shard metadata and cache identity retain their published v2 contract. +NATIVE_IMPORT_FORMAT_VERSION = 2 IMPORTER_NAME = "pt-snap-cli" HASH_CHUNK_SIZE = 1024 * 1024 diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py index 7e8428f..0f5722d 100644 --- a/src/pt_snap_cli/core/import_service.py +++ b/src/pt_snap_cli/core/import_service.py @@ -20,7 +20,7 @@ SourceChangedError, ) from pt_snap_cli.core.focus_service import FocusService -from pt_snap_cli.core.import_metadata import ImportMetadataService +from pt_snap_cli.core.import_metadata import NATIVE_IMPORT_FORMAT_VERSION, ImportMetadataService from pt_snap_cli.core.models import ( CacheMissReason, FocusState, @@ -229,8 +229,11 @@ def _import_dataset(self, options: ImportOptions) -> ImportResult: def finalize(result: ShardedReplayResult) -> None: nonlocal completed, generated - metadata = self._metadata_service.build_metadata( - options.snapshot_file, source_hash, options.device + metadata = replace( + self._metadata_service.build_metadata( + options.snapshot_file, source_hash, options.device + ), + import_format_version=NATIVE_IMPORT_FORMAT_VERSION, ) for item in result.slices: _require_readonly_member(result.directory, item.file) diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index 6a823a5..b319d73 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -42,6 +42,27 @@ def close(self) -> None: """Release connections owned by this report service; allow later reuse.""" self._query_service.close() + def memory_tree_report( + self, + event_id: int, + db_path: Path | str | None = None, + device_id: int | None = None, + include_static: bool = True, + min_size: int = 0, + *, + start_dir: Path | None = None, + timeout_s: float | None = None, + ) -> QueryResult: + """Return frame occupancy through the shared standalone query contract.""" + return self._query_service.execute_query( + "active_memory_frame_tree_at_event", + params={"event_id": event_id, "include_static": include_static, "min_size": min_size}, + db_path=db_path, + device_id=device_id, + start_dir=start_dir, + timeout_s=timeout_s, + ) + def event_attribution( self, event_id: int, diff --git a/src/pt_snap_cli/memory_tree_render.py b/src/pt_snap_cli/memory_tree_render.py new file mode 100644 index 0000000..a03c055 --- /dev/null +++ b/src/pt_snap_cli/memory_tree_render.py @@ -0,0 +1,178 @@ +"""Presentation of complete frame trees; occupancy comes from the query layer.""" + +from __future__ import annotations + +import json +from typing import cast + +from pt_snap_cli.core.models import QueryResult + + +def render_memory_tree(result: QueryResult, event_id: int, *, html: bool = False) -> str: + """Render an indented breakdown or a self-contained interactive flamegraph.""" + if result.truncated or result.has_more: + raise ValueError("A complete frame tree is required to render a memory breakdown.") + _validate_complete_tree(result.rows) + title = f"Active memory after event {event_id} · device {result.device_id}" + if html: + payload = json.dumps({"title": title, "nodes": result.rows}, ensure_ascii=False) + # Frame text is untrusted. Escape HTML script delimiters in the data; + # the renderer inserts labels using textContent, never innerHTML. + payload = payload.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + return _HTML.replace("__TREE_DATA__", payload) + + children: dict[str, list[dict[str, object]]] = {} + for row in result.rows: + children.setdefault(str(row["parent_id"]), []).append(row) + lines = [title, "Inclusive bytes / self bytes / live blocks"] + pending = list(reversed(children.get("None", []))) + while pending: + row = pending.pop() + depth = cast(int, row["depth"]) + # Keep each captured frame on one terminal line, even with unusual names. + label = json.dumps(str(row["label"]), ensure_ascii=False)[1:-1] + lines.append( + f"{' ' * depth}{label}: {row['size_bytes']} / {row['self_bytes']} / {row['block_count']}" + ) + pending.extend(reversed(children.get(str(row["node_id"]), []))) + return "\n".join(lines) + + +def _validate_complete_tree(rows: list[dict[str, object]]) -> None: + """Never display a partial/orphaned graph as a complete occupancy tree.""" + nodes = {str(row["node_id"]): row for row in rows} + error = "A complete frame tree is required to render a memory breakdown." + if len(nodes) != len(rows) or "root" not in nodes: + raise ValueError(error) + totals: dict[str, list[int]] = {node_id: [0, 0, 0] for node_id in nodes} + fields = ( + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ) + for node_id, row in nodes.items(): + parent_id = row["parent_id"] + if node_id == "root": + if parent_id is not None or row["depth"] != 0: + raise ValueError(error) + elif not isinstance(parent_id, str) or parent_id not in nodes: + raise ValueError(error) + elif cast(int, row["depth"]) != cast(int, nodes[parent_id]["depth"]) + 1: + raise ValueError(error) + for index, (total, own) in enumerate(fields): + if type(row[total]) is not int or type(row[own]) is not int: + raise ValueError(error) + if cast(int, row[total]) < 0 or cast(int, row[own]) < 0: + raise ValueError(error) + if isinstance(parent_id, str): + totals[parent_id][index] += cast(int, row[total]) + for node_id, row in nodes.items(): + if any( + row[total] != cast(int, row[own]) + totals[node_id][index] + for index, (total, own) in enumerate(fields) + ): + raise ValueError(error) + + +_HTML = """ + + +pt-snap memory flamegraph +

+

Width = live block bytes, not execution time. Each level groups allocation frames by caller path. +Inclusive = self + direct children. Pending-free blocks stay live until free_completed. +Static, preexisting, empty and missing stacks are separate leaves. Click a frame to zoom; hover for details. +Use the lists below to reach frames too small to click, including zero-byte frames. Buttons support Enter and Space.

+ + + + +

+

+ + + +""" diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 1704c3d..3fdee9d 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -246,6 +246,14 @@ def _timeout_error(self, timeout_s: float) -> QueryTimeoutError: def _execution_error(self, error: sqlite3.OperationalError) -> QueryExecutionError: detail = str(error) + if re.search( + r"no such table: (?:frame|callstack_frame|callstack_frame_manifest)\b", detail + ): + return QueryExecutionError( + "Query execution failed: structured callstack frames are unavailable. " + "Re-import the original snapshot with this version of pt-snap; " + "legacy callstack text cannot reconstruct original frame fields reliably." + ) if "no such table: callstack" in detail or re.search( r"no such column: (?:[\w]+\.)?callstackId", detail ): diff --git a/src/pt_snap_cli/query/templates/basic/event_frames.yaml b/src/pt_snap_cli/query/templates/basic/event_frames.yaml new file mode 100644 index 0000000..3bb992b --- /dev/null +++ b/src/pt_snap_cli/query/templates/basic/event_frames.yaml @@ -0,0 +1,55 @@ +version: "1.0" +queries: + event_frames: + description: Read an event's structured callstack frames in the original snapshot order + semantics_version: 1 + interpretation_limits: + - "Requires an import with structured frame tables (import_format_version >= 3). Legacy text is not split or guessed." + - "position is the zero-based original snapshot index. root_first reverses traversal without changing position. Empty captured stacks, missing events/stacks and invalid frame associations return no frame rows. Corrupt arrays are never returned partially." + parameters: + event_id: + type: int + required: true + description: Event whose ordered frame array is requested + frame_order: + type: str + default: original + choices: [original, root_first] + description: Original snapshot order or outermost caller first + query: | + WITH valid_stacks AS ( + SELECT m.callstackId, m.frameCount - 1 AS last_position + FROM callstack_frame_manifest m + JOIN callstack cs ON cs.id = m.callstackId + LEFT JOIN callstack_frame cf ON cf.callstackId = m.callstackId + LEFT JOIN frame f ON f.id = cf.frameId + WHERE m.callstackId = (SELECT callstackId FROM {{ device_trace_table }} WHERE id = {{ event_id|int }}) + GROUP BY m.callstackId + HAVING typeof(m.callstackId) = 'integer' + AND typeof(m.frameCount) = 'integer' AND m.frameCount >= 0 + AND COUNT(cf.callstackId) = m.frameCount + AND (m.frameCount = 0 OR ( + MIN(cf.position) = 0 AND MAX(cf.position) = m.frameCount - 1 + AND COUNT(DISTINCT cf.position) = m.frameCount + AND SUM(CASE WHEN typeof(cf.position) = 'integer' + AND typeof(cf.frameId) = 'integer' AND typeof(f.id) = 'integer' + AND typeof(f.filename) = 'text' AND typeof(f.line) = 'integer' + AND typeof(f.name) = 'text' THEN 1 ELSE 0 END) = m.frameCount + )) + ) + SELECT t.id AS event_id, t.callstackId AS stack_id, + cf.position, f.id AS frame_id, f.filename, f.line, f.name + FROM {{ device_trace_table }} t + JOIN valid_stacks valid ON valid.callstackId = t.callstackId + JOIN callstack_frame cf ON cf.callstackId = t.callstackId + JOIN frame f ON f.id = cf.frameId + WHERE t.id = {{ event_id|int }} + ORDER BY cf.position {% if frame_order == 'root_first' %}DESC{% else %}ASC{% endif %} + output_schema: + - {column: event_id, type: int, units: event_id} + - {column: stack_id, type: int, metric_semantics: identifier} + - {column: position, type: int, metric_semantics: ordering_marker} + - {column: frame_id, type: int, metric_semantics: identifier} + - {column: filename, type: str} + - {column: line, type: int} + - {column: name, type: str} diff --git a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml new file mode 100644 index 0000000..1c8d7f1 --- /dev/null +++ b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml @@ -0,0 +1,144 @@ +version: "1.0" +queries: + active_memory_frame_tree_at_event: + description: Decompose live memory after an event into a caller-first frame tree + semantics_version: 1 + interpretation_limits: + - "Requires structured frame tables from import_format_version >= 3; old text-only databases must be re-imported." + - "Uses allocation stacks of blocks live after event_id: allocEventId <= event_id and freeEventId > event_id (or absent). freeEventId is free_completed, so pending-free memory remains included." + - "Traversal reverses the original innermost-first frame array. Node identity is the whole ordered caller path, preserving recursion and separating identical frames under different callers." + - "size_bytes includes descendant occupancy; self_bytes counts only allocations ending at this node. size_bytes = self_bytes + sum(direct child size_bytes). Do not sum inclusive bytes across depths." + - "Root totals and percentages cover every selected live block before any caller-side row cap. There is no top_n pruning. A truncated query is not a complete tree." + - "Static, preexisting, known-empty and missing/corrupt stacks remain separate synthetic leaves. This is active-block occupancy, not reserved memory or allocation traffic. Event IDs are ordering markers, not elapsed time." + parameters: + event_id: + type: int + required: true + description: Point in time after this trace event + include_static: + type: bool + default: true + description: Include static and preexisting live blocks as synthetic leaves + min_size: + type: int + default: 0 + description: Minimum individual live block size in bytes + query: | + WITH RECURSIVE active_blocks AS ( + SELECT b.size, COALESCE(b.requestedSize, 0) AS requested_size, + CASE WHEN b.allocEventId = -1 AND b.freeEventId = -1 THEN 'static' + WHEN b.allocEventId = -1 THEN 'preexisting_live_at_event' + ELSE 'dynamic_live_at_event' END AS category, + t.callstackId AS stack_id + FROM {{ device_block_table }} b + LEFT JOIN {{ device_trace_table }} t ON t.id = b.allocEventId + WHERE b.size >= {{ min_size|int }} + AND (b.freeEventId IS NULL OR b.freeEventId < 0 OR b.freeEventId > {{ event_id|int }}) + AND ( + {% if include_static %}b.allocEventId = -1 OR{% endif %} + (b.allocEventId != -1 AND b.allocEventId <= {{ event_id|int }}) + ) + ), stack_shape AS ( + SELECT m.callstackId, m.frameCount - 1 AS last_position + FROM callstack_frame_manifest m + JOIN callstack cs ON cs.id = m.callstackId + LEFT JOIN callstack_frame cf ON cf.callstackId = m.callstackId + LEFT JOIN frame f ON f.id = cf.frameId + GROUP BY m.callstackId + HAVING typeof(m.callstackId) = 'integer' + AND typeof(m.frameCount) = 'integer' AND m.frameCount >= 0 + AND COUNT(cf.callstackId) = m.frameCount + AND (m.frameCount = 0 OR ( + MIN(cf.position) = 0 AND MAX(cf.position) = m.frameCount - 1 + AND COUNT(DISTINCT cf.position) = m.frameCount + AND SUM(CASE WHEN typeof(cf.position) = 'integer' + AND typeof(cf.frameId) = 'integer' AND typeof(f.id) = 'integer' + AND typeof(f.filename) = 'text' AND typeof(f.line) = 'integer' + AND typeof(f.name) = 'text' THEN 1 ELSE 0 END) = m.frameCount + )) + ), groups AS ( + SELECT CASE WHEN b.category = 'dynamic_live_at_event' AND s.last_position = -1 + THEN 'empty_callstack' ELSE b.category END AS category, + CASE WHEN s.last_position >= 0 THEN b.stack_id END AS stack_id, + s.last_position, + COUNT(*) AS block_count, SUM(b.size) AS size_bytes, + SUM(b.requested_size) AS requested_bytes + FROM active_blocks b + LEFT JOIN stack_shape s ON s.callstackId = b.stack_id + AND b.category = 'dynamic_live_at_event' + GROUP BY CASE WHEN b.category = 'dynamic_live_at_event' AND s.last_position = -1 + THEN 'empty_callstack' ELSE b.category END, + CASE WHEN s.last_position >= 0 THEN b.stack_id END + ), paths AS ( + SELECT g.stack_id, cf.position, cf.frameId AS frame_id, + 'frame/' || cf.frameId AS node_id, 'root' AS parent_id, 1 AS depth, + g.block_count, g.size_bytes, g.requested_bytes + FROM groups g + JOIN callstack_frame cf ON cf.callstackId = g.stack_id + AND cf.position = g.last_position + UNION ALL + SELECT p.stack_id, cf.position, cf.frameId, + p.node_id || '/' || cf.frameId, p.node_id, p.depth + 1, + p.block_count, p.size_bytes, p.requested_bytes + FROM paths p + JOIN callstack_frame cf ON cf.callstackId = p.stack_id + AND cf.position = p.position - 1 + ), nodes AS ( + SELECT 'root' AS node_id, NULL AS parent_id, 0 AS depth, NULL AS frame_id, + 'mixed' AS category, '[all active blocks]' AS label, + COALESCE(SUM(block_count), 0) AS block_count, + COALESCE(SUM(size_bytes), 0) AS size_bytes, + COALESCE(SUM(requested_bytes), 0) AS requested_bytes, + 0 AS self_block_count, 0 AS self_bytes, 0 AS self_requested_bytes + FROM groups + UNION ALL + SELECT p.node_id, p.parent_id, p.depth, p.frame_id, + 'dynamic_live_at_event', f.filename || ':' || f.line || ' ' || f.name, + SUM(p.block_count), SUM(p.size_bytes), SUM(p.requested_bytes), + SUM(CASE WHEN p.position = 0 THEN p.block_count ELSE 0 END), + SUM(CASE WHEN p.position = 0 THEN p.size_bytes ELSE 0 END), + SUM(CASE WHEN p.position = 0 THEN p.requested_bytes ELSE 0 END) + FROM paths p JOIN frame f ON f.id = p.frame_id + GROUP BY p.node_id + UNION ALL + SELECT CASE WHEN category = 'dynamic_live_at_event' THEN 'missing' ELSE category END, + 'root', 1, NULL, category, + CASE WHEN category = 'static' THEN '[static]' + WHEN category = 'preexisting_live_at_event' THEN '[preexisting live]' + WHEN category = 'empty_callstack' THEN '[empty captured callstack]' + ELSE '[missing callstack]' END, + block_count, size_bytes, requested_bytes, + block_count, size_bytes, requested_bytes + FROM groups WHERE stack_id IS NULL + ) + SELECT n.node_id, n.parent_id, n.depth, n.frame_id, n.category, n.label, + f.filename, f.line, f.name, + n.block_count, n.size_bytes, n.requested_bytes, + n.self_block_count, n.self_bytes, n.self_requested_bytes, + ROUND(n.size_bytes * 100.0 / NULLIF(r.size_bytes, 0), 4) AS percent_of_total + FROM nodes n + LEFT JOIN frame f ON f.id = n.frame_id + CROSS JOIN nodes r + WHERE r.node_id = 'root' + ORDER BY n.depth ASC, n.parent_id ASC, n.size_bytes DESC, n.node_id ASC + output_schema: + - {column: node_id, type: str, metric_semantics: identifier} + - {column: parent_id, type: str, metric_semantics: identifier} + - {column: depth, type: int, units: count} + - {column: frame_id, type: int, metric_semantics: identifier} + - {column: category, type: str, metric_semantics: classification, scope: mixed} + - {column: label, type: str} + - {column: filename, type: str} + - {column: line, type: int} + - {column: name, type: str} + - {column: block_count, type: int, units: count} + - {column: size_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: requested_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: self_block_count, type: int, units: count} + - {column: self_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: self_requested_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - column: percent_of_total + type: float + units: percent + metric_semantics: share_of_included_rows + denominator: Root size_bytes for all live blocks after include_static and min_size filtering, before caller row caps diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index a84aa16..40df201 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -286,3 +286,16 @@ rejection across resolver, focus, CLI and API. No repair, checkpoint, journal-mo change, artifact deletion or standalone-DB policy change is introduced. This is not concurrent-producer locking. Existing attribution, source mappings, fixtures and license records are unchanged; no live upstream artifact acceptance is claimed. + + +## 2026-10-10: standalone ordered frames and event-time memory trees + +First-party changes extend only standalone snapshot imports to format 3 with +lossless ordered frame identities and expected per-stack frame counts. Native +sharded replay retains format 2, text interning and its existing cache identity; +compatible export retains format 1. Standalone frame evidence is written by the +existing replay adaptor, without changing allocator transitions or event IDs. +Read-only queries preserve path identity, classify absent/corrupt evidence, and +conserve live block, requested-byte and occupancy totals. No upstream code or +new executable fixtures were incorporated; source mappings and licenses remain +unchanged. This supersedes the unmerged implementation proposed in PR #207. diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py index df9c46e..c85e812 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py @@ -1,26 +1,27 @@ +from collections.abc import Mapping, Sequence from typing import Any -from ....base import TraceEntry +from ....base import Frame, TraceEntry from .defs import CallstackFieldDefs class CallstackInterner: - """Assign one integer id per distinct callstack text. + """Intern callstacks by text, or opt into standalone structured identity. - PyTorch snapshots repeat a small number of callstacks across a very large - number of events, and the pickle memo already shares the frame containers - behind them. Interning therefore turns the per-event callstack into an - integer reference and keeps a single text copy per distinct callstack. - - Lookups use the identities of the individual frames and only render text - on a miss. Distinct containers that render to the same text collapse onto - one id, so the id-to-text mapping stays one-to-one and grouping by id - matches grouping by text. + Native writers retain one id per distinct rendered text. Standalone v3 + writers explicitly use original frame fields and order instead, since text + can contain delimiters that make different frame sequences look identical. + Identity fast paths retain frame containers to prevent object-id reuse. """ - def __init__(self) -> None: + def __init__(self, *, structured_frames: bool = False) -> None: + self._structured_frames: bool = structured_frames self._by_frames: dict[tuple[int, ...], tuple[object, int]] = {} self._by_text: dict[str, int] = {} + self._by_stack: dict[tuple[int, ...], int] = {} + self._by_frame: dict[tuple[str, int, str], int] = {} + self._frames: list[dict[str, object]] = [] + self._stacks: list[tuple[int, ...]] = [] self._texts: list[str] = [] def intern(self, event: TraceEntry) -> int: @@ -29,7 +30,9 @@ def intern(self, event: TraceEntry) -> int: frame_key = tuple(map(id, frames)) by_frames = self._by_frames.get(frame_key) if by_frames is not None: - callstack_id = by_frames[1] + return by_frames[1] + if self._structured_frames: + callstack_id = self._intern_structured(event, frames) else: text = event.get_callstack() existing = self._by_text.get(text) @@ -39,8 +42,47 @@ def intern(self, event: TraceEntry) -> int: self._by_text[text] = callstack_id else: callstack_id = existing - self._by_frames[frame_key] = (frames, callstack_id) + self._by_frames[frame_key] = (frames, callstack_id) + return callstack_id + + @staticmethod + def _frame_key(frame: Frame | Mapping[str, object]) -> tuple[str, int, str]: + filename, line, name = ( + (frame.filename, frame.line, frame.name) + if isinstance(frame, Frame) + else (frame["filename"], frame["line"], frame["name"]) + ) + # SQLite INTEGER affinity coerces bool/float/string values. Reject those + # here rather than lose their types or conflate True, 1 and 1.0 as keys. + # Keep this stricter standalone-v3 contract out of native text imports. + if not isinstance(filename, str) or type(line) is not int or not isinstance(name, str): + raise ValueError( + "Structured frames require string filename/name and integer line (not bool)." + ) + return filename, line, name + def _intern_structured( + self, event: TraceEntry, frames: Sequence[Frame | Mapping[str, object]] + ) -> int: + keys = [self._frame_key(frame) for frame in frames] + frame_ids: list[int] = [] + for key in keys: + frame_id = self._by_frame.get(key) + if frame_id is None: + frame_id = len(self._frames) + self._by_frame[key] = frame_id + self._frames.append( + {"id": frame_id, "filename": key[0], "line": key[1], "name": key[2]} + ) + frame_ids.append(frame_id) + stack = tuple(frame_ids) + existing = self._by_stack.get(stack) + if existing is not None: + return existing + callstack_id = len(self._texts) + self._texts.append(event.get_callstack()) + self._stacks.append(stack) + self._by_stack[stack] = callstack_id return callstack_id def records(self) -> list[dict[str, Any]]: @@ -52,3 +94,22 @@ def records(self) -> list[dict[str, Any]]: def __len__(self) -> int: return len(self._texts) + + def frame_records(self) -> list[dict[str, object]]: + """Return deduplicated original frame fields for a structured interner.""" + return self._frames.copy() + + def stack_frame_records(self) -> list[dict[str, int]]: + """Return original-order references, including repeated recursive frames.""" + return [ + {"callstackId": stack_id, "position": position, "frameId": frame_id} + for stack_id, stack in enumerate(self._stacks) + for position, frame_id in enumerate(stack) + ] + + def stack_manifest_records(self) -> list[dict[str, int]]: + """Attest every captured stack's size, including genuinely empty stacks.""" + return [ + {"callstackId": stack_id, "frameCount": len(stack)} + for stack_id, stack in enumerate(self._stacks) + ] diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py index ef71010..49bc753 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py @@ -40,6 +40,24 @@ SqliteColumn(name=CallstackFieldDefs.CALLSTACK), ] +_FRAME_TABLE_COLUMNS = [ + SqliteColumn(name="id", data_type=int, primary_key=True), + SqliteColumn(name="filename", not_null=True), + SqliteColumn(name="line", data_type=int, not_null=True), + SqliteColumn(name="name", not_null=True), +] + +_CALLSTACK_FRAME_TABLE_COLUMNS = [ + SqliteColumn(name="callstackId", data_type=int, not_null=True), + SqliteColumn(name="position", data_type=int, not_null=True), + SqliteColumn(name="frameId", data_type=int, not_null=True), +] + +_CALLSTACK_FRAME_MANIFEST_COLUMNS = [ + SqliteColumn(name="callstackId", data_type=int, primary_key=True), + SqliteColumn(name="frameCount", data_type=int, not_null=True), +] + _BLOCK_TABLE_COLUMNS = [ SqliteColumn(name=BlockFieldDefs.ID, data_type=int, primary_key=True), SqliteColumn(name=BlockFieldDefs.ADDR, data_type=int), @@ -103,11 +121,26 @@ def create_block_table(self, device: int = 0): ): table.create_index(self.conn, column) - def create_callstack_table(self): + def create_callstack_table(self, *, structured_frames: bool = False): self.create_table( SqliteTable(self.CALLSTACK_TABLE_NAME, _CALLSTACK_TABLE_COLUMNS), delete_if_exists=True, ) + if not structured_frames: + return + self.create_table(SqliteTable("frame", _FRAME_TABLE_COLUMNS), delete_if_exists=True) + self.create_table( + SqliteTable("callstack_frame", _CALLSTACK_FRAME_TABLE_COLUMNS), delete_if_exists=True + ) + self.create_table( + SqliteTable("callstack_frame_manifest", _CALLSTACK_FRAME_MANIFEST_COLUMNS), + delete_if_exists=True, + ) + self.conn.execute( + "CREATE UNIQUE INDEX idx_callstack_frame_position " + "ON callstack_frame (callstackId, position)" + ) + self.conn.commit() def get_trace_entry_table(self, device: int = 0): return self.get_table_by_name(self.get_trace_table_name_by_device(device)) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index 282280a..1f3bcad 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -19,7 +19,12 @@ class SnapshotDbHandler: def __init__( - self, db_path: str, devices: list[int], insert_cache_size: int = DEFAULT_INSERT_CACHE_SIZE + self, + db_path: str, + devices: list[int], + insert_cache_size: int = DEFAULT_INSERT_CACHE_SIZE, + *, + structured_frames: bool = False, ): self._closed = True self.db_path = db_path @@ -29,7 +34,7 @@ def __init__( self._device_block_cache = {} self._insert_cache_size = insert_cache_size try: - self.db.create_callstack_table() + self.db.create_callstack_table(structured_frames=structured_frames) for device in devices: self._device_block_cache[device] = [] self._device_event_cache[device] = [] @@ -105,9 +110,12 @@ class DumpEventHooker(SimulateHooker, AllocatorHooker): def __init__( self, db_path: str, devices: list[int], dump_cache_size: int = DEFAULT_INSERT_CACHE_SIZE ): - self.db_handler = SnapshotDbHandler(db_path, devices, insert_cache_size=dump_cache_size) + self.db_handler = SnapshotDbHandler( + db_path, devices, insert_cache_size=dump_cache_size, structured_frames=True + ) + # Standalone v3 opts in; native shard writers keep the default text contract. # One interner for the whole dump so devices share identical callstacks. - self.callstacks = CallstackInterner() + self.callstacks = CallstackInterner(structured_frames=True) def post_undo_event( self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot @@ -162,6 +170,14 @@ def flush(self, device: int = 0): def flush_callstacks(self): self.db_handler.insert_callstacks(self.callstacks.records()) + db = self.db_handler.db + db.get_table_by_name("frame").insert_records(db.conn, self.callstacks.frame_records()) + db.get_table_by_name("callstack_frame").insert_records( + db.conn, self.callstacks.stack_frame_records() + ) + db.get_table_by_name("callstack_frame_manifest").insert_records( + db.conn, self.callstacks.stack_manifest_records() + ) def close(self, *, commit: bool = True): self.db_handler.close(commit=commit) diff --git a/tests/frame_tree_helpers.py b/tests/frame_tree_helpers.py new file mode 100644 index 0000000..589649b --- /dev/null +++ b/tests/frame_tree_helpers.py @@ -0,0 +1,66 @@ +"""Small explicit allocator lifetimes and structured stacks for contract tests.""" + +from pathlib import Path + +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database import CallstackInterner, SnapshotDb + + +def create_frame_tree_db(path: Path) -> Path: + root = {"filename": "train.py", "line": 1, "name": "train"} + shared = {"filename": "model.py", "line": 20, "name": "forward"} + a = {"filename": "tensor.py", "line": 30, "name": "a"} + b = {"filename": "tensor.py", "line": 40, "name": "b"} + other = {"filename": "other.py", "line": 1, "name": "other"} + stacks = [ + [a, shared, root], + [b, shared, root], + [shared, root], + [shared, shared, root], + [a, other], + [], + ] + db = SnapshotDb(str(path)) + interner = CallstackInterner(structured_frames=True) + try: + db.create_callstack_table(structured_frames=True) + for device in (0, 1): + db.create_trace_entry_table(device) + db.create_block_table(device) + for event_id, frames in enumerate(stacks, start=1): + event = TraceEntry.from_dict({"action": "alloc", "frames": frames}) + stack_id = interner.intern(event) + db.conn.execute( + f"INSERT INTO trace_entry_{device} (id, action, callstackId) VALUES (?, 4, ?)", + (event_id, stack_id), + ) + db.conn.execute(f"INSERT INTO trace_entry_{device} (id, action) VALUES (8, 5)") + db.conn.executemany( + f"INSERT INTO block_{device} " + "(id, size, requestedSize, allocEventId, freeEventId) VALUES (?, ?, ?, ?, ?)", + [ + (1, 100, 90, 1, 10), # Pending free at 8; completed at 10. + (2, 200, 180, 2, -1), + (3, 50, 45, 3, None), # Allocation ends at an internal node. + (4, 25, 20, 4, -1), # Recursive frame, same caller identity twice. + (5, 75, 60, 5, -1), # Same leaf under a different caller. + (6, 10, 9, 6, -1), # Empty captured stack. + (7, 5, 4, 7, -1), # Missing allocation event. + (8, 40, 32, -1, -1), + (9, 30, 24, -1, 10), + (10, 999, 900, 1, 8), # Already completed at selected boundary. + (11, 888, 800, 9, -1), # Future allocation. + ], + ) + db.get_callstack_table().insert_records(db.conn, interner.records()) + db.get_table_by_name("frame").insert_records(db.conn, interner.frame_records()) + db.get_table_by_name("callstack_frame").insert_records( + db.conn, interner.stack_frame_records() + ) + db.get_table_by_name("callstack_frame_manifest").insert_records( + db.conn, interner.stack_manifest_records() + ) + db.conn.commit() + finally: + db.conn.close() + return path diff --git a/tests/query/test_callstack_schema_compat.py b/tests/query/test_callstack_schema_compat.py index 9363e7a..7280feb 100644 --- a/tests/query/test_callstack_schema_compat.py +++ b/tests/query/test_callstack_schema_compat.py @@ -409,7 +409,7 @@ def test_v1_layout_detected_with_or_without_metadata( assert Context(db_path).callstack_layout == "v1" -@pytest.mark.parametrize("metadata_version", [None, 2]) +@pytest.mark.parametrize("metadata_version", [None, 2, 3]) def test_v2_layout_detected_with_or_without_metadata( tmp_path: Path, metadata_version: int | None ) -> None: @@ -442,7 +442,7 @@ def test_metadata_conflict_fails_only_variant_templates(tmp_path: Path) -> None: def test_unknown_metadata_version_does_not_override_structure(tmp_path: Path) -> None: - db_path = create_v2_db(tmp_path / "future.db", metadata_version=3) + db_path = create_v2_db(tmp_path / "future.db", metadata_version=4) assert Context(db_path).callstack_layout == "v2" diff --git a/tests/query/test_frame_tree.py b/tests/query/test_frame_tree.py new file mode 100644 index 0000000..98e5b82 --- /dev/null +++ b/tests/query/test_frame_tree.py @@ -0,0 +1,196 @@ +import hashlib +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.frame_tree_helpers import create_frame_tree_db + + +@pytest.fixture +def frame_db(tmp_path: Path) -> Path: + return create_frame_tree_db(tmp_path / "frames.db") + + +def query(path: Path, name: str, params: dict, **kwargs): + service = QueryService() + try: + return service.execute_query(name, params=params, db_path=path, **kwargs) + finally: + service.close() + + +def tree(path: Path, **params) -> list[dict[str, object]]: + return query(path, "active_memory_frame_tree_at_event", {"event_id": 8, **params}).rows + + +def test_event_frames_preserve_original_order_and_reverse_only_on_request(frame_db: Path): + original = query(frame_db, "event_frames", {"event_id": 1}).rows + assert [(r["position"], r["filename"], r["name"]) for r in original] == [ + (0, "tensor.py", "a"), + (1, "model.py", "forward"), + (2, "train.py", "train"), + ] + reverse = query(frame_db, "event_frames", {"event_id": 1, "frame_order": "root_first"}).rows + assert reverse == original[::-1] + assert query(frame_db, "event_frames", {"event_id": 6}).rows == [] + assert query(frame_db, "event_frames", {"event_id": 999}).rows == [] + + +def test_tree_conserves_bytes_counts_and_requested_memory_at_every_node(frame_db: Path): + rows = tree(frame_db) + assert rows[0]["node_id"] == "root" + assert (rows[0]["size_bytes"], rows[0]["requested_bytes"], rows[0]["block_count"]) == ( + 535, + 464, + 9, + ) + assert sum(r["self_bytes"] for r in rows) == 535 + assert sum(r["self_requested_bytes"] for r in rows) == 464 + assert sum(r["self_block_count"] for r in rows) == 9 + for row in rows: + children = [r for r in rows if r["parent_id"] == row["node_id"]] + for total, own in [ + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ]: + assert row[total] == row[own] + sum(r[total] for r in children) + assert row["percent_of_total"] == round(row["size_bytes"] * 100 / 535, 4) + forward = [r for r in rows if r["name"] == "forward"] + assert [(r["depth"], r["size_bytes"], r["self_bytes"]) for r in forward] == [ + (2, 375, 50), + (3, 25, 25), + ] + leaves = [r for r in rows if r["name"] == "a"] + assert sorted(r["size_bytes"] for r in leaves) == [75, 100] + assert len({r["node_id"] for r in leaves}) == 2 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == 5 + assert next(r for r in rows if r["node_id"] == "empty_callstack")["size_bytes"] == 10 + + +def test_completion_boundary_and_filters_preserve_exact_denominator(frame_db: Path): + after = tree(frame_db, event_id=10) + # 100-byte dynamic and 30-byte preexisting allocations complete at 10; + # the 888-byte allocation at 9 has now become live (without a stack). + assert after[0]["size_bytes"] == 535 - 130 + 888 + filtered = tree(frame_db, include_static=False, min_size=50) + assert filtered[0]["size_bytes"] == 425 + assert {r["category"] for r in filtered} == {"mixed", "dynamic_live_at_event"} + empty = tree(frame_db, min_size=9999) + assert len(empty) == 1 + assert empty[0]["node_id"] == "root" and empty[0]["parent_id"] is None + assert empty[0]["size_bytes"] == empty[0]["requested_bytes"] == empty[0]["block_count"] == 0 + assert empty[0]["percent_of_total"] is None + + +def test_row_cap_is_explicit_and_does_not_recompute_total(frame_db: Path): + result = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, max_rows=2) + assert result.truncated and result.has_more + assert result.rows[0]["size_bytes"] == 535 + assert result.rows[1]["percent_of_total"] == round(375 * 100 / 535, 4) + full = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, exact_total=True) + assert full.total == full.returned and full.total_is_exact and not full.truncated + + +def test_multi_device_analysis_is_read_only(frame_db: Path): + before = hashlib.sha256(frame_db.read_bytes()).digest() + first = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, device_id=0) + second = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, device_id=1) + assert first.rows == second.rows + assert hashlib.sha256(frame_db.read_bytes()).digest() == before + + +@pytest.mark.parametrize( + "damage", + [ + "gap", + "dangling_frame", + "dangling_stack", + "fractional_position", + "tail", + "all", + "bad_line", + "bad_name", + ], +) +def test_corrupt_stack_links_preserve_occupancy_as_missing(frame_db: Path, damage: str): + with closing(sqlite3.connect(frame_db)) as conn, conn: + stack = conn.execute("SELECT callstackId FROM trace_entry_0 WHERE id=1").fetchone()[0] + if damage == "gap": + conn.execute("DELETE FROM callstack_frame WHERE callstackId=? AND position=1", (stack,)) + elif damage == "dangling_frame": + conn.execute( + "UPDATE callstack_frame SET frameId=999 WHERE callstackId=? AND position=0", + (stack,), + ) + elif damage == "fractional_position": + conn.execute( + "UPDATE callstack_frame SET position=0.5 WHERE callstackId=? AND position=1", + (stack,), + ) + elif damage == "tail": + conn.execute("DELETE FROM callstack_frame WHERE callstackId=? AND position=2", (stack,)) + elif damage == "all": + conn.execute("DELETE FROM callstack_frame WHERE callstackId=?", (stack,)) + elif damage in {"bad_line", "bad_name"}: + column = "line" if damage == "bad_line" else "name" + conn.execute( + f"UPDATE frame SET {column}=? WHERE id=(SELECT frameId FROM callstack_frame WHERE callstackId=? AND position=0)", + (0.5 if damage == "bad_line" else b"bad", stack), + ) + else: + conn.execute("DELETE FROM callstack WHERE id=?", (stack,)) + rows = tree(frame_db) + assert rows[0]["size_bytes"] == 535 + expected_missing = 180 if damage in {"bad_line", "bad_name"} else 105 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == expected_missing + assert query(frame_db, "event_frames", {"event_id": 1}).rows == [] + for row in rows: + children = [child for child in rows if child["parent_id"] == row["node_id"]] + for total, own in [ + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ]: + assert row[total] == row[own] + sum(child[total] for child in children) + + +@pytest.mark.parametrize( + "name,params", + [("event_frames", {"event_id": 1}), ("active_memory_frame_tree_at_event", {"event_id": 8})], +) +def test_legacy_database_reports_actionable_reimport_error(frame_db: Path, name: str, params: dict): + with closing(sqlite3.connect(frame_db)) as conn, conn: + conn.execute("DROP TABLE callstack_frame") + conn.execute("DROP TABLE frame") + with pytest.raises(QueryExecutionError, match="Re-import the original snapshot"): + query(frame_db, name, params) + assert query(frame_db, "event", {"id": 1}).rows[0]["id"] == 1 + + +def test_inline_text_database_retains_queries_but_cannot_supply_original_frames(tmp_path: Path): + from tests.query.test_callstack_schema_compat import create_v1_db + + path = create_v1_db(tmp_path / "inline.db") + for name in ("event_frames", "active_memory_frame_tree_at_event"): + with pytest.raises(QueryExecutionError, match="Re-import the original snapshot"): + query(path, name, {"event_id": 1}) + assert query(path, "event", {"id": 1}).rows[0]["callstack"] == "train.py:10" + + +@pytest.mark.parametrize("name", ["event_frames", "active_memory_frame_tree_at_event"]) +@pytest.mark.parametrize("slice_index", [None, 0]) +def test_dataset_root_never_implicitly_selects_a_shard(tmp_path: Path, name: str, slice_index): + from pt_snap_cli.core.dataset_support import DATASET_SUPPORT + from tests.test_dataset_focus import hashes, make_dataset + + root = make_dataset(tmp_path / "dataset") + before = hashes(root) + assert name not in DATASET_SUPPORT + with pytest.raises(QueryExecutionError, match="support|standalone|available"): + query(root, name, {"event_id": 1}, slice_index=slice_index) + assert hashes(root) == before diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py index be917e0..6e93130 100644 --- a/tests/snapshot/test_callstack_interner.py +++ b/tests/snapshot/test_callstack_interner.py @@ -1,6 +1,8 @@ import gc import weakref +import pytest + from pt_snap_cli.snapshot.base import TraceEntry from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner @@ -120,3 +122,92 @@ def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): assert raw_id == eager_id == 0 assert len(interner) == 1 + + +def test_original_order_and_recursive_frames_survive_interning(): + interner = CallstackInterner(structured_frames=True) + interner.intern(_raw_event([INNER, INNER, OUTER])) + interner.intern(_raw_event([dict(INNER), dict(OUTER)])) + assert interner.frame_records() == [{"id": 0, **INNER}, {"id": 1, **OUTER}] + assert interner.stack_frame_records() == [ + {"callstackId": 0, "position": 0, "frameId": 0}, + {"callstackId": 0, "position": 1, "frameId": 0}, + {"callstackId": 0, "position": 2, "frameId": 1}, + {"callstackId": 1, "position": 0, "frameId": 0}, + {"callstackId": 1, "position": 1, "frameId": 1}, + ] + + +@pytest.mark.parametrize("structured", [False, True]) +def test_display_text_collisions_only_remain_distinct_for_structured_stacks(structured): + interner = CallstackInterner(structured_frames=structured) + first = _raw_event([{"filename": "x", "line": 1, "name": "f\ny:2 g"}]) + second = _raw_event( + [ + {"filename": "y", "line": 2, "name": "g"}, + {"filename": "x", "line": 1, "name": "f"}, + ] + ) + assert first.get_callstack() == second.get_callstack() + assert (interner.intern(first) != interner.intern(second)) is structured + assert len(interner) == (2 if structured else 1) + + +@pytest.mark.parametrize("structured", [False, True]) +def test_empty_stack_manifest_only_attests_structured_capture(structured): + interner = CallstackInterner(structured_frames=structured) + interner.intern(_raw_event([])) + interner.intern(_raw_event([INNER, INNER, OUTER])) + assert interner.stack_manifest_records() == ( + [{"callstackId": 0, "frameCount": 0}, {"callstackId": 1, "frameCount": 3}] + if structured + else [] + ) + if not structured: + assert interner.frame_records() == interner.stack_frame_records() == [] + + +@pytest.mark.parametrize( + "field,value", + [ + ("line", True), + ("line", False), + ("line", 1.0), + ("line", "1"), + ("line", None), + ("filename", 42), + ("name", False), + ], +) +@pytest.mark.parametrize("eager", [False, True]) +def test_structured_fields_reject_lossy_types_without_changing_native_text(field, value, eager): + invalid = {**INNER, field: value} + event = _raw_event([INNER, invalid]) + if eager: + event = TraceEntry.from_dict(event.to_dict()) + structured = CallstackInterner(structured_frames=True) + with pytest.raises(ValueError, match="Structured frames require"): + structured.intern(event) + assert structured.records() == structured.frame_records() == [] + native = CallstackInterner() + assert native.intern(event) == 0 + assert native.records() == [{"id": 0, "callstack": event.get_callstack()}] + + +@pytest.mark.parametrize("eager", [False, True]) +def test_structured_identity_preserves_exact_fields_without_normalizing(eager): + frames = [ + {"filename": "训练:模型.py", "line": -1, "name": "内部\n分配"}, + {"filename": "", "line": 0, "name": ""}, + {"filename": "训练:模型.py", "line": 1, "name": "内部\n分配"}, + ] + first = _raw_event(frames) + equal = _raw_event([dict(frame) for frame in frames]) + if eager: + first = TraceEntry.from_dict(first.to_dict()) + interner = CallstackInterner(structured_frames=True) + assert interner.intern(first) == interner.intern(equal) == 0 + assert interner.frame_records() == [ + {"id": index, **frame} for index, frame in enumerate(frames) + ] + assert interner.stack_manifest_records() == [{"callstackId": 0, "frameCount": 3}] diff --git a/tests/snapshot/test_snapshot2db_runtime.py b/tests/snapshot/test_snapshot2db_runtime.py index a7bac9a..55bcdf7 100644 --- a/tests/snapshot/test_snapshot2db_runtime.py +++ b/tests/snapshot/test_snapshot2db_runtime.py @@ -79,6 +79,44 @@ def replay(representation, device, **kwargs): assert calls == {"load": 1, "replay": 1, "raw_frames": True} +def test_dump_persists_original_frame_fields_and_positions(tmp_path): + frames = [ + {"filename": "训练:模型.py", "line": 17, "name": "内部\n分配"}, + {"filename": "outer.py", "line": 20, "name": "outer"}, + ] + representation = { + "segments": [], + "device_traces": [ + [ + { + "id": 17, + "action": "segment_free", + "addr": 0x1000, + "size": 64, + "stream": 0, + "frames": frames, + } + ] + ], + } + snapshot = tmp_path / "structured.pkl" + database = tmp_path / "structured.db" + with snapshot.open("wb") as stream: + pickle.dump(representation, stream) + assert snapshot2db.dump(snapshot, database) + with closing(sqlite3.connect(database)) as conn: + rows = conn.execute( + "SELECT f.filename, f.line, f.name FROM trace_entry_0 t " + "JOIN callstack_frame cf ON cf.callstackId=t.callstackId " + "JOIN frame f ON f.id=cf.frameId WHERE t.id=17 ORDER BY cf.position" + ).fetchall() + assert [dict(zip(("filename", "line", "name"), row, strict=True)) for row in rows] == frames + assert conn.execute( + "SELECT m.frameCount FROM trace_entry_0 t " + "JOIN callstack_frame_manifest m ON m.callstackId=t.callstackId WHERE t.id=17" + ).fetchone() == (2,) + + def test_fast_frame_import_matches_eager_database_exactly(monkeypatch, tmp_path): representation = { "segments": [], @@ -328,6 +366,9 @@ def test_dump_all_multiple_device_snapshot(dump_database): } assert tables == { "callstack", + "frame", + "callstack_frame", + "callstack_frame_manifest", "dictionary", "trace_entry_0", "block_0", diff --git a/tests/test_frame_import_isolation.py b/tests/test_frame_import_isolation.py new file mode 100644 index 0000000..03d7b6f --- /dev/null +++ b/tests/test_frame_import_isolation.py @@ -0,0 +1,157 @@ +"""Standalone structured v3 must not alter native-v2 or compatible-v1 imports.""" + +import json +import pickle +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import import_metadata +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.core.native_dataset_contract import NATIVE_FORMAT +from pt_snap_cli.snapshot.tools.adaptors import snapshot2db +from tests.snapshot.test_sharded_replay import lifecycle_data, write_source + +FRAME_TABLES = {"frame", "callstack_frame", "callstack_frame_manifest"} + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +def source(tmp_path): + data = lifecycle_data() + events = [event for event in data["device_traces"][0] if event["action"] != "oom"] + data["device_traces"][0] = events + events[0]["frames"] = [{"filename": "x", "line": 1, "name": "f\ny:2 g"}] + events[1]["frames"] = [ + {"filename": "y", "line": 2, "name": "g"}, + {"filename": "x", "line": 1, "name": "f"}, + ] + events[2]["frames"] = [] + return write_source(tmp_path, data) + + +def tables(conn): + return {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} + + +@pytest.mark.parametrize("fmt,version", [("single-db", 3), (NATIVE_FORMAT, 2), ("msinsight", 1)]) +def test_import_formats_keep_their_own_metadata_schema_and_cache( + tmp_path, monkeypatch, fmt, version +): + options = ImportOptions( + source(tmp_path), + tmp_path / fmt, + set_focus=False, + format=fmt, + events_per_slice=None if fmt == "single-db" else 100, + ) + service = ImportService() + result = service.import_snapshot(options) + assert result.metadata.import_format_version == version + members = [result.db_path] if version == 3 else list(result.db_path.rglob("*.db")) + assert members + for member in members: + with closing(sqlite3.connect(member)) as conn: + assert conn.execute( + "SELECT import_format_version FROM pt_snap_metadata" + ).fetchone() == (version,) + assert tables(conn) & FRAME_TABLES == (FRAME_TABLES if version == 3 else set()) + if version == 3: + rows = conn.execute( + "SELECT t.id, m.frameCount FROM trace_entry_0 t " + "JOIN callstack_frame_manifest m ON m.callstackId=t.callstackId " + "WHERE t.id IN (0,1,2) ORDER BY t.id" + ).fetchall() + assert rows == [(0, 1), (1, 2), (2, 0)] + counts = conn.execute( + "SELECT (SELECT COUNT(*) FROM callstack), " + "(SELECT COUNT(*) FROM callstack_frame_manifest)" + ).fetchone() + assert counts[0] == counts[1] + if version in (2, 3): + rows = conn.execute( + "SELECT t.callstackId, c.callstack FROM trace_entry_0 t " + "JOIN callstack c ON c.id=t.callstackId WHERE t.id IN (0,1) ORDER BY t.id" + ).fetchall() + assert rows[0][1] == rows[1][1] + assert (rows[0][0] == rows[1][0]) is (version == 2) + if version == 2: + manifest = json.loads((result.db_path / "manifest.json").read_text()) + assert manifest["identity"]["importFormatVersion"] == 2 + assert manifest["metadata"]["import_format_version"] == 2 + + def fail(*args, **kwargs): + pytest.fail("reusable cache loaded or replayed source") + + monkeypatch.setattr(service._backend, "dump_to_db", fail) + monkeypatch.setattr(service._dataset_backend.replay, "stage", fail) + monkeypatch.setattr(service._msinsight_backend.replay, "stage", fail) + if version != 3: + monkeypatch.setattr(import_metadata, "IMPORT_FORMAT_VERSION", 99) + cached = service.import_snapshot(options) + assert cached.reused and cached.metadata == result.metadata + + +def test_standalone_v2_cache_is_rebuilt_to_structured_v3(tmp_path): + options = ImportOptions(source(tmp_path), tmp_path / "standalone", set_focus=False) + service = ImportService() + first = service.import_snapshot(options) + with closing(sqlite3.connect(first.db_path)) as conn, conn: + conn.execute("UPDATE pt_snap_metadata SET import_format_version=2") + for table in FRAME_TABLES: + conn.execute(f"DROP TABLE {table}") + rebuilt = service.import_snapshot(options) + assert not rebuilt.reused + assert rebuilt.cache_miss_reason == "import_format_changed" + assert rebuilt.metadata.import_format_version == 3 + with closing(sqlite3.connect(rebuilt.db_path)) as conn: + assert FRAME_TABLES <= tables(conn) + assert service.import_snapshot(options).reused + + +@pytest.mark.parametrize("line", [True, 1.0, "1"]) +def test_invalid_structured_frame_rejects_standalone_but_preserves_native(tmp_path, line): + data = lifecycle_data() + data["device_traces"][0][0]["frames"][0]["line"] = line + path = write_source(tmp_path, data) + service = ImportService() + native = service.import_snapshot( + ImportOptions(path, tmp_path / "native", set_focus=False, events_per_slice=100) + ) + assert native.metadata.import_format_version == 2 + options = ImportOptions(path, tmp_path / "standalone", set_focus=False) + with pytest.raises(ImportExecutionError, match="Structured frames require"): + service.import_snapshot(options) + assert not service._backend.target_db_path(path, options.output_dir).exists() + + +def test_native_handler_default_never_creates_standalone_frame_tables(tmp_path): + handler = snapshot2db.SnapshotDbHandler(str(tmp_path / "native.db"), [0]) + try: + assert not (tables(handler.db.conn) & FRAME_TABLES) + finally: + handler.close() + + +def test_malformed_structured_reimport_preserves_existing_database(tmp_path): + options = ImportOptions(source(tmp_path), tmp_path / "standalone", set_focus=False) + service = ImportService() + first = service.import_snapshot(options) + before = first.db_path.read_bytes() + data = lifecycle_data() + data["device_traces"][0][0]["frames"][0]["line"] = True + options.snapshot_file.write_bytes(pickle.dumps(data)) + with pytest.raises(ImportExecutionError, match="Structured frames require"): + service.import_snapshot(options) + assert first.db_path.read_bytes() == before + assert list(first.db_path.parent.iterdir()) == [first.db_path] diff --git a/tests/test_memory_tree_browser.py b/tests/test_memory_tree_browser.py new file mode 100644 index 0000000..baef121 --- /dev/null +++ b/tests/test_memory_tree_browser.py @@ -0,0 +1,501 @@ +"""Exercise frame navigation with real Chromium events, including invisible frames. + +Set PT_SNAP_TEST_CHROMIUM to use an existing Chromium executable. CI sets +PT_SNAP_REQUIRE_BROWSER=1 so a missing browser cannot silently skip these tests. +""" + +from __future__ import annotations + +import importlib +import os +import re +from collections.abc import Iterator + +import pytest + +from pt_snap_cli.core.models import QueryResult +from pt_snap_cli.memory_tree_render import render_memory_tree + +playwright = ( + importlib.import_module("playwright.sync_api") + if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1" + else pytest.importorskip("playwright.sync_api") +) + +ROOT_BYTES = 471_498_368 +GIANT_BYTES = 283_057_288 +MALICIOUS_LABEL = ( + '路径/中文 ' + "" +) + + +def _complete_rows(specifications): + """Build an internally consistent tree from preorder (id, parent, label, self).""" + rows = [] + by_id = {} + for node_id, parent_id, label, own_bytes in specifications: + own_blocks = int(own_bytes > 0) + row = { + "node_id": node_id, + "parent_id": parent_id, + "label": label, + "depth": 0 if parent_id is None else by_id[parent_id]["depth"] + 1, + "frame_id": len(rows), + "category": "dynamic_live_at_event", + "size_bytes": own_bytes, + "self_bytes": own_bytes, + "requested_bytes": own_bytes, + "self_requested_bytes": own_bytes, + "block_count": own_blocks, + "self_block_count": own_blocks, + } + rows.append(row) + by_id[node_id] = row + for row in reversed(rows): + if row["parent_id"] is not None: + parent = by_id[row["parent_id"]] + for field in ("size_bytes", "requested_bytes", "block_count"): + parent[field] += row[field] + total = by_id["root"]["size_bytes"] + for row in rows: + row["percent_of_total"] = 100 * row["size_bytes"] / total if total else None + return rows + + +@pytest.fixture +def frame_rows(): + rows = _complete_rows( + [ + ("root", None, "All active memory", 0), + ("frame:giant", "root", "giant_branch", GIANT_BYTES - 40), + ("frame:giant/tiny", "frame:giant", "shared_function", 40), + ("frame:giant/zero", "frame:giant", "zero_parent", 0), + ("frame:giant/zero/leaf", "frame:giant/zero", "zero_leaf", 0), + ("frame:other", "root", "other_branch", ROOT_BYTES - GIANT_BYTES - 40), + ("frame:other/tiny", "frame:other", "shared_function", 40), + ("frame:malicious", "root", MALICIOUS_LABEL, 0), + ] + ) + assert rows[0]["size_bytes"] == ROOT_BYTES + assert rows[1]["size_bytes"] == GIANT_BYTES + return rows + + +@pytest.fixture(scope="module") +def chromium_browser(): + with playwright.sync_playwright() as runtime: + options = {"headless": True} + executable = os.environ.get("PT_SNAP_TEST_CHROMIUM") + if executable: + options["executable_path"] = executable + try: + browser = runtime.chromium.launch(**options) + except playwright.Error as exc: + # A broken running browser is a test failure, not an optional dependency. + missing = "executable doesn't exist" in str(exc).lower() + if missing and os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1": + pytest.skip("Chromium is not installed; run python -m playwright install chromium") + pytest.fail(f"Cannot launch Chromium for frame navigation tests: {exc}") + try: + yield browser + finally: + browser.close() + + +@pytest.fixture +def browser_page(chromium_browser) -> Iterator: + context = chromium_browser.new_context(viewport={"width": 1440, "height": 1000}, locale="en-US") + errors = [] + try: + page = context.new_page() + page.on("pageerror", lambda error: errors.append(str(error))) + yield page + finally: + context.close() + assert not errors, f"Uncaught browser errors: {errors}" + + +def _render(page, rows): + result = QueryResult( + total=len(rows), + returned=len(rows), + device_id=0, + rows=rows, + template="active_memory_frame_tree_at_event", + total_is_exact=True, + ) + page.set_content(render_memory_tree(result, event_id=17, html=True), wait_until="load") + playwright.expect(page.locator("#chart .node").first).to_be_attached() + + +def _button(page, container, node_id): + return page.locator(f'#{container} button[data-node-id="{node_id}"]') + + +def _rect(page, node_id): + return page.locator(f'#chart .node[data-node-id="{node_id}"] > rect').first + + +def _geometry(page): + return page.locator("#chart .node").evaluate_all("""groups => groups.map(group => { + const rect = group.querySelector('rect'); + return [group.dataset.nodeId, rect.getAttribute('x'), + rect.getAttribute('y'), rect.getAttribute('width')]; + })""") + + +def _assert_path(locator, *labels): + text = locator.inner_text() + positions = [text.index(label) for label in labels] + assert positions == sorted(positions), text + + +def _assert_selected(page, node_id, *labels): + playwright.expect(page.locator("#chart .node").first).to_have_attribute("data-node-id", node_id) + playwright.expect(page.locator("#detail")).to_contain_text(node_id) + _assert_path(page.locator("#focus-path"), *labels) + + +def _tab_to(page, target): + """Reach the control through the browser's native sequential focus order.""" + page.locator("#search").focus() + for _ in range(350): + page.keyboard.press("Tab") + if target.evaluate("element => element === document.activeElement"): + return + pytest.fail(f"Control is not reachable with Tab: {target}") + + +def test_search_keeps_true_proportional_geometry_until_navigation(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + playwright.expect(page.locator("#search-status")).not_to_be_empty() + playwright.expect(page.locator("#search-results button")).to_have_count(0) + _assert_selected(page, "root", "All active memory") + + giant_width = float(_rect(page, "frame:giant").get_attribute("width")) + assert giant_width == pytest.approx(1200 * GIANT_BYTES / ROOT_BYTES - 0.6) + assert float(_rect(page, "frame:giant/tiny").get_attribute("width")) == 0 + assert float(_rect(page, "frame:giant/zero/leaf").get_attribute("width")) == 0 + original_geometry = _geometry(page) + + page.locator("#search").fill("SHARED_FUNCTION") + playwright.expect(page.locator("#search-results button")).to_have_count(2) + assert _geometry(page) == original_geometry + _assert_selected(page, "root", "All active memory") + + _button(page, "search-results", "frame:giant/tiny").click() + _assert_selected( + page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" + ) + assert float(_rect(page, "frame:giant/tiny").get_attribute("width")) == pytest.approx(1199.4) + page.locator("#reset").click() + assert _geometry(page) == original_geometry + + +def test_search_disambiguates_duplicate_paths_and_supports_native_keyboard( + browser_page, frame_rows +): + page = browser_page + _render(page, frame_rows) + page.locator("#search").fill("sHaReD_fUnCtIoN") + playwright.expect(page.locator("#search-results button")).to_have_count(2) + playwright.expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b2\b")) + for node_id, branch in ( + ("frame:giant/tiny", "giant_branch"), + ("frame:other/tiny", "other_branch"), + ): + button = _button(page, "search-results", node_id) + _assert_path(button, "All active memory", branch, "shared_function") + playwright.expect(button).to_contain_text(node_id) + playwright.expect(button).to_contain_text(re.compile(r"40\s+bytes", re.IGNORECASE)) + target = _button(page, "search-results", "frame:giant/tiny") + _tab_to(page, target) + page.keyboard.press("Enter") + _assert_selected( + page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" + ) + playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 40 bytes") + + # The other same-named frame remains available outside the focused subtree. + playwright.expect(page.locator("#search-results button")).to_have_count(2) + target = _button(page, "search-results", "frame:other/tiny") + _tab_to(page, target) + page.keyboard.press("Space") + _assert_selected( + page, "frame:other/tiny", "All active memory", "other_branch", "shared_function" + ) + + +def test_node_id_search_is_case_insensitive_and_global_after_focus(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + _button(page, "children-list", "frame:giant").click() + page.locator("#search").fill("FRAME:OTHER/TINY") + playwright.expect(page.locator("#search-results button")).to_have_count(1) + _button(page, "search-results", "frame:other/tiny").click() + _assert_selected( + page, "frame:other/tiny", "All active memory", "other_branch", "shared_function" + ) + page.locator("#search").fill("ZERO_LEAF") + playwright.expect(page.locator("#search-results button")).to_have_count(1) + _button(page, "search-results", "frame:giant/zero/leaf").click() + _assert_selected( + page, + "frame:giant/zero/leaf", + "All active memory", + "giant_branch", + "zero_parent", + "zero_leaf", + ) + playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") + + +def test_children_reach_tiny_and_zero_frames_and_parent_retraces_path(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + playwright.expect(page.locator("#children-list button")).to_have_count(3) + giant = _button(page, "children-list", "frame:giant") + _assert_path(giant, "All active memory", "giant_branch") + playwright.expect(giant).to_contain_text("frame:giant") + playwright.expect(giant).to_contain_text("283,057,288 bytes") + giant.click() + playwright.expect(page.locator("#children-list button")).to_have_count(2) + + tiny = _button(page, "children-list", "frame:giant/tiny") + _assert_path(tiny, "All active memory", "giant_branch", "shared_function") + playwright.expect(tiny).to_contain_text("40 bytes") + _tab_to(page, tiny) + page.keyboard.press("Enter") + _assert_selected( + page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" + ) + playwright.expect(page.locator("#children-list button")).to_have_count(0) + playwright.expect(page.locator("#children-status")).not_to_be_empty() + page.get_by_role("button", name="Parent frame", exact=True).click() + + zero = _button(page, "children-list", "frame:giant/zero") + playwright.expect(zero).to_contain_text("0 bytes") + _tab_to(page, zero) + page.keyboard.press("Space") + _assert_selected(page, "frame:giant/zero", "All active memory", "giant_branch", "zero_parent") + _button(page, "children-list", "frame:giant/zero/leaf").click() + _assert_selected( + page, + "frame:giant/zero/leaf", + "All active memory", + "giant_branch", + "zero_parent", + "zero_leaf", + ) + playwright.expect(page.locator("#children-list button")).to_have_count(0) + playwright.expect(page.locator("#children-status")).not_to_be_empty() + for node_id, labels in ( + ("frame:giant/zero", ("All active memory", "giant_branch", "zero_parent")), + ("frame:giant", ("All active memory", "giant_branch")), + ("root", ("All active memory",)), + ): + page.get_by_role("button", name="Parent frame", exact=True).click() + _assert_selected(page, node_id, *labels) + playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + + +def test_reset_is_repeatable_and_preserves_global_search(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + original_geometry = _geometry(page) + page.locator("#search").fill("shared_function") + for node_id in ("frame:giant/tiny", "frame:other/tiny", "frame:giant/tiny"): + _button(page, "search-results", node_id).click() + page.locator("#reset").click() + _assert_selected(page, "root", "All active memory") + playwright.expect(page.locator("#search")).to_have_value("shared_function") + playwright.expect(page.locator("#search-results button")).to_have_count(2) + playwright.expect(page.locator("#children-list button")).to_have_count(3) + playwright.expect( + page.get_by_role("button", name="Parent frame", exact=True) + ).to_be_disabled() + assert _geometry(page) == original_geometry + page.locator("#reset").click() + _assert_selected(page, "root", "All active memory") + + +def test_no_results_and_clearing_search_leave_navigation_usable(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + instructions = page.locator("#search-status").inner_text() + _button(page, "children-list", "frame:giant").click() + geometry = _geometry(page) + page.locator("#search").fill("there is no such frame") + playwright.expect(page.locator("#search-results button")).to_have_count(0) + playwright.expect(page.locator("#search-status")).to_have_text("No matching frames.") + assert _geometry(page) == geometry + _assert_selected(page, "frame:giant", "All active memory", "giant_branch") + playwright.expect(page.locator("#children-list button")).to_have_count(2) + page.locator("#search").fill("") + playwright.expect(page.locator("#search-status")).to_have_text(instructions) + playwright.expect(page.locator("#search-results button")).to_have_count(0) + _button(page, "children-list", "frame:giant/tiny").click() + _assert_selected( + page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" + ) + + +def test_unicode_and_html_like_labels_remain_plain_text_everywhere(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + root_child = _button(page, "children-list", "frame:malicious") + playwright.expect(root_child).to_contain_text(MALICIOUS_LABEL) + page.locator("#search").fill("中文") + playwright.expect(page.locator("#search-results button")).to_have_count(1) + result = _button(page, "search-results", "frame:malicious") + playwright.expect(result).to_contain_text(MALICIOUS_LABEL) + result.click() + _assert_selected(page, "frame:malicious", "All active memory", MALICIOUS_LABEL) + playwright.expect(page.locator("#detail")).to_contain_text(MALICIOUS_LABEL) + playwright.expect(page.locator("#chart text")).to_have_text(MALICIOUS_LABEL) + playwright.expect(page.locator("img")).to_have_count(0) + playwright.expect(page.locator("script")).to_have_count(2) + assert page.evaluate("window.__frameInjected === undefined") + page.locator("#reset").click() + playwright.expect(_button(page, "children-list", "frame:malicious")).to_contain_text( + MALICIOUS_LABEL + ) + + +def test_search_does_not_silently_truncate_more_than_100_matches(browser_page): + page = browser_page + rows = _complete_rows( + [("root", None, "All active memory", 0)] + + [(f"batch:{index:03d}", "root", f"batch frame {index:03d}", 1) for index in range(151)] + ) + _render(page, rows) + page.locator("#search").fill("BATCH FRAME") + playwright.expect(page.locator("#search-results button")).to_have_count(151) + playwright.expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b151\b")) + _button(page, "search-results", "batch:150").click() + _assert_selected(page, "batch:150", "All active memory", "batch frame 150") + playwright.expect(page.locator("#search-results button")).to_have_count(151) + page.locator("#reset").click() + playwright.expect(page.locator("#children-list button")).to_have_count(151) + + +def test_zero_total_tree_can_be_explored_and_reset(browser_page): + page = browser_page + rows = _complete_rows( + [ + ("root", None, "Empty active memory", 0), + ("empty:branch", "root", "empty branch", 0), + ("empty:leaf", "empty:branch", "empty leaf", 0), + ("empty:sibling", "root", "empty sibling", 0), + ] + ) + _render(page, rows) + original_geometry = _geometry(page) + assert float(_rect(page, "empty:branch").get_attribute("width")) == 0 + playwright.expect(page.locator("#children-list button")).to_have_count(2) + _button(page, "children-list", "empty:branch").click() + leaf = _button(page, "children-list", "empty:leaf") + _tab_to(page, leaf) + page.keyboard.press("Enter") + _assert_selected(page, "empty:leaf", "Empty active memory", "empty branch", "empty leaf") + playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") + playwright.expect(page.locator("#detail")).to_contain_text("n/a") + playwright.expect(page.locator("#children-status")).not_to_be_empty() + page.locator("#search").fill("empty sibling") + _button(page, "search-results", "empty:sibling").click() + _assert_selected(page, "empty:sibling", "Empty active memory", "empty sibling") + page.locator("#reset").click() + _assert_selected(page, "root", "Empty active memory") + assert _geometry(page) == original_geometry + assert "NaN" not in page.locator("#chart").inner_html() + assert "Infinity" not in page.locator("#chart").inner_html() + + +def test_existing_svg_click_and_hover_keep_working(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + _rect(page, "frame:giant").hover() + playwright.expect(page.locator("#detail")).to_contain_text("283,057,288 bytes") + _rect(page, "frame:giant").click() + _assert_selected(page, "frame:giant", "All active memory", "giant_branch") + playwright.expect(page.locator("#children-list button")).to_have_count(2) + page.locator("#reset").click() + _assert_selected(page, "root", "All active memory") + + +def test_keyboard_focus_survives_continuous_descent_parent_and_reset(browser_page, frame_rows): + page = browser_page + _render(page, frame_rows) + path = page.locator("#focus-path") + _button(page, "children-list", "frame:giant").click() + playwright.expect(path).to_be_focused() + + # No focus() helper between navigations: the next Tab reaches the new children. + page.keyboard.press("Tab") + playwright.expect(_button(page, "children-list", "frame:giant/tiny")).to_be_focused() + page.keyboard.press("Tab") + playwright.expect(_button(page, "children-list", "frame:giant/zero")).to_be_focused() + page.keyboard.press("Space") + playwright.expect(path).to_be_focused() + page.keyboard.press("Tab") + playwright.expect(_button(page, "children-list", "frame:giant/zero/leaf")).to_be_focused() + page.keyboard.press("Enter") + _assert_selected( + page, + "frame:giant/zero/leaf", + "All active memory", + "giant_branch", + "zero_parent", + "zero_leaf", + ) + playwright.expect(path).to_be_focused() + + page.keyboard.press("Shift+Tab") + playwright.expect(page.locator("#search")).to_be_focused() + page.keyboard.press("Shift+Tab") + playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_focused() + page.keyboard.press("Enter") + _assert_selected(page, "frame:giant/zero", "All active memory", "giant_branch", "zero_parent") + playwright.expect(path).to_be_focused() + + for _ in range(3): + page.keyboard.press("Shift+Tab") + playwright.expect(page.locator("#reset")).to_be_focused() + page.keyboard.press("Space") + _assert_selected(page, "root", "All active memory") + playwright.expect(path).to_be_focused() + playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + page.keyboard.press("Tab") + playwright.expect(_button(page, "children-list", "frame:giant")).to_be_focused() + + +def test_special_allocation_categories_remain_reachable(browser_page): + page = browser_page + categories = ( + ("static", "[static]", "static"), + ("preexisting_live_at_event", "[preexisting live]", "preexisting_live_at_event"), + ("empty_callstack", "[empty captured callstack]", "empty_callstack"), + ("missing", "[missing callstack]", "dynamic_live_at_event"), + ) + rows = _complete_rows( + [("root", None, "All active memory", ROOT_BYTES)] + + [(node_id, "root", label, 0) for node_id, label, _ in categories] + ) + for row, (_, _, category) in zip(rows[1:], categories, strict=True): + row["category"] = category + row["frame_id"] = None + _render(page, rows) + for node_id, label, _ in categories: + assert float(_rect(page, node_id).get_attribute("width")) == 0 + _button(page, "children-list", node_id).click() + _assert_selected(page, node_id, "All active memory", label) + playwright.expect(page.locator("#children-status")).not_to_be_empty() + page.locator("#reset").click() + page.locator("#search").fill(node_id) + playwright.expect(page.locator("#search-results button")).to_have_count(1) + _button(page, "search-results", node_id).click() + _assert_selected(page, node_id, "All active memory", label) + page.locator("#reset").click() diff --git a/tests/test_memory_tree_report.py b/tests/test_memory_tree_report.py new file mode 100644 index 0000000..0b4a106 --- /dev/null +++ b/tests/test_memory_tree_report.py @@ -0,0 +1,148 @@ +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.report_service import ReportService +from pt_snap_cli.memory_tree_render import render_memory_tree +from tests.frame_tree_helpers import create_frame_tree_db + + +@pytest.fixture +def frame_db(tmp_path: Path) -> Path: + return create_frame_tree_db(tmp_path / "frames.db") + + +def test_report_cli_query_and_analyzer_share_exact_tree(frame_db: Path): + runner = CliRunner() + report = runner.invoke( + app, ["report", "memory-tree", str(frame_db), "--event-id", "8", "--json"] + ) + assert report.exit_code == 0, report.output + query = runner.invoke( + app, + [ + "query", + str(frame_db), + "--template-use", + "active_memory_frame_tree_at_event", + "--params", + '{"event_id":8}', + "--json", + ], + ) + assert query.exit_code == 0, query.output + with SnapshotAnalyzer(db_path=frame_db) as analyzer: + api = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 8}) + assert api["rows"] == json.loads(report.output)["rows"] == json.loads(query.output)["rows"] + assert api["total_is_exact"] and not api["truncated"] + assert "size_bytes = self_bytes" in " ".join( + analyzer.get_template_info("active_memory_frame_tree_at_event")["interpretation_limits"] + ) + + +def test_report_text_and_html_are_usable(frame_db: Path): + runner = CliRunner() + args = ["report", "memory-tree", str(frame_db), "--event-id", "8"] + text = runner.invoke(app, args) + assert text.exit_code == 0, text.output + assert "[all active blocks]: 535 / 0 / 9" in text.output + assert "model.py:20 forward: 375 / 50 / 4" in text.output + html = runner.invoke(app, [*args, "--format", "html"]) + assert html.exit_code == 0, html.output + assert html.output.startswith("") + assert "Reset zoom" in html.output and '"size_bytes": 535' in html.output + assert "__TREE_DATA__" not in html.output + assert runner.invoke(app, ["report", "memory-tree", str(frame_db)]).exit_code != 0 + conflict = runner.invoke(app, [*args, "--format", "html", "--json"]) + assert conflict.exit_code == 1 + assert json.loads(conflict.output)["error"]["code"] == "INVALID_PARAMETER" + + +def test_renderer_rejects_partial_tree_and_escapes_frame_text(frame_db: Path): + service = ReportService() + try: + result = service.memory_tree_report(event_id=8, db_path=frame_db) + finally: + service.close() + with pytest.raises(ValueError, match="complete frame tree"): + render_memory_tree(replace(result, truncated=True), 8) + result.rows[1]["label"] = '&训练' + html = render_memory_tree(result, 8, html=True) + assert '")[0] + ) + assert payload["nodes"][1]["label"] == result.rows[1]["label"] + + +def test_report_legacy_schema_error_is_json_and_actionable(frame_db: Path): + import sqlite3 + from contextlib import closing + + with closing(sqlite3.connect(frame_db)) as conn, conn: + conn.execute("DROP TABLE callstack_frame") + result = CliRunner().invoke( + app, ["report", "memory-tree", str(frame_db), "--event-id", "8", "--json"] + ) + assert result.exit_code == 1 + assert "Re-import the original snapshot" in json.loads(result.output)["error"]["message"] + + +@pytest.mark.parametrize( + "damage", + [ + "empty", + "duplicate", + "orphan", + "depth", + "root_parent", + "negative", + "noninteger", + "conservation", + ], +) +@pytest.mark.parametrize("html", [False, True]) +def test_renderer_rejects_invalid_complete_claims(frame_db: Path, damage: str, html: bool): + service = ReportService() + try: + result = service.memory_tree_report(event_id=8, db_path=frame_db) + finally: + service.close() + if damage == "empty": + result.rows.clear() + elif damage == "duplicate": + result.rows.append(result.rows[-1].copy()) + elif damage == "orphan": + result.rows[-1]["parent_id"] = "nonexistent" + elif damage == "depth": + result.rows[-1]["depth"] = 100 + elif damage == "root_parent": + result.rows[0]["parent_id"] = "root" + elif damage == "negative": + result.rows[0]["size_bytes"] = -1 + elif damage == "noninteger": + result.rows[0]["size_bytes"] = 535.0 + else: + result.rows[0]["size_bytes"] = 536 + with pytest.raises(ValueError, match="complete frame tree"): + render_memory_tree(result, 8, html=html) + + +def test_renderer_rejects_has_more_without_truncated(frame_db: Path): + service = ReportService() + try: + result = service.memory_tree_report(event_id=8, db_path=frame_db) + finally: + service.close() + with pytest.raises(ValueError, match="complete frame tree"): + render_memory_tree(replace(result, has_more=True), 8, html=True) + result.rows[1]["label"] = "line\nnext\r\x1b[31m" + text = render_memory_tree(result, 8) + assert "\x1b" not in text + assert r"line\nnext\r\u001b[31m" in text