From 668ec3eb7f17ceb131471c09daa2153e2d1ccfd3 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:11:20 +0800 Subject: [PATCH] feat(callstacks): preserve ordered frames and add memory tree reports Store original frame fields and positions, then aggregate active memory by caller path with inclusive and self occupancy. Add shared queries plus text, JSON, and interactive HTML reports; rebuild old import caches. Fixes #206 --- README.md | 8 + README_zh.md | 7 + docs/en/database.md | 45 +++++- docs/en/querying.md | 56 +++++++ docs/zh/database.md | 40 ++++- docs/zh/querying.md | 49 ++++++ src/pt_snap_cli/cli_reports.py | 45 ++++++ src/pt_snap_cli/context.py | 2 +- src/pt_snap_cli/core/import_metadata.py | 4 +- src/pt_snap_cli/core/report_service.py | 22 ++- src/pt_snap_cli/memory_tree_render.py | 102 +++++++++++++ src/pt_snap_cli/query/executor.py | 6 + .../query/templates/basic/event_frames.yaml | 34 +++++ .../active_memory_frame_tree_at_event.yaml | 131 ++++++++++++++++ src/pt_snap_cli/snapshot/PROVENANCE.md | 14 ++ .../tools/adaptors/database/callstack.py | 53 +++++-- .../tools/adaptors/database/snapshot_db.py | 22 +++ .../snapshot/tools/adaptors/snapshot2db.py | 5 + tests/frame_tree_helpers.py | 63 ++++++++ tests/query/test_callstack_schema_compat.py | 4 +- tests/query/test_frame_tree.py | 144 ++++++++++++++++++ tests/snapshot/test_callstack_interner.py | 28 ++++ tests/snapshot/test_snapshot2db_runtime.py | 36 +++++ tests/test_memory_tree_report.py | 94 ++++++++++++ 24 files changed, 991 insertions(+), 23 deletions(-) create mode 100644 src/pt_snap_cli/memory_tree_render.py create mode 100644 src/pt_snap_cli/query/templates/basic/event_frames.yaml create mode 100644 src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml create mode 100644 tests/frame_tree_helpers.py create mode 100644 tests/query/test_frame_tree.py create mode 100644 tests/test_memory_tree_report.py diff --git a/README.md b/README.md index 2f28e83..dab1e28 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ See the [full quick start guide](docs/en/quickstart.md) for a walkthrough. | `pt-snap overview [database.db]` | Read-only device list, per-device event-id bounds, and import-metadata status | | `pt-snap query` | Run memory analysis queries | | `pt-snap report` | Generate higher-level memory analysis reports | +| `pt-snap report memory-tree --event-id ` | Frame-level live-memory breakdown; `--format html` emits an interactive flamegraph | | `pt-snap config` | Manage global configuration | | `pt-snap skill` | List and install bundled agent skills | @@ -73,6 +74,13 @@ Peak reports expose attribution completeness and same-event active-byte coverage group percentages use included bytes after filtering and ranking. See the [report guide](docs/en/querying.md#report-command) before interpreting a capped breakdown. +Imports preserve ordered structured callstack frames. Use `event_frames` to +inspect them, or `report memory-tree --event-id 100 --format html > memory-tree.html` +to explore event-time occupancy by caller path. See +[frame memory breakdown](docs/en/querying.md#frame-memory-breakdown); older +text-only databases require re-importing the original snapshot. +`report memory-tree` also accepts `--json`. + ## Agent Skills For long diagnostic stacks, opt into `stack_bytes` on `event` or diff --git a/README_zh.md b/README_zh.md index 264544d..68b8ad7 100644 --- a/README_zh.md +++ b/README_zh.md @@ -59,6 +59,7 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices | `pt-snap overview [database.db]` | 只读输出设备列表、各设备 event id 边界和导入 metadata 状态 | | `pt-snap query` | 运行内存分析查询 | | `pt-snap report` | 生成高层内存分析报告 | +| `pt-snap report memory-tree --event-id ` | 按帧分层拆解存活内存;`--format html` 输出交互式火焰图 | | `pt-snap config` | 管理全局配置 | | `pt-snap skill` | 列出并安装随包 agent skill | @@ -68,6 +69,12 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices included bytes 为分母。解读有上限的分组结果前,请参阅 [报告指南](docs/zh/querying.md#report-命令)。 +导入时保留结构化调用栈帧及原始顺序。用 `event_frames` 查看帧,或执行 +`report memory-tree --event-id 100 --format html > memory-tree.html` +按调用路径查看事件时刻的占用。详见[调用栈帧内存拆解](docs/zh/querying.md#调用栈帧内存拆解); +旧文本数据库需要从原始快照重新导入。 +`report memory-tree` 同样支持 `--json`。 + ## Agent Skills 诊断长调用栈时,可在 `event` 或 `active_memory_callstack_at_event` 上显式设置 diff --git a/docs/en/database.md b/docs/en/database.md index 14c32cc..192fa53 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -327,15 +327,53 @@ CREATE TABLE callstack ( | Property | Guarantee | |----------|-----------| -| Uniqueness | One row per distinct callstack text, so grouping by `callstackId` equals grouping by text | +| Uniqueness | One row per distinct ordered structured frame array; display text alone is not a lossless identity | | Scope | Shared by every device in the database; the table has no device suffix | | Coverage | Every event produced by import resolves to a row; `callstackId` is `NULL` only in externally generated databases | -Current `pt-snap import` writes this v2 layout (`import_format_version = 2`). +Current `pt-snap import` retains this v2 text layout and adds structured frame +tables (`import_format_version = 3`). Format 2 databases have the same text +layout but no original frame records. Databases produced before callstack deduplication store inline `callstack` TEXT on each `trace_entry_` table and have no shared `callstack` table (`import_format_version = 1` when metadata is present). +### Structured frames and ordered arrays (format 3) + +`frame` stores each distinct original `(filename, line, name)` tuple once. +`callstack_frame` associates a stack with an ordered array of frame IDs: + +```sql +CREATE TABLE frame ( + id INTEGER PRIMARY KEY, + filename TEXT NOT NULL, + line INTEGER NOT NULL, + name TEXT NOT NULL +); +CREATE TABLE callstack_frame ( + callstackId INTEGER NOT NULL, + position INTEGER NOT NULL, + frameId INTEGER NOT NULL +); +CREATE UNIQUE INDEX idx_callstack_frame_position + ON callstack_frame (callstackId, position); +``` + +Both tables are shared across devices. An event refers to the array through +`trace_entry_.callstackId`; read it in `position ASC` order to reproduce +the original snapshot array. Position 0 is the innermost frame. The frame tree +reverses traversal to start with the outermost caller. Recursive occurrences +retain distinct positions even when they share a frame ID. Empty arrays have +no association rows. Structured fields preserve colons, whitespace, Unicode +and embedded newlines without parsing display strings. + +Use `event_frames` to read an event's ordered frames and +`active_memory_frame_tree_at_event` to obtain parent/child occupancy. +See [Frame memory breakdown](querying.md#frame-memory-breakdown). +Text-only databases remain readable by existing queries, but these new +queries require re-importing the original snapshot. Import detects the changed +format and rebuilds the cache; read-only analysis never migrates a database. + ### Callstack layout compatibility `pt-snap` reads both layouts without rewriting the file: @@ -347,7 +385,8 @@ on each `trace_entry_` table and have no shared `callstack` table `event`, `callstack_analysis`, and `active_memory_callstack_at_event` keep one template name, parameter set, and output schema. The query engine selects v1 or -v2 SQL from the detected layout. Other templates do not depend on this split. +v2 SQL from the detected layout. Structured-frame queries additionally require +the format 3 tables; their frame evidence is not reconstructed from legacy text. Detection is read-only: `focus`, `query`, reports, and the Python API all open the database with SQLite `mode=ro`. Layout is not stored in diff --git a/docs/en/querying.md b/docs/en/querying.md index 074496f..6032920 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -441,6 +441,62 @@ explanations; look up `semantics_version` and interpretation limits with rows. `SnapshotAnalyzer.execute_query()` results also include `template` and `semantics_version` so that lookup stays tied to the rows. +## Frame Memory Breakdown + +After importing with this version, inspect the original ordered frame array or +decompose live memory after an event: + +```bash +pt-snap query --template-use event_frames --params '{"event_id":100}' --json +pt-snap query --template-use active_memory_frame_tree_at_event --params '{"event_id":100}' --json +pt-snap report memory-tree --event-id 100 +pt-snap report memory-tree --event-id 100 --json +pt-snap report memory-tree --event-id 100 --format html > memory-tree.html +``` + +The HTML file is self-contained: open it locally, click a frame to zoom, reset +to show the whole tree, or search filenames/functions. Width measures live +block bytes, not CPU time or allocation traffic. No static server is needed. +`--device` selects the device; `--exclude-static` omits static/preexisting +memory; `--min-size` filters individual blocks. `--event-id` is required. +The report requests a complete tree without a ranked or row cap. + +| Field | Meaning | +| --- | --- | +| `node_id`, `parent_id` | Caller-path identity and direct parent; `root` is the total | +| `depth`, `frame_id` | Caller-first level and structured frame identity | +| `size_bytes`, `requested_bytes`, `block_count` | Inclusive occupancy/count for this node and descendants | +| `self_bytes`, `self_requested_bytes`, `self_block_count` | Allocations whose recorded stack ends at this node | +| `percent_of_total` | 0–100 byte share of root occupancy after block/category filters, before output row caps | + +For each node, inclusive occupancy equals self occupancy plus the inclusive +occupancy of its direct children. Sum `self_bytes` across all nodes, or sum +inclusive bytes at one complete cut through the tree; summing inclusive bytes +across levels counts allocations repeatedly. An internal node can have self +bytes when a recorded stack ends there. Recursion and identical frames under +different callers remain separate paths. + +Only blocks with `allocEventId <= event_id` and a later or absent +`freeEventId` are attributed dynamically. `freeEventId` is `free_completed`, +so pending-free memory is still included. Static, preexisting, and missing or +damaged stack links remain explicit synthetic leaves. This is active-block +occupancy and does not decompose reserved bytes. Event IDs mark ordering, not +elapsed time. `min_size` and `include_static` change the root denominator. +If a direct query uses `-n`/`max_rows`, inspect its completeness flags: a +truncated set of node rows is not a complete tree. Old text-only databases +must be re-imported; the analyzer does not reconstruct frames from text. + +The Python API uses the same template and contracts: + +```python +from pathlib import Path +from pt_snap_cli import SnapshotAnalyzer + +with SnapshotAnalyzer(db_path=Path("snapshot.pkl.db"), device_id=0) as analyzer: + frames = analyzer.execute_query("event_frames", params={"event_id": 100}) + tree = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 100}) +``` + ## Template Architecture Query templates are defined in YAML format with: diff --git a/docs/zh/database.md b/docs/zh/database.md index 94fdec2..7c113fa 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -312,15 +312,47 @@ CREATE TABLE callstack ( | 属性 | 保证 | |------|------| -| 唯一性 | 每条不同的调用栈文本只有一行,因此按 `callstackId` 分组等价于按文本分组 | +| 唯一性 | 每组不同的有序结构化帧数组只有一行;显示文本本身不能无损标识调用栈 | | 作用域 | 数据库内所有设备共享,表名不带设备后缀 | | 覆盖度 | 导入产生的每个事件都能解析到一行;只有外部生成的数据库才可能出现 `callstackId` 为 `NULL` | -当前 `pt-snap import` 写入的是这种 v2 布局(`import_format_version = 2`)。 +当前 `pt-snap import` 保留这种 v2 文本布局,同时增加结构化帧表 +(`import_format_version = 3`)。格式 2 数据库的文本布局相同,但没有原始帧记录。 调用栈去重之前生成的数据库在每张 `trace_entry_` 表中内联存储 `callstack` TEXT,没有共享 `callstack` 表(有 metadata 时为 `import_format_version = 1`)。 +### 结构化帧与有序数组(格式 3) + +`frame` 对原始 `(filename, line, name)` 三元组去重;`callstack_frame` +把调用栈关联到有序的帧 ID 数组: + +```sql +CREATE TABLE frame ( + id INTEGER PRIMARY KEY, + filename TEXT NOT NULL, + line INTEGER NOT NULL, + name TEXT NOT NULL +); +CREATE TABLE callstack_frame ( + callstackId INTEGER NOT NULL, + position INTEGER NOT NULL, + frameId INTEGER NOT NULL +); +CREATE UNIQUE INDEX idx_callstack_frame_position + ON callstack_frame (callstackId, position); +``` + +两张表在设备之间共享。事件通过 `trace_entry_.callstackId` 关联数组, +按 `position ASC` 读取即可还原快照原始顺序;位置 0 是最内层帧。帧树分析反向 +遍历,从最外层调用者开始。递归中的同一帧保留各自位置,即使帧 ID 相同。 +空数组没有关联行。结构化字段直接保留冒号、空白、中文和内嵌换行,无需拆解显示文本。 + +使用 `event_frames` 读取事件有序帧,使用 `active_memory_frame_tree_at_event` +读取各父子节点的占用。详见[调用栈帧内存拆解](querying.md#调用栈帧内存拆解)。 +旧文本数据库仍可用于原有查询,新查询需要从原始快照重新导入。导入器检测格式变化 +后重建缓存;只读分析不会迁移数据库。 + ### 调用栈布局兼容 `pt-snap` 可以读取两种布局,且不会改写数据库文件: @@ -331,8 +363,8 @@ CREATE TABLE callstack ( | v1(旧版) | `trace_entry_.callstack` TEXT,无 `callstackId`,无共享表 | 直接读该列 | `event`、`callstack_analysis` 和 `active_memory_callstack_at_event` 保持相同的 -模板名、参数和输出契约。查询引擎按识别到的布局选择 v1 或 v2 SQL。其他模板不依赖 -这次拆分。 +模板名、参数和输出契约。查询引擎按识别到的布局选择 v1 或 v2 SQL。 +结构化帧查询额外要求格式 3 的两张帧表,不会从旧显示文本猜测原始帧。 识别过程是只读的:`focus`、`query`、报告和 Python API 都以 SQLite `mode=ro` 打开数据库。布局不会写入 `.pt-snap/focus.json`。列冲突、设备布局不一致、 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 6ad5ad8..7f4404f 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -396,6 +396,55 @@ CLI 和 Python API 的查询结果包含原始 SQLite 值。模板的 `output_sc `SnapshotAnalyzer.execute_query()` 的结果还包含 `template` 与 `semantics_version`, 便于把行与契约对应起来。 +## 调用栈帧内存拆解 + +使用当前版本导入后,可以读取原始有序帧数组,或拆解指定事件后的存活内存: + +```bash +pt-snap query --template-use event_frames --params '{"event_id":100}' --json +pt-snap query --template-use active_memory_frame_tree_at_event --params '{"event_id":100}' --json +pt-snap report memory-tree --event-id 100 +pt-snap report memory-tree --event-id 100 --json +pt-snap report memory-tree --event-id 100 --format html > memory-tree.html +``` + +HTML 文件可直接在本地打开:点击帧放大、重置查看整棵树、搜索文件或函数。 +宽度表示存活内存块的字节数,不是 CPU 时间或累计分配量,无需启动服务器。 +`--device` 选择设备,`--exclude-static` 排除静态和采集前已存在的内存, +`--min-size` 按单个块大小筛选;`--event-id` 必填。报告不做排名或行数截断。 + +| 字段 | 含义 | +| --- | --- | +| `node_id`、`parent_id` | 完整调用路径身份及直接父节点;`root` 为总占用 | +| `depth`、`frame_id` | 从调用者开始的层级和结构化帧身份 | +| `size_bytes`、`requested_bytes`、`block_count` | 当前节点及所有后代的占用和块数 | +| `self_bytes`、`self_requested_bytes`、`self_block_count` | 记录的分配栈终止于当前节点的自身占用和块数 | +| `percent_of_total` | 根节点总占用的 0–100 字节百分比;分母在块和类别筛选后、输出行截断前计算 | + +每个节点满足“总占用 = 自身占用 + 各直接子节点总占用”。可以对所有节点的 +`self_bytes` 求和,或对树的一组完整切面求和;不能跨层累加 `size_bytes`, +否则同一分配会重复计算。记录的调用栈可能止于中间节点,所以中间节点也可能有 +自身占用。递归帧以及不同调用者下的相同帧保留为不同路径。 + +动态内存只统计 `allocEventId <= event_id` 且 `freeEventId` 晚于该事件或未记录 +的块。`freeEventId` 对应 `free_completed`,因此待释放内存仍计入。 +静态内存、采集前已存在的内存、缺失或损坏调用栈分别作为显式虚拟叶节点。 +这表示 active 内存块占用,不是 reserved 内存拆解;事件 ID 是顺序标记,不是 +耗时。`min_size` 和 `include_static` 会改变根节点分母。直接查询若设置 +`-n`/`max_rows`,需检查完整性字段,截断结果不能当作完整树。 +旧文本数据库需要从原始快照重新导入,分析器不会从文本猜测原始帧。 + +Python API 使用相同模板和契约: + +```python +from pathlib import Path +from pt_snap_cli import SnapshotAnalyzer + +with SnapshotAnalyzer(db_path=Path("snapshot.pkl.db"), device_id=0) as analyzer: + frames = analyzer.execute_query("event_frames", params={"event_id": 100}) + tree = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 100}) +``` + ## 模板架构 查询模板使用 YAML 格式定义,包含: diff --git a/src/pt_snap_cli/cli_reports.py b/src/pt_snap_cli/cli_reports.py index b9abd25..3b02e7a 100644 --- a/src/pt_snap_cli/cli_reports.py +++ b/src/pt_snap_cli/cli_reports.py @@ -35,10 +35,55 @@ TemplateRenderError, ) from pt_snap_cli.core.error_codes import DATABASE_NOT_FOUND, INVALID_PARAMETER +from pt_snap_cli.core.errors import PtSnapCoreError +from pt_snap_cli.memory_tree_render import render_memory_tree report_app = typer.Typer(help="Generate memory analysis reports") +@report_app.command("memory-tree") +def report_memory_tree( + event_id: Annotated[ + int, typer.Option("--event-id", help="Analyze live memory after this event") + ], + db_path: Annotated[ + Path | None, typer.Argument(help="Path to database (optional if configured)") + ] = None, + device: Annotated[ + int | None, typer.Option("--device", "-d", autocompletion=complete_device_ids) + ] = None, + include_static: Annotated[bool, typer.Option("--include-static/--exclude-static")] = True, + min_size: Annotated[int, typer.Option(help="Minimum individual block size in bytes")] = 0, + output_format: Annotated[ + Literal["text", "html"], + typer.Option("--format", help="Text breakdown or interactive HTML flamegraph on stdout"), + ] = "text", + json_output: Annotated[bool, _json_flag()] = False, +) -> None: + """Decompose event-time active memory by ordered allocation frames.""" + service = ReportService() + try: + if json_output and output_format != "text": + raise ValueError("--json cannot be combined with --format html.") + result = service.memory_tree_report( + event_id=event_id, + db_path=db_path, + device_id=device, + include_static=include_static, + min_size=min_size, + ) + if json_output: + typer.echo(json.dumps({"event_id": event_id, **asdict(result)}, indent=2)) + else: + typer.echo(render_memory_tree(result, event_id, html=output_format == "html")) + except PtSnapCoreError as exc: + _error_from_exc(exc) + except ValueError as exc: + _error(str(exc), code=INVALID_PARAMETER) + finally: + service.close() + + @report_app.command("peak-memory") def report_peak_memory( db_path: Annotated[ diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index c64b772..434b002 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -169,7 +169,7 @@ def _detect_callstack_layout( ) metadata_version = _import_format_version(cursor) - if metadata_version in (1, 2): + if metadata_version in (1, 2, 3): expected: CallstackLayout = "v1" if metadata_version == 1 else "v2" if layout is None: return None, ( diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index b2f9922..973945f 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -22,7 +22,9 @@ METADATA_SCHEMA_VERSION = 1 # 2: trace_entry_ stores `callstackId` referencing the shared `callstack` # table instead of an inlined `callstack` text column. -IMPORT_FORMAT_VERSION = 2 +# 3: retains the v2 text layout and adds shared structured frame records plus +# original-order callstack_frame references. Old caches must be re-imported. +IMPORT_FORMAT_VERSION = 3 IMPORTER_NAME = "pt-snap-cli" HASH_CHUNK_SIZE = 1024 * 1024 diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index 7761eb7..a43d8eb 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -4,7 +4,7 @@ from typing import Literal, cast from pt_snap_cli.core.focus_service import FocusService -from pt_snap_cli.core.models import PeakMemoryReport +from pt_snap_cli.core.models import PeakMemoryReport, QueryResult from pt_snap_cli.core.query_service import QueryService PeakMetric = Literal["active", "allocated", "reserved"] @@ -31,6 +31,26 @@ def close(self) -> None: """Release connections owned by this report service; allow later reuse.""" self._query_service.close() + def memory_tree_report( + self, + event_id: int, + db_path: Path | str | None = None, + device_id: int | None = None, + include_static: bool = True, + min_size: int = 0, + ) -> QueryResult: + """Return a complete frame tree using the shared query contract.""" + return self._query_service.execute_query( + "active_memory_frame_tree_at_event", + params={ + "event_id": event_id, + "include_static": include_static, + "min_size": min_size, + }, + db_path=db_path, + device_id=device_id, + ) + def peak_memory_report( self, db_path: Path | str | None = None, diff --git a/src/pt_snap_cli/memory_tree_render.py b/src/pt_snap_cli/memory_tree_render.py new file mode 100644 index 0000000..2af6ba5 --- /dev/null +++ b/src/pt_snap_cli/memory_tree_render.py @@ -0,0 +1,102 @@ +"""Presentation of complete frame trees; occupancy comes from the query layer.""" + +from __future__ import annotations + +import json +from typing import cast + +from pt_snap_cli.core.models import QueryResult + + +def render_memory_tree(result: QueryResult, event_id: int, *, html: bool = False) -> str: + """Render an indented breakdown or a self-contained interactive flamegraph.""" + if result.truncated or result.has_more: + raise ValueError("A complete frame tree is required to render a memory breakdown.") + title = f"Active memory after event {event_id} · device {result.device_id}" + if html: + payload = json.dumps({"title": title, "nodes": result.rows}, ensure_ascii=False) + # Frame text is untrusted. Escape HTML script delimiters in the data; + # the renderer inserts labels using textContent, never innerHTML. + payload = payload.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") + return _HTML.replace("__TREE_DATA__", payload) + + children: dict[str, list[dict[str, object]]] = {} + for row in result.rows: + children.setdefault(str(row["parent_id"]), []).append(row) + lines = [title, "Inclusive bytes / self bytes / live blocks"] + pending = list(reversed(children.get("None", []))) + while pending: + row = pending.pop() + depth = cast(int, row["depth"]) + # Keep each captured frame on one terminal line, even with unusual names. + label = str(row["label"]).replace("\n", "\\n").replace("\r", "\\r") + lines.append( + f"{' ' * depth}{label}: {row['size_bytes']} / {row['self_bytes']} / {row['block_count']}" + ) + pending.extend(reversed(children.get(str(row["node_id"]), []))) + return "\n".join(lines) + + +_HTML = """ + + +pt-snap memory flamegraph +

+

Width = live block bytes, not execution time. Each level groups allocation frames by caller path. +Inclusive = self + direct children. Pending-free blocks stay live until free_completed. +Static, preexisting and missing stacks are separate leaves. Click a frame to zoom; hover for details.

+ + + +

Hover over a frame to inspect its memory.

+ + +""" diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 7537bc0..619c806 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -246,6 +246,12 @@ def _timeout_error(self, timeout_s: float) -> QueryTimeoutError: def _execution_error(self, error: sqlite3.OperationalError) -> QueryExecutionError: detail = str(error) + if re.search(r"no such table: (?:frame|callstack_frame)\b", detail): + return QueryExecutionError( + "Query execution failed: structured callstack frames are unavailable. " + "Re-import the original snapshot with this version of pt-snap; " + "legacy callstack text cannot reconstruct original frame fields reliably." + ) if "no such table: callstack" in detail or re.search( r"no such column: (?:[\w]+\.)?callstackId", detail ): diff --git a/src/pt_snap_cli/query/templates/basic/event_frames.yaml b/src/pt_snap_cli/query/templates/basic/event_frames.yaml new file mode 100644 index 0000000..0fdc099 --- /dev/null +++ b/src/pt_snap_cli/query/templates/basic/event_frames.yaml @@ -0,0 +1,34 @@ +version: "1.0" +queries: + event_frames: + description: Read an event's structured callstack frames in the original snapshot order + semantics_version: 1 + interpretation_limits: + - "Requires an import with structured frame tables (import_format_version >= 3). Legacy text is not split or guessed." + - "position is the zero-based original snapshot index. root_first reverses traversal without changing position. Empty captured stacks return no frame rows." + parameters: + event_id: + type: int + required: true + description: Event whose ordered frame array is requested + frame_order: + type: str + default: original + choices: [original, root_first] + description: Original snapshot order or outermost caller first + query: | + SELECT t.id AS event_id, t.callstackId AS stack_id, + cf.position, f.id AS frame_id, f.filename, f.line, f.name + FROM {{ device_trace_table }} t + JOIN callstack_frame cf ON cf.callstackId = t.callstackId + JOIN frame f ON f.id = cf.frameId + WHERE t.id = {{ event_id|int }} + ORDER BY cf.position {% if frame_order == 'root_first' %}DESC{% else %}ASC{% endif %} + output_schema: + - {column: event_id, type: int, units: event_id} + - {column: stack_id, type: int, metric_semantics: identifier} + - {column: position, type: int, metric_semantics: ordering_marker} + - {column: frame_id, type: int, metric_semantics: identifier} + - {column: filename, type: str} + - {column: line, type: int} + - {column: name, type: str} diff --git a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml new file mode 100644 index 0000000..7aaa8f4 --- /dev/null +++ b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml @@ -0,0 +1,131 @@ +version: "1.0" +queries: + active_memory_frame_tree_at_event: + description: Decompose live memory after an event into a caller-first frame tree + semantics_version: 1 + interpretation_limits: + - "Requires structured frame tables from import_format_version >= 3; old text-only databases must be re-imported." + - "Uses allocation stacks of blocks live after event_id: allocEventId <= event_id and freeEventId > event_id (or absent). freeEventId is free_completed, so pending-free memory remains included." + - "Traversal reverses the original innermost-first frame array. Node identity is the whole ordered caller path, preserving recursion and separating identical frames under different callers." + - "size_bytes includes descendant occupancy; self_bytes counts only allocations ending at this node. size_bytes = self_bytes + sum(direct child size_bytes). Do not sum inclusive bytes across depths." + - "Root totals and percentages cover every selected live block before any caller-side row cap. There is no top_n pruning. A truncated query is not a complete tree." + - "Static, preexisting and missing/corrupt stacks remain separate synthetic leaves. This is active-block occupancy, not reserved memory or allocation traffic. Event IDs are ordering markers, not elapsed time." + parameters: + event_id: + type: int + required: true + description: Point in time after this trace event + include_static: + type: bool + default: true + description: Include static and preexisting live blocks as synthetic leaves + min_size: + type: int + default: 0 + description: Minimum individual live block size in bytes + query: | + WITH RECURSIVE active_blocks AS ( + SELECT b.size, COALESCE(b.requestedSize, 0) AS requested_size, + CASE WHEN b.allocEventId = -1 AND b.freeEventId = -1 THEN 'static' + WHEN b.allocEventId = -1 THEN 'preexisting_live_at_event' + ELSE 'dynamic_live_at_event' END AS category, + t.callstackId AS stack_id + FROM {{ device_block_table }} b + LEFT JOIN {{ device_trace_table }} t ON t.id = b.allocEventId + WHERE b.size >= {{ min_size|int }} + AND (b.freeEventId IS NULL OR b.freeEventId < 0 OR b.freeEventId > {{ event_id|int }}) + AND ( + {% if include_static %}b.allocEventId = -1 OR{% endif %} + (b.allocEventId != -1 AND b.allocEventId <= {{ event_id|int }}) + ) + ), stack_shape AS ( + SELECT cf.callstackId, MAX(cf.position) AS last_position + FROM callstack_frame cf + JOIN callstack cs ON cs.id = cf.callstackId + LEFT JOIN frame f ON f.id = cf.frameId + GROUP BY cf.callstackId + HAVING MIN(cf.position) = 0 AND MAX(cf.position) = COUNT(*) - 1 + AND COUNT(DISTINCT cf.position) = COUNT(*) AND COUNT(f.id) = COUNT(*) + ), groups AS ( + SELECT b.category, + CASE WHEN s.callstackId IS NOT NULL THEN b.stack_id END AS stack_id, + s.last_position, + COUNT(*) AS block_count, SUM(b.size) AS size_bytes, + SUM(b.requested_size) AS requested_bytes + FROM active_blocks b + LEFT JOIN stack_shape s ON s.callstackId = b.stack_id + AND b.category = 'dynamic_live_at_event' + GROUP BY b.category, + CASE WHEN s.callstackId IS NOT NULL THEN b.stack_id END + ), paths AS ( + SELECT g.stack_id, cf.position, cf.frameId AS frame_id, + 'frame/' || cf.frameId AS node_id, 'root' AS parent_id, 1 AS depth, + g.block_count, g.size_bytes, g.requested_bytes + FROM groups g + JOIN callstack_frame cf ON cf.callstackId = g.stack_id + AND cf.position = g.last_position + UNION ALL + SELECT p.stack_id, cf.position, cf.frameId, + p.node_id || '/' || cf.frameId, p.node_id, p.depth + 1, + p.block_count, p.size_bytes, p.requested_bytes + FROM paths p + JOIN callstack_frame cf ON cf.callstackId = p.stack_id + AND cf.position = p.position - 1 + ), nodes AS ( + SELECT 'root' AS node_id, NULL AS parent_id, 0 AS depth, NULL AS frame_id, + 'mixed' AS category, '[all active blocks]' AS label, + COALESCE(SUM(block_count), 0) AS block_count, + COALESCE(SUM(size_bytes), 0) AS size_bytes, + COALESCE(SUM(requested_bytes), 0) AS requested_bytes, + 0 AS self_block_count, 0 AS self_bytes, 0 AS self_requested_bytes + FROM groups + UNION ALL + SELECT p.node_id, p.parent_id, p.depth, p.frame_id, + 'dynamic_live_at_event', f.filename || ':' || f.line || ' ' || f.name, + SUM(p.block_count), SUM(p.size_bytes), SUM(p.requested_bytes), + SUM(CASE WHEN p.position = 0 THEN p.block_count ELSE 0 END), + SUM(CASE WHEN p.position = 0 THEN p.size_bytes ELSE 0 END), + SUM(CASE WHEN p.position = 0 THEN p.requested_bytes ELSE 0 END) + FROM paths p JOIN frame f ON f.id = p.frame_id + GROUP BY p.node_id + UNION ALL + SELECT CASE WHEN category = 'dynamic_live_at_event' THEN 'missing' ELSE category END, + 'root', 1, NULL, category, + CASE WHEN category = 'static' THEN '[static]' + WHEN category = 'preexisting_live_at_event' THEN '[preexisting live]' + ELSE '[missing callstack]' END, + block_count, size_bytes, requested_bytes, + block_count, size_bytes, requested_bytes + FROM groups WHERE stack_id IS NULL + ) + SELECT n.node_id, n.parent_id, n.depth, n.frame_id, n.category, n.label, + f.filename, f.line, f.name, + n.block_count, n.size_bytes, n.requested_bytes, + n.self_block_count, n.self_bytes, n.self_requested_bytes, + ROUND(n.size_bytes * 100.0 / NULLIF(r.size_bytes, 0), 4) AS percent_of_total + FROM nodes n + LEFT JOIN frame f ON f.id = n.frame_id + CROSS JOIN nodes r + WHERE r.node_id = 'root' + ORDER BY n.depth ASC, n.parent_id ASC, n.size_bytes DESC, n.node_id ASC + output_schema: + - {column: node_id, type: str, metric_semantics: identifier} + - {column: parent_id, type: str, metric_semantics: identifier} + - {column: depth, type: int, units: count} + - {column: frame_id, type: int, metric_semantics: identifier} + - {column: category, type: str, metric_semantics: classification, scope: mixed} + - {column: label, type: str} + - {column: filename, type: str} + - {column: line, type: int} + - {column: name, type: str} + - {column: block_count, type: int, units: count} + - {column: size_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: requested_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: self_block_count, type: int, units: count} + - {column: self_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - {column: self_requested_bytes, type: int, units: bytes, metric_semantics: instantaneous_occupancy} + - column: percent_of_total + type: float + units: percent + metric_semantics: share_of_included_rows + denominator: Root size_bytes for all live blocks after include_static and min_size filtering, before caller row caps diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 75f3fd7..e8bc0b2 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -228,3 +228,17 @@ discarded side-effect return values explicit. Database/slice adaptor consumers annotate the existing per-device trace-list and segment-list format assumptions. No serialization, validation, replay, or database layout behavior changes. Audited upstream source mappings and license terms remain unchanged. + +## 2026-10-05 — Structured frames and ordered callstack storage + +Local first-party changes in `tools/adaptors/database/callstack.py`, +`snapshot_db.py`, and `tools/adaptors/snapshot2db.py`: intern original frame +fields (`filename`, `line`, `name`) in a shared `frame` table and preserve +zero-based original positions, including recursive repetitions, in +`callstack_frame`. Events retain `callstackId` and the existing display text. +Stack identity now follows ordered structured frames rather than rendered text, +so ambiguous display strings no longer collapse distinct frame arrays. +Allocator replay, block lifetimes, synthetic events and counters are unchanged. +The importer records format version 3 so old text-only caches are rebuilt. +This provides lossless frame-field evidence for caller-path memory attribution; +audited upstream source mappings and license terms are unchanged. diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py index df9c46e..80240ba 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py @@ -1,26 +1,29 @@ from typing import Any -from ....base import TraceEntry +from ....base import Frame, TraceEntry from .defs import CallstackFieldDefs class CallstackInterner: - """Assign one integer id per distinct callstack text. + """Intern structured frames and their ordered allocation callstacks. PyTorch snapshots repeat a small number of callstacks across a very large number of events, and the pickle memo already shares the frame containers behind them. Interning therefore turns the per-event callstack into an integer reference and keeps a single text copy per distinct callstack. - Lookups use the identities of the individual frames and only render text - on a miss. Distinct containers that render to the same text collapse onto - one id, so the id-to-text mapping stays one-to-one and grouping by id - matches grouping by text. + The identity fast path retains frame containers to prevent object-id reuse. + Value keys preserve filename, line and name independently: display text is + not a lossless identity. Frame positions retain the snapshot's original + order; the existing outermost-first text remains available to old readers. """ def __init__(self) -> None: self._by_frames: dict[tuple[int, ...], tuple[object, int]] = {} - self._by_text: dict[str, int] = {} + self._by_stack: dict[tuple[int, ...], int] = {} + self._by_frame: dict[tuple[str, int, str], int] = {} + self._frames: list[dict[str, Any]] = [] + self._stacks: list[tuple[int, ...]] = [] self._texts: list[str] = [] def intern(self, event: TraceEntry) -> int: @@ -31,12 +34,28 @@ def intern(self, event: TraceEntry) -> int: if by_frames is not None: callstack_id = by_frames[1] else: - text = event.get_callstack() - existing = self._by_text.get(text) + frame_ids = [] + for frame in frames: + key: tuple[str, int, str] = ( + (frame.filename, frame.line, frame.name) + if isinstance(frame, Frame) + else (frame["filename"], frame["line"], frame["name"]) + ) + frame_id = self._by_frame.get(key) + if frame_id is None: + frame_id = len(self._frames) + self._by_frame[key] = frame_id + self._frames.append( + {"id": frame_id, "filename": key[0], "line": key[1], "name": key[2]} + ) + frame_ids.append(frame_id) + stack = tuple(frame_ids) + existing = self._by_stack.get(stack) if existing is None: callstack_id = len(self._texts) - self._texts.append(text) - self._by_text[text] = callstack_id + self._texts.append(event.get_callstack()) + self._stacks.append(stack) + self._by_stack[stack] = callstack_id else: callstack_id = existing self._by_frames[frame_key] = (frames, callstack_id) @@ -52,3 +71,15 @@ def records(self) -> list[dict[str, Any]]: def __len__(self) -> int: return len(self._texts) + + def frame_records(self) -> list[dict[str, Any]]: + """Return deduplicated original frame fields shared by all devices.""" + return self._frames.copy() + + def stack_frame_records(self) -> list[dict[str, Any]]: + """Return ordered frame references, including repeated recursive frames.""" + return [ + {"callstackId": stack_id, "position": position, "frameId": frame_id} + for stack_id, stack in enumerate(self._stacks) + for position, frame_id in enumerate(stack) + ] diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py index 6fd43b2..a2e0a9a 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py @@ -40,6 +40,19 @@ SqliteColumn(name=CallstackFieldDefs.CALLSTACK), ] +_FRAME_TABLE_COLUMNS = [ + SqliteColumn(name="id", data_type=int, primary_key=True), + SqliteColumn(name="filename", not_null=True), + SqliteColumn(name="line", data_type=int, not_null=True), + SqliteColumn(name="name", not_null=True), +] + +_CALLSTACK_FRAME_TABLE_COLUMNS = [ + SqliteColumn(name="callstackId", data_type=int, not_null=True), + SqliteColumn(name="position", data_type=int, not_null=True), + SqliteColumn(name="frameId", data_type=int, not_null=True), +] + _BLOCK_TABLE_COLUMNS = [ SqliteColumn(name=BlockFieldDefs.ID, data_type=int, primary_key=True), SqliteColumn(name=BlockFieldDefs.ADDR, data_type=int), @@ -103,6 +116,15 @@ def create_callstack_table(self): SqliteTable(self.CALLSTACK_TABLE_NAME, _CALLSTACK_TABLE_COLUMNS), delete_if_exists=True, ) + self.create_table(SqliteTable("frame", _FRAME_TABLE_COLUMNS), delete_if_exists=True) + self.create_table( + SqliteTable("callstack_frame", _CALLSTACK_FRAME_TABLE_COLUMNS), delete_if_exists=True + ) + self.conn.execute( + "CREATE UNIQUE INDEX idx_callstack_frame_position " + "ON callstack_frame (callstackId, position)" + ) + self.conn.commit() def get_trace_entry_table(self, device: int = 0): return self.get_table_by_name(self.get_trace_table_name_by_device(device)) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index f9b84c5..c9bbe57 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -155,6 +155,11 @@ def flush(self, device: int = 0): def flush_callstacks(self): self.db_handler.insert_callstacks(self.callstacks.records()) + db = self.db_handler.db + db.get_table_by_name("frame").insert_records(db.conn, self.callstacks.frame_records()) + db.get_table_by_name("callstack_frame").insert_records( + db.conn, self.callstacks.stack_frame_records() + ) def close(self, *, commit: bool = True): self.db_handler.close(commit=commit) diff --git a/tests/frame_tree_helpers.py b/tests/frame_tree_helpers.py new file mode 100644 index 0000000..f791901 --- /dev/null +++ b/tests/frame_tree_helpers.py @@ -0,0 +1,63 @@ +"""Small explicit allocator lifetimes and structured stacks for contract tests.""" + +from pathlib import Path + +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database import CallstackInterner, SnapshotDb + + +def create_frame_tree_db(path: Path) -> Path: + root = {"filename": "train.py", "line": 1, "name": "train"} + shared = {"filename": "model.py", "line": 20, "name": "forward"} + a = {"filename": "tensor.py", "line": 30, "name": "a"} + b = {"filename": "tensor.py", "line": 40, "name": "b"} + other = {"filename": "other.py", "line": 1, "name": "other"} + stacks = [ + [a, shared, root], + [b, shared, root], + [shared, root], + [shared, shared, root], + [a, other], + [], + ] + db = SnapshotDb(str(path)) + interner = CallstackInterner() + try: + db.create_callstack_table() + for device in (0, 1): + db.create_trace_entry_table(device) + db.create_block_table(device) + for event_id, frames in enumerate(stacks, start=1): + event = TraceEntry.from_dict({"action": "alloc", "frames": frames}) + stack_id = interner.intern(event) + db.conn.execute( + f"INSERT INTO trace_entry_{device} (id, action, callstackId) VALUES (?, 4, ?)", + (event_id, stack_id), + ) + db.conn.execute(f"INSERT INTO trace_entry_{device} (id, action) VALUES (8, 5)") + db.conn.executemany( + f"INSERT INTO block_{device} " + "(id, size, requestedSize, allocEventId, freeEventId) VALUES (?, ?, ?, ?, ?)", + [ + (1, 100, 90, 1, 10), # Pending free at 8; completed at 10. + (2, 200, 180, 2, -1), + (3, 50, 45, 3, None), # Allocation ends at an internal node. + (4, 25, 20, 4, -1), # Recursive frame, same caller identity twice. + (5, 75, 60, 5, -1), # Same leaf under a different caller. + (6, 10, 9, 6, -1), # Empty captured stack. + (7, 5, 4, 7, -1), # Missing allocation event. + (8, 40, 32, -1, -1), + (9, 30, 24, -1, 10), + (10, 999, 900, 1, 8), # Already completed at selected boundary. + (11, 888, 800, 9, -1), # Future allocation. + ], + ) + db.get_callstack_table().insert_records(db.conn, interner.records()) + db.get_table_by_name("frame").insert_records(db.conn, interner.frame_records()) + db.get_table_by_name("callstack_frame").insert_records( + db.conn, interner.stack_frame_records() + ) + db.conn.commit() + finally: + db.conn.close() + return path diff --git a/tests/query/test_callstack_schema_compat.py b/tests/query/test_callstack_schema_compat.py index 2cbfa53..c2ae135 100644 --- a/tests/query/test_callstack_schema_compat.py +++ b/tests/query/test_callstack_schema_compat.py @@ -408,7 +408,7 @@ def test_v1_layout_detected_with_or_without_metadata( assert Context(db_path).callstack_layout == "v1" -@pytest.mark.parametrize("metadata_version", [None, 2]) +@pytest.mark.parametrize("metadata_version", [None, 2, 3]) def test_v2_layout_detected_with_or_without_metadata( tmp_path: Path, metadata_version: int | None ) -> None: @@ -441,7 +441,7 @@ def test_metadata_conflict_fails_only_variant_templates(tmp_path: Path) -> None: def test_unknown_metadata_version_does_not_override_structure(tmp_path: Path) -> None: - db_path = create_v2_db(tmp_path / "future.db", metadata_version=3) + db_path = create_v2_db(tmp_path / "future.db", metadata_version=4) assert Context(db_path).callstack_layout == "v2" diff --git a/tests/query/test_frame_tree.py b/tests/query/test_frame_tree.py new file mode 100644 index 0000000..92e8d29 --- /dev/null +++ b/tests/query/test_frame_tree.py @@ -0,0 +1,144 @@ +import hashlib +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.frame_tree_helpers import create_frame_tree_db + + +@pytest.fixture +def frame_db(tmp_path: Path) -> Path: + return create_frame_tree_db(tmp_path / "frames.db") + + +def query(path: Path, name: str, params: dict, **kwargs): + service = QueryService() + try: + return service.execute_query(name, params=params, db_path=path, **kwargs) + finally: + service.close() + + +def tree(path: Path, **params) -> list[dict[str, object]]: + return query(path, "active_memory_frame_tree_at_event", {"event_id": 8, **params}).rows + + +def test_event_frames_preserve_original_order_and_reverse_only_on_request(frame_db: Path): + original = query(frame_db, "event_frames", {"event_id": 1}).rows + assert [(r["position"], r["filename"], r["name"]) for r in original] == [ + (0, "tensor.py", "a"), + (1, "model.py", "forward"), + (2, "train.py", "train"), + ] + reverse = query(frame_db, "event_frames", {"event_id": 1, "frame_order": "root_first"}).rows + assert reverse == original[::-1] + assert query(frame_db, "event_frames", {"event_id": 6}).rows == [] + assert query(frame_db, "event_frames", {"event_id": 999}).rows == [] + + +def test_tree_conserves_bytes_counts_and_requested_memory_at_every_node(frame_db: Path): + rows = tree(frame_db) + assert rows[0]["node_id"] == "root" + assert (rows[0]["size_bytes"], rows[0]["requested_bytes"], rows[0]["block_count"]) == ( + 535, + 464, + 9, + ) + assert sum(r["self_bytes"] for r in rows) == 535 + assert sum(r["self_requested_bytes"] for r in rows) == 464 + assert sum(r["self_block_count"] for r in rows) == 9 + for row in rows: + children = [r for r in rows if r["parent_id"] == row["node_id"]] + for total, own in [ + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ]: + assert row[total] == row[own] + sum(r[total] for r in children) + assert row["percent_of_total"] == round(row["size_bytes"] * 100 / 535, 4) + forward = [r for r in rows if r["name"] == "forward"] + assert [(r["depth"], r["size_bytes"], r["self_bytes"]) for r in forward] == [ + (2, 375, 50), + (3, 25, 25), + ] + leaves = [r for r in rows if r["name"] == "a"] + assert sorted(r["size_bytes"] for r in leaves) == [75, 100] + assert len({r["node_id"] for r in leaves}) == 2 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == 15 + + +def test_completion_boundary_and_filters_preserve_exact_denominator(frame_db: Path): + after = tree(frame_db, event_id=10) + # 100-byte dynamic and 30-byte preexisting allocations complete at 10; + # the 888-byte allocation at 9 has now become live (without a stack). + assert after[0]["size_bytes"] == 535 - 130 + 888 + filtered = tree(frame_db, include_static=False, min_size=50) + assert filtered[0]["size_bytes"] == 425 + assert {r["category"] for r in filtered} == {"mixed", "dynamic_live_at_event"} + empty = tree(frame_db, min_size=9999) + assert len(empty) == 1 + assert empty[0]["node_id"] == "root" and empty[0]["parent_id"] is None + assert empty[0]["size_bytes"] == empty[0]["requested_bytes"] == empty[0]["block_count"] == 0 + assert empty[0]["percent_of_total"] is None + + +def test_row_cap_is_explicit_and_does_not_recompute_total(frame_db: Path): + result = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, max_rows=2) + assert result.truncated and result.has_more + assert result.rows[0]["size_bytes"] == 535 + assert result.rows[1]["percent_of_total"] == round(375 * 100 / 535, 4) + full = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, exact_total=True) + assert full.total == full.returned and full.total_is_exact and not full.truncated + + +def test_multi_device_analysis_is_read_only(frame_db: Path): + before = hashlib.sha256(frame_db.read_bytes()).digest() + first = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, device_id=0) + second = query(frame_db, "active_memory_frame_tree_at_event", {"event_id": 8}, device_id=1) + assert first.rows == second.rows + assert hashlib.sha256(frame_db.read_bytes()).digest() == before + + +@pytest.mark.parametrize("damage", ["gap", "dangling_frame", "dangling_stack"]) +def test_corrupt_stack_links_preserve_occupancy_as_missing(frame_db: Path, damage: str): + with closing(sqlite3.connect(frame_db)) as conn, conn: + stack = conn.execute("SELECT callstackId FROM trace_entry_0 WHERE id=1").fetchone()[0] + if damage == "gap": + conn.execute("DELETE FROM callstack_frame WHERE callstackId=? AND position=1", (stack,)) + elif damage == "dangling_frame": + conn.execute( + "UPDATE callstack_frame SET frameId=999 WHERE callstackId=? AND position=0", + (stack,), + ) + else: + conn.execute("DELETE FROM callstack WHERE id=?", (stack,)) + rows = tree(frame_db) + assert rows[0]["size_bytes"] == 535 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == 115 + + +@pytest.mark.parametrize( + "name,params", + [("event_frames", {"event_id": 1}), ("active_memory_frame_tree_at_event", {"event_id": 8})], +) +def test_legacy_database_reports_actionable_reimport_error(frame_db: Path, name: str, params: dict): + with closing(sqlite3.connect(frame_db)) as conn, conn: + conn.execute("DROP TABLE callstack_frame") + conn.execute("DROP TABLE frame") + with pytest.raises(QueryExecutionError, match="Re-import the original snapshot"): + query(frame_db, name, params) + assert query(frame_db, "event", {"id": 1}).rows[0]["id"] == 1 + + +def test_inline_text_database_retains_queries_but_cannot_supply_original_frames(tmp_path: Path): + from tests.query.test_callstack_schema_compat import create_v1_db + + path = create_v1_db(tmp_path / "inline.db") + for name in ("event_frames", "active_memory_frame_tree_at_event"): + with pytest.raises(QueryExecutionError, match="Re-import the original snapshot"): + query(path, name, {"event_id": 1}) + assert query(path, "event", {"id": 1}).rows[0]["callstack"] == "train.py:10" diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py index be917e0..76bc9b2 100644 --- a/tests/snapshot/test_callstack_interner.py +++ b/tests/snapshot/test_callstack_interner.py @@ -120,3 +120,31 @@ def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): assert raw_id == eager_id == 0 assert len(interner) == 1 + + +def test_original_order_and_recursive_frames_survive_interning(): + interner = CallstackInterner() + interner.intern(_raw_event([INNER, INNER, OUTER])) + interner.intern(_raw_event([dict(INNER), dict(OUTER)])) + assert interner.frame_records() == [{"id": 0, **INNER}, {"id": 1, **OUTER}] + assert interner.stack_frame_records() == [ + {"callstackId": 0, "position": 0, "frameId": 0}, + {"callstackId": 0, "position": 1, "frameId": 0}, + {"callstackId": 0, "position": 2, "frameId": 1}, + {"callstackId": 1, "position": 0, "frameId": 0}, + {"callstackId": 1, "position": 1, "frameId": 1}, + ] + + +def test_display_text_collisions_do_not_merge_distinct_structured_stacks(): + interner = CallstackInterner() + first = _raw_event([{"filename": "x", "line": 1, "name": "f\ny:2 g"}]) + second = _raw_event( + [ + {"filename": "y", "line": 2, "name": "g"}, + {"filename": "x", "line": 1, "name": "f"}, + ] + ) + assert first.get_callstack() == second.get_callstack() + assert interner.intern(first) != interner.intern(second) + assert len(interner) == 2 diff --git a/tests/snapshot/test_snapshot2db_runtime.py b/tests/snapshot/test_snapshot2db_runtime.py index a7bac9a..dafacac 100644 --- a/tests/snapshot/test_snapshot2db_runtime.py +++ b/tests/snapshot/test_snapshot2db_runtime.py @@ -79,6 +79,40 @@ def replay(representation, device, **kwargs): assert calls == {"load": 1, "replay": 1, "raw_frames": True} +def test_dump_persists_original_frame_fields_and_positions(tmp_path): + frames = [ + {"filename": "训练:模型.py", "line": 17, "name": "内部\n分配"}, + {"filename": "outer.py", "line": 20, "name": "outer"}, + ] + representation = { + "segments": [], + "device_traces": [ + [ + { + "id": 17, + "action": "segment_free", + "addr": 0x1000, + "size": 64, + "stream": 0, + "frames": frames, + } + ] + ], + } + snapshot = tmp_path / "structured.pkl" + database = tmp_path / "structured.db" + with snapshot.open("wb") as stream: + pickle.dump(representation, stream) + assert snapshot2db.dump(snapshot, database) + with closing(sqlite3.connect(database)) as conn: + rows = conn.execute( + "SELECT f.filename, f.line, f.name FROM trace_entry_0 t " + "JOIN callstack_frame cf ON cf.callstackId=t.callstackId " + "JOIN frame f ON f.id=cf.frameId WHERE t.id=17 ORDER BY cf.position" + ).fetchall() + assert [dict(zip(("filename", "line", "name"), row, strict=True)) for row in rows] == frames + + def test_fast_frame_import_matches_eager_database_exactly(monkeypatch, tmp_path): representation = { "segments": [], @@ -328,6 +362,8 @@ def test_dump_all_multiple_device_snapshot(dump_database): } assert tables == { "callstack", + "frame", + "callstack_frame", "dictionary", "trace_entry_0", "block_0", diff --git a/tests/test_memory_tree_report.py b/tests/test_memory_tree_report.py new file mode 100644 index 0000000..4db8068 --- /dev/null +++ b/tests/test_memory_tree_report.py @@ -0,0 +1,94 @@ +import json +from dataclasses import replace +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.report_service import ReportService +from pt_snap_cli.memory_tree_render import render_memory_tree +from tests.frame_tree_helpers import create_frame_tree_db + + +@pytest.fixture +def frame_db(tmp_path: Path) -> Path: + return create_frame_tree_db(tmp_path / "frames.db") + + +def test_report_cli_query_and_analyzer_share_exact_tree(frame_db: Path): + runner = CliRunner() + report = runner.invoke( + app, ["report", "memory-tree", str(frame_db), "--event-id", "8", "--json"] + ) + assert report.exit_code == 0, report.output + query = runner.invoke( + app, + [ + "query", + str(frame_db), + "--template-use", + "active_memory_frame_tree_at_event", + "--params", + '{"event_id":8}', + "--json", + ], + ) + assert query.exit_code == 0, query.output + with SnapshotAnalyzer(db_path=frame_db) as analyzer: + api = analyzer.execute_query("active_memory_frame_tree_at_event", params={"event_id": 8}) + assert api["rows"] == json.loads(report.output)["rows"] == json.loads(query.output)["rows"] + assert api["total_is_exact"] and not api["truncated"] + assert "size_bytes = self_bytes" in " ".join( + analyzer.get_template_info("active_memory_frame_tree_at_event")["interpretation_limits"] + ) + + +def test_report_text_and_html_are_usable(frame_db: Path): + runner = CliRunner() + args = ["report", "memory-tree", str(frame_db), "--event-id", "8"] + text = runner.invoke(app, args) + assert text.exit_code == 0, text.output + assert "[all active blocks]: 535 / 0 / 9" in text.output + assert "model.py:20 forward: 375 / 50 / 4" in text.output + html = runner.invoke(app, [*args, "--format", "html"]) + assert html.exit_code == 0, html.output + assert html.output.startswith("") + assert "Reset zoom" in html.output and '"size_bytes": 535' in html.output + assert "__TREE_DATA__" not in html.output + assert runner.invoke(app, ["report", "memory-tree", str(frame_db)]).exit_code != 0 + conflict = runner.invoke(app, [*args, "--format", "html", "--json"]) + assert conflict.exit_code == 1 + assert json.loads(conflict.output)["error"]["code"] == "INVALID_PARAMETER" + + +def test_renderer_rejects_partial_tree_and_escapes_frame_text(frame_db: Path): + service = ReportService() + try: + result = service.memory_tree_report(event_id=8, db_path=frame_db) + finally: + service.close() + with pytest.raises(ValueError, match="complete frame tree"): + render_memory_tree(replace(result, truncated=True), 8) + result.rows[1]["label"] = '&训练' + html = render_memory_tree(result, 8, html=True) + assert '")[0] + ) + assert payload["nodes"][1]["label"] == result.rows[1]["label"] + + +def test_report_legacy_schema_error_is_json_and_actionable(frame_db: Path): + import sqlite3 + from contextlib import closing + + with closing(sqlite3.connect(frame_db)) as conn, conn: + conn.execute("DROP TABLE callstack_frame") + result = CliRunner().invoke( + app, ["report", "memory-tree", str(frame_db), "--event-id", "8", "--json"] + ) + assert result.exit_code == 1 + assert "Re-import the original snapshot" in json.loads(result.output)["error"]["message"]