From a7e9f976102f5ef6c583619e8176ea606aadcb4c Mon Sep 17 00:00:00 2001 From: codeasier Date: Sun, 6 Sep 2026 02:14:40 +0800 Subject: [PATCH 1/3] perf: deduplicate SnapshotDB callstacks and skip Frame rebuild in split (#113) Import stored the full callstack text on every trace event, so a snapshot with a few thousand distinct callstacks inflated into gigabytes of duplicated text and made SQLite writing dominate import time. Interning the text and storing an integer reference removes that cost without touching allocator replay. `trace_entry_` now stores `callstackId` referencing a shared `callstack` table. Replay, synthetic event and block identities, totals, and callstack text are unchanged: every trace and block row matches the pre-change output on all five deserializable fixtures once `callstackId` is resolved. `SplitService` also replayed with eager `Frame` construction even though slices serialize the original frame dictionaries, so it rebuilt objects only to discard them. Measured on the sanitized benchmark fixtures (628k events / 175 MB source): import 29.1s -> 8.6s, database 5.95 GB -> 177 MB, peak RSS 5.8 GB -> 814 MB. Splitting the 131k fixture into 4 slices goes 12.5s -> 2.5s with identical canonical slice hashes. `IMPORT_FORMAT_VERSION` moves to 2 so cached databases in the old layout are rebuilt rather than reused. --- docs/en/database.md | 54 ++++++++- docs/zh/database.md | 53 ++++++++- src/pt_snap_cli/core/import_metadata.py | 4 +- src/pt_snap_cli/core/split_service.py | 6 +- .../query/templates/basic/event.yaml | 15 ++- .../active_memory_callstack_at_event.yaml | 4 +- .../statistical/callstack_analysis.yaml | 29 +++-- src/pt_snap_cli/snapshot/PROVENANCE.md | 21 ++++ src/pt_snap_cli/snapshot/base/entities.py | 10 +- .../tools/adaptors/database/__init__.py | 2 + .../tools/adaptors/database/callstack.py | 62 ++++++++++ .../snapshot/tools/adaptors/database/defs.py | 5 + .../tools/adaptors/database/entity2record.py | 12 +- .../tools/adaptors/database/snapshot_db.py | 21 +++- .../snapshot/tools/adaptors/snapshot2db.py | 34 +++++- src/pt_snap_cli/snapshot/util/sqlite_meta.py | 13 ++- tests/core/test_report_service.py | 31 ++++- tests/core/test_split_service.py | 7 +- tests/query/test_peak_memory_templates.py | 37 ++++-- tests/query/test_query_max_rows_pushdown.py | 18 ++- tests/skills/harness/fixtures.py | 27 ++++- tests/snapshot/golden_observations.py | 7 +- tests/snapshot/test_callstack_interner.py | 109 ++++++++++++++++++ tests/snapshot/test_entity2record.py | 11 +- tests/snapshot/test_snapshot2db_runtime.py | 43 ++++++- tests/test_cli.py | 25 +++- 26 files changed, 587 insertions(+), 73 deletions(-) create mode 100644 src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py create mode 100644 tests/snapshot/test_callstack_interner.py diff --git a/docs/en/database.md b/docs/en/database.md index e19f28d..24c2921 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -75,11 +75,12 @@ the supported query surface. See [Querying](querying.md) for the complete workfl | Table | Description | Records (example) | |-------|-------------|-------------------| | `dictionary` | Enum mapping dictionary | - | +| `callstack` | Deduplicated callstack text shared by all devices | 4,369 | | `trace_entry_0` | Device 0 event trace table | 8,094 | | `block_0` | Device 0 memory block table | - | | `pt_snap_metadata` | First-party import provenance and cache metadata | 1 | -> **Naming convention**: For multi-device scenarios, table names are suffixed with the device ID, e.g., `trace_entry_1` and `block_1` for device 1. +> **Naming convention**: For multi-device scenarios, table names are suffixed with the device ID, e.g., `trace_entry_1` and `block_1` for device 1. The `callstack` table carries no device suffix because callstacks are shared across devices. --- @@ -101,7 +102,7 @@ CREATE TABLE trace_entry_0 ( `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, - `callstack` TEXT + `callstackId` INTEGER ); ``` @@ -117,7 +118,7 @@ CREATE TABLE trace_entry_0 ( | allocated | INTEGER | — | Total allocated bytes | `136426496` | | active | INTEGER | — | Total active bytes | `136426496` | | reserved | INTEGER | — | Memory pool reserved bytes | `155189248` | -| callstack | TEXT | NULL | Callstack information (multi-line text) | See example | +| callstackId | INTEGER | NULL | Reference to `callstack.id`; join to read the text | `12` | #### Action Type Codes @@ -144,13 +145,24 @@ CREATE TABLE trace_entry_0 ( ``` id=1, action=4, address=20697535234048, size=41943040, stream=1276474240 allocated=136426496, active=136426496, reserved=155189248 -callstack: +callstackId=12 -> callstack.callstack: /home/liuyekang/dev/projects/test/memory_leaks_demo.py:60 /home/liuyekang/dev/projects/test/memory_leaks_demo.py:34 main /home/liuyekang/dev/projects/test/memory_leaks_demo.py:24 train /home/liuyekang/dev/projects/test/memory_leaks_demo.py:12 train_one_step ``` +#### Reading Callstack Text + +Join `callstack` to resolve `callstackId`: + +```sql +SELECT t.id, t.size, c.callstack +FROM trace_entry_0 t +LEFT JOIN callstack c ON c.id = t.callstackId +WHERE t.id = 1; +``` + --- ### 2. block_{device} — Memory Block Table @@ -264,7 +276,39 @@ table=trace_entry_0, column=action, key=4, value=alloc table=block_0, column=state, key=1, value=active_allocated ``` -### 4. pt_snap_metadata — Import Metadata Table +--- + +### 4. callstack — Deduplicated Callstack Text + +Snapshots repeat a small number of callstacks across a very large number of +events, so the text is stored once per distinct callstack and referenced by +`trace_entry_.callstackId`. + +#### Schema + +```sql +CREATE TABLE callstack ( + `id` INTEGER PRIMARY KEY, + `callstack` TEXT +); +``` + +#### Columns + +| Column | Type | Description | +|--------|------|-------------| +| id | INTEGER | Callstack ID referenced by `trace_entry_.callstackId` | +| callstack | TEXT | Callstack text, outermost frame first, one `file:line name` per line | + +#### Properties + +| Property | Guarantee | +|----------|-----------| +| Uniqueness | One row per distinct callstack text, so grouping by `callstackId` equals grouping by text | +| Scope | Shared by every device in the database; the table has no device suffix | +| Coverage | Every event produced by import resolves to a row; `callstackId` is `NULL` only in externally generated databases | + +### 5. pt_snap_metadata — Import Metadata Table First-party imports create one metadata row for provenance and cache compatibility checks: diff --git a/docs/zh/database.md b/docs/zh/database.md index 5c084f9..e9d306b 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -70,11 +70,12 @@ pt-snap query --template-use block --params '{"min_size": 1048576}' | 表名 | 说明 | 记录数(示例) | |------|------|---------------| | `dictionary` | 字典映射表 | - | +| `callstack` | 去重后的调用栈文本,所有设备共享 | 4,369 | | `trace_entry_0` | 设备 0 的事件跟踪表 | 8,094 | | `block_0` | 设备 0 的内存块表 | - | | `pt_snap_metadata` | 第一方导入来源与缓存 metadata | 1 | -> **命名规则**: 多设备场景下,表名后缀为设备 ID,如 `trace_entry_1`、`block_1` 表示设备 1 的数据。 +> **命名规则**: 多设备场景下,表名后缀为设备 ID,如 `trace_entry_1`、`block_1` 表示设备 1 的数据。`callstack` 表没有设备后缀,因为调用栈在设备之间共享。 --- @@ -96,7 +97,7 @@ CREATE TABLE trace_entry_0 ( `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, - `callstack` TEXT + `callstackId` INTEGER ); ``` @@ -112,7 +113,7 @@ CREATE TABLE trace_entry_0 ( | allocated | INTEGER | — | 当前已分配总量 | `136426496` | | active | INTEGER | — | 当前活跃总量 | `136426496` | | reserved | INTEGER | — | 内存池保留总量 | `155189248` | -| callstack | TEXT | NULL | 调用栈,多行文本格式 | 见示例 | +| callstackId | INTEGER | NULL | 指向 `callstack.id`,需 join 才能取到文本 | `12` | #### 动作类型编码(action 字段) @@ -139,13 +140,24 @@ CREATE TABLE trace_entry_0 ( ``` id=1, action=4, address=20697535234048, size=41943040, stream=1276474240 allocated=136426496, active=136426496, reserved=155189248 -callstack: +callstackId=12 -> callstack.callstack: /home/liuyekang/dev/projects/test/memory_leaks_demo.py:60 /home/liuyekang/dev/projects/test/memory_leaks_demo.py:34 main /home/liuyekang/dev/projects/test/memory_leaks_demo.py:24 train /home/liuyekang/dev/projects/test/memory_leaks_demo.py:12 train_one_step ``` +#### 读取调用栈文本 + +join `callstack` 表解析 `callstackId`: + +```sql +SELECT t.id, t.size, c.callstack +FROM trace_entry_0 t +LEFT JOIN callstack c ON c.id = t.callstackId +WHERE t.id = 1; +``` + --- ### 2. block_{device} — 内存块表 @@ -257,7 +269,38 @@ table=trace_entry_0, column=action, key=4, value=alloc table=block_0, column=state, key=1, value=active_allocated ``` -### 4. pt_snap_metadata — 导入 Metadata 表 +--- + +### 4. callstack — 去重调用栈表 + +快照中少量调用栈会在极多事件上重复出现,因此文本按不同调用栈只存一份,由 +`trace_entry_.callstackId` 引用。 + +#### Schema + +```sql +CREATE TABLE callstack ( + `id` INTEGER PRIMARY KEY, + `callstack` TEXT +); +``` + +#### 字段说明 + +| 字段名 | 类型 | 说明 | +|--------|------|------| +| id | INTEGER | 调用栈 ID,被 `trace_entry_.callstackId` 引用 | +| callstack | TEXT | 调用栈文本,最外层帧在前,每行一个 `文件:行号 函数名` | + +#### 表属性 + +| 属性 | 保证 | +|------|------| +| 唯一性 | 每条不同的调用栈文本只有一行,因此按 `callstackId` 分组等价于按文本分组 | +| 作用域 | 数据库内所有设备共享,表名不带设备后缀 | +| 覆盖度 | 导入产生的每个事件都能解析到一行;只有外部生成的数据库才可能出现 `callstackId` 为 `NULL` | + +### 5. pt_snap_metadata — 导入 Metadata 表 第一方导入会创建一条 metadata 记录,用于来源追踪和缓存兼容性检查: diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index c1d1584..b2f9922 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -20,7 +20,9 @@ METADATA_TABLE = "pt_snap_metadata" METADATA_SCHEMA_VERSION = 1 -IMPORT_FORMAT_VERSION = 1 +# 2: trace_entry_ stores `callstackId` referencing the shared `callstack` +# table instead of an inlined `callstack` text column. +IMPORT_FORMAT_VERSION = 2 IMPORTER_NAME = "pt-snap-cli" HASH_CHUNK_SIZE = 1024 * 1024 diff --git a/src/pt_snap_cli/core/split_service.py b/src/pt_snap_cli/core/split_service.py index 0689239..520043a 100644 --- a/src/pt_snap_cli/core/split_service.py +++ b/src/pt_snap_cli/core/split_service.py @@ -145,7 +145,9 @@ def _slice_device( max_entries=max_entries if max_entries is not None else event_count, dump_type="pkl", ) - _, replayed = replay_snapshot(representation, device, hooker=hooker) + # Slices serialize the original frame dictionaries, so eager Frame + # objects would be rebuilt only to be discarded. + _, replayed = replay_snapshot(representation, device, hooker=hooker, _raw_frames=True) except Exception as exc: self._fail("load/engine", source, f"device {device}: {exc}", exc) if not replayed: @@ -174,7 +176,7 @@ def _slice_device( or not traces[device] ): raise ValueError(f"generated slice has no trace entries for device {device}") - _, replayed = replay_snapshot(generated, device) + _, replayed = replay_snapshot(generated, device, _raw_frames=True) if not replayed: raise ValueError("replay returned false") except Exception as exc: diff --git a/src/pt_snap_cli/query/templates/basic/event.yaml b/src/pt_snap_cli/query/templates/basic/event.yaml index 6a85602..559e262 100644 --- a/src/pt_snap_cli/query/templates/basic/event.yaml +++ b/src/pt_snap_cli/query/templates/basic/event.yaml @@ -76,7 +76,20 @@ queries: required: false description: Number of rows to skip query: | - SELECT * FROM {{ device_trace_table }} + SELECT + id, + action, + address, + size, + stream, + allocated, + active, + reserved, + ( + SELECT callstack FROM callstack + WHERE callstack.id = {{ device_trace_table }}.callstackId + ) AS callstack + FROM {{ device_trace_table }} WHERE 1=1 {% if id is not none %}AND id = {{ id }}{% endif %} {% if min_id is not none %}AND id >= {{ min_id }}{% endif %} diff --git a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml index 2ec1de1..fa13749 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml @@ -48,7 +48,7 @@ queries: CASE WHEN b.category = 'static' THEN '[static] allocEventId=-1, freeEventId=-1' WHEN b.category = 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' - ELSE COALESCE(t.callstack, '[missing callstack]') + ELSE COALESCE(cs.callstack, '[missing callstack]') END AS callstack, b.category, COUNT(*) AS block_count, @@ -59,6 +59,8 @@ queries: FROM active_blocks b LEFT JOIN {{ device_trace_table }} t ON b.allocEventId = t.id + LEFT JOIN callstack cs + ON t.callstackId = cs.id GROUP BY callstack, b.category ), ranked_dynamic AS ( diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index cba6f69..4f1765c 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -24,16 +24,25 @@ queries: description: Maximum rows to return (-1 for unlimited). Mirrored from the caller-side ``max_rows`` for SQL pushdown. query: | SELECT - callstack, - COUNT(*) as alloc_count, - SUM(size) as total_size, - AVG(size) as avg_size, - MAX(size) as max_size - FROM {{ device_trace_table }} - WHERE callstack IS NOT NULL - GROUP BY callstack - HAVING alloc_count >= {{ min_count }} - AND total_size >= {{ min_size }} + c.callstack AS callstack, + grouped.alloc_count AS alloc_count, + grouped.total_size AS total_size, + grouped.avg_size AS avg_size, + grouped.max_size AS max_size + FROM ( + SELECT + callstackId, + COUNT(*) as alloc_count, + SUM(size) as total_size, + AVG(size) as avg_size, + MAX(size) as max_size + FROM {{ device_trace_table }} + WHERE callstackId IS NOT NULL + GROUP BY callstackId + HAVING alloc_count >= {{ min_count }} + AND total_size >= {{ min_size }} + ) AS grouped + JOIN callstack c ON c.id = grouped.callstackId ORDER BY total_size DESC, callstack ASC LIMIT {{ limit|int }} output_schema: diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 1b18c57..82c5f2b 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -158,6 +158,27 @@ The upstream standalone split frontend is deliberately excluded, not deferred: split maps it to `unsafe pickle:` so allowlist rejection is distinct from corrupt input. `dump()` re-raises `UnsafePickleError` instead of returning false. +- On 2026-09-06, issue #113 deduplicated database callstacks. A new local + `snapshot/tools/adaptors/database/callstack.py` interns one id per distinct + callstack text, `trace_entry_` stores `callstackId` instead of an + inlined `callstack` text column, and a shared device-suffix-free `callstack` + table holds the text. `TraceEntry.callstack_frames()` exposes the frame + container that already backed `get_callstack()` so interning can key on + container identity; the interner holds a reference to every keyed container so + an `id()` key cannot be reused by an unrelated object. `CallstackInterner` is + not present in audited MemSnapDump @87ea207; it is a local addition. + `sqlite_meta.SqliteTable.get_insert_values_by_records` now resolves per-column + value maps once per batch instead of once per cell, and the default insert + batch size moved from 1000 to 10000 because import stages rows in a temporary + database and only publishes after a successful replay. Allocator replay, + synthetic event and block identities, totals, and callstack text are + unchanged: on `snapshot_with_multi_devices.pkl`, `snapshot_expandable.pkl`, + `snapshot_with_empty_cache_expandable.pkl`, + `snapshot_import_131k_sanitized.pickle`, and + `snapshot_import_628k_sanitized.pickle`, every trace and block row matches the + pre-change output after resolving `callstackId` through the join. This is a + local performance and schema change; audited upstream source mappings and + license terms are unchanged. ## Migration toolchain diff --git a/src/pt_snap_cli/snapshot/base/entities.py b/src/pt_snap_cli/snapshot/base/entities.py index fe4f8ac..edc636f 100644 --- a/src/pt_snap_cli/snapshot/base/entities.py +++ b/src/pt_snap_cli/snapshot/base/entities.py @@ -98,8 +98,16 @@ def from_dict(cls, trace_dict: dict, _raw_frames: bool = False): ) return trace_entry + def callstack_frames(self) -> list[Frame] | list[dict]: + """Return the frame container that backs ``get_callstack()``. + + Callers that deduplicate callstacks key on this container's identity, + so the same object must be returned for repeated calls. + """ + return self._raw_frames if self._raw_frames is not None else self.frames + def get_callstack(self): - return _format_callstack(self._raw_frames if self._raw_frames is not None else self.frames) + return _format_callstack(self.callstack_frames()) def to_dict(self, include_id: bool = False): if self._origin and not include_id: diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py index d635e29..c919b4d 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py @@ -1,9 +1,11 @@ +from .callstack import CallstackInterner from .entity2record import block2record, event2record from .snapshot_db import BLOCK_STATE_VALUE_MAP, TRACE_ENTRY_ACTION_VALUE_MAP, SnapshotDb __all__ = [ "BLOCK_STATE_VALUE_MAP", "TRACE_ENTRY_ACTION_VALUE_MAP", + "CallstackInterner", "SnapshotDb", "block2record", "event2record", diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py new file mode 100644 index 0000000..6740196 --- /dev/null +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py @@ -0,0 +1,62 @@ +from ....base import TraceEntry +from .defs import CallstackFieldDefs + + +class CallstackInterner: + """Assign one integer id per distinct callstack text. + + PyTorch snapshots repeat a small number of callstacks across a very large + number of events, and the pickle memo already shares the frame containers + behind them. Interning therefore turns the per-event callstack into an + integer reference and keeps a single text copy per distinct callstack. + + Lookups try the frame container identity first, then the identities of the + individual frames, and only render text on a miss. Distinct containers that + render to the same text collapse onto one id, so the id-to-text mapping + stays one-to-one and grouping by id matches grouping by text. + """ + + def __init__(self) -> None: + # The id()-keyed caches store their keyed container alongside the id. + # Holding that reference keeps the container alive, which is what makes + # the id() key meaningful: a freed container could otherwise be + # replaced by an unrelated object at the same address. + self._by_container: dict[int, tuple[object, int]] = {} + self._by_frames: dict[tuple[int, ...], tuple[object, int]] = {} + self._by_text: dict[str, int] = {} + self._texts: list[str] = [] + + def intern(self, event: TraceEntry) -> int: + """Return the callstack id for ``event``, rendering text only on a miss.""" + frames = event.callstack_frames() + by_container = self._by_container.get(id(frames)) + if by_container is not None: + return by_container[1] + + frame_key = tuple(map(id, frames)) + by_frames = self._by_frames.get(frame_key) + if by_frames is not None: + callstack_id = by_frames[1] + else: + text = event.get_callstack() + existing = self._by_text.get(text) + if existing is None: + callstack_id = len(self._texts) + self._texts.append(text) + self._by_text[text] = callstack_id + else: + callstack_id = existing + self._by_frames[frame_key] = (frames, callstack_id) + + self._by_container[id(frames)] = (frames, callstack_id) + return callstack_id + + def records(self) -> list[dict]: + """Return one record per distinct callstack, ordered by id.""" + return [ + {CallstackFieldDefs.ID: callstack_id, CallstackFieldDefs.CALLSTACK: text} + for callstack_id, text in enumerate(self._texts) + ] + + def __len__(self) -> int: + return len(self._texts) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py index e6ad711..f18ad8d 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py @@ -7,6 +7,11 @@ class EventFieldDefs: ALLOCATED = "allocated" ACTIVE = "active" RESERVED = "reserved" + CALLSTACK_ID = "callstackId" + + +class CallstackFieldDefs: + ID = "id" CALLSTACK = "callstack" diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py index 8f9affc..9014054 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py @@ -1,4 +1,5 @@ from ....base import Block, TraceEntry +from .callstack import CallstackInterner from .defs import BlockFieldDefs, EventFieldDefs @@ -22,7 +23,14 @@ def get_timestamp_by_event_idx(event_idx: int) -> int: next_default_event_id = make_default_id_counter() -def event2record(event: TraceEntry, allocated: int = 0, active: int = 0, reserved: int = 0) -> dict: +def event2record( + event: TraceEntry, + allocated: int = 0, + active: int = 0, + reserved: int = 0, + *, + callstacks: CallstackInterner, +) -> dict: return { EventFieldDefs.ID: (event.idx if event.idx is not None else next_default_event_id()), EventFieldDefs.ACTION: event.action, @@ -32,7 +40,7 @@ def event2record(event: TraceEntry, allocated: int = 0, active: int = 0, reserve EventFieldDefs.ALLOCATED: allocated, EventFieldDefs.ACTIVE: active, EventFieldDefs.RESERVED: reserved, - EventFieldDefs.CALLSTACK: event.get_callstack(), + EventFieldDefs.CALLSTACK_ID: callstacks.intern(event), } diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py index a18dbd5..6fd43b2 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py @@ -1,5 +1,5 @@ from ....util.sqlite_meta import SqliteColumn, SqliteDB, SqliteTable -from .defs import BlockFieldDefs, EventFieldDefs +from .defs import BlockFieldDefs, CallstackFieldDefs, EventFieldDefs TRACE_ENTRY_ACTION_VALUE_MAP = { "segment_map": 0, @@ -32,7 +32,12 @@ SqliteColumn(name=EventFieldDefs.ALLOCATED, data_type=int), SqliteColumn(name=EventFieldDefs.ACTIVE, data_type=int), SqliteColumn(name=EventFieldDefs.RESERVED, data_type=int), - SqliteColumn(name=EventFieldDefs.CALLSTACK), + SqliteColumn(name=EventFieldDefs.CALLSTACK_ID, data_type=int), +] + +_CALLSTACK_TABLE_COLUMNS = [ + SqliteColumn(name=CallstackFieldDefs.ID, data_type=int, primary_key=True), + SqliteColumn(name=CallstackFieldDefs.CALLSTACK), ] _BLOCK_TABLE_COLUMNS = [ @@ -54,6 +59,9 @@ class SnapshotDb(SqliteDB): TRACE_ENTRY_TABLE_NAME = "trace_entry" BLOCK_TABLE_NAME = "block" + # Callstacks are shared across devices, so this table carries no device suffix. + # The name must stay outside the `trace_entry_%` space that device discovery scans. + CALLSTACK_TABLE_NAME = "callstack" def __init__(self, path: str): super().__init__(path, auto_create=True, with_dictionary_table=True) @@ -90,12 +98,21 @@ def create_block_table(self, device: int = 0): ): table.create_index(self.conn, column) + def create_callstack_table(self): + self.create_table( + SqliteTable(self.CALLSTACK_TABLE_NAME, _CALLSTACK_TABLE_COLUMNS), + delete_if_exists=True, + ) + def get_trace_entry_table(self, device: int = 0): return self.get_table_by_name(self.get_trace_table_name_by_device(device)) def get_block_table(self, device: int = 0): return self.get_table_by_name(self.get_block_table_name_by_device(device)) + def get_callstack_table(self): + return self.get_table_by_name(self.CALLSTACK_TABLE_NAME) + def _clear_old_tables(self): """ 旧版本中创建的db不带device前缀,如果通过旧db打开,需要清理旧表 diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index 3127cfd..380d15c 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -6,19 +6,27 @@ from ...representation import UnsafePickleError, load_snapshot_representation, replay_snapshot from ...simulate import AllocatorHooker, SimulateHooker from ...util.logger import get_logger, set_global_log_file -from .database import SnapshotDb, block2record, event2record +from .database import CallstackInterner, SnapshotDb, block2record, event2record dump_logger = get_logger("DatabaseDump") +# Import stages rows in a temporary database and only publishes it after a +# successful replay, so batches are sized for throughput rather than durability. +DEFAULT_INSERT_CACHE_SIZE = 10000 + + class SnapshotDbHandler: - def __init__(self, db_path: str, devices: list[int], insert_cache_size: int = 1000): + def __init__( + self, db_path: str, devices: list[int], insert_cache_size: int = DEFAULT_INSERT_CACHE_SIZE + ): self._closed = False self.db_path = db_path self.db = SnapshotDb(db_path) self._device_event_cache = {} self._device_block_cache = {} self._insert_cache_size = insert_cache_size + self.db.create_callstack_table() for device in devices: self._device_block_cache[device] = [] self._device_event_cache[device] = [] @@ -45,6 +53,13 @@ def flush(self, device: int = 0): if self._device_block_cache.get(device, None): self._do_insert_blocks(device) + def insert_callstacks(self, records: list[dict]): + """Write the interned callstack table once every device has replayed.""" + if not records: + return + self.db.get_callstack_table().insert_records(self.db.conn, records) + self.db.conn.commit() + def _do_insert_events(self, device: int = 0): if device not in self._device_event_cache: self._device_event_cache[device] = [] @@ -79,8 +94,12 @@ def __del__(self): class DumpEventHooker(SimulateHooker, AllocatorHooker): - def __init__(self, db_path: str, devices: list[int], dump_cache_size: int = 1000): + def __init__( + self, db_path: str, devices: list[int], dump_cache_size: int = DEFAULT_INSERT_CACHE_SIZE + ): self.db_handler = SnapshotDbHandler(db_path, devices, insert_cache_size=dump_cache_size) + # One interner for the whole dump so devices share identical callstacks. + self.callstacks = CallstackInterner() def post_undo_event( self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot @@ -107,6 +126,7 @@ def post_undo_event( allocated=current_snapshot.total_allocated, active=current_snapshot.total_activated, reserved=current_snapshot.total_reserved, + callstacks=self.callstacks, ), current_snapshot.device, ) @@ -120,6 +140,7 @@ def pre_undo_event(self, wait4undo_event: TraceEntry, current_snapshot: DeviceSn allocated=current_snapshot.total_allocated, active=current_snapshot.total_activated, reserved=current_snapshot.total_reserved, + callstacks=self.callstacks, ), current_snapshot.device, ) @@ -131,6 +152,9 @@ def post_replay_free_block(self, released_block: Block, current_snapshot: Device def flush(self, device: int = 0): self.db_handler.flush(device) + def flush_callstacks(self): + self.db_handler.insert_callstacks(self.callstacks.records()) + def close(self, *, commit: bool = True): self.db_handler.close(commit=commit) @@ -179,8 +203,10 @@ def dump(pickle_file: str, dump_file: str, device=None) -> bool: return False dump_logger.info(f"Finished dump the snapshot to database for device {device}.") hooker.flush(device) + hooker.flush_callstacks() dump_logger.info( - f"Successfully dump the snapshot to database for devices {need_dump_devices}." + f"Successfully dump the snapshot to database for devices {need_dump_devices}, " + f"{len(hooker.callstacks)} distinct callstacks." ) hooker.close() os.replace(temporary_path, dump_path) diff --git a/src/pt_snap_cli/snapshot/util/sqlite_meta.py b/src/pt_snap_cli/snapshot/util/sqlite_meta.py index cef2a82..e291d6d 100644 --- a/src/pt_snap_cli/snapshot/util/sqlite_meta.py +++ b/src/pt_snap_cli/snapshot/util/sqlite_meta.py @@ -336,9 +336,18 @@ def get_insert_values_by_records(self, records: list[dict[str, Any]]): """ if not records: return [] + # Resolve the per-column value maps once instead of per cell; batches of + # snapshot rows are large enough that the per-cell lookup dominated. + keys = list(records[0].keys()) + mapped = [(key, self._column_value_map.get(key)) for key in keys] + if not any(value_map for _, value_map in mapped): + return [tuple(record[key] for key in keys) for record in records] return [ - tuple(self._column_value_map.get(k, {}).get(r[k], r[k]) for k in records[0].keys()) - for r in records + tuple( + record[key] if value_map is None else value_map.get(record[key], record[key]) + for key, value_map in mapped + ) + for record in records ] diff --git a/tests/core/test_report_service.py b/tests/core/test_report_service.py index 558591e..8f01856 100644 --- a/tests/core/test_report_service.py +++ b/tests/core/test_report_service.py @@ -37,6 +37,12 @@ def report_db(tmp_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -51,17 +57,26 @@ def report_db(tmp_path: Path) -> Path: freeEventId INTEGER ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "block.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "block.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), ], ) @@ -123,6 +138,12 @@ def test_peak_memory_report_handles_empty_trace(tmp_path: Path) -> None: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) diff --git a/tests/core/test_split_service.py b/tests/core/test_split_service.py index 356c765..7206888 100644 --- a/tests/core/test_split_service.py +++ b/tests/core/test_split_service.py @@ -128,10 +128,13 @@ def _database_observation( ) assert replayed hooker.flush(device) + hooker.flush_callstacks() connection = hooker.db_handler.db.conn trace_rows = connection.execute( - f"SELECT id, action, address, size, stream, allocated, active, reserved, callstack " - f"FROM trace_entry_{device} ORDER BY id" + f"SELECT t.id, t.action, t.address, t.size, t.stream, " + f"t.allocated, t.active, t.reserved, c.callstack " + f"FROM trace_entry_{device} t JOIN callstack c ON c.id = t.callstackId " + f"ORDER BY t.id" ).fetchall() block_rows = connection.execute( f"SELECT id, address, size, requestedSize, state, allocEventId, freeEventId " diff --git a/tests/query/test_peak_memory_templates.py b/tests/query/test_peak_memory_templates.py index 6a012a9..61101a5 100644 --- a/tests/query/test_peak_memory_templates.py +++ b/tests/query/test_peak_memory_templates.py @@ -43,9 +43,24 @@ def peak_memory_db(tmp_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "freed.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.execute(""" CREATE TABLE block_0 ( id INTEGER PRIMARY KEY, @@ -61,14 +76,14 @@ def peak_memory_db(tmp_path: Path) -> Path: conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "freed.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), ], ) @@ -308,16 +323,20 @@ def test_active_memory_callstack_at_event_keeps_special_groups_beyond_top_n( peak_memory_db: Path, ) -> None: conn = sqlite3.connect(str(peak_memory_db)) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [(10, "a.py:1"), (11, "b.py:2"), (12, "c.py:3")], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (-101, 2, 0x11000, 20000, 0, 0, 0, 0, "a.py:1"), - (-102, 2, 0x12000, 18000, 0, 0, 0, 0, "b.py:2"), - (-103, 2, 0x13000, 16000, 0, 0, 0, 0, "c.py:3"), + (-101, 2, 0x11000, 20000, 0, 0, 0, 0, 10), + (-102, 2, 0x12000, 18000, 0, 0, 0, 0, 11), + (-103, 2, 0x13000, 16000, 0, 0, 0, 0, 12), ], ) conn.executemany( diff --git a/tests/query/test_query_max_rows_pushdown.py b/tests/query/test_query_max_rows_pushdown.py index d8fb6f1..cb82454 100644 --- a/tests/query/test_query_max_rows_pushdown.py +++ b/tests/query/test_query_max_rows_pushdown.py @@ -82,6 +82,12 @@ def callstack_db(tmp_path: Path) -> Path: CREATE TABLE trace_entry_0 ( id INTEGER PRIMARY KEY, size INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -89,7 +95,7 @@ def callstack_db(tmp_path: Path) -> Path: # actually has something to drop: # * 4 callstacks have 10+ entries (pass min_count=5) # * 2 callstacks have 4 entries each (fail min_count=5) - rows: list[tuple[int, int, str | None]] = [] + rows: list[tuple[int, int, int]] = [] callstacks = [ ("train.py:10", 12), ("train.py:20", 11), @@ -98,11 +104,15 @@ def callstack_db(tmp_path: Path) -> Path: ("noise.py:50", 4), ("noise.py:60", 4), ] - for cs, count in callstacks: + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [(index, text) for index, (text, _) in enumerate(callstacks)], + ) + for index, (_, count) in enumerate(callstacks): for _ in range(count): - rows.append((len(rows) + 1, 100, cs)) + rows.append((len(rows) + 1, 100, index)) conn.executemany( - "INSERT INTO trace_entry_0 (id, size, callstack) VALUES (?, ?, ?)", + "INSERT INTO trace_entry_0 (id, size, callstackId) VALUES (?, ?, ?)", rows, ) conn.commit() diff --git a/tests/skills/harness/fixtures.py b/tests/skills/harness/fixtures.py index 07f2bb6..2f93090 100644 --- a/tests/skills/harness/fixtures.py +++ b/tests/skills/harness/fixtures.py @@ -9,6 +9,8 @@ from .descriptors import DescriptorError +# Fixture YAML keeps authoring the callstack as text per trace row; the builder +# interns it into the shared `callstack` table that SnapshotDB now emits. TRACE_COLUMNS = ( "id", "action", @@ -20,6 +22,8 @@ "reserved", "callstack", ) +_TRACE_TABLE_COLUMNS = (*TRACE_COLUMNS[:-1], "callstackId") +_CALLSTACK_INDEX = TRACE_COLUMNS.index("callstack") BLOCK_COLUMNS = ( "id", "address", @@ -77,6 +81,8 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo connection.execute( "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" ) + connection.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + callstack_ids: dict[str, int] = {} for raw_device_id, device in definition["devices"].items(): device_id = int(raw_device_id) connection.execute(f""" @@ -89,7 +95,7 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo allocated INTEGER, active INTEGER, reserved INTEGER, - callstack TEXT + callstackId INTEGER ) """) connection.execute(f""" @@ -105,9 +111,24 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo """) trace_rows = _rows(device["trace"], TRACE_COLUMNS, f"device {device_id}.trace") block_rows = _rows(device["blocks"], BLOCK_COLUMNS, f"device {device_id}.blocks") - placeholders = ", ".join("?" for _ in TRACE_COLUMNS) + interned_rows: list[tuple[Any, ...]] = [] + for row in trace_rows: + callstack = row[_CALLSTACK_INDEX] + if callstack is None: + callstack_id = None + else: + callstack_id = callstack_ids.get(callstack) + if callstack_id is None: + callstack_id = len(callstack_ids) + callstack_ids[callstack] = callstack_id + connection.execute( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + (callstack_id, callstack), + ) + interned_rows.append((*row[:_CALLSTACK_INDEX], callstack_id)) + placeholders = ", ".join("?" for _ in _TRACE_TABLE_COLUMNS) connection.executemany( - f"INSERT INTO trace_entry_{device_id} VALUES ({placeholders})", trace_rows + f"INSERT INTO trace_entry_{device_id} VALUES ({placeholders})", interned_rows ) placeholders = ", ".join("?" for _ in BLOCK_COLUMNS) connection.executemany( diff --git a/tests/snapshot/golden_observations.py b/tests/snapshot/golden_observations.py index 7cc11b8..e2e740d 100644 --- a/tests/snapshot/golden_observations.py +++ b/tests/snapshot/golden_observations.py @@ -126,7 +126,12 @@ (5, "allocated", "INTEGER", 0, None, 0), (6, "active", "INTEGER", 0, None, 0), (7, "reserved", "INTEGER", 0, None, 0), - (8, "callstack", "TEXT", 0, None, 0), + (8, "callstackId", "INTEGER", 0, None, 0), +] + +CALLSTACK_SCHEMA = [ + (0, "id", "INTEGER", 0, None, 1), + (1, "callstack", "TEXT", 0, None, 0), ] BLOCK_SCHEMA = [ diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py new file mode 100644 index 0000000..2d1f48a --- /dev/null +++ b/tests/snapshot/test_callstack_interner.py @@ -0,0 +1,109 @@ +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner + +INNER = {"filename": "inner.py", "line": 10, "name": "inner"} +OUTER = {"filename": "outer.py", "line": 20, "name": "outer"} +EXPECTED_TEXT = "outer.py:20 outer\ninner.py:10 inner" + + +def _raw_event(frames: list[dict], idx: int = 0) -> TraceEntry: + return TraceEntry.from_dict( + {"action": "alloc", "addr": 0x1000, "size": 16, "stream": 0, "id": idx, "frames": frames}, + _raw_frames=True, + ) + + +def test_interned_text_matches_get_callstack(): + interner = CallstackInterner() + event = _raw_event([INNER, OUTER]) + + assert interner.intern(event) == 0 + assert interner.records() == [{"id": 0, "callstack": EXPECTED_TEXT}] + assert interner.records()[0]["callstack"] == event.get_callstack() + + +def test_shared_frame_container_reuses_one_id(): + interner = CallstackInterner() + frames = [INNER, OUTER] + + first = interner.intern(_raw_event(frames, idx=1)) + second = interner.intern(_raw_event(frames, idx=2)) + + assert first == second == 0 + assert len(interner) == 1 + + +def test_distinct_containers_with_equal_frames_reuse_one_id(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([INNER, OUTER], idx=1)) + second = interner.intern(_raw_event([dict(INNER), dict(OUTER)], idx=2)) + + assert first == second == 0 + assert len(interner) == 1 + + +def test_different_callstacks_get_different_ids(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([INNER], idx=1)) + second = interner.intern(_raw_event([OUTER], idx=2)) + + assert {first, second} == {0, 1} + assert len(interner) == 2 + assert [record["callstack"] for record in interner.records()] == [ + "inner.py:10 inner", + "outer.py:20 outer", + ] + + +def test_empty_frames_intern_to_empty_text(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([], idx=1)) + second = interner.intern(_raw_event([], idx=2)) + + assert first == second == 0 + assert interner.records() == [{"id": 0, "callstack": ""}] + + +def test_reclaimed_container_address_does_not_produce_a_false_hit(): + """A freed container must not let a new object inherit its callstack id.""" + interner = CallstackInterner() + addresses = [] + + for frames in ([INNER], [OUTER]): + event = _raw_event(frames) + addresses.append(id(event.callstack_frames())) + interner.intern(event) + del event + + # The interner pins each keyed container, so a later container cannot land + # on a pinned address and inherit the wrong id. + assert len(interner) == 2 + assert [record["callstack"] for record in interner.records()] == [ + "inner.py:10 inner", + "outer.py:20 outer", + ] + assert len(set(addresses)) == 2 + + +def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): + interner = CallstackInterner() + raw = _raw_event([INNER, OUTER], idx=1) + eager = TraceEntry.from_dict( + { + "action": "alloc", + "addr": 0x1000, + "size": 16, + "stream": 0, + "id": 2, + "frames": [INNER, OUTER], + }, + ) + + raw_id = interner.intern(raw) + eager_id = interner.intern(eager) + + assert raw_id == eager_id == 0 + assert len(interner) == 1 diff --git a/tests/snapshot/test_entity2record.py b/tests/snapshot/test_entity2record.py index a2e9e9f..15ae9d0 100644 --- a/tests/snapshot/test_entity2record.py +++ b/tests/snapshot/test_entity2record.py @@ -1,4 +1,5 @@ from pt_snap_cli.snapshot.base import Block, Frame, TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner from pt_snap_cli.snapshot.tools.adaptors.database.defs import ( BlockFieldDefs, EventFieldDefs, @@ -26,7 +27,7 @@ def test_get_timestamp_by_event_idx_handles_none(): def test_event2record_uses_generated_default_id_when_missing(): event = TraceEntry(action="alloc", addr=0x1000, size=16, stream=0, idx=None) - record = event2record(event, allocated=1, active=2, reserved=3) + record = event2record(event, allocated=1, active=2, reserved=3, callstacks=CallstackInterner()) assert record[EventFieldDefs.ID] < 0 assert record[EventFieldDefs.ALLOCATED] == 1 @@ -65,7 +66,10 @@ def test_records_preserve_totals_state_ids_and_callstack_serialization(): free_event_idx=12, ) - event_record = event2record(event, allocated=100, active=120, reserved=256) + callstacks = CallstackInterner() + event_record = event2record( + event, allocated=100, active=120, reserved=256, callstacks=callstacks + ) block_record = block2record(block) assert event_record == { @@ -77,8 +81,9 @@ def test_records_preserve_totals_state_ids_and_callstack_serialization(): "allocated": 100, "active": 120, "reserved": 256, - "callstack": "outer.py:20 outer\ninner.py:10 inner", + "callstackId": 0, } + assert callstacks.records() == [{"id": 0, "callstack": "outer.py:20 outer\ninner.py:10 inner"}] assert block_record == { "id": 9, "address": 0x2000, diff --git a/tests/snapshot/test_snapshot2db_runtime.py b/tests/snapshot/test_snapshot2db_runtime.py index 8eb6f36..0c17a7e 100644 --- a/tests/snapshot/test_snapshot2db_runtime.py +++ b/tests/snapshot/test_snapshot2db_runtime.py @@ -18,6 +18,7 @@ ACTION_VALUE_MAP, BLOCK_SCHEMA, BLOCK_STATE_VALUE_MAP, + CALLSTACK_SCHEMA, DATABASE_GOLDEN, TRACE_SCHEMA, ) @@ -128,8 +129,10 @@ def dump_rows(database, *, eager): assert snapshot2db.dump(snapshot, database, 0) with sqlite3.connect(database) as connection: return connection.execute( - "SELECT id, action, address, size, stream, allocated, active, reserved, callstack " - "FROM trace_entry_0 ORDER BY id" + "SELECT t.id, t.action, t.address, t.size, t.stream, " + "t.allocated, t.active, t.reserved, c.callstack " + "FROM trace_entry_0 t JOIN callstack c ON c.id = t.callstackId " + "ORDER BY t.id" ).fetchall() eager_rows = dump_rows(tmp_path / "eager.db", eager=True) @@ -322,7 +325,14 @@ def test_dump_all_multiple_device_snapshot(dump_database): "SELECT name FROM sqlite_master WHERE type = 'table'" ).fetchall() } - assert tables == {"dictionary", "trace_entry_0", "block_0", "trace_entry_1", "block_1"} + assert tables == { + "callstack", + "dictionary", + "trace_entry_0", + "block_0", + "trace_entry_1", + "block_1", + } def negative_id_observation(connection, table): @@ -360,7 +370,9 @@ def test_database_matches_pre_relocation_golden(dump_database, fixture_name, dev "WHERE id >= 0 ORDER BY id DESC LIMIT 1" ).fetchone() callstacks = connection.execute( - f"SELECT COUNT(*) FROM {trace_table} WHERE callstack != ''" + f"SELECT COUNT(*) FROM {trace_table} t " + "JOIN callstack c ON c.id = t.callstackId " + "WHERE c.callstack != ''" ).fetchone()[0] assert rows == golden["rows"] @@ -384,6 +396,7 @@ def test_database_schema_and_action_state_mapping_are_exact(dump_database): assert RUNTIME_ACTION_VALUE_MAP == ACTION_VALUE_MAP assert RUNTIME_BLOCK_STATE_VALUE_MAP == BLOCK_STATE_VALUE_MAP with sqlite3.connect(database) as connection: + assert connection.execute("PRAGMA table_info(callstack)").fetchall() == CALLSTACK_SCHEMA for device in (0, 1): assert ( connection.execute(f"PRAGMA table_info(trace_entry_{device})").fetchall() @@ -427,9 +440,29 @@ def test_callstack_serialization_preserves_frame_order(dump_database): _, database = dump_database("snapshot_1768383987920985470.pkl", 0) with sqlite3.connect(database) as connection: callstack = connection.execute( - "SELECT callstack FROM trace_entry_0 WHERE callstack LIKE '%\n%' LIMIT 1" + "SELECT c.callstack FROM trace_entry_0 t " + "JOIN callstack c ON c.id = t.callstackId " + "WHERE c.callstack LIKE '%\n%' LIMIT 1" ).fetchone()[0] lines = callstack.splitlines() assert len(lines) >= 2 assert all(":" in line and " " in line for line in lines) + + +def test_callstack_table_is_deduplicated_and_covers_every_event(dump_database): + _, database = dump_database("snapshot_1768383987920985470.pkl", 0) + with sqlite3.connect(database) as connection: + rows, distinct_ids, distinct_texts = connection.execute( + "SELECT COUNT(*), COUNT(DISTINCT id), COUNT(DISTINCT callstack) FROM callstack" + ).fetchone() + events, resolved = connection.execute( + "SELECT COUNT(*), COUNT(c.id) FROM trace_entry_0 t " + "LEFT JOIN callstack c ON c.id = t.callstackId" + ).fetchone() + + # One row per distinct text keeps grouping by id equivalent to grouping by text. + assert rows == distinct_ids == distinct_texts + # Every event resolves to a callstack row, and interning actually collapses rows. + assert resolved == events + assert rows < events diff --git a/tests/test_cli.py b/tests/test_cli.py index d83d68b..dbf3012 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -97,6 +97,12 @@ def create_peak_report_db(db_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -111,17 +117,26 @@ def create_peak_report_db(db_path: Path) -> Path: freeEventId INTEGER ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "block.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "block.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), ], ) conn.executemany( From 6143228dd46e10e20a91a877bde455e0eae7de9e Mon Sep 17 00:00:00 2001 From: codeasier Date: Sun, 6 Sep 2026 10:08:09 +0800 Subject: [PATCH 2/3] fix: address PR 114 review feedback --- docs/en/database.md | 7 +++++-- docs/zh/database.md | 5 +++-- src/pt_snap_cli/query/executor.py | 15 +++++++++++++ .../statistical/callstack_analysis.yaml | 2 +- src/pt_snap_cli/snapshot/simulate/simulate.py | 1 + .../tools/adaptors/database/callstack.py | 18 ++++------------ tests/query/test_executor.py | 21 ++++++++++++++++++- tests/snapshot/test_callstack_interner.py | 10 ++------- tests/snapshot/test_simulate.py | 12 +++++++++++ 9 files changed, 63 insertions(+), 28 deletions(-) diff --git a/docs/en/database.md b/docs/en/database.md index 24c2921..3a2ff03 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -48,8 +48,11 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -Legacy or externally generated compatible databases without this table remain queryable. Their -metadata status is reported as unavailable, and a later import rebuilds them once before reuse. +Legacy or externally generated compatible databases without this table remain queryable for +templates that do not require the deduplicated callstack schema. Callstack templates require a +database produced by the current importer; otherwise the query reports that the snapshot must be +re-imported. Their metadata status is reported as unavailable, and a later import rebuilds them +once before reuse. ### Query workflow diff --git a/docs/zh/database.md b/docs/zh/database.md index e9d306b..8e43f08 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -44,8 +44,9 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -旧版或外部生成且结构兼容的 DB 仍可正常查询。若没有 metadata,查询会返回 unavailable, -下一次执行对应 pickle 导入时会重建一次。 +旧版或外部生成且结构兼容的 DB,仍可用于不依赖去重调用栈结构的查询。调用栈相关模板要求 +数据库由当前导入器生成;否则会提示需要重新导入快照。若没有 metadata,查询会返回 +unavailable,下一次执行对应 pickle 导入时会重建一次。 ### 查询工作流 diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index ae96a69..3181e07 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import sqlite3 import warnings from pathlib import Path from typing import Any @@ -101,6 +102,16 @@ def _append_limit(self, sql: str, limit: int) -> str: trimmed = trimmed[:-1].rstrip() return f"{trimmed} LIMIT {int(limit)}" + @staticmethod + def _execution_error(error: sqlite3.OperationalError) -> QueryExecutionError: + detail = str(error) + if "no such table: callstack" in detail or "no such column: callstackId" in detail: + return QueryExecutionError( + "Query execution failed: database uses the legacy inline-callstack layout; " + "re-import the snapshot to use callstack queries" + ) + return QueryExecutionError(f"Query execution failed: {error}") + def _load_templates(self) -> None: """Load all YAML templates from template directory.""" if not self._template_dir: @@ -199,6 +210,8 @@ def execute( cursor.execute(sql) columns = [desc[0] for desc in cursor.description] return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + except sqlite3.OperationalError as e: + raise self._execution_error(e) from e except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e @@ -212,6 +225,8 @@ def count(self, sql: str) -> int: cursor.execute(count_sql) row = cursor.fetchone() return int(row[0]) if row is not None else 0 + except sqlite3.OperationalError as e: + raise self._execution_error(e) from e except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index 4f1765c..93a56b7 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -42,7 +42,7 @@ queries: HAVING alloc_count >= {{ min_count }} AND total_size >= {{ min_size }} ) AS grouped - JOIN callstack c ON c.id = grouped.callstackId + LEFT JOIN callstack c ON c.id = grouped.callstackId ORDER BY total_size DESC, callstack ASC LIMIT {{ limit|int }} output_schema: diff --git a/src/pt_snap_cli/snapshot/simulate/simulate.py b/src/pt_snap_cli/snapshot/simulate/simulate.py index 6968d50..ed9e12c 100644 --- a/src/pt_snap_cli/snapshot/simulate/simulate.py +++ b/src/pt_snap_cli/snapshot/simulate/simulate.py @@ -127,6 +127,7 @@ def _adapt_workspace_snapshot(self): address=existed_seg.address, state=BlockState.ACTIVE_ALLOCATED, frames=existed_seg.frames, + _raw_frames=existed_seg._raw_frames, segment_ptr=existed_seg, ), 0, diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py index 6740196..04599fb 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py @@ -10,18 +10,13 @@ class CallstackInterner: behind them. Interning therefore turns the per-event callstack into an integer reference and keeps a single text copy per distinct callstack. - Lookups try the frame container identity first, then the identities of the - individual frames, and only render text on a miss. Distinct containers that - render to the same text collapse onto one id, so the id-to-text mapping - stays one-to-one and grouping by id matches grouping by text. + Lookups use the identities of the individual frames and only render text + on a miss. Distinct containers that render to the same text collapse onto + one id, so the id-to-text mapping stays one-to-one and grouping by id + matches grouping by text. """ def __init__(self) -> None: - # The id()-keyed caches store their keyed container alongside the id. - # Holding that reference keeps the container alive, which is what makes - # the id() key meaningful: a freed container could otherwise be - # replaced by an unrelated object at the same address. - self._by_container: dict[int, tuple[object, int]] = {} self._by_frames: dict[tuple[int, ...], tuple[object, int]] = {} self._by_text: dict[str, int] = {} self._texts: list[str] = [] @@ -29,10 +24,6 @@ def __init__(self) -> None: def intern(self, event: TraceEntry) -> int: """Return the callstack id for ``event``, rendering text only on a miss.""" frames = event.callstack_frames() - by_container = self._by_container.get(id(frames)) - if by_container is not None: - return by_container[1] - frame_key = tuple(map(id, frames)) by_frames = self._by_frames.get(frame_key) if by_frames is not None: @@ -48,7 +39,6 @@ def intern(self, event: TraceEntry) -> int: callstack_id = existing self._by_frames[frame_key] = (frames, callstack_id) - self._by_container[id(frames)] = (frames, callstack_id) return callstack_id def records(self) -> list[dict]: diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 1849002..95aab8e 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -1,9 +1,12 @@ """Tests for query executor.""" +import sqlite3 + import pytest +from pt_snap_cli.context import Context from pt_snap_cli.query.config import QueryParameter, QueryTemplate -from pt_snap_cli.query.executor import QueryExecutor, TemplateRenderError +from pt_snap_cli.query.executor import QueryExecutionError, QueryExecutor, TemplateRenderError class TestQueryExecutor: @@ -337,6 +340,22 @@ def test_execute_on_all_devices_empty_device_list(self): results = executor.execute_on_all_devices("empty_test") assert results == {} + def test_legacy_callstack_schema_error_is_actionable(self, tmp_path): + db_path = tmp_path / "legacy.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dictionary (table_name TEXT)") + conn.execute("CREATE TABLE trace_entry_0 " "(id INTEGER, callstack TEXT)") + executor = QueryExecutor(Context(db_path)) + executor.register_template( + QueryTemplate( + name="legacy_callstack", + query="SELECT callstack FROM callstack", + ) + ) + + with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + executor.execute_template("legacy_callstack", device_id=0) + def _make_mock_context(device_ids: list[int]): """Create a mock Context with the given device IDs.""" diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py index 2d1f48a..c71c37f 100644 --- a/tests/snapshot/test_callstack_interner.py +++ b/tests/snapshot/test_callstack_interner.py @@ -67,25 +67,19 @@ def test_empty_frames_intern_to_empty_text(): assert interner.records() == [{"id": 0, "callstack": ""}] -def test_reclaimed_container_address_does_not_produce_a_false_hit(): - """A freed container must not let a new object inherit its callstack id.""" +def test_distinct_frame_identities_do_not_produce_a_false_hit(): + """Distinct frame identities must not inherit an unrelated callstack id.""" interner = CallstackInterner() - addresses = [] for frames in ([INNER], [OUTER]): event = _raw_event(frames) - addresses.append(id(event.callstack_frames())) interner.intern(event) - del event - # The interner pins each keyed container, so a later container cannot land - # on a pinned address and inherit the wrong id. assert len(interner) == 2 assert [record["callstack"] for record in interner.records()] == [ "inner.py:10 inner", "outer.py:20 outer", ] - assert len(set(addresses)) == 2 def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): diff --git a/tests/snapshot/test_simulate.py b/tests/snapshot/test_simulate.py index bb4171a..7eb364e 100644 --- a/tests/snapshot/test_simulate.py +++ b/tests/snapshot/test_simulate.py @@ -265,6 +265,18 @@ def test_simulate_device_snapshot_sets_workspace_flag_from_first_event(): assert device.total_reserved == 4096 +def test_raw_workspace_adaptation_preserves_segment_frames(): + frame = {"filename": "workspace.py", "line": 7, "name": "reserve"} + data = make_torch_npu_workspace_snapshot() + data["segments"][0]["frames"] = [frame] + + snapshot = SimulateDeviceSnapshot(data, 0, _raw_frames=True) + + block = snapshot.device_snapshot.segments[0].blocks[0] + assert block._raw_frames == [frame] + assert block.to_dict()["frames"] == [frame] + + def test_simulate_register_unregister_hookers_and_allocator_hookers(): snapshot = SimulateDeviceSnapshot(make_snapshot_dict(), 0) hooker = Hooker() From be098dbb7fd6e73f18554d7b386f6202cd7bd40b Mon Sep 17 00:00:00 2001 From: codeasier Date: Sun, 6 Sep 2026 10:33:45 +0800 Subject: [PATCH 3/3] fix: harden callstack follow-up --- src/pt_snap_cli/query/executor.py | 4 +++- .../statistical/callstack_analysis.yaml | 2 +- src/pt_snap_cli/snapshot/PROVENANCE.md | 9 +++++++++ tests/query/test_executor.py | 16 ++++++++++++++++ tests/query/test_query_max_rows_pushdown.py | 16 ++++++++++++++++ tests/snapshot/test_callstack_interner.py | 19 +++++++++++++++++++ 6 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 3181e07..053af51 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -105,7 +105,9 @@ def _append_limit(self, sql: str, limit: int) -> str: @staticmethod def _execution_error(error: sqlite3.OperationalError) -> QueryExecutionError: detail = str(error) - if "no such table: callstack" in detail or "no such column: callstackId" in detail: + if "no such table: callstack" in detail or re.search( + r"no such column: (?:[\w]+\.)?callstackId", detail + ): return QueryExecutionError( "Query execution failed: database uses the legacy inline-callstack layout; " "re-import the snapshot to use callstack queries" diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index 93a56b7..26d37cf 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -24,7 +24,7 @@ queries: description: Maximum rows to return (-1 for unlimited). Mirrored from the caller-side ``max_rows`` for SQL pushdown. query: | SELECT - c.callstack AS callstack, + COALESCE(c.callstack, '[missing callstack]') AS callstack, grouped.alloc_count AS alloc_count, grouped.total_size AS total_size, grouped.avg_size AS avg_size, diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 82c5f2b..98b4f47 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -179,6 +179,15 @@ The upstream standalone split frontend is deliberately excluded, not deferred: pre-change output after resolving `callstackId` through the join. This is a local performance and schema change; audited upstream source mappings and license terms are unchanged. +- On 2026-09-06, PR #114 follow-up corrected the preceding callstack entry's + implementation description: the interner keys on individual frame object + identities and retains one exemplar frame container per distinct frame-key + tuple, rather than keying and retaining every outer frame container. The + follow-up also propagates `_raw_frames` through NPU workspace adaptation, + maps legacy callstack-schema SQLite errors to an actionable re-import + message, and uses a left join with a missing-callstack placeholder in + statistical analysis. These are local fixes to the issue #113 runtime; + audited upstream source mappings and license terms remain unchanged. ## Migration toolchain diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 95aab8e..3663575 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -356,6 +356,22 @@ def test_legacy_callstack_schema_error_is_actionable(self, tmp_path): with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): executor.execute_template("legacy_callstack", device_id=0) + def test_qualified_legacy_callstack_schema_error_is_actionable(self, tmp_path): + db_path = tmp_path / "legacy-qualified.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dictionary (table_name TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER)") + executor = QueryExecutor(Context(db_path)) + executor.register_template( + QueryTemplate( + name="qualified_legacy_callstack", + query="SELECT t.callstackId FROM trace_entry_0 t", + ) + ) + + with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + executor.execute_template("qualified_legacy_callstack", device_id=0) + def _make_mock_context(device_ids: list[int]): """Create a mock Context with the given device IDs.""" diff --git a/tests/query/test_query_max_rows_pushdown.py b/tests/query/test_query_max_rows_pushdown.py index cb82454..80e4da2 100644 --- a/tests/query/test_query_max_rows_pushdown.py +++ b/tests/query/test_query_max_rows_pushdown.py @@ -181,3 +181,19 @@ def test_min_size_filter_combines_with_limit(self, callstack_db: Path) -> None: {"min_size": 0, "min_count": 1, "limit": 2}, ) assert len(rows) == 2 + + def test_missing_callstack_uses_placeholder(self, callstack_db: Path) -> None: + with sqlite3.connect(callstack_db) as conn: + conn.execute( + "INSERT INTO trace_entry_0 (id, size, callstackId) VALUES (?, ?, ?)", + (100, 1, 99), + ) + conn.commit() + + rows = _execute_template( + callstack_db, + "callstack_analysis", + {"min_count": 1, "min_size": 0}, + ) + + assert rows[-1]["callstack"] == "[missing callstack]" diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py index c71c37f..be917e0 100644 --- a/tests/snapshot/test_callstack_interner.py +++ b/tests/snapshot/test_callstack_interner.py @@ -1,3 +1,6 @@ +import gc +import weakref + from pt_snap_cli.snapshot.base import TraceEntry from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner @@ -6,6 +9,10 @@ EXPECTED_TEXT = "outer.py:20 outer\ninner.py:10 inner" +class WeakrefableFrames(list): + """List-shaped frame container used to test exemplar retention.""" + + def _raw_event(frames: list[dict], idx: int = 0) -> TraceEntry: return TraceEntry.from_dict( {"action": "alloc", "addr": 0x1000, "size": 16, "stream": 0, "id": idx, "frames": frames}, @@ -82,6 +89,18 @@ def test_distinct_frame_identities_do_not_produce_a_false_hit(): ] +def test_frame_container_exemplar_stays_alive_for_id_reuse_safety(): + interner = CallstackInterner() + frames = WeakrefableFrames([INNER]) + frames_ref = weakref.ref(frames) + + interner.intern(_raw_event(frames)) + del frames + gc.collect() + + assert frames_ref() is not None + + def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): interner = CallstackInterner() raw = _raw_event([INNER, OUTER], idx=1)