diff --git a/docs/en/database.md b/docs/en/database.md index e19f28d..3a2ff03 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -48,8 +48,11 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -Legacy or externally generated compatible databases without this table remain queryable. Their -metadata status is reported as unavailable, and a later import rebuilds them once before reuse. +Legacy or externally generated compatible databases without this table remain queryable for +templates that do not require the deduplicated callstack schema. Callstack templates require a +database produced by the current importer; otherwise the query reports that the snapshot must be +re-imported. Their metadata status is reported as unavailable, and a later import rebuilds them +once before reuse. ### Query workflow @@ -75,11 +78,12 @@ the supported query surface. See [Querying](querying.md) for the complete workfl | Table | Description | Records (example) | |-------|-------------|-------------------| | `dictionary` | Enum mapping dictionary | - | +| `callstack` | Deduplicated callstack text shared by all devices | 4,369 | | `trace_entry_0` | Device 0 event trace table | 8,094 | | `block_0` | Device 0 memory block table | - | | `pt_snap_metadata` | First-party import provenance and cache metadata | 1 | -> **Naming convention**: For multi-device scenarios, table names are suffixed with the device ID, e.g., `trace_entry_1` and `block_1` for device 1. +> **Naming convention**: For multi-device scenarios, table names are suffixed with the device ID, e.g., `trace_entry_1` and `block_1` for device 1. The `callstack` table carries no device suffix because callstacks are shared across devices. --- @@ -101,7 +105,7 @@ CREATE TABLE trace_entry_0 ( `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, - `callstack` TEXT + `callstackId` INTEGER ); ``` @@ -117,7 +121,7 @@ CREATE TABLE trace_entry_0 ( | allocated | INTEGER | — | Total allocated bytes | `136426496` | | active | INTEGER | — | Total active bytes | `136426496` | | reserved | INTEGER | — | Memory pool reserved bytes | `155189248` | -| callstack | TEXT | NULL | Callstack information (multi-line text) | See example | +| callstackId | INTEGER | NULL | Reference to `callstack.id`; join to read the text | `12` | #### Action Type Codes @@ -144,13 +148,24 @@ CREATE TABLE trace_entry_0 ( ``` id=1, action=4, address=20697535234048, size=41943040, stream=1276474240 allocated=136426496, active=136426496, reserved=155189248 -callstack: +callstackId=12 -> callstack.callstack: /home/liuyekang/dev/projects/test/memory_leaks_demo.py:60 /home/liuyekang/dev/projects/test/memory_leaks_demo.py:34 main /home/liuyekang/dev/projects/test/memory_leaks_demo.py:24 train /home/liuyekang/dev/projects/test/memory_leaks_demo.py:12 train_one_step ``` +#### Reading Callstack Text + +Join `callstack` to resolve `callstackId`: + +```sql +SELECT t.id, t.size, c.callstack +FROM trace_entry_0 t +LEFT JOIN callstack c ON c.id = t.callstackId +WHERE t.id = 1; +``` + --- ### 2. block_{device} — Memory Block Table @@ -264,7 +279,39 @@ table=trace_entry_0, column=action, key=4, value=alloc table=block_0, column=state, key=1, value=active_allocated ``` -### 4. pt_snap_metadata — Import Metadata Table +--- + +### 4. callstack — Deduplicated Callstack Text + +Snapshots repeat a small number of callstacks across a very large number of +events, so the text is stored once per distinct callstack and referenced by +`trace_entry_.callstackId`. + +#### Schema + +```sql +CREATE TABLE callstack ( + `id` INTEGER PRIMARY KEY, + `callstack` TEXT +); +``` + +#### Columns + +| Column | Type | Description | +|--------|------|-------------| +| id | INTEGER | Callstack ID referenced by `trace_entry_.callstackId` | +| callstack | TEXT | Callstack text, outermost frame first, one `file:line name` per line | + +#### Properties + +| Property | Guarantee | +|----------|-----------| +| Uniqueness | One row per distinct callstack text, so grouping by `callstackId` equals grouping by text | +| Scope | Shared by every device in the database; the table has no device suffix | +| Coverage | Every event produced by import resolves to a row; `callstackId` is `NULL` only in externally generated databases | + +### 5. pt_snap_metadata — Import Metadata Table First-party imports create one metadata row for provenance and cache compatibility checks: diff --git a/docs/zh/database.md b/docs/zh/database.md index 5c084f9..8e43f08 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -44,8 +44,9 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -旧版或外部生成且结构兼容的 DB 仍可正常查询。若没有 metadata,查询会返回 unavailable, -下一次执行对应 pickle 导入时会重建一次。 +旧版或外部生成且结构兼容的 DB,仍可用于不依赖去重调用栈结构的查询。调用栈相关模板要求 +数据库由当前导入器生成;否则会提示需要重新导入快照。若没有 metadata,查询会返回 +unavailable,下一次执行对应 pickle 导入时会重建一次。 ### 查询工作流 @@ -70,11 +71,12 @@ pt-snap query --template-use block --params '{"min_size": 1048576}' | 表名 | 说明 | 记录数(示例) | |------|------|---------------| | `dictionary` | 字典映射表 | - | +| `callstack` | 去重后的调用栈文本,所有设备共享 | 4,369 | | `trace_entry_0` | 设备 0 的事件跟踪表 | 8,094 | | `block_0` | 设备 0 的内存块表 | - | | `pt_snap_metadata` | 第一方导入来源与缓存 metadata | 1 | -> **命名规则**: 多设备场景下,表名后缀为设备 ID,如 `trace_entry_1`、`block_1` 表示设备 1 的数据。 +> **命名规则**: 多设备场景下,表名后缀为设备 ID,如 `trace_entry_1`、`block_1` 表示设备 1 的数据。`callstack` 表没有设备后缀,因为调用栈在设备之间共享。 --- @@ -96,7 +98,7 @@ CREATE TABLE trace_entry_0 ( `allocated` INTEGER, `active` INTEGER, `reserved` INTEGER, - `callstack` TEXT + `callstackId` INTEGER ); ``` @@ -112,7 +114,7 @@ CREATE TABLE trace_entry_0 ( | allocated | INTEGER | — | 当前已分配总量 | `136426496` | | active | INTEGER | — | 当前活跃总量 | `136426496` | | reserved | INTEGER | — | 内存池保留总量 | `155189248` | -| callstack | TEXT | NULL | 调用栈,多行文本格式 | 见示例 | +| callstackId | INTEGER | NULL | 指向 `callstack.id`,需 join 才能取到文本 | `12` | #### 动作类型编码(action 字段) @@ -139,13 +141,24 @@ CREATE TABLE trace_entry_0 ( ``` id=1, action=4, address=20697535234048, size=41943040, stream=1276474240 allocated=136426496, active=136426496, reserved=155189248 -callstack: +callstackId=12 -> callstack.callstack: /home/liuyekang/dev/projects/test/memory_leaks_demo.py:60 /home/liuyekang/dev/projects/test/memory_leaks_demo.py:34 main /home/liuyekang/dev/projects/test/memory_leaks_demo.py:24 train /home/liuyekang/dev/projects/test/memory_leaks_demo.py:12 train_one_step ``` +#### 读取调用栈文本 + +join `callstack` 表解析 `callstackId`: + +```sql +SELECT t.id, t.size, c.callstack +FROM trace_entry_0 t +LEFT JOIN callstack c ON c.id = t.callstackId +WHERE t.id = 1; +``` + --- ### 2. block_{device} — 内存块表 @@ -257,7 +270,38 @@ table=trace_entry_0, column=action, key=4, value=alloc table=block_0, column=state, key=1, value=active_allocated ``` -### 4. pt_snap_metadata — 导入 Metadata 表 +--- + +### 4. callstack — 去重调用栈表 + +快照中少量调用栈会在极多事件上重复出现,因此文本按不同调用栈只存一份,由 +`trace_entry_.callstackId` 引用。 + +#### Schema + +```sql +CREATE TABLE callstack ( + `id` INTEGER PRIMARY KEY, + `callstack` TEXT +); +``` + +#### 字段说明 + +| 字段名 | 类型 | 说明 | +|--------|------|------| +| id | INTEGER | 调用栈 ID,被 `trace_entry_.callstackId` 引用 | +| callstack | TEXT | 调用栈文本,最外层帧在前,每行一个 `文件:行号 函数名` | + +#### 表属性 + +| 属性 | 保证 | +|------|------| +| 唯一性 | 每条不同的调用栈文本只有一行,因此按 `callstackId` 分组等价于按文本分组 | +| 作用域 | 数据库内所有设备共享,表名不带设备后缀 | +| 覆盖度 | 导入产生的每个事件都能解析到一行;只有外部生成的数据库才可能出现 `callstackId` 为 `NULL` | + +### 5. pt_snap_metadata — 导入 Metadata 表 第一方导入会创建一条 metadata 记录,用于来源追踪和缓存兼容性检查: diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index c1d1584..b2f9922 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -20,7 +20,9 @@ METADATA_TABLE = "pt_snap_metadata" METADATA_SCHEMA_VERSION = 1 -IMPORT_FORMAT_VERSION = 1 +# 2: trace_entry_ stores `callstackId` referencing the shared `callstack` +# table instead of an inlined `callstack` text column. +IMPORT_FORMAT_VERSION = 2 IMPORTER_NAME = "pt-snap-cli" HASH_CHUNK_SIZE = 1024 * 1024 diff --git a/src/pt_snap_cli/core/split_service.py b/src/pt_snap_cli/core/split_service.py index 0689239..520043a 100644 --- a/src/pt_snap_cli/core/split_service.py +++ b/src/pt_snap_cli/core/split_service.py @@ -145,7 +145,9 @@ def _slice_device( max_entries=max_entries if max_entries is not None else event_count, dump_type="pkl", ) - _, replayed = replay_snapshot(representation, device, hooker=hooker) + # Slices serialize the original frame dictionaries, so eager Frame + # objects would be rebuilt only to be discarded. + _, replayed = replay_snapshot(representation, device, hooker=hooker, _raw_frames=True) except Exception as exc: self._fail("load/engine", source, f"device {device}: {exc}", exc) if not replayed: @@ -174,7 +176,7 @@ def _slice_device( or not traces[device] ): raise ValueError(f"generated slice has no trace entries for device {device}") - _, replayed = replay_snapshot(generated, device) + _, replayed = replay_snapshot(generated, device, _raw_frames=True) if not replayed: raise ValueError("replay returned false") except Exception as exc: diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index ae96a69..053af51 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +import sqlite3 import warnings from pathlib import Path from typing import Any @@ -101,6 +102,18 @@ def _append_limit(self, sql: str, limit: int) -> str: trimmed = trimmed[:-1].rstrip() return f"{trimmed} LIMIT {int(limit)}" + @staticmethod + def _execution_error(error: sqlite3.OperationalError) -> QueryExecutionError: + detail = str(error) + if "no such table: callstack" in detail or re.search( + r"no such column: (?:[\w]+\.)?callstackId", detail + ): + return QueryExecutionError( + "Query execution failed: database uses the legacy inline-callstack layout; " + "re-import the snapshot to use callstack queries" + ) + return QueryExecutionError(f"Query execution failed: {error}") + def _load_templates(self) -> None: """Load all YAML templates from template directory.""" if not self._template_dir: @@ -199,6 +212,8 @@ def execute( cursor.execute(sql) columns = [desc[0] for desc in cursor.description] return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + except sqlite3.OperationalError as e: + raise self._execution_error(e) from e except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e @@ -212,6 +227,8 @@ def count(self, sql: str) -> int: cursor.execute(count_sql) row = cursor.fetchone() return int(row[0]) if row is not None else 0 + except sqlite3.OperationalError as e: + raise self._execution_error(e) from e except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e diff --git a/src/pt_snap_cli/query/templates/basic/event.yaml b/src/pt_snap_cli/query/templates/basic/event.yaml index 6a85602..559e262 100644 --- a/src/pt_snap_cli/query/templates/basic/event.yaml +++ b/src/pt_snap_cli/query/templates/basic/event.yaml @@ -76,7 +76,20 @@ queries: required: false description: Number of rows to skip query: | - SELECT * FROM {{ device_trace_table }} + SELECT + id, + action, + address, + size, + stream, + allocated, + active, + reserved, + ( + SELECT callstack FROM callstack + WHERE callstack.id = {{ device_trace_table }}.callstackId + ) AS callstack + FROM {{ device_trace_table }} WHERE 1=1 {% if id is not none %}AND id = {{ id }}{% endif %} {% if min_id is not none %}AND id >= {{ min_id }}{% endif %} diff --git a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml index 2ec1de1..fa13749 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml @@ -48,7 +48,7 @@ queries: CASE WHEN b.category = 'static' THEN '[static] allocEventId=-1, freeEventId=-1' WHEN b.category = 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' - ELSE COALESCE(t.callstack, '[missing callstack]') + ELSE COALESCE(cs.callstack, '[missing callstack]') END AS callstack, b.category, COUNT(*) AS block_count, @@ -59,6 +59,8 @@ queries: FROM active_blocks b LEFT JOIN {{ device_trace_table }} t ON b.allocEventId = t.id + LEFT JOIN callstack cs + ON t.callstackId = cs.id GROUP BY callstack, b.category ), ranked_dynamic AS ( diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index cba6f69..26d37cf 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -24,16 +24,25 @@ queries: description: Maximum rows to return (-1 for unlimited). Mirrored from the caller-side ``max_rows`` for SQL pushdown. query: | SELECT - callstack, - COUNT(*) as alloc_count, - SUM(size) as total_size, - AVG(size) as avg_size, - MAX(size) as max_size - FROM {{ device_trace_table }} - WHERE callstack IS NOT NULL - GROUP BY callstack - HAVING alloc_count >= {{ min_count }} - AND total_size >= {{ min_size }} + COALESCE(c.callstack, '[missing callstack]') AS callstack, + grouped.alloc_count AS alloc_count, + grouped.total_size AS total_size, + grouped.avg_size AS avg_size, + grouped.max_size AS max_size + FROM ( + SELECT + callstackId, + COUNT(*) as alloc_count, + SUM(size) as total_size, + AVG(size) as avg_size, + MAX(size) as max_size + FROM {{ device_trace_table }} + WHERE callstackId IS NOT NULL + GROUP BY callstackId + HAVING alloc_count >= {{ min_count }} + AND total_size >= {{ min_size }} + ) AS grouped + LEFT JOIN callstack c ON c.id = grouped.callstackId ORDER BY total_size DESC, callstack ASC LIMIT {{ limit|int }} output_schema: diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 1b18c57..98b4f47 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -158,6 +158,36 @@ The upstream standalone split frontend is deliberately excluded, not deferred: split maps it to `unsafe pickle:` so allowlist rejection is distinct from corrupt input. `dump()` re-raises `UnsafePickleError` instead of returning false. +- On 2026-09-06, issue #113 deduplicated database callstacks. A new local + `snapshot/tools/adaptors/database/callstack.py` interns one id per distinct + callstack text, `trace_entry_` stores `callstackId` instead of an + inlined `callstack` text column, and a shared device-suffix-free `callstack` + table holds the text. `TraceEntry.callstack_frames()` exposes the frame + container that already backed `get_callstack()` so interning can key on + container identity; the interner holds a reference to every keyed container so + an `id()` key cannot be reused by an unrelated object. `CallstackInterner` is + not present in audited MemSnapDump @87ea207; it is a local addition. + `sqlite_meta.SqliteTable.get_insert_values_by_records` now resolves per-column + value maps once per batch instead of once per cell, and the default insert + batch size moved from 1000 to 10000 because import stages rows in a temporary + database and only publishes after a successful replay. Allocator replay, + synthetic event and block identities, totals, and callstack text are + unchanged: on `snapshot_with_multi_devices.pkl`, `snapshot_expandable.pkl`, + `snapshot_with_empty_cache_expandable.pkl`, + `snapshot_import_131k_sanitized.pickle`, and + `snapshot_import_628k_sanitized.pickle`, every trace and block row matches the + pre-change output after resolving `callstackId` through the join. This is a + local performance and schema change; audited upstream source mappings and + license terms are unchanged. +- On 2026-09-06, PR #114 follow-up corrected the preceding callstack entry's + implementation description: the interner keys on individual frame object + identities and retains one exemplar frame container per distinct frame-key + tuple, rather than keying and retaining every outer frame container. The + follow-up also propagates `_raw_frames` through NPU workspace adaptation, + maps legacy callstack-schema SQLite errors to an actionable re-import + message, and uses a left join with a missing-callstack placeholder in + statistical analysis. These are local fixes to the issue #113 runtime; + audited upstream source mappings and license terms remain unchanged. ## Migration toolchain diff --git a/src/pt_snap_cli/snapshot/base/entities.py b/src/pt_snap_cli/snapshot/base/entities.py index fe4f8ac..edc636f 100644 --- a/src/pt_snap_cli/snapshot/base/entities.py +++ b/src/pt_snap_cli/snapshot/base/entities.py @@ -98,8 +98,16 @@ def from_dict(cls, trace_dict: dict, _raw_frames: bool = False): ) return trace_entry + def callstack_frames(self) -> list[Frame] | list[dict]: + """Return the frame container that backs ``get_callstack()``. + + Callers that deduplicate callstacks key on this container's identity, + so the same object must be returned for repeated calls. + """ + return self._raw_frames if self._raw_frames is not None else self.frames + def get_callstack(self): - return _format_callstack(self._raw_frames if self._raw_frames is not None else self.frames) + return _format_callstack(self.callstack_frames()) def to_dict(self, include_id: bool = False): if self._origin and not include_id: diff --git a/src/pt_snap_cli/snapshot/simulate/simulate.py b/src/pt_snap_cli/snapshot/simulate/simulate.py index 6968d50..ed9e12c 100644 --- a/src/pt_snap_cli/snapshot/simulate/simulate.py +++ b/src/pt_snap_cli/snapshot/simulate/simulate.py @@ -127,6 +127,7 @@ def _adapt_workspace_snapshot(self): address=existed_seg.address, state=BlockState.ACTIVE_ALLOCATED, frames=existed_seg.frames, + _raw_frames=existed_seg._raw_frames, segment_ptr=existed_seg, ), 0, diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py index d635e29..c919b4d 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/__init__.py @@ -1,9 +1,11 @@ +from .callstack import CallstackInterner from .entity2record import block2record, event2record from .snapshot_db import BLOCK_STATE_VALUE_MAP, TRACE_ENTRY_ACTION_VALUE_MAP, SnapshotDb __all__ = [ "BLOCK_STATE_VALUE_MAP", "TRACE_ENTRY_ACTION_VALUE_MAP", + "CallstackInterner", "SnapshotDb", "block2record", "event2record", diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py new file mode 100644 index 0000000..04599fb --- /dev/null +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/callstack.py @@ -0,0 +1,52 @@ +from ....base import TraceEntry +from .defs import CallstackFieldDefs + + +class CallstackInterner: + """Assign one integer id per distinct callstack text. + + PyTorch snapshots repeat a small number of callstacks across a very large + number of events, and the pickle memo already shares the frame containers + behind them. Interning therefore turns the per-event callstack into an + integer reference and keeps a single text copy per distinct callstack. + + Lookups use the identities of the individual frames and only render text + on a miss. Distinct containers that render to the same text collapse onto + one id, so the id-to-text mapping stays one-to-one and grouping by id + matches grouping by text. + """ + + def __init__(self) -> None: + self._by_frames: dict[tuple[int, ...], tuple[object, int]] = {} + self._by_text: dict[str, int] = {} + self._texts: list[str] = [] + + def intern(self, event: TraceEntry) -> int: + """Return the callstack id for ``event``, rendering text only on a miss.""" + frames = event.callstack_frames() + frame_key = tuple(map(id, frames)) + by_frames = self._by_frames.get(frame_key) + if by_frames is not None: + callstack_id = by_frames[1] + else: + text = event.get_callstack() + existing = self._by_text.get(text) + if existing is None: + callstack_id = len(self._texts) + self._texts.append(text) + self._by_text[text] = callstack_id + else: + callstack_id = existing + self._by_frames[frame_key] = (frames, callstack_id) + + return callstack_id + + def records(self) -> list[dict]: + """Return one record per distinct callstack, ordered by id.""" + return [ + {CallstackFieldDefs.ID: callstack_id, CallstackFieldDefs.CALLSTACK: text} + for callstack_id, text in enumerate(self._texts) + ] + + def __len__(self) -> int: + return len(self._texts) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py index e6ad711..f18ad8d 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/defs.py @@ -7,6 +7,11 @@ class EventFieldDefs: ALLOCATED = "allocated" ACTIVE = "active" RESERVED = "reserved" + CALLSTACK_ID = "callstackId" + + +class CallstackFieldDefs: + ID = "id" CALLSTACK = "callstack" diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py index 8f9affc..9014054 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/entity2record.py @@ -1,4 +1,5 @@ from ....base import Block, TraceEntry +from .callstack import CallstackInterner from .defs import BlockFieldDefs, EventFieldDefs @@ -22,7 +23,14 @@ def get_timestamp_by_event_idx(event_idx: int) -> int: next_default_event_id = make_default_id_counter() -def event2record(event: TraceEntry, allocated: int = 0, active: int = 0, reserved: int = 0) -> dict: +def event2record( + event: TraceEntry, + allocated: int = 0, + active: int = 0, + reserved: int = 0, + *, + callstacks: CallstackInterner, +) -> dict: return { EventFieldDefs.ID: (event.idx if event.idx is not None else next_default_event_id()), EventFieldDefs.ACTION: event.action, @@ -32,7 +40,7 @@ def event2record(event: TraceEntry, allocated: int = 0, active: int = 0, reserve EventFieldDefs.ALLOCATED: allocated, EventFieldDefs.ACTIVE: active, EventFieldDefs.RESERVED: reserved, - EventFieldDefs.CALLSTACK: event.get_callstack(), + EventFieldDefs.CALLSTACK_ID: callstacks.intern(event), } diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py index a18dbd5..6fd43b2 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py @@ -1,5 +1,5 @@ from ....util.sqlite_meta import SqliteColumn, SqliteDB, SqliteTable -from .defs import BlockFieldDefs, EventFieldDefs +from .defs import BlockFieldDefs, CallstackFieldDefs, EventFieldDefs TRACE_ENTRY_ACTION_VALUE_MAP = { "segment_map": 0, @@ -32,7 +32,12 @@ SqliteColumn(name=EventFieldDefs.ALLOCATED, data_type=int), SqliteColumn(name=EventFieldDefs.ACTIVE, data_type=int), SqliteColumn(name=EventFieldDefs.RESERVED, data_type=int), - SqliteColumn(name=EventFieldDefs.CALLSTACK), + SqliteColumn(name=EventFieldDefs.CALLSTACK_ID, data_type=int), +] + +_CALLSTACK_TABLE_COLUMNS = [ + SqliteColumn(name=CallstackFieldDefs.ID, data_type=int, primary_key=True), + SqliteColumn(name=CallstackFieldDefs.CALLSTACK), ] _BLOCK_TABLE_COLUMNS = [ @@ -54,6 +59,9 @@ class SnapshotDb(SqliteDB): TRACE_ENTRY_TABLE_NAME = "trace_entry" BLOCK_TABLE_NAME = "block" + # Callstacks are shared across devices, so this table carries no device suffix. + # The name must stay outside the `trace_entry_%` space that device discovery scans. + CALLSTACK_TABLE_NAME = "callstack" def __init__(self, path: str): super().__init__(path, auto_create=True, with_dictionary_table=True) @@ -90,12 +98,21 @@ def create_block_table(self, device: int = 0): ): table.create_index(self.conn, column) + def create_callstack_table(self): + self.create_table( + SqliteTable(self.CALLSTACK_TABLE_NAME, _CALLSTACK_TABLE_COLUMNS), + delete_if_exists=True, + ) + def get_trace_entry_table(self, device: int = 0): return self.get_table_by_name(self.get_trace_table_name_by_device(device)) def get_block_table(self, device: int = 0): return self.get_table_by_name(self.get_block_table_name_by_device(device)) + def get_callstack_table(self): + return self.get_table_by_name(self.CALLSTACK_TABLE_NAME) + def _clear_old_tables(self): """ 旧版本中创建的db不带device前缀,如果通过旧db打开,需要清理旧表 diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index 3127cfd..380d15c 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -6,19 +6,27 @@ from ...representation import UnsafePickleError, load_snapshot_representation, replay_snapshot from ...simulate import AllocatorHooker, SimulateHooker from ...util.logger import get_logger, set_global_log_file -from .database import SnapshotDb, block2record, event2record +from .database import CallstackInterner, SnapshotDb, block2record, event2record dump_logger = get_logger("DatabaseDump") +# Import stages rows in a temporary database and only publishes it after a +# successful replay, so batches are sized for throughput rather than durability. +DEFAULT_INSERT_CACHE_SIZE = 10000 + + class SnapshotDbHandler: - def __init__(self, db_path: str, devices: list[int], insert_cache_size: int = 1000): + def __init__( + self, db_path: str, devices: list[int], insert_cache_size: int = DEFAULT_INSERT_CACHE_SIZE + ): self._closed = False self.db_path = db_path self.db = SnapshotDb(db_path) self._device_event_cache = {} self._device_block_cache = {} self._insert_cache_size = insert_cache_size + self.db.create_callstack_table() for device in devices: self._device_block_cache[device] = [] self._device_event_cache[device] = [] @@ -45,6 +53,13 @@ def flush(self, device: int = 0): if self._device_block_cache.get(device, None): self._do_insert_blocks(device) + def insert_callstacks(self, records: list[dict]): + """Write the interned callstack table once every device has replayed.""" + if not records: + return + self.db.get_callstack_table().insert_records(self.db.conn, records) + self.db.conn.commit() + def _do_insert_events(self, device: int = 0): if device not in self._device_event_cache: self._device_event_cache[device] = [] @@ -79,8 +94,12 @@ def __del__(self): class DumpEventHooker(SimulateHooker, AllocatorHooker): - def __init__(self, db_path: str, devices: list[int], dump_cache_size: int = 1000): + def __init__( + self, db_path: str, devices: list[int], dump_cache_size: int = DEFAULT_INSERT_CACHE_SIZE + ): self.db_handler = SnapshotDbHandler(db_path, devices, insert_cache_size=dump_cache_size) + # One interner for the whole dump so devices share identical callstacks. + self.callstacks = CallstackInterner() def post_undo_event( self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot @@ -107,6 +126,7 @@ def post_undo_event( allocated=current_snapshot.total_allocated, active=current_snapshot.total_activated, reserved=current_snapshot.total_reserved, + callstacks=self.callstacks, ), current_snapshot.device, ) @@ -120,6 +140,7 @@ def pre_undo_event(self, wait4undo_event: TraceEntry, current_snapshot: DeviceSn allocated=current_snapshot.total_allocated, active=current_snapshot.total_activated, reserved=current_snapshot.total_reserved, + callstacks=self.callstacks, ), current_snapshot.device, ) @@ -131,6 +152,9 @@ def post_replay_free_block(self, released_block: Block, current_snapshot: Device def flush(self, device: int = 0): self.db_handler.flush(device) + def flush_callstacks(self): + self.db_handler.insert_callstacks(self.callstacks.records()) + def close(self, *, commit: bool = True): self.db_handler.close(commit=commit) @@ -179,8 +203,10 @@ def dump(pickle_file: str, dump_file: str, device=None) -> bool: return False dump_logger.info(f"Finished dump the snapshot to database for device {device}.") hooker.flush(device) + hooker.flush_callstacks() dump_logger.info( - f"Successfully dump the snapshot to database for devices {need_dump_devices}." + f"Successfully dump the snapshot to database for devices {need_dump_devices}, " + f"{len(hooker.callstacks)} distinct callstacks." ) hooker.close() os.replace(temporary_path, dump_path) diff --git a/src/pt_snap_cli/snapshot/util/sqlite_meta.py b/src/pt_snap_cli/snapshot/util/sqlite_meta.py index cef2a82..e291d6d 100644 --- a/src/pt_snap_cli/snapshot/util/sqlite_meta.py +++ b/src/pt_snap_cli/snapshot/util/sqlite_meta.py @@ -336,9 +336,18 @@ def get_insert_values_by_records(self, records: list[dict[str, Any]]): """ if not records: return [] + # Resolve the per-column value maps once instead of per cell; batches of + # snapshot rows are large enough that the per-cell lookup dominated. + keys = list(records[0].keys()) + mapped = [(key, self._column_value_map.get(key)) for key in keys] + if not any(value_map for _, value_map in mapped): + return [tuple(record[key] for key in keys) for record in records] return [ - tuple(self._column_value_map.get(k, {}).get(r[k], r[k]) for k in records[0].keys()) - for r in records + tuple( + record[key] if value_map is None else value_map.get(record[key], record[key]) + for key, value_map in mapped + ) + for record in records ] diff --git a/tests/core/test_report_service.py b/tests/core/test_report_service.py index 558591e..8f01856 100644 --- a/tests/core/test_report_service.py +++ b/tests/core/test_report_service.py @@ -37,6 +37,12 @@ def report_db(tmp_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -51,17 +57,26 @@ def report_db(tmp_path: Path) -> Path: freeEventId INTEGER ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "block.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "block.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), ], ) @@ -123,6 +138,12 @@ def test_peak_memory_report_handles_empty_trace(tmp_path: Path) -> None: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) diff --git a/tests/core/test_split_service.py b/tests/core/test_split_service.py index 356c765..7206888 100644 --- a/tests/core/test_split_service.py +++ b/tests/core/test_split_service.py @@ -128,10 +128,13 @@ def _database_observation( ) assert replayed hooker.flush(device) + hooker.flush_callstacks() connection = hooker.db_handler.db.conn trace_rows = connection.execute( - f"SELECT id, action, address, size, stream, allocated, active, reserved, callstack " - f"FROM trace_entry_{device} ORDER BY id" + f"SELECT t.id, t.action, t.address, t.size, t.stream, " + f"t.allocated, t.active, t.reserved, c.callstack " + f"FROM trace_entry_{device} t JOIN callstack c ON c.id = t.callstackId " + f"ORDER BY t.id" ).fetchall() block_rows = connection.execute( f"SELECT id, address, size, requestedSize, state, allocEventId, freeEventId " diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 1849002..3663575 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -1,9 +1,12 @@ """Tests for query executor.""" +import sqlite3 + import pytest +from pt_snap_cli.context import Context from pt_snap_cli.query.config import QueryParameter, QueryTemplate -from pt_snap_cli.query.executor import QueryExecutor, TemplateRenderError +from pt_snap_cli.query.executor import QueryExecutionError, QueryExecutor, TemplateRenderError class TestQueryExecutor: @@ -337,6 +340,38 @@ def test_execute_on_all_devices_empty_device_list(self): results = executor.execute_on_all_devices("empty_test") assert results == {} + def test_legacy_callstack_schema_error_is_actionable(self, tmp_path): + db_path = tmp_path / "legacy.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dictionary (table_name TEXT)") + conn.execute("CREATE TABLE trace_entry_0 " "(id INTEGER, callstack TEXT)") + executor = QueryExecutor(Context(db_path)) + executor.register_template( + QueryTemplate( + name="legacy_callstack", + query="SELECT callstack FROM callstack", + ) + ) + + with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + executor.execute_template("legacy_callstack", device_id=0) + + def test_qualified_legacy_callstack_schema_error_is_actionable(self, tmp_path): + db_path = tmp_path / "legacy-qualified.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dictionary (table_name TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER)") + executor = QueryExecutor(Context(db_path)) + executor.register_template( + QueryTemplate( + name="qualified_legacy_callstack", + query="SELECT t.callstackId FROM trace_entry_0 t", + ) + ) + + with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + executor.execute_template("qualified_legacy_callstack", device_id=0) + def _make_mock_context(device_ids: list[int]): """Create a mock Context with the given device IDs.""" diff --git a/tests/query/test_peak_memory_templates.py b/tests/query/test_peak_memory_templates.py index 6a012a9..61101a5 100644 --- a/tests/query/test_peak_memory_templates.py +++ b/tests/query/test_peak_memory_templates.py @@ -43,9 +43,24 @@ def peak_memory_db(tmp_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "freed.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.execute(""" CREATE TABLE block_0 ( id INTEGER PRIMARY KEY, @@ -61,14 +76,14 @@ def peak_memory_db(tmp_path: Path) -> Path: conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "freed.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), ], ) @@ -308,16 +323,20 @@ def test_active_memory_callstack_at_event_keeps_special_groups_beyond_top_n( peak_memory_db: Path, ) -> None: conn = sqlite3.connect(str(peak_memory_db)) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [(10, "a.py:1"), (11, "b.py:2"), (12, "c.py:3")], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (-101, 2, 0x11000, 20000, 0, 0, 0, 0, "a.py:1"), - (-102, 2, 0x12000, 18000, 0, 0, 0, 0, "b.py:2"), - (-103, 2, 0x13000, 16000, 0, 0, 0, 0, "c.py:3"), + (-101, 2, 0x11000, 20000, 0, 0, 0, 0, 10), + (-102, 2, 0x12000, 18000, 0, 0, 0, 0, 11), + (-103, 2, 0x13000, 16000, 0, 0, 0, 0, 12), ], ) conn.executemany( diff --git a/tests/query/test_query_max_rows_pushdown.py b/tests/query/test_query_max_rows_pushdown.py index d8fb6f1..80e4da2 100644 --- a/tests/query/test_query_max_rows_pushdown.py +++ b/tests/query/test_query_max_rows_pushdown.py @@ -82,6 +82,12 @@ def callstack_db(tmp_path: Path) -> Path: CREATE TABLE trace_entry_0 ( id INTEGER PRIMARY KEY, size INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -89,7 +95,7 @@ def callstack_db(tmp_path: Path) -> Path: # actually has something to drop: # * 4 callstacks have 10+ entries (pass min_count=5) # * 2 callstacks have 4 entries each (fail min_count=5) - rows: list[tuple[int, int, str | None]] = [] + rows: list[tuple[int, int, int]] = [] callstacks = [ ("train.py:10", 12), ("train.py:20", 11), @@ -98,11 +104,15 @@ def callstack_db(tmp_path: Path) -> Path: ("noise.py:50", 4), ("noise.py:60", 4), ] - for cs, count in callstacks: + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [(index, text) for index, (text, _) in enumerate(callstacks)], + ) + for index, (_, count) in enumerate(callstacks): for _ in range(count): - rows.append((len(rows) + 1, 100, cs)) + rows.append((len(rows) + 1, 100, index)) conn.executemany( - "INSERT INTO trace_entry_0 (id, size, callstack) VALUES (?, ?, ?)", + "INSERT INTO trace_entry_0 (id, size, callstackId) VALUES (?, ?, ?)", rows, ) conn.commit() @@ -171,3 +181,19 @@ def test_min_size_filter_combines_with_limit(self, callstack_db: Path) -> None: {"min_size": 0, "min_count": 1, "limit": 2}, ) assert len(rows) == 2 + + def test_missing_callstack_uses_placeholder(self, callstack_db: Path) -> None: + with sqlite3.connect(callstack_db) as conn: + conn.execute( + "INSERT INTO trace_entry_0 (id, size, callstackId) VALUES (?, ?, ?)", + (100, 1, 99), + ) + conn.commit() + + rows = _execute_template( + callstack_db, + "callstack_analysis", + {"min_count": 1, "min_size": 0}, + ) + + assert rows[-1]["callstack"] == "[missing callstack]" diff --git a/tests/skills/harness/fixtures.py b/tests/skills/harness/fixtures.py index 07f2bb6..2f93090 100644 --- a/tests/skills/harness/fixtures.py +++ b/tests/skills/harness/fixtures.py @@ -9,6 +9,8 @@ from .descriptors import DescriptorError +# Fixture YAML keeps authoring the callstack as text per trace row; the builder +# interns it into the shared `callstack` table that SnapshotDB now emits. TRACE_COLUMNS = ( "id", "action", @@ -20,6 +22,8 @@ "reserved", "callstack", ) +_TRACE_TABLE_COLUMNS = (*TRACE_COLUMNS[:-1], "callstackId") +_CALLSTACK_INDEX = TRACE_COLUMNS.index("callstack") BLOCK_COLUMNS = ( "id", "address", @@ -77,6 +81,8 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo connection.execute( "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" ) + connection.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + callstack_ids: dict[str, int] = {} for raw_device_id, device in definition["devices"].items(): device_id = int(raw_device_id) connection.execute(f""" @@ -89,7 +95,7 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo allocated INTEGER, active INTEGER, reserved INTEGER, - callstack TEXT + callstackId INTEGER ) """) connection.execute(f""" @@ -105,9 +111,24 @@ def build_snapshotdb(definition_path: Path, output_path: Path, *, read_only: boo """) trace_rows = _rows(device["trace"], TRACE_COLUMNS, f"device {device_id}.trace") block_rows = _rows(device["blocks"], BLOCK_COLUMNS, f"device {device_id}.blocks") - placeholders = ", ".join("?" for _ in TRACE_COLUMNS) + interned_rows: list[tuple[Any, ...]] = [] + for row in trace_rows: + callstack = row[_CALLSTACK_INDEX] + if callstack is None: + callstack_id = None + else: + callstack_id = callstack_ids.get(callstack) + if callstack_id is None: + callstack_id = len(callstack_ids) + callstack_ids[callstack] = callstack_id + connection.execute( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + (callstack_id, callstack), + ) + interned_rows.append((*row[:_CALLSTACK_INDEX], callstack_id)) + placeholders = ", ".join("?" for _ in _TRACE_TABLE_COLUMNS) connection.executemany( - f"INSERT INTO trace_entry_{device_id} VALUES ({placeholders})", trace_rows + f"INSERT INTO trace_entry_{device_id} VALUES ({placeholders})", interned_rows ) placeholders = ", ".join("?" for _ in BLOCK_COLUMNS) connection.executemany( diff --git a/tests/snapshot/golden_observations.py b/tests/snapshot/golden_observations.py index 7cc11b8..e2e740d 100644 --- a/tests/snapshot/golden_observations.py +++ b/tests/snapshot/golden_observations.py @@ -126,7 +126,12 @@ (5, "allocated", "INTEGER", 0, None, 0), (6, "active", "INTEGER", 0, None, 0), (7, "reserved", "INTEGER", 0, None, 0), - (8, "callstack", "TEXT", 0, None, 0), + (8, "callstackId", "INTEGER", 0, None, 0), +] + +CALLSTACK_SCHEMA = [ + (0, "id", "INTEGER", 0, None, 1), + (1, "callstack", "TEXT", 0, None, 0), ] BLOCK_SCHEMA = [ diff --git a/tests/snapshot/test_callstack_interner.py b/tests/snapshot/test_callstack_interner.py new file mode 100644 index 0000000..be917e0 --- /dev/null +++ b/tests/snapshot/test_callstack_interner.py @@ -0,0 +1,122 @@ +import gc +import weakref + +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner + +INNER = {"filename": "inner.py", "line": 10, "name": "inner"} +OUTER = {"filename": "outer.py", "line": 20, "name": "outer"} +EXPECTED_TEXT = "outer.py:20 outer\ninner.py:10 inner" + + +class WeakrefableFrames(list): + """List-shaped frame container used to test exemplar retention.""" + + +def _raw_event(frames: list[dict], idx: int = 0) -> TraceEntry: + return TraceEntry.from_dict( + {"action": "alloc", "addr": 0x1000, "size": 16, "stream": 0, "id": idx, "frames": frames}, + _raw_frames=True, + ) + + +def test_interned_text_matches_get_callstack(): + interner = CallstackInterner() + event = _raw_event([INNER, OUTER]) + + assert interner.intern(event) == 0 + assert interner.records() == [{"id": 0, "callstack": EXPECTED_TEXT}] + assert interner.records()[0]["callstack"] == event.get_callstack() + + +def test_shared_frame_container_reuses_one_id(): + interner = CallstackInterner() + frames = [INNER, OUTER] + + first = interner.intern(_raw_event(frames, idx=1)) + second = interner.intern(_raw_event(frames, idx=2)) + + assert first == second == 0 + assert len(interner) == 1 + + +def test_distinct_containers_with_equal_frames_reuse_one_id(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([INNER, OUTER], idx=1)) + second = interner.intern(_raw_event([dict(INNER), dict(OUTER)], idx=2)) + + assert first == second == 0 + assert len(interner) == 1 + + +def test_different_callstacks_get_different_ids(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([INNER], idx=1)) + second = interner.intern(_raw_event([OUTER], idx=2)) + + assert {first, second} == {0, 1} + assert len(interner) == 2 + assert [record["callstack"] for record in interner.records()] == [ + "inner.py:10 inner", + "outer.py:20 outer", + ] + + +def test_empty_frames_intern_to_empty_text(): + interner = CallstackInterner() + + first = interner.intern(_raw_event([], idx=1)) + second = interner.intern(_raw_event([], idx=2)) + + assert first == second == 0 + assert interner.records() == [{"id": 0, "callstack": ""}] + + +def test_distinct_frame_identities_do_not_produce_a_false_hit(): + """Distinct frame identities must not inherit an unrelated callstack id.""" + interner = CallstackInterner() + + for frames in ([INNER], [OUTER]): + event = _raw_event(frames) + interner.intern(event) + + assert len(interner) == 2 + assert [record["callstack"] for record in interner.records()] == [ + "inner.py:10 inner", + "outer.py:20 outer", + ] + + +def test_frame_container_exemplar_stays_alive_for_id_reuse_safety(): + interner = CallstackInterner() + frames = WeakrefableFrames([INNER]) + frames_ref = weakref.ref(frames) + + interner.intern(_raw_event(frames)) + del frames + gc.collect() + + assert frames_ref() is not None + + +def test_eager_frame_objects_intern_to_the_same_text_as_raw_frames(): + interner = CallstackInterner() + raw = _raw_event([INNER, OUTER], idx=1) + eager = TraceEntry.from_dict( + { + "action": "alloc", + "addr": 0x1000, + "size": 16, + "stream": 0, + "id": 2, + "frames": [INNER, OUTER], + }, + ) + + raw_id = interner.intern(raw) + eager_id = interner.intern(eager) + + assert raw_id == eager_id == 0 + assert len(interner) == 1 diff --git a/tests/snapshot/test_entity2record.py b/tests/snapshot/test_entity2record.py index a2e9e9f..15ae9d0 100644 --- a/tests/snapshot/test_entity2record.py +++ b/tests/snapshot/test_entity2record.py @@ -1,4 +1,5 @@ from pt_snap_cli.snapshot.base import Block, Frame, TraceEntry +from pt_snap_cli.snapshot.tools.adaptors.database.callstack import CallstackInterner from pt_snap_cli.snapshot.tools.adaptors.database.defs import ( BlockFieldDefs, EventFieldDefs, @@ -26,7 +27,7 @@ def test_get_timestamp_by_event_idx_handles_none(): def test_event2record_uses_generated_default_id_when_missing(): event = TraceEntry(action="alloc", addr=0x1000, size=16, stream=0, idx=None) - record = event2record(event, allocated=1, active=2, reserved=3) + record = event2record(event, allocated=1, active=2, reserved=3, callstacks=CallstackInterner()) assert record[EventFieldDefs.ID] < 0 assert record[EventFieldDefs.ALLOCATED] == 1 @@ -65,7 +66,10 @@ def test_records_preserve_totals_state_ids_and_callstack_serialization(): free_event_idx=12, ) - event_record = event2record(event, allocated=100, active=120, reserved=256) + callstacks = CallstackInterner() + event_record = event2record( + event, allocated=100, active=120, reserved=256, callstacks=callstacks + ) block_record = block2record(block) assert event_record == { @@ -77,8 +81,9 @@ def test_records_preserve_totals_state_ids_and_callstack_serialization(): "allocated": 100, "active": 120, "reserved": 256, - "callstack": "outer.py:20 outer\ninner.py:10 inner", + "callstackId": 0, } + assert callstacks.records() == [{"id": 0, "callstack": "outer.py:20 outer\ninner.py:10 inner"}] assert block_record == { "id": 9, "address": 0x2000, diff --git a/tests/snapshot/test_simulate.py b/tests/snapshot/test_simulate.py index bb4171a..7eb364e 100644 --- a/tests/snapshot/test_simulate.py +++ b/tests/snapshot/test_simulate.py @@ -265,6 +265,18 @@ def test_simulate_device_snapshot_sets_workspace_flag_from_first_event(): assert device.total_reserved == 4096 +def test_raw_workspace_adaptation_preserves_segment_frames(): + frame = {"filename": "workspace.py", "line": 7, "name": "reserve"} + data = make_torch_npu_workspace_snapshot() + data["segments"][0]["frames"] = [frame] + + snapshot = SimulateDeviceSnapshot(data, 0, _raw_frames=True) + + block = snapshot.device_snapshot.segments[0].blocks[0] + assert block._raw_frames == [frame] + assert block.to_dict()["frames"] == [frame] + + def test_simulate_register_unregister_hookers_and_allocator_hookers(): snapshot = SimulateDeviceSnapshot(make_snapshot_dict(), 0) hooker = Hooker() diff --git a/tests/snapshot/test_snapshot2db_runtime.py b/tests/snapshot/test_snapshot2db_runtime.py index 8eb6f36..0c17a7e 100644 --- a/tests/snapshot/test_snapshot2db_runtime.py +++ b/tests/snapshot/test_snapshot2db_runtime.py @@ -18,6 +18,7 @@ ACTION_VALUE_MAP, BLOCK_SCHEMA, BLOCK_STATE_VALUE_MAP, + CALLSTACK_SCHEMA, DATABASE_GOLDEN, TRACE_SCHEMA, ) @@ -128,8 +129,10 @@ def dump_rows(database, *, eager): assert snapshot2db.dump(snapshot, database, 0) with sqlite3.connect(database) as connection: return connection.execute( - "SELECT id, action, address, size, stream, allocated, active, reserved, callstack " - "FROM trace_entry_0 ORDER BY id" + "SELECT t.id, t.action, t.address, t.size, t.stream, " + "t.allocated, t.active, t.reserved, c.callstack " + "FROM trace_entry_0 t JOIN callstack c ON c.id = t.callstackId " + "ORDER BY t.id" ).fetchall() eager_rows = dump_rows(tmp_path / "eager.db", eager=True) @@ -322,7 +325,14 @@ def test_dump_all_multiple_device_snapshot(dump_database): "SELECT name FROM sqlite_master WHERE type = 'table'" ).fetchall() } - assert tables == {"dictionary", "trace_entry_0", "block_0", "trace_entry_1", "block_1"} + assert tables == { + "callstack", + "dictionary", + "trace_entry_0", + "block_0", + "trace_entry_1", + "block_1", + } def negative_id_observation(connection, table): @@ -360,7 +370,9 @@ def test_database_matches_pre_relocation_golden(dump_database, fixture_name, dev "WHERE id >= 0 ORDER BY id DESC LIMIT 1" ).fetchone() callstacks = connection.execute( - f"SELECT COUNT(*) FROM {trace_table} WHERE callstack != ''" + f"SELECT COUNT(*) FROM {trace_table} t " + "JOIN callstack c ON c.id = t.callstackId " + "WHERE c.callstack != ''" ).fetchone()[0] assert rows == golden["rows"] @@ -384,6 +396,7 @@ def test_database_schema_and_action_state_mapping_are_exact(dump_database): assert RUNTIME_ACTION_VALUE_MAP == ACTION_VALUE_MAP assert RUNTIME_BLOCK_STATE_VALUE_MAP == BLOCK_STATE_VALUE_MAP with sqlite3.connect(database) as connection: + assert connection.execute("PRAGMA table_info(callstack)").fetchall() == CALLSTACK_SCHEMA for device in (0, 1): assert ( connection.execute(f"PRAGMA table_info(trace_entry_{device})").fetchall() @@ -427,9 +440,29 @@ def test_callstack_serialization_preserves_frame_order(dump_database): _, database = dump_database("snapshot_1768383987920985470.pkl", 0) with sqlite3.connect(database) as connection: callstack = connection.execute( - "SELECT callstack FROM trace_entry_0 WHERE callstack LIKE '%\n%' LIMIT 1" + "SELECT c.callstack FROM trace_entry_0 t " + "JOIN callstack c ON c.id = t.callstackId " + "WHERE c.callstack LIKE '%\n%' LIMIT 1" ).fetchone()[0] lines = callstack.splitlines() assert len(lines) >= 2 assert all(":" in line and " " in line for line in lines) + + +def test_callstack_table_is_deduplicated_and_covers_every_event(dump_database): + _, database = dump_database("snapshot_1768383987920985470.pkl", 0) + with sqlite3.connect(database) as connection: + rows, distinct_ids, distinct_texts = connection.execute( + "SELECT COUNT(*), COUNT(DISTINCT id), COUNT(DISTINCT callstack) FROM callstack" + ).fetchone() + events, resolved = connection.execute( + "SELECT COUNT(*), COUNT(c.id) FROM trace_entry_0 t " + "LEFT JOIN callstack c ON c.id = t.callstackId" + ).fetchone() + + # One row per distinct text keeps grouping by id equivalent to grouping by text. + assert rows == distinct_ids == distinct_texts + # Every event resolves to a callstack row, and interning actually collapses rows. + assert resolved == events + assert rows < events diff --git a/tests/test_cli.py b/tests/test_cli.py index d83d68b..dbf3012 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -97,6 +97,12 @@ def create_peak_report_db(db_path: Path) -> Path: allocated INTEGER, active INTEGER, reserved INTEGER, + callstackId INTEGER + ) + """) + conn.execute(""" + CREATE TABLE callstack ( + id INTEGER PRIMARY KEY, callstack TEXT ) """) @@ -111,17 +117,26 @@ def create_peak_report_db(db_path: Path) -> Path: freeEventId INTEGER ) """) + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + [ + (0, "train.py:10"), + (1, "block.py:20"), + (2, "free.py:30"), + (3, "after.py:40"), + ], + ) conn.executemany( """ INSERT INTO trace_entry_0 - (id, action, address, size, stream, allocated, active, reserved, callstack) + (id, action, address, size, stream, allocated, active, reserved, callstackId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), - (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "block.py:20"), - (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), - (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, 0), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, 1), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, 2), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, 3), ], ) conn.executemany(