From de1b385cb5b00abd0ed05d944a86413d63c0dbd2 Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 17 Sep 2026 22:55:11 +0800 Subject: [PATCH 1/2] feat: read v1 and v2 SnapshotDB callstack layouts without migrating Legacy inline-callstack databases became unusable for callstack templates after schema v2. Detect the layout read-only and select matching SQL so focus and query keep working on both schemas. Fixes #134 --- AGENTS.md | 7 + docs/en/database.md | 52 ++- docs/en/focus-management.md | 2 +- docs/en/querying.md | 4 + docs/en/snapshot-analyzer-api.md | 2 + docs/zh/database.md | 45 ++- docs/zh/focus-management.md | 2 +- docs/zh/querying.md | 4 + docs/zh/snapshot-analyzer-api.md | 2 + src/pt_snap_cli/AGENTS.md | 3 + src/pt_snap_cli/api.py | 2 + src/pt_snap_cli/cli.py | 9 + src/pt_snap_cli/context.py | 132 +++++++ src/pt_snap_cli/core/focus_service.py | 11 +- src/pt_snap_cli/core/models.py | 1 + src/pt_snap_cli/mcp/server.py | 3 + src/pt_snap_cli/query/AGENTS.md | 9 +- src/pt_snap_cli/query/config.py | 54 ++- src/pt_snap_cli/query/executor.py | 38 +- src/pt_snap_cli/query/templates/AGENTS.md | 4 + .../query/templates/basic/event.yaml | 85 +++-- .../active_memory_callstack_at_event.yaml | 224 ++++++++---- .../statistical/callstack_analysis.yaml | 45 ++- tests/core/test_context_cache.py | 29 ++ tests/core/test_focus_service.py | 19 + tests/core/test_report_service.py | 63 ++++ tests/query/AGENTS.md | 1 + tests/query/test_callstack_schema_compat.py | 343 ++++++++++++++++++ tests/query/test_config.py | 25 ++ tests/query/test_executor.py | 22 ++ tests/test_api.py | 13 +- tests/test_cli.py | 16 + tests/test_context.py | 29 ++ tests/test_contract_cli_mcp.py | 45 +++ tests/test_mcp_server.py | 1 + 35 files changed, 1210 insertions(+), 136 deletions(-) create mode 100644 tests/query/test_callstack_schema_compat.py diff --git a/AGENTS.md b/AGENTS.md index 1b31db2..d408104 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,6 +81,13 @@ templates under category subdirectories are included by `query/registry.py` recursively loads templates into the singleton registry; `query/executor.py` renders with Jinja2 `StrictUndefined` and injects device table names before executing through `Context`. +- Templates that differ by SnapshotDB callstack layout declare + `query_variants.v1` / `query_variants.v2` in the same YAML entry and share + description, parameters, and `output_schema`. Unaffected templates keep a + single `query`. `Context` detects the layout from read-only column/table + checks (with `pt_snap_metadata` as an auxiliary check) and + `QueryExecutor` selects the matching SQL. Do not persist layout in + `focus.json` or migrate databases on open. - When template metadata or behavior changes, review the YAML, config, registry, executor, `core/query_service.py`, CLI/MCP presentation, and focused tests together. diff --git a/docs/en/database.md b/docs/en/database.md index 3a2ff03..a737427 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -48,11 +48,16 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -Legacy or externally generated compatible databases without this table remain queryable for -templates that do not require the deduplicated callstack schema. Callstack templates require a -database produced by the current importer; otherwise the query reports that the snapshot must be -re-imported. Their metadata status is reported as unavailable, and a later import rebuilds them -once before reuse. +Legacy or externally generated compatible databases without this table remain +queryable. `pt-snap` detects the callstack layout from table columns (and treats +`pt_snap_metadata.import_format_version` as an auxiliary check, not the only +source of truth). Callstack templates use the matching SQL automatically. + +Opening, focusing, and querying a database never writes the `.db` file and never +migrates it in place. Import cache invalidation still rebuilds a destination +when the current importer's format version does not match; that is separate from +read compatibility. `pt-snap metadata` reports missing metadata as unavailable, +and a later import of the same pickle rebuilds the database once before reuse. ### Query workflow @@ -67,7 +72,11 @@ pt-snap query --template-use block --params '{"min_size": 1048576}' ``` Use `pt-snap query --list` and `pt-snap query --template-info ` to inspect -the supported query surface. See [Querying](querying.md) for the complete workflow. +the supported query surface. `event`, `callstack_analysis`, and +`active_memory_callstack_at_event` work on both the current deduplicated +callstack schema and older inline-text databases; see +[Callstack layout compatibility](database.md#callstack-layout-compatibility). +See [Querying](querying.md) for the complete workflow. --- @@ -311,6 +320,37 @@ CREATE TABLE callstack ( | Scope | Shared by every device in the database; the table has no device suffix | | Coverage | Every event produced by import resolves to a row; `callstackId` is `NULL` only in externally generated databases | +Current `pt-snap import` writes this v2 layout (`import_format_version = 2`). +Databases produced before callstack deduplication store inline `callstack` TEXT +on each `trace_entry_` table and have no shared `callstack` table +(`import_format_version = 1` when metadata is present). + +### Callstack layout compatibility + +`pt-snap` reads both layouts without rewriting the file: + +| Layout | How to recognize it | Callstack text | +|--------|---------------------|----------------| +| v2 (current import) | `trace_entry_.callstackId` plus shared `callstack` | Join `callstack.id` | +| v1 (legacy) | `trace_entry_.callstack` TEXT, no `callstackId`, no shared table | Read the column directly | + +`event`, `callstack_analysis`, and `active_memory_callstack_at_event` keep one +template name, parameter set, and output schema. The query engine selects v1 or +v2 SQL from the detected layout. Other templates do not depend on this split. + +Detection is read-only: `focus`, `query`, reports, the Python API, and MCP all +open the database with SQLite `mode=ro`. Layout is not stored in +`.pt-snap/focus.json`. Conflicting columns, mixed devices, a damaged +`callstack` table, or metadata that disagrees with the physical schema raise a +schema error instead of guessing. There is no implicit v1→v2 migration. + +```bash +pt-snap focus legacy.db +# Callstack layout: v1 (inline text) +pt-snap query --template-use event +pt-snap query --template-use callstack_analysis +``` + ### 5. pt_snap_metadata — Import Metadata Table First-party imports create one metadata row for provenance and cache diff --git a/docs/en/focus-management.md b/docs/en/focus-management.md index f1a9716..42e0461 100644 --- a/docs/en/focus-management.md +++ b/docs/en/focus-management.md @@ -56,7 +56,7 @@ its device, becomes effective again only after the session override is unset. pt-snap focus ``` -This shows the resolved database path, device ID, and where they came from (project focus, session env, or global config). +This shows the resolved database path, device ID, and where they came from (project focus, session env, or global config). When the database uses a recognized callstack schema, focus also prints `Callstack layout: v1 (inline text)` or `v2 (deduplicated)`. That label is detected read-only from the database; it is not stored in `.pt-snap/focus.json`. ## Override Focus diff --git a/docs/en/querying.md b/docs/en/querying.md index 830a551..14346d9 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -39,6 +39,10 @@ Raw data lookup. | `event` | Query memory events with flexible field filters | | `allocation` | Memory allocation timeline (id, allocated, active, reserved) | +`event`, `callstack_analysis`, and `active_memory_callstack_at_event` share one +public contract and select v1 or v2 SQL from the database layout. See +[Callstack layout compatibility](database.md#callstack-layout-compatibility). + ### Statistical Queries Aggregation and analysis. diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index 416d3ec..5ac4bd3 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -31,6 +31,7 @@ state = analyzer.get_focus() print(state.db_path) print(state.device_id) print(state.available_devices) +print(state.callstack_layout) print(state.source) state = analyzer.set_focus( @@ -61,6 +62,7 @@ explicit `db_path`, a validated device-only analyzer override is reported by | `device_id` | Analyzer device override, or the device attached to the resolved focus | | `source` | Resolution source such as `explicit`, `env`, `project`, `global`, or `none` | | `available_devices` | Device IDs discovered from `trace_entry_` tables | +| `callstack_layout` | `"v1"` for inline callstack text, `"v2"` for deduplicated `callstackId`, or `None` when unrecognized | See [Focus Management](focus-management.md) for the complete resolution and persistence model. diff --git a/docs/zh/database.md b/docs/zh/database.md index 8e43f08..a0871c9 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -44,9 +44,14 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -旧版或外部生成且结构兼容的 DB,仍可用于不依赖去重调用栈结构的查询。调用栈相关模板要求 -数据库由当前导入器生成;否则会提示需要重新导入快照。若没有 metadata,查询会返回 -unavailable,下一次执行对应 pickle 导入时会重建一次。 +旧版或外部生成且结构兼容的 DB 仍可查询。`pt-snap` 根据表列识别调用栈布局,并把 +`pt_snap_metadata.import_format_version` 当作辅助校验,而不是唯一依据。调用栈相关 +模板会自动选择匹配的 SQL。 + +打开、focus 和查询都不会写入 `.db` 文件,也不会原地迁移 schema。导入缓存仍会在 +当前导入器格式版本不匹配时重建目标库,这与读取兼容是分开的。没有 metadata 时, +`pt-snap metadata` 会报告 unavailable;之后对同一 pickle 再导入会先重建一次再参与 +缓存复用。 ### 查询工作流 @@ -60,7 +65,9 @@ pt-snap query --template-use block --params '{"min_size": 1048576}' ``` 使用 `pt-snap query --list` 和 `pt-snap query --template-info ` 查看受支持的 -查询入口。完整流程见[运行查询](querying.md)。 +查询入口。`event`、`callstack_analysis` 和 `active_memory_callstack_at_event` +同时支持当前去重调用栈结构和旧版内联文本库;见 +[调用栈布局兼容](database.md#调用栈布局兼容)。完整流程见[运行查询](querying.md)。 --- @@ -301,6 +308,36 @@ CREATE TABLE callstack ( | 作用域 | 数据库内所有设备共享,表名不带设备后缀 | | 覆盖度 | 导入产生的每个事件都能解析到一行;只有外部生成的数据库才可能出现 `callstackId` 为 `NULL` | +当前 `pt-snap import` 写入的是这种 v2 布局(`import_format_version = 2`)。 +调用栈去重之前生成的数据库在每张 `trace_entry_` 表中内联存储 +`callstack` TEXT,没有共享 `callstack` 表(有 metadata 时为 +`import_format_version = 1`)。 + +### 调用栈布局兼容 + +`pt-snap` 可以读取两种布局,且不会改写数据库文件: + +| 布局 | 识别方式 | 调用栈文本 | +|------|----------|------------| +| v2(当前导入) | `trace_entry_.callstackId` 加共享 `callstack` 表 | join `callstack.id` | +| v1(旧版) | `trace_entry_.callstack` TEXT,无 `callstackId`,无共享表 | 直接读该列 | + +`event`、`callstack_analysis` 和 `active_memory_callstack_at_event` 保持相同的 +模板名、参数和输出契约。查询引擎按识别到的布局选择 v1 或 v2 SQL。其他模板不依赖 +这次拆分。 + +识别过程是只读的:`focus`、`query`、报告、Python API 和 MCP 都以 SQLite +`mode=ro` 打开数据库。布局不会写入 `.pt-snap/focus.json`。列冲突、设备布局不一致、 +损坏的 `callstack` 表,或 metadata 与物理结构不一致时会明确报错,而不是猜测。 +默认不会做 v1→v2 迁移。 + +```bash +pt-snap focus legacy.db +# Callstack layout: v1 (inline text) +pt-snap query --template-use event +pt-snap query --template-use callstack_analysis +``` + ### 5. pt_snap_metadata — 导入 Metadata 表 第一方导入会创建一条 metadata 记录,用于来源追踪和缓存兼容性检查: diff --git a/docs/zh/focus-management.md b/docs/zh/focus-management.md index d1180f4..35fb2b2 100644 --- a/docs/zh/focus-management.md +++ b/docs/zh/focus-management.md @@ -55,7 +55,7 @@ Session focus 只导出 `PT_SNAP_DB_PATH`,因此 `--session` 不能与 `--devi pt-snap focus ``` -显示已解析的数据库路径、设备 ID 及其来源(项目 focus、session 环境变量或全局配置)。 +显示已解析的数据库路径、设备 ID 及其来源(项目 focus、session 环境变量或全局配置)。若数据库使用可识别的调用栈 schema,还会打印 `Callstack layout: v1 (inline text)` 或 `v2 (deduplicated)`。该信息从数据库只读识别,不会写入 `.pt-snap/focus.json`。 ## 覆盖焦点 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index bf45551..c564c13 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -39,6 +39,10 @@ pt-snap query [DB_PATH] [--template-use ] [--params ] \ | `event` | 灵活字段过滤的内存事件查询 | | `allocation` | 内存分配时间线(id, allocated, active, reserved) | +`event`、`callstack_analysis` 和 `active_memory_callstack_at_event` 对外保持同一契约, +并根据数据库布局选择 v1 或 v2 SQL。见 +[调用栈布局兼容](database.md#调用栈布局兼容)。 + ### Statistical Queries 聚合分析。 diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index 1a66193..c596d33 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -29,6 +29,7 @@ state = analyzer.get_focus() print(state.db_path) print(state.device_id) print(state.available_devices) +print(state.callstack_layout) print(state.source) state = analyzer.set_focus( @@ -56,6 +57,7 @@ focus 中的设备。没有显式 `db_path` 时,已验证的 analyzer 设备 | `device_id` | analyzer 设备覆盖;未设置覆盖时为已解析 focus 所附带的设备 | | `source` | `explicit`、`env`、`project`、`global`、`none` 等解析来源 | | `available_devices` | 从 `trace_entry_` 表发现的设备 ID | +| `callstack_layout` | 内联调用栈文本为 `"v1"`,去重 `callstackId` 为 `"v2"`,无法识别时为 `None` | 完整解析和持久化模型见 [Focus 管理](focus-management.md)。 diff --git a/src/pt_snap_cli/AGENTS.md b/src/pt_snap_cli/AGENTS.md index e8ebff6..bb00d9b 100644 --- a/src/pt_snap_cli/AGENTS.md +++ b/src/pt_snap_cli/AGENTS.md @@ -35,6 +35,9 @@ - Preserve focus precedence: explicit CLI/API path, `PT_SNAP_DB_PATH`, nearest project `.pt-snap/focus.json`, then legacy global config. - Keep SQLite access read-only for analysis paths. - When adding or changing a repository skill under `skills/`, copy the same `SKILL.md` into `bundled_skills//`. +- `Context.callstack_layout` is `v1`, `v2`, or `None`; conflicting layouts raise + `SchemaVersionError` during construction. Query SQL variants are selected by + `QueryExecutor`, not by adapters. ### Testing Requirements - CLI changes should update/run `tests/test_cli.py` and any affected config/focus tests. diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index f16127d..03eb61e 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -27,6 +27,7 @@ class FocusState: device_id: int | None source: str available_devices: list[int] + callstack_layout: str | None = None class SnapshotAnalyzer: @@ -71,6 +72,7 @@ def get_focus(self) -> FocusState: device_id=self._device_id if self._device_id is not None else state.device_id, source=state.source, available_devices=state.available_devices, + callstack_layout=state.callstack_layout, ) def set_focus(self, db_path: str | None = None, device_id: int | None = None) -> FocusState: diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 6160dce..3854e62 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -151,6 +151,7 @@ def focus_database( typer.echo(f"Focus file: {state.focus_file}") if state.device_id is not None: typer.echo(f"Focused device: {state.device_id}") + _echo_callstack_layout(state.callstack_layout) if not state.db_path.exists(): typer.secho("Warning: Database file does not exist!", fg=typer.colors.YELLOW) else: @@ -203,6 +204,7 @@ def focus_database( typer.echo(f"Available devices: {', '.join(map(str, state.available_devices))}") else: typer.echo("No devices found in database.") + _echo_callstack_layout(state.callstack_layout) except ( DatabaseMissingError, DatabaseSchemaError, @@ -971,6 +973,13 @@ def show_config( typer.echo(f" {key}: {value}") +def _echo_callstack_layout(layout: str | None) -> None: + if layout == "v1": + typer.echo("Callstack layout: v1 (inline text)") + elif layout == "v2": + typer.echo("Callstack layout: v2 (deduplicated)") + + def _error(message: str) -> NoReturn: typer.secho(f"Error: {message}", fg=typer.colors.RED) raise typer.Exit(1) from None diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index 51e224f..f47c194 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -6,8 +6,12 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path +from typing import Literal from urllib.parse import quote +CallstackLayout = Literal["v1", "v2"] +_METADATA_TABLE = "pt_snap_metadata" + class DatabaseNotFoundError(FileNotFoundError): """Raised when database file does not exist.""" @@ -60,6 +64,7 @@ def __init__( self._conn: sqlite3.Connection | None = None self._connect_depth = 0 self._device_ids: list[int] | None = None + self._callstack_layout: CallstackLayout | None = None if not self.db_path.exists(): raise DatabaseNotFoundError(f"Database not found: {self.db_path}") @@ -78,6 +83,86 @@ def _validate_schema(self) -> None: "Invalid database schema: 'dictionary' table not found. " "This may not be a valid PyTorch memory snapshot database." ) + self._callstack_layout = self._detect_callstack_layout(conn) + + @property + def callstack_layout(self) -> CallstackLayout | None: + """Return the detected callstack schema, or None when it is unrecognized. + + ``v1`` is inline ``callstack`` text on each ``trace_entry_`` + table. ``v2`` stores ``callstackId`` and a shared ``callstack`` table. + Detection is read-only and cached on this context; it is not written + to focus files. Conflicting or damaged layouts raise + :class:`SchemaVersionError` during construction. + """ + return self._callstack_layout + + def _detect_callstack_layout(self, conn: sqlite3.Connection) -> CallstackLayout | None: + """Identify v1/v2 callstack layout from table columns and metadata.""" + cursor = conn.cursor() + layouts: dict[int, CallstackLayout] = {} + unrecognized: list[int] = [] + for device_id, table_name in _trace_entry_tables(cursor): + columns = _table_columns(cursor, table_name) + has_id = "callstackid" in columns + has_text = "callstack" in columns + if has_id and has_text: + raise SchemaVersionError( + "Incompatible callstack layout: " + f"'{table_name}' has both callstack and callstackId columns." + ) + if has_id: + layouts[device_id] = "v2" + elif has_text: + layouts[device_id] = "v1" + else: + unrecognized.append(device_id) + + if layouts and unrecognized: + raise SchemaVersionError( + "Incompatible callstack layout: some devices have a recognized " + "callstack schema but others do not " + f"(unrecognized devices: {unrecognized})." + ) + + distinct = set(layouts.values()) + if len(distinct) > 1: + raise SchemaVersionError( + "Incompatible callstack layout: devices disagree " + f"({', '.join(sorted(distinct))})." + ) + + layout = next(iter(distinct), None) + has_callstack_table = _has_table(cursor, "callstack") + if layout == "v1" and has_callstack_table: + raise SchemaVersionError( + "Incompatible callstack layout: inline callstack text coexists " + "with a shared callstack table." + ) + if layout == "v2": + if not has_callstack_table: + raise SchemaVersionError( + "Incompatible callstack layout: callstackId is present but " + "the shared callstack table is missing." + ) + callstack_columns = _table_columns(cursor, "callstack") + if not {"id", "callstack"}.issubset(callstack_columns): + raise SchemaVersionError( + "Incompatible callstack layout: shared callstack table is " + "missing id or callstack columns." + ) + + metadata_version = _import_format_version(cursor) + if metadata_version in (1, 2) and layout is not None: + expected: CallstackLayout = "v1" if metadata_version == 1 else "v2" + if layout != expected: + raise SchemaVersionError( + "Incompatible callstack layout: " + f"pt_snap_metadata.import_format_version is {metadata_version} " + f"but the database structure is {layout}." + ) + + return layout @property def device_ids(self) -> list[int]: @@ -160,3 +245,50 @@ def close(self) -> None: self._conn.close() self._conn = None self._connect_depth = 0 + + +def _quote_ident(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _has_table(cursor: sqlite3.Cursor, name: str) -> bool: + cursor.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)) + return cursor.fetchone() is not None + + +def _table_columns(cursor: sqlite3.Cursor, table_name: str) -> set[str]: + cursor.execute(f"PRAGMA table_info({_quote_ident(table_name)})") + return {str(row[1]).lower() for row in cursor.fetchall()} + + +def _trace_entry_tables(cursor: sqlite3.Cursor) -> list[tuple[int, str]]: + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'trace_entry_%'" + ) + tables: list[tuple[int, str]] = [] + for row in cursor.fetchall(): + table_name = row[0] + suffix = table_name.rsplit("_", 1)[-1] + try: + tables.append((int(suffix), table_name)) + except ValueError: + continue + return tables + + +def _import_format_version(cursor: sqlite3.Cursor) -> int | None: + if not _has_table(cursor, _METADATA_TABLE): + return None + columns = _table_columns(cursor, _METADATA_TABLE) + if "import_format_version" not in columns: + return None + try: + rows = cursor.execute(f"SELECT import_format_version FROM {_METADATA_TABLE}").fetchall() + except sqlite3.DatabaseError: + return None + if len(rows) != 1: + return None + try: + return int(rows[0][0]) + except (TypeError, ValueError): + return None diff --git a/src/pt_snap_cli/core/focus_service.py b/src/pt_snap_cli/core/focus_service.py index 5ebc46f..8eb3f5a 100644 --- a/src/pt_snap_cli/core/focus_service.py +++ b/src/pt_snap_cli/core/focus_service.py @@ -50,10 +50,13 @@ def get_focus( ) -> FocusState: resolved = self.resolve_focus(explicit_db_path, explicit_device_id, start_dir) available_devices: list[int] = [] + callstack_layout: str | None = None if resolved.db_path is not None and resolved.db_path.exists(): try: - available_devices = Context(resolved.db_path).device_ids + ctx = Context(resolved.db_path) + available_devices = ctx.device_ids + callstack_layout = ctx.callstack_layout except (DatabaseNotFoundError, SchemaVersionError): available_devices = [] @@ -63,6 +66,7 @@ def get_focus( available_devices=available_devices, source=resolved.source, focus_file=resolved.focus_file, + callstack_layout=callstack_layout, ) def set_project_focus( @@ -81,6 +85,7 @@ def set_project_focus( available_devices=ctx.device_ids, source="project", focus_file=self._config.project_focus_path(base_dir), + callstack_layout=ctx.callstack_layout, ) def set_global_focus(self, db_path: Path | str, device_id: int | None = None) -> FocusState: @@ -94,6 +99,7 @@ def set_global_focus(self, db_path: Path | str, device_id: int | None = None) -> available_devices=ctx.device_ids, source="global", focus_file=self._config.config_file, + callstack_layout=ctx.callstack_layout, ) def set_device( @@ -117,6 +123,7 @@ def set_device( available_devices=ctx.device_ids, source="global", focus_file=self._config.config_file, + callstack_layout=ctx.callstack_layout, ) focus_file = self._config.write_project_focus( @@ -130,6 +137,7 @@ def set_device( available_devices=ctx.device_ids, source="project", focus_file=focus_file, + callstack_layout=ctx.callstack_layout, ) def clear_global_focus(self) -> None: @@ -155,6 +163,7 @@ def validate_session_db( available_devices=ctx.device_ids, source="explicit", focus_file=None, + callstack_layout=ctx.callstack_layout, ) def _validated_context(self, db_path: Path) -> Context: diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index c28cb2e..577d2f0 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -61,6 +61,7 @@ class FocusState: available_devices: list[int] = field(default_factory=list) source: FocusSource = "none" focus_file: Path | None = None + callstack_layout: str | None = None @dataclass(frozen=True) diff --git a/src/pt_snap_cli/mcp/server.py b/src/pt_snap_cli/mcp/server.py index 9784c8f..986c8ae 100644 --- a/src/pt_snap_cli/mcp/server.py +++ b/src/pt_snap_cli/mcp/server.py @@ -22,6 +22,7 @@ def get_focus() -> dict[str, Any]: "device_id": state.device_id, "source": state.source, "available_devices": state.available_devices, + "callstack_layout": state.callstack_layout, } @@ -32,6 +33,7 @@ def set_focus(db_path: str | None = None, device_id: int | None = None) -> dict[ "db_path": state.db_path, "device_id": state.device_id, "available_devices": state.available_devices, + "callstack_layout": state.callstack_layout, } @@ -71,6 +73,7 @@ def focus_resource() -> dict[str, Any]: "device_id": state.device_id, "source": state.source, "available_devices": state.available_devices, + "callstack_layout": state.callstack_layout, } diff --git a/src/pt_snap_cli/query/AGENTS.md b/src/pt_snap_cli/query/AGENTS.md index ba1595a..47f4e36 100644 --- a/src/pt_snap_cli/query/AGENTS.md +++ b/src/pt_snap_cli/query/AGENTS.md @@ -29,9 +29,12 @@ - Keep SQL value filters parameterized where using builder/condition APIs. - Device-specific template SQL should use injected table names such as `device_trace_table` and `device_block_table`. - Keep direct `QueryParameter` validation errors local; `QueryExecutor` must normalize them to its `TemplateRenderError` before the core boundary. +- Variant SQL is selected from `Context.callstack_layout`; unknown layouts must + fail variant templates with a layout-specific error rather than a generic + re-import message. ### Testing Requirements -- Run `pytest tests/query` for query subsystem changes; real SQLite template semantics live in `test_peak_memory_templates.py` and `test_query_max_rows_pushdown.py`. +- Run `pytest tests/query` for query subsystem changes; real SQLite template semantics live in `test_peak_memory_templates.py`, `test_callstack_schema_compat.py`, and `test_query_max_rows_pushdown.py`. - Run `pytest tests/test_cli.py` when template listing/info/output behavior changes. ### Common Patterns @@ -52,3 +55,7 @@ - `jinja2` for SQL template rendering. + +- Callstack-dependent templates may declare `query_variants.v1` / `query_variants.v2`. + `QueryExecutor` selects the body from `Context.callstack_layout`. Keep v2 SQL + grouped by `callstackId`; do not regress it to grouping by callstack text. diff --git a/src/pt_snap_cli/query/config.py b/src/pt_snap_cli/query/config.py index 7ba56c3..99975a7 100644 --- a/src/pt_snap_cli/query/config.py +++ b/src/pt_snap_cli/query/config.py @@ -8,6 +8,8 @@ import yaml +_CALLSTACK_VARIANTS = ("v1", "v2") + @dataclass class QueryParameter: @@ -62,9 +64,23 @@ class QueryTemplate: devices: list[str] = field(default_factory=lambda: ["all"]) parameters: dict[str, QueryParameter] = field(default_factory=dict) query: str = "" + query_variants: dict[str, str] = field(default_factory=dict) output_schema: list[dict[str, str]] = field(default_factory=list) category: str = "basic" + def sql_for_layout(self, layout: str | None) -> str | None: + """Return SQL for a detected callstack layout. + + Templates without variants always use ``query``. Variant templates + return the matching body, or ``None`` when the layout is missing or + unknown. + """ + if not self.query_variants: + return self.query + if layout is None: + return None + return self.query_variants.get(layout) + def validate_params(self, params: dict[str, Any]) -> dict[str, Any]: """Validate and convert parameters. @@ -111,17 +127,53 @@ def from_dict(cls, data: dict[str, Any], default_category: str | None = None) -> description=param_data.get("description", ""), ) + query_variants = _parse_query_variants(data.get("name", ""), data.get("query_variants")) + query = data.get("query") or "" + if query_variants: + query = query_variants.get("v2") or query + return cls( name=data.get("name", ""), description=data.get("description", ""), devices=data.get("devices", ["all"]), parameters=parameters, - query=data.get("query", ""), + query=query, + query_variants=query_variants, output_schema=data.get("output_schema", []), category=data.get("category") or default_category or "basic", ) +def _parse_query_variants(template_name: str, raw: Any) -> dict[str, str]: + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError(f"query_variants for '{template_name}' must be a mapping of layout to SQL") + if not raw: + return {} + + unknown = set(raw) - set(_CALLSTACK_VARIANTS) + if unknown: + raise ValueError( + f"query_variants for '{template_name}' has unsupported layouts: " + f"{', '.join(sorted(str(key) for key in unknown))}" + ) + missing = [name for name in _CALLSTACK_VARIANTS if name not in raw] + if missing: + raise ValueError( + f"query_variants for '{template_name}' must include {_CALLSTACK_VARIANTS}, " + f"missing: {', '.join(missing)}" + ) + + variants: dict[str, str] = {} + for name in _CALLSTACK_VARIANTS: + sql = raw[name] + if not isinstance(sql, str) or not sql.strip(): + raise ValueError(f"query_variants.{name} for '{template_name}' must be a SQL string") + variants[name] = sql + return variants + + @dataclass class QueryConfig: """Query configuration containing multiple templates.""" diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 053af51..23a9d77 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -63,23 +63,49 @@ def __init__( if self._template_dir and self._template_dir.exists(): self._load_templates() - def _compiled_template(self, template: QueryTemplate) -> Template: - """Return the cached compiled Jinja template for ``template``. + def _compiled_template(self, template: QueryTemplate, sql: str | None = None) -> Template: + """Return the cached compiled Jinja template for ``sql``. Compiles and caches on first access so repeat renders skip the - ``Environment.from_string`` parse step. + ``Environment.from_string`` parse step. Variant templates cache + each layout body separately. """ - cache_key = (template.name, template.query) + body = sql if sql is not None else template.query + cache_key = (template.name, body) cached = self._compiled_cache.get(cache_key) if cached is not None: return cached try: - compiled = self._env.from_string(template.query) + compiled = self._env.from_string(body) except TemplateSyntaxError as e: raise TemplateRenderError(f"Template syntax error in '{template.name}': {e}") from e self._compiled_cache[cache_key] = compiled return compiled + def _sql_for_layout(self, template: QueryTemplate) -> str: + """Select the SQL body for the current database callstack layout.""" + sql = template.sql_for_layout(self._callstack_layout()) + if sql is not None: + return sql + layout = self._callstack_layout() + if layout is None: + raise QueryExecutionError( + f"Query execution failed: template '{template.name}' needs a v1 " + "or v2 callstack layout, but this database's trace tables have " + "neither an inline callstack column nor callstackId" + ) + raise QueryExecutionError( + f"Query execution failed: template '{template.name}' has no SQL " + f"variant for callstack layout {layout}" + ) + + def _callstack_layout(self) -> str | None: + context = getattr(self, "_context", None) + if context is None: + return None + layout = getattr(context, "callstack_layout", None) + return layout if isinstance(layout, str) else None + def _apply_limit(self, sql: str, limit: int) -> str: """Tighten a trailing numeric LIMIT or append one when absent.""" match = _TRAILING_LIMIT_RE.search(sql) @@ -175,7 +201,7 @@ def render( effective_limit = max_rows render_context["limit"] = effective_limit - jinja_template = self._compiled_template(template) + jinja_template = self._compiled_template(template, self._sql_for_layout(template)) try: rendered_sql = jinja_template.render(render_context) except Exception as e: diff --git a/src/pt_snap_cli/query/templates/AGENTS.md b/src/pt_snap_cli/query/templates/AGENTS.md index 2699728..bb0f3dc 100644 --- a/src/pt_snap_cli/query/templates/AGENTS.md +++ b/src/pt_snap_cli/query/templates/AGENTS.md @@ -45,3 +45,7 @@ - YAML syntax via `pyyaml` and Jinja2 expressions inside SQL strings. + +- For callstack-dependent templates, put v1 inline SQL and v2 `callstackId` SQL + under `query_variants` instead of duplicating the template file. Keep the + public name, parameters, and `output_schema` identical. diff --git a/src/pt_snap_cli/query/templates/basic/event.yaml b/src/pt_snap_cli/query/templates/basic/event.yaml index 559e262..8d7ad5f 100644 --- a/src/pt_snap_cli/query/templates/basic/event.yaml +++ b/src/pt_snap_cli/query/templates/basic/event.yaml @@ -75,35 +75,62 @@ queries: default: 0 required: false description: Number of rows to skip - query: | - SELECT - id, - action, - address, - size, - stream, - allocated, - active, - reserved, - ( - SELECT callstack FROM callstack - WHERE callstack.id = {{ device_trace_table }}.callstackId - ) AS callstack - FROM {{ device_trace_table }} - WHERE 1=1 - {% if id is not none %}AND id = {{ id }}{% endif %} - {% if min_id is not none %}AND id >= {{ min_id }}{% endif %} - {% if max_id is not none %}AND id <= {{ max_id }}{% endif %} - {% if action is not none %}AND action = {{ action }}{% endif %} - {% if address is not none %}AND address = {{ address }}{% endif %} - {% if size is not none %}AND size = {{ size }}{% endif %} - {% if stream is not none %}AND stream = {{ stream }}{% endif %} - {% if min_allocated is not none %}AND allocated >= {{ min_allocated }}{% endif %} - {% if min_active is not none %}AND active >= {{ min_active }}{% endif %} - {% if min_reserved is not none %}AND reserved >= {{ min_reserved }}{% endif %} - ORDER BY {{ order_by }} {{ order_dir }} - LIMIT {{ limit }} - {% if offset > 0 %}OFFSET {{ offset }}{% endif %} + query_variants: + v1: | + SELECT + id, + action, + address, + size, + stream, + allocated, + active, + reserved, + callstack + FROM {{ device_trace_table }} + WHERE 1=1 + {% if id is not none %}AND id = {{ id }}{% endif %} + {% if min_id is not none %}AND id >= {{ min_id }}{% endif %} + {% if max_id is not none %}AND id <= {{ max_id }}{% endif %} + {% if action is not none %}AND action = {{ action }}{% endif %} + {% if address is not none %}AND address = {{ address }}{% endif %} + {% if size is not none %}AND size = {{ size }}{% endif %} + {% if stream is not none %}AND stream = {{ stream }}{% endif %} + {% if min_allocated is not none %}AND allocated >= {{ min_allocated }}{% endif %} + {% if min_active is not none %}AND active >= {{ min_active }}{% endif %} + {% if min_reserved is not none %}AND reserved >= {{ min_reserved }}{% endif %} + ORDER BY {{ order_by }} {{ order_dir }} + LIMIT {{ limit }} + {% if offset > 0 %}OFFSET {{ offset }}{% endif %} + v2: | + SELECT + id, + action, + address, + size, + stream, + allocated, + active, + reserved, + ( + SELECT callstack FROM callstack + WHERE callstack.id = {{ device_trace_table }}.callstackId + ) AS callstack + FROM {{ device_trace_table }} + WHERE 1=1 + {% if id is not none %}AND id = {{ id }}{% endif %} + {% if min_id is not none %}AND id >= {{ min_id }}{% endif %} + {% if max_id is not none %}AND id <= {{ max_id }}{% endif %} + {% if action is not none %}AND action = {{ action }}{% endif %} + {% if address is not none %}AND address = {{ address }}{% endif %} + {% if size is not none %}AND size = {{ size }}{% endif %} + {% if stream is not none %}AND stream = {{ stream }}{% endif %} + {% if min_allocated is not none %}AND allocated >= {{ min_allocated }}{% endif %} + {% if min_active is not none %}AND active >= {{ min_active }}{% endif %} + {% if min_reserved is not none %}AND reserved >= {{ min_reserved }}{% endif %} + ORDER BY {{ order_by }} {{ order_dir }} + LIMIT {{ limit }} + {% if offset > 0 %}OFFSET {{ offset }}{% endif %} output_schema: - column: id type: int diff --git a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml index fa13749..723cba8 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_callstack_at_event.yaml @@ -24,46 +24,58 @@ queries: default: 20 required: false description: Maximum number of callstack groups to return (-1 for unlimited) - query: | - WITH active_blocks AS ( - SELECT - id, - size, - requestedSize, - allocEventId, - freeEventId, - CASE - WHEN allocEventId = -1 AND freeEventId = -1 THEN 'static' - WHEN allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }}) THEN 'preexisting_live_at_event' - ELSE 'dynamic_live_at_event' - END AS category - FROM {{ device_block_table }} - WHERE size >= {{ min_size|int }} - AND ( - {% if include_static %}(allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) OR{% endif %} - (allocEventId != -1 AND allocEventId <= {{ event_id|int }} AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) - ) - ), grouped AS ( - SELECT - CASE - WHEN b.category = 'static' THEN '[static] allocEventId=-1, freeEventId=-1' - WHEN b.category = 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' - ELSE COALESCE(cs.callstack, '[missing callstack]') - END AS callstack, - b.category, - COUNT(*) AS block_count, - SUM(b.size) AS size_bytes, - ROUND(SUM(b.size) / 1073741824.0, 6) AS size_gib, - SUM(b.requestedSize) AS requested_bytes, - ROUND(SUM(b.requestedSize) / 1073741824.0, 6) AS requested_gib - FROM active_blocks b - LEFT JOIN {{ device_trace_table }} t - ON b.allocEventId = t.id - LEFT JOIN callstack cs - ON t.callstackId = cs.id - GROUP BY callstack, b.category - ), - ranked_dynamic AS ( + query_variants: + v1: | + WITH active_blocks AS ( + SELECT + id, + size, + requestedSize, + allocEventId, + freeEventId, + CASE + WHEN allocEventId = -1 AND freeEventId = -1 THEN 'static' + WHEN allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }}) THEN 'preexisting_live_at_event' + ELSE 'dynamic_live_at_event' + END AS category + FROM {{ device_block_table }} + WHERE size >= {{ min_size|int }} + AND ( + {% if include_static %}(allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) OR{% endif %} + (allocEventId != -1 AND allocEventId <= {{ event_id|int }} AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) + ) + ), grouped AS ( + SELECT + CASE + WHEN b.category = 'static' THEN '[static] allocEventId=-1, freeEventId=-1' + WHEN b.category = 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' + ELSE COALESCE(t.callstack, '[missing callstack]') + END AS callstack, + b.category, + COUNT(*) AS block_count, + SUM(b.size) AS size_bytes, + ROUND(SUM(b.size) / 1073741824.0, 6) AS size_gib, + SUM(b.requestedSize) AS requested_bytes, + ROUND(SUM(b.requestedSize) / 1073741824.0, 6) AS requested_gib + FROM active_blocks b + LEFT JOIN {{ device_trace_table }} t + ON b.allocEventId = t.id + GROUP BY callstack, b.category + ), + ranked_dynamic AS ( + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM grouped + WHERE category = 'dynamic_live_at_event' + ORDER BY size_bytes DESC, block_count DESC, callstack ASC + LIMIT {{ top_n|int }} + ) SELECT callstack, category, @@ -71,33 +83,84 @@ queries: size_bytes, size_gib, requested_bytes, - requested_gib - FROM grouped - WHERE category = 'dynamic_live_at_event' + requested_gib, + ROUND(size_bytes * 100.0 / NULLIF(SUM(size_bytes) OVER (), 0), 4) AS percent_of_active_blocks + FROM ( + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM grouped + WHERE category IN ('static', 'preexisting_live_at_event') + UNION ALL + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM ranked_dynamic + ) ORDER BY size_bytes DESC, block_count DESC, callstack ASC - LIMIT {{ top_n|int }} - ) - SELECT - callstack, - category, - block_count, - size_bytes, - size_gib, - requested_bytes, - requested_gib, - ROUND(size_bytes * 100.0 / NULLIF(SUM(size_bytes) OVER (), 0), 4) AS percent_of_active_blocks - FROM ( - SELECT - callstack, - category, - block_count, - size_bytes, - size_gib, - requested_bytes, - requested_gib - FROM grouped - WHERE category IN ('static', 'preexisting_live_at_event') - UNION ALL + v2: | + WITH active_blocks AS ( + SELECT + id, + size, + requestedSize, + allocEventId, + freeEventId, + CASE + WHEN allocEventId = -1 AND freeEventId = -1 THEN 'static' + WHEN allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }}) THEN 'preexisting_live_at_event' + ELSE 'dynamic_live_at_event' + END AS category + FROM {{ device_block_table }} + WHERE size >= {{ min_size|int }} + AND ( + {% if include_static %}(allocEventId = -1 AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) OR{% endif %} + (allocEventId != -1 AND allocEventId <= {{ event_id|int }} AND (freeEventId IS NULL OR freeEventId < 0 OR freeEventId > {{ event_id|int }})) + ) + ), grouped AS ( + SELECT + CASE + WHEN b.category = 'static' THEN '[static] allocEventId=-1, freeEventId=-1' + WHEN b.category = 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' + ELSE COALESCE(cs.callstack, '[missing callstack]') + END AS callstack, + b.category, + COUNT(*) AS block_count, + SUM(b.size) AS size_bytes, + ROUND(SUM(b.size) / 1073741824.0, 6) AS size_gib, + SUM(b.requestedSize) AS requested_bytes, + ROUND(SUM(b.requestedSize) / 1073741824.0, 6) AS requested_gib + FROM active_blocks b + LEFT JOIN {{ device_trace_table }} t + ON b.allocEventId = t.id + LEFT JOIN callstack cs + ON t.callstackId = cs.id + GROUP BY callstack, b.category + ), + ranked_dynamic AS ( + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM grouped + WHERE category = 'dynamic_live_at_event' + ORDER BY size_bytes DESC, block_count DESC, callstack ASC + LIMIT {{ top_n|int }} + ) SELECT callstack, category, @@ -105,10 +168,31 @@ queries: size_bytes, size_gib, requested_bytes, - requested_gib - FROM ranked_dynamic - ) - ORDER BY size_bytes DESC, block_count DESC, callstack ASC + requested_gib, + ROUND(size_bytes * 100.0 / NULLIF(SUM(size_bytes) OVER (), 0), 4) AS percent_of_active_blocks + FROM ( + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM grouped + WHERE category IN ('static', 'preexisting_live_at_event') + UNION ALL + SELECT + callstack, + category, + block_count, + size_bytes, + size_gib, + requested_bytes, + requested_gib + FROM ranked_dynamic + ) + ORDER BY size_bytes DESC, block_count DESC, callstack ASC output_schema: - column: callstack type: str diff --git a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml index 26d37cf..7cdb3a8 100644 --- a/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml +++ b/src/pt_snap_cli/query/templates/statistical/callstack_analysis.yaml @@ -22,29 +22,44 @@ queries: default: -1 required: false description: Maximum rows to return (-1 for unlimited). Mirrored from the caller-side ``max_rows`` for SQL pushdown. - query: | - SELECT - COALESCE(c.callstack, '[missing callstack]') AS callstack, - grouped.alloc_count AS alloc_count, - grouped.total_size AS total_size, - grouped.avg_size AS avg_size, - grouped.max_size AS max_size - FROM ( + query_variants: + v1: | SELECT - callstackId, + callstack, COUNT(*) as alloc_count, SUM(size) as total_size, AVG(size) as avg_size, MAX(size) as max_size FROM {{ device_trace_table }} - WHERE callstackId IS NOT NULL - GROUP BY callstackId + WHERE callstack IS NOT NULL + GROUP BY callstack HAVING alloc_count >= {{ min_count }} AND total_size >= {{ min_size }} - ) AS grouped - LEFT JOIN callstack c ON c.id = grouped.callstackId - ORDER BY total_size DESC, callstack ASC - LIMIT {{ limit|int }} + ORDER BY total_size DESC, callstack ASC + LIMIT {{ limit|int }} + v2: | + SELECT + COALESCE(c.callstack, '[missing callstack]') AS callstack, + grouped.alloc_count AS alloc_count, + grouped.total_size AS total_size, + grouped.avg_size AS avg_size, + grouped.max_size AS max_size + FROM ( + SELECT + callstackId, + COUNT(*) as alloc_count, + SUM(size) as total_size, + AVG(size) as avg_size, + MAX(size) as max_size + FROM {{ device_trace_table }} + WHERE callstackId IS NOT NULL + GROUP BY callstackId + HAVING alloc_count >= {{ min_count }} + AND total_size >= {{ min_size }} + ) AS grouped + LEFT JOIN callstack c ON c.id = grouped.callstackId + ORDER BY total_size DESC, callstack ASC + LIMIT {{ limit|int }} output_schema: - column: callstack type: str diff --git a/tests/core/test_context_cache.py b/tests/core/test_context_cache.py index cf9617f..c87b105 100644 --- a/tests/core/test_context_cache.py +++ b/tests/core/test_context_cache.py @@ -94,6 +94,35 @@ def test_same_mtime_replacement_refreshes_cached_context(self, tmp_path: Path) - assert second is not first assert second.device_ids == [1] + def test_replaced_database_redetects_callstack_layout(self, tmp_path: Path) -> None: + db_path = tmp_path / "layout.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + + cache = ContextCache(maxsize=4) + first = cache.get(db_path) + assert first.callstack_layout == "v1" + + replacement = tmp_path / "v2.db" + conn = sqlite3.connect(str(replacement)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstackId INTEGER)") + conn.commit() + conn.close() + os.replace(replacement, db_path) + + second = cache.get(db_path) + assert second is not first + assert second.callstack_layout == "v2" + def test_invalidate_path_drops_entry(self, tmp_path: Path) -> None: db_path = _make_db(tmp_path) cache = ContextCache(maxsize=4) diff --git a/tests/core/test_focus_service.py b/tests/core/test_focus_service.py index 98f17ce..8e8871c 100644 --- a/tests/core/test_focus_service.py +++ b/tests/core/test_focus_service.py @@ -107,3 +107,22 @@ def test_validate_session_db_returns_devices(self, sample_db: Path) -> None: def test_validate_session_db_rejects_invalid_device(self, sample_db: Path) -> None: with pytest.raises(InvalidDeviceError, match="Device 99 not found"): FocusService().validate_session_db(sample_db, device_id=99) + + def test_set_project_focus_reports_v1_layout_without_persisting_it( + self, tmp_path: Path + ) -> None: + db_path = tmp_path / "v1.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + + state = FocusService().set_project_focus(db_path) + focus_data = json.loads((Path.cwd() / ".pt-snap" / "focus.json").read_text()) + + assert state.callstack_layout == "v1" + assert "callstack_layout" not in focus_data + assert set(focus_data) == {"db_path"} diff --git a/tests/core/test_report_service.py b/tests/core/test_report_service.py index 8f01856..e0c9d2f 100644 --- a/tests/core/test_report_service.py +++ b/tests/core/test_report_service.py @@ -166,3 +166,66 @@ def test_peak_memory_report_handles_empty_trace(tmp_path: Path) -> None: assert report.event_id is None assert report.allocator_gap is None assert report.callstack_groups == [] + + +def test_peak_memory_report_reads_v1_inline_callstack(tmp_path: Path) -> None: + db_path = tmp_path / "v1-report.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)") + conn.execute(""" + CREATE TABLE trace_entry_0 ( + id INTEGER PRIMARY KEY, + action INTEGER, + address INTEGER, + size INTEGER, + stream INTEGER, + allocated INTEGER, + active INTEGER, + reserved INTEGER, + callstack TEXT + ) + """) + conn.execute(""" + CREATE TABLE block_0 ( + id INTEGER PRIMARY KEY, + address INTEGER, + size INTEGER, + requestedSize INTEGER, + state INTEGER, + allocEventId INTEGER, + freeEventId INTEGER + ) + """) + conn.executemany( + """ + INSERT INTO trace_entry_0 + (id, action, address, size, stream, allocated, active, reserved, callstack) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), + (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), + ], + ) + conn.executemany( + """ + INSERT INTO block_0 + (id, address, size, requestedSize, state, allocEventId, freeEventId) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + (-10, 0xA000, 8192, 8000, 1, -1, -1), + (1, 0x1000, 1024, 1000, 1, 1, -1), + (5, 0x5000, 512, 500, 1, 5, -1), + ], + ) + conn.commit() + conn.close() + + report = ReportService().peak_memory_report(db_path, metric="active") + + assert report.event_id == 5 + by_callstack = {row["callstack"]: row for row in report.callstack_groups} + assert by_callstack["[static] allocEventId=-1, freeEventId=-1"]["size_bytes"] == 8192 + assert by_callstack["train.py:10"]["size_bytes"] == 1024 + assert by_callstack["[missing callstack]"]["size_bytes"] == 512 diff --git a/tests/query/AGENTS.md b/tests/query/AGENTS.md index 24a0819..2ad49e2 100644 --- a/tests/query/AGENTS.md +++ b/tests/query/AGENTS.md @@ -17,6 +17,7 @@ | `test_memory_peak_cte.py` | Tests generated memory-peak SQL CTE structure and semantics. | | `test_mapper.py` | Tests result type conversion and model factory mapping. | | `test_peak_memory_templates.py` | Tests active-at-event, allocator-gap, and callstack attribution template semantics. | +| `test_callstack_schema_compat.py` | Tests v1/v2 callstack layout detection and dual-schema template results. | | `test_query_max_rows_pushdown.py` | Executes packaged leak/callstack templates against SQLite and tests SQL limit pushdown and stable ordering. | | `test_registry.py` | Tests template registration, lookup, category listing, and registry reset behavior. | diff --git a/tests/query/test_callstack_schema_compat.py b/tests/query/test_callstack_schema_compat.py new file mode 100644 index 0000000..b41b990 --- /dev/null +++ b/tests/query/test_callstack_schema_compat.py @@ -0,0 +1,343 @@ +"""Read-only v1/v2 callstack schema compatibility for packaged templates.""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from pathlib import Path + +import pytest + +from pt_snap_cli.context import Context, SchemaVersionError +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.query.executor import QueryExecutionError, QueryExecutor +from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates, get_query + + +@pytest.fixture(autouse=True) +def _reload_query_templates() -> None: + QueryRegistry.reset() + _load_all_templates() + + +_BLOCKS = [ + (-10, 0xA000, 8192, 8000, 1, -1, -1), + (1, 0x1000, 1024, 1000, 1, 1, -1), + (2, 0x2000, 2048, 2000, 0, 2, 3), + (4, 0x4000, 4096, 4000, 1, 4, -1), + (5, 0x5000, 512, 500, 1, 5, -1), +] +_EVENTS = [ + (1, 2, 0x1000, 1024, 0, 1024, 1024, 4096, "train.py:10"), + (2, 2, 0x2000, 2048, 0, 3072, 3072, 4096, "freed.py:20"), + (3, 3, 0x2000, 2048, 0, 1024, 1024, 8192, "free.py:30"), + (4, 2, 0x4000, 4096, 0, 5120, 5120, 8192, "after.py:40"), + (5, 2, 0x5000, 512, 0, 5632, 5632, 8192, None), +] + + +def _fingerprint(path: Path) -> tuple[str, int]: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + return digest, path.stat().st_mtime_ns + + +def _create_dictionary(conn: sqlite3.Connection) -> None: + conn.execute("CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)") + + +def _create_block_table(conn: sqlite3.Connection, device_id: int = 0) -> None: + conn.execute(f""" + CREATE TABLE block_{device_id} ( + id INTEGER PRIMARY KEY, + address INTEGER, + size INTEGER, + requestedSize INTEGER, + state INTEGER, + allocEventId INTEGER, + freeEventId INTEGER + ) + """) + conn.executemany( + f""" + INSERT INTO block_{device_id} + (id, address, size, requestedSize, state, allocEventId, freeEventId) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + _BLOCKS, + ) + + +def _insert_metadata(conn: sqlite3.Connection, import_format_version: int) -> None: + conn.execute(""" + CREATE TABLE pt_snap_metadata ( + id INTEGER PRIMARY KEY CHECK (id = 1), + metadata_schema_version INTEGER NOT NULL, + import_format_version INTEGER NOT NULL, + source_sha256 TEXT NOT NULL, + source_size INTEGER NOT NULL, + source_name TEXT NOT NULL, + requested_device INTEGER, + importer_name TEXT NOT NULL, + importer_version TEXT NOT NULL, + completed_at TEXT NOT NULL + ) + """) + conn.execute( + """ + INSERT INTO pt_snap_metadata ( + id, metadata_schema_version, import_format_version, source_sha256, + source_size, source_name, requested_device, importer_name, + importer_version, completed_at + ) VALUES (1, 1, ?, ?, 1, 'snapshot.pkl', NULL, 'pt-snap-cli', '0.2.0', + '2026-01-01T00:00:00+00:00') + """, + (import_format_version, "a" * 64), + ) + + +def create_v1_db( + path: Path, + *, + metadata_version: int | None = None, + extra_device: bool = False, +) -> Path: + conn = sqlite3.connect(str(path)) + _create_dictionary(conn) + devices = [0, 1] if extra_device else [0] + for device_id in devices: + conn.execute(f""" + CREATE TABLE trace_entry_{device_id} ( + id INTEGER PRIMARY KEY, + action INTEGER, + address INTEGER, + size INTEGER, + stream INTEGER, + allocated INTEGER, + active INTEGER, + reserved INTEGER, + callstack TEXT + ) + """) + conn.executemany( + f""" + INSERT INTO trace_entry_{device_id} + (id, action, address, size, stream, allocated, active, reserved, callstack) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + _EVENTS, + ) + _create_block_table(conn, device_id) + if metadata_version is not None: + _insert_metadata(conn, metadata_version) + conn.commit() + conn.close() + return path + + +def create_v2_db(path: Path, *, metadata_version: int | None = None) -> Path: + conn = sqlite3.connect(str(path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + texts = ["train.py:10", "freed.py:20", "free.py:30", "after.py:40"] + conn.executemany( + "INSERT INTO callstack (id, callstack) VALUES (?, ?)", + list(enumerate(texts)), + ) + conn.execute(""" + CREATE TABLE trace_entry_0 ( + id INTEGER PRIMARY KEY, + action INTEGER, + address INTEGER, + size INTEGER, + stream INTEGER, + allocated INTEGER, + active INTEGER, + reserved INTEGER, + callstackId INTEGER + ) + """) + conn.executemany( + """ + INSERT INTO trace_entry_0 + (id, action, address, size, stream, allocated, active, reserved, callstackId) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [(*event[:-1], None if event[-1] is None else texts.index(event[-1])) for event in _EVENTS], + ) + _create_block_table(conn) + if metadata_version is not None: + _insert_metadata(conn, metadata_version) + conn.commit() + conn.close() + return path + + +def _execute(db_path: Path, name: str, params: dict | None = None) -> list[dict]: + context = Context(db_path) + try: + return QueryExecutor(context).execute_template(name, params or {}, device_id=0) + finally: + context.close() + + +@pytest.fixture +def v1_db(tmp_path: Path) -> Path: + return create_v1_db(tmp_path / "v1.db") + + +@pytest.fixture +def v2_db(tmp_path: Path) -> Path: + return create_v2_db(tmp_path / "v2.db") + + +def test_packaged_callstack_templates_declare_both_variants() -> None: + for name in ("event", "callstack_analysis", "active_memory_callstack_at_event"): + template = get_query(name) + assert template is not None + assert set(template.query_variants) == {"v1", "v2"} + assert "callstackId" in template.query_variants["v2"] + assert "callstackId" not in template.query_variants["v1"] + + +def test_event_results_match_across_v1_and_v2(v1_db: Path, v2_db: Path) -> None: + v1_rows = _execute(v1_db, "event", {"order_by": "id", "order_dir": "ASC"}) + v2_rows = _execute(v2_db, "event", {"order_by": "id", "order_dir": "ASC"}) + assert [row["id"] for row in v1_rows] == [1, 2, 3, 4, 5] + assert [row["callstack"] for row in v1_rows] == [row["callstack"] for row in v2_rows] + assert v1_rows[0]["callstack"] == "train.py:10" + assert v1_rows[-1]["callstack"] is None + + +def test_callstack_analysis_aggregates_match_across_layouts(v1_db: Path, v2_db: Path) -> None: + params = {"min_count": 1, "min_size": 0} + v1_rows = _execute(v1_db, "callstack_analysis", params) + v2_rows = _execute(v2_db, "callstack_analysis", params) + assert [(row["callstack"], row["alloc_count"], row["total_size"]) for row in v1_rows] == [ + (row["callstack"], row["alloc_count"], row["total_size"]) for row in v2_rows + ] + assert [row["callstack"] for row in v1_rows] == [ + "after.py:40", + "free.py:30", + "freed.py:20", + "train.py:10", + ] + + +def test_active_memory_callstack_groups_match_across_layouts(v1_db: Path, v2_db: Path) -> None: + params = {"event_id": 5, "include_static": True, "top_n": -1} + v1_rows = _execute(v1_db, "active_memory_callstack_at_event", params) + v2_rows = _execute(v2_db, "active_memory_callstack_at_event", params) + assert {row["callstack"]: row["size_bytes"] for row in v1_rows} == { + row["callstack"]: row["size_bytes"] for row in v2_rows + } + by_callstack = {row["callstack"]: row for row in v1_rows} + assert by_callstack["[static] allocEventId=-1, freeEventId=-1"]["size_bytes"] == 8192 + assert by_callstack["after.py:40"]["size_bytes"] == 4096 + assert by_callstack["[missing callstack]"]["requested_bytes"] == 500 + + +def test_v1_callstack_analysis_pagination_and_total(v1_db: Path) -> None: + result = QueryService().execute_query( + "callstack_analysis", + params={"min_count": 1, "min_size": 0}, + db_path=v1_db, + max_rows=2, + ) + assert result.total == 4 + assert result.returned == 2 + assert [row["callstack"] for row in result.rows] == ["after.py:40", "free.py:30"] + + +def test_v2_callstack_analysis_keeps_id_aggregation(v2_db: Path) -> None: + context = Context(v2_db) + sql = QueryExecutor(context).render( + get_query("callstack_analysis"), + {"min_count": 1, "min_size": 0, "limit": -1}, + device_id=0, + ) + normalized = sql.replace("GROUP BY callstackId", "") + assert "GROUP BY callstackId" in sql + assert "GROUP BY callstack" not in normalized + context.close() + + +def test_v1_callstack_analysis_groups_by_text(v1_db: Path) -> None: + context = Context(v1_db) + sql = QueryExecutor(context).render( + get_query("callstack_analysis"), + {"min_count": 1, "min_size": 0, "limit": -1}, + device_id=0, + ) + assert "GROUP BY callstack" in sql + assert "callstackId" not in sql + context.close() + + +def test_query_and_focus_do_not_write_v1_database(v1_db: Path) -> None: + before = _fingerprint(v1_db) + QueryService().execute_query("event", db_path=v1_db, params={"limit": 1}) + Context(v1_db) + assert _fingerprint(v1_db) == before + + +@pytest.mark.parametrize("metadata_version", [None, 1]) +def test_v1_layout_detected_with_or_without_metadata( + tmp_path: Path, metadata_version: int | None +) -> None: + db_path = create_v1_db(tmp_path / "legacy.db", metadata_version=metadata_version) + assert Context(db_path).callstack_layout == "v1" + + +@pytest.mark.parametrize("metadata_version", [None, 2]) +def test_v2_layout_detected_with_or_without_metadata( + tmp_path: Path, metadata_version: int | None +) -> None: + db_path = create_v2_db(tmp_path / "current.db", metadata_version=metadata_version) + assert Context(db_path).callstack_layout == "v2" + + +def test_multi_device_v1_layout_is_consistent(tmp_path: Path) -> None: + db_path = create_v1_db(tmp_path / "multi.db", extra_device=True) + ctx = Context(db_path) + assert ctx.device_ids == [0, 1] + assert ctx.callstack_layout == "v1" + rows = QueryExecutor(ctx).execute_template("event", {"id": 1}, device_id=1) + assert rows[0]["callstack"] == "train.py:10" + + +def test_metadata_conflict_is_a_schema_error(tmp_path: Path) -> None: + db_path = create_v1_db(tmp_path / "conflict.db", metadata_version=2) + with pytest.raises(SchemaVersionError, match="import_format_version is 2"): + Context(db_path) + + +def test_unknown_metadata_version_does_not_override_structure(tmp_path: Path) -> None: + db_path = create_v2_db(tmp_path / "future.db", metadata_version=3) + assert Context(db_path).callstack_layout == "v2" + + +def test_mixed_device_layouts_are_a_schema_error(tmp_path: Path) -> None: + db_path = tmp_path / "mixed.db" + conn = sqlite3.connect(str(db_path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("CREATE TABLE trace_entry_1 (id INTEGER PRIMARY KEY, callstackId INTEGER)") + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + with pytest.raises(SchemaVersionError, match="devices disagree"): + Context(db_path) + + +def test_unknown_layout_errors_only_for_variant_templates(tmp_path: Path) -> None: + db_path = tmp_path / "unknown.db" + conn = sqlite3.connect(str(db_path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, size INTEGER)") + conn.commit() + conn.close() + + ctx = Context(db_path) + assert ctx.callstack_layout is None + with pytest.raises(QueryExecutionError, match="needs a v1 or v2 callstack layout"): + QueryExecutor(ctx).execute_template("event", device_id=0) diff --git a/tests/query/test_config.py b/tests/query/test_config.py index e464f1e..4af0edf 100644 --- a/tests/query/test_config.py +++ b/tests/query/test_config.py @@ -75,6 +75,7 @@ def test_from_dict(self): assert "min_size" in template.parameters assert template.parameters["min_size"].type == "int" assert template.category == "basic" + assert template.query_variants == {} def test_from_dict_with_category(self): data = { @@ -113,6 +114,30 @@ def test_validate_params_missing_required(self): with pytest.raises(ValueError): template.validate_params({}) + def test_from_dict_loads_query_variants(self): + template = QueryTemplate.from_dict( + { + "name": "event", + "query_variants": { + "v1": "SELECT callstack FROM {{ device_trace_table }}", + "v2": "SELECT callstackId FROM {{ device_trace_table }}", + }, + } + ) + assert template.query_variants["v1"].startswith("SELECT callstack") + assert template.query == template.query_variants["v2"] + assert template.sql_for_layout("v1") == template.query_variants["v1"] + assert template.sql_for_layout(None) is None + + def test_from_dict_rejects_incomplete_query_variants(self): + with pytest.raises(ValueError, match="must include"): + QueryTemplate.from_dict( + { + "name": "event", + "query_variants": {"v2": "SELECT 1"}, + } + ) + class TestQueryConfig: def test_load_yaml_from_string(self): diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 3663575..981f8b0 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -372,6 +372,28 @@ def test_qualified_legacy_callstack_schema_error_is_actionable(self, tmp_path): with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): executor.execute_template("qualified_legacy_callstack", device_id=0) + def test_variant_template_uses_inline_sql_on_v1_database(self, tmp_path): + db_path = tmp_path / "v1.db" + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE dictionary (table_name TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER, size INTEGER, callstack TEXT)") + conn.execute( + "INSERT INTO trace_entry_0 (id, size, callstack) VALUES (1, 10, 'train.py:1')" + ) + executor = QueryExecutor(Context(db_path)) + executor.register_template( + QueryTemplate( + name="variant_event", + query_variants={ + "v1": "SELECT callstack FROM {{ device_trace_table }}", + "v2": "SELECT callstackId FROM {{ device_trace_table }}", + }, + ) + ) + + rows = executor.execute_template("variant_event", device_id=0) + assert rows == [{"callstack": "train.py:1"}] + def _make_mock_context(device_ids: list[int]): """Create a mock Context with the given device IDs.""" diff --git a/tests/test_api.py b/tests/test_api.py index 6c079ec..e656e27 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -78,6 +78,11 @@ def test_focus_state_fields(self) -> None: assert state.source == "explicit" assert state.available_devices == [0, 1] + def test_get_focus_reports_v1_callstack_layout(self, valid_db: Path) -> None: + analyzer = SnapshotAnalyzer(db_path=valid_db) + state = analyzer.get_focus() + assert state.callstack_layout == "v1" + class TestSnapshotAnalyzerWithDB: def test_get_focus_with_explicit_db(self, valid_db: Path) -> None: @@ -86,6 +91,7 @@ def test_get_focus_with_explicit_db(self, valid_db: Path) -> None: assert state.db_path == str(valid_db) assert state.source == "explicit" assert 0 in state.available_devices + assert state.callstack_layout == "v1" def test_set_focus_changes_db(self, valid_db: Path, tmp_path: Path) -> None: """Create a second DB and switch focus.""" @@ -224,13 +230,18 @@ def test_get_database_metadata_unavailable(self, valid_db: Path) -> None: assert result["reason"] == "metadata_missing" def test_get_database_metadata_available(self, valid_db: Path) -> None: + from dataclasses import replace + from pt_snap_cli.core.import_metadata import ImportMetadataService source = valid_db.parent / "snapshot.pkl" source.write_bytes(b"snapshot") service = ImportMetadataService() digest = service.calculate_sha256(source) - service.write(valid_db, service.build_metadata(source, digest, None)) + service.write( + valid_db, + replace(service.build_metadata(source, digest, None), import_format_version=1), + ) analyzer = SnapshotAnalyzer(db_path=valid_db) result = analyzer.get_database_metadata() diff --git a/tests/test_cli.py b/tests/test_cli.py index 9d4e52b..4d6f445 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -497,6 +497,22 @@ def test_focus_database_success(self, sample_db: Path) -> None: focus_file = Path.cwd() / ".pt-snap" / "focus.json" assert json.loads(focus_file.read_text())["db_path"] == str(sample_db.resolve()) + def test_focus_database_reports_v1_callstack_layout(self, tmp_path: Path) -> None: + db_path = tmp_path / "v1.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + + result = runner.invoke(app, ["focus", str(db_path)]) + assert result.exit_code == 0 + assert "Callstack layout: v1 (inline text)" in result.stdout + focus_data = json.loads((Path.cwd() / ".pt-snap" / "focus.json").read_text()) + assert "callstack_layout" not in focus_data + def test_focus_database_with_device(self, sample_db: Path) -> None: """Test 'focus --device' sets both database and device.""" result = runner.invoke(app, ["focus", str(sample_db), "--device", "0"]) diff --git a/tests/test_context.py b/tests/test_context.py index 94391fc..3e9d40e 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -84,6 +84,7 @@ def test_context_with_valid_database(self, valid_db: Path) -> None: ctx = Context(valid_db) assert ctx.db_path == valid_db assert ctx.device_ids == [0] + assert ctx.callstack_layout == "v1" def test_context_database_not_found(self, tmp_path: Path) -> None: """Test creating context with non-existent database.""" @@ -126,6 +127,34 @@ def test_device_filter(self, valid_db: Path) -> None: ctx2 = Context(valid_db, devices=[1]) assert ctx2.device_ids == [] + def test_v2_callstack_layout_detection(self, tmp_path: Path) -> None: + db_path = tmp_path / "v2.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstackId INTEGER)") + conn.commit() + conn.close() + + assert Context(db_path).callstack_layout == "v2" + + def test_both_callstack_columns_are_incompatible(self, tmp_path: Path) -> None: + db_path = tmp_path / "both.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute( + "CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT, callstackId INTEGER)" + ) + conn.commit() + conn.close() + + with pytest.raises(SchemaVersionError, match="both callstack and callstackId"): + Context(db_path) + def test_close_connection(self, valid_db: Path) -> None: """Test closing connection.""" ctx = Context(valid_db) diff --git a/tests/test_contract_cli_mcp.py b/tests/test_contract_cli_mcp.py index cd574f3..9ee14c6 100644 --- a/tests/test_contract_cli_mcp.py +++ b/tests/test_contract_cli_mcp.py @@ -126,8 +126,11 @@ def _normalize_cli_focus(output: str) -> dict[str, object]: int(device.strip()) for device in line.removeprefix("Available devices: ").split(",") ] + elif line.startswith("Callstack layout: "): + focus["callstack_layout"] = line.removeprefix("Callstack layout: ").split(" ", 1)[0] focus.setdefault("device_id", None) focus.setdefault("available_devices", []) + focus.setdefault("callstack_layout", None) return focus @@ -230,7 +233,9 @@ def test_focus_contract_matches_cli_and_mcp_semantics( "db_path": mcp_focus["db_path"], "device_id": mcp_focus["device_id"], "available_devices": mcp_focus["available_devices"], + "callstack_layout": mcp_focus["callstack_layout"], } + assert cli_focus["callstack_layout"] == "v1" def test_invalid_focus_device_contract_matches_cli_and_mcp_semantics( @@ -333,6 +338,46 @@ def test_query_execution_contract_matches_cli_and_mcp_semantics( assert cli_query == mcp_query +def test_event_query_contract_reads_v1_inline_callstack( + contract_db: Path, mcp_server: ModuleType +) -> None: + conn = sqlite3.connect(str(contract_db)) + conn.execute( + """ + INSERT INTO trace_entry_1 + (id, action, address, size, stream, allocated, active, reserved, callstack) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (1, 4, 0x1000, 2048, 0, 2048, 2048, 4096, "train.py:10"), + ) + conn.commit() + conn.close() + + cli_result = runner.invoke( + app, + [ + "query", + str(contract_db), + "--template-use", + "event", + "--params", + json.dumps({"id": 1}), + "--device", + "1", + "-n", + "0", + ], + ) + assert cli_result.exit_code == 0 + + server = _set_mcp_focus(mcp_server, contract_db) + cli_query = _normalize_cli_query_result(cli_result.stdout, device_id=1) + mcp_query = server.execute_query("event", params={"id": 1}, device_id=1, max_rows=0) + + assert cli_query == mcp_query + assert mcp_query["rows"][0]["callstack"] == "train.py:10" + + def test_query_parameter_error_contract_matches_cli_and_mcp_semantics( contract_db: Path, mcp_server: ModuleType ) -> None: diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index d0dfba8..9a49b78 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -140,6 +140,7 @@ def test_set_focus_returns_dict(self, valid_db: Path) -> None: assert isinstance(result, dict) assert result["db_path"] == str(valid_db) assert result["device_id"] == 0 + assert result["callstack_layout"] == "v1" def test_set_focus_rejects_invalid_device(self, valid_db: Path) -> None: import pt_snap_cli.mcp.server as server_mod From 0ddd7ebb72784a1bd3200cab509580541d4e78b1 Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 17 Sep 2026 23:34:12 +0800 Subject: [PATCH 2/2] fix: keep non-callstack analysis open when callstack layout conflicts Construction-time rejection blocked focus, metadata, and every template for conflicting external databases. Detect the conflict, surface it on focus, and fail only variant templates. --- AGENTS.md | 5 +- docs/en/database.md | 9 ++- docs/en/focus-management.md | 2 +- docs/en/snapshot-analyzer-api.md | 3 +- docs/zh/database.md | 9 ++- docs/zh/focus-management.md | 2 +- docs/zh/snapshot-analyzer-api.md | 3 +- src/pt_snap_cli/AGENTS.md | 7 +- src/pt_snap_cli/api.py | 2 + src/pt_snap_cli/cli.py | 9 ++- src/pt_snap_cli/context.py | 71 ++++++++++++------ src/pt_snap_cli/core/focus_service.py | 8 ++ src/pt_snap_cli/core/models.py | 1 + src/pt_snap_cli/mcp/server.py | 3 + src/pt_snap_cli/query/AGENTS.md | 6 +- src/pt_snap_cli/query/config.py | 13 +++- src/pt_snap_cli/query/executor.py | 28 ++++++- tests/core/test_focus_service.py | 22 ++++++ tests/query/test_callstack_schema_compat.py | 83 +++++++++++++++++++-- tests/query/test_config.py | 13 ++++ tests/query/test_executor.py | 4 +- tests/test_api.py | 20 +++++ tests/test_cli.py | 39 ++++++++++ tests/test_context.py | 6 +- tests/test_contract_cli_mcp.py | 5 ++ 25 files changed, 311 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d408104..48fa9aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,8 +86,9 @@ templates under category subdirectories are included by description, parameters, and `output_schema`. Unaffected templates keep a single `query`. `Context` detects the layout from read-only column/table checks (with `pt_snap_metadata` as an auxiliary check) and - `QueryExecutor` selects the matching SQL. Do not persist layout in - `focus.json` or migrate databases on open. + `QueryExecutor` selects the matching SQL. Conflicting or damaged layouts + leave `callstack_layout` unset and fail only variant templates. Do not + persist layout in `focus.json` or migrate databases on open. - When template metadata or behavior changes, review the YAML, config, registry, executor, `core/query_service.py`, CLI/MCP presentation, and focused tests together. diff --git a/docs/en/database.md b/docs/en/database.md index a737427..0a1c150 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -49,7 +49,8 @@ pt-snap import snapshot.pkl --force ``` Legacy or externally generated compatible databases without this table remain -queryable. `pt-snap` detects the callstack layout from table columns (and treats +queryable for templates that do not require a recognized callstack layout. +`pt-snap` detects the callstack layout from table columns (and treats `pt_snap_metadata.import_format_version` as an auxiliary check, not the only source of truth). Callstack templates use the matching SQL automatically. @@ -341,8 +342,10 @@ v2 SQL from the detected layout. Other templates do not depend on this split. Detection is read-only: `focus`, `query`, reports, the Python API, and MCP all open the database with SQLite `mode=ro`. Layout is not stored in `.pt-snap/focus.json`. Conflicting columns, mixed devices, a damaged -`callstack` table, or metadata that disagrees with the physical schema raise a -schema error instead of guessing. There is no implicit v1→v2 migration. +`callstack` table, or metadata that disagrees with the physical schema leave +the layout unrecognized and fail only callstack templates, instead of guessing +or blocking focus, metadata, and other queries. There is no implicit v1→v2 +migration. ```bash pt-snap focus legacy.db diff --git a/docs/en/focus-management.md b/docs/en/focus-management.md index 42e0461..183b859 100644 --- a/docs/en/focus-management.md +++ b/docs/en/focus-management.md @@ -56,7 +56,7 @@ its device, becomes effective again only after the session override is unset. pt-snap focus ``` -This shows the resolved database path, device ID, and where they came from (project focus, session env, or global config). When the database uses a recognized callstack schema, focus also prints `Callstack layout: v1 (inline text)` or `v2 (deduplicated)`. That label is detected read-only from the database; it is not stored in `.pt-snap/focus.json`. +This shows the resolved database path, device ID, and where they came from (project focus, session env, or global config). `pt-snap focus`, `pt-snap focus `, `pt-snap focus --global`, and `pt-snap focus --device` also print `Callstack layout: v1 (inline text)` or `v2 (deduplicated)` when the schema is recognized, or a warning when the layout conflicts. `pt-snap focus --session` prints only the `export PT_SNAP_DB_PATH=...` line so it can be evaluated by the shell. The layout is detected read-only from the database; it is not stored in `.pt-snap/focus.json`. ## Override Focus diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index 5ac4bd3..5aaa25e 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -62,7 +62,8 @@ explicit `db_path`, a validated device-only analyzer override is reported by | `device_id` | Analyzer device override, or the device attached to the resolved focus | | `source` | Resolution source such as `explicit`, `env`, `project`, `global`, or `none` | | `available_devices` | Device IDs discovered from `trace_entry_` tables | -| `callstack_layout` | `"v1"` for inline callstack text, `"v2"` for deduplicated `callstackId`, or `None` when unrecognized | +| `callstack_layout` | `"v1"` for inline callstack text, `"v2"` for deduplicated `callstackId`, or `None` when unrecognized or conflicting | +| `callstack_layout_error` | Conflict reason when the layout cannot be used, otherwise `None` | See [Focus Management](focus-management.md) for the complete resolution and persistence model. diff --git a/docs/zh/database.md b/docs/zh/database.md index a0871c9..9a37e6e 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -44,9 +44,9 @@ pt-snap metadata snapshot.pkl.db --json pt-snap import snapshot.pkl --force ``` -旧版或外部生成且结构兼容的 DB 仍可查询。`pt-snap` 根据表列识别调用栈布局,并把 -`pt_snap_metadata.import_format_version` 当作辅助校验,而不是唯一依据。调用栈相关 -模板会自动选择匹配的 SQL。 +旧版或外部生成且结构兼容的 DB 仍可查询不依赖可识别调用栈布局的模板。`pt-snap` +根据表列识别调用栈布局,并把 `pt_snap_metadata.import_format_version` 当作辅助 +校验,而不是唯一依据。调用栈相关模板会自动选择匹配的 SQL。 打开、focus 和查询都不会写入 `.db` 文件,也不会原地迁移 schema。导入缓存仍会在 当前导入器格式版本不匹配时重建目标库,这与读取兼容是分开的。没有 metadata 时, @@ -328,7 +328,8 @@ CREATE TABLE callstack ( 识别过程是只读的:`focus`、`query`、报告、Python API 和 MCP 都以 SQLite `mode=ro` 打开数据库。布局不会写入 `.pt-snap/focus.json`。列冲突、设备布局不一致、 -损坏的 `callstack` 表,或 metadata 与物理结构不一致时会明确报错,而不是猜测。 +损坏的 `callstack` 表,或 metadata 与物理结构不一致时,布局保持无法识别,并且 +只有调用栈模板会报错;不会猜测布局,也不会阻止 focus、metadata 和其他查询。 默认不会做 v1→v2 迁移。 ```bash diff --git a/docs/zh/focus-management.md b/docs/zh/focus-management.md index 35fb2b2..faee6e3 100644 --- a/docs/zh/focus-management.md +++ b/docs/zh/focus-management.md @@ -55,7 +55,7 @@ Session focus 只导出 `PT_SNAP_DB_PATH`,因此 `--session` 不能与 `--devi pt-snap focus ``` -显示已解析的数据库路径、设备 ID 及其来源(项目 focus、session 环境变量或全局配置)。若数据库使用可识别的调用栈 schema,还会打印 `Callstack layout: v1 (inline text)` 或 `v2 (deduplicated)`。该信息从数据库只读识别,不会写入 `.pt-snap/focus.json`。 +显示已解析的数据库路径、设备 ID 及其来源(项目 focus、session 环境变量或全局配置)。`pt-snap focus`、`pt-snap focus `、`pt-snap focus --global` 和 `pt-snap focus --device` 在识别到调用栈 schema 时还会打印 `Callstack layout: v1 (inline text)` 或 `v2 (deduplicated)`;布局冲突时打印警告。`pt-snap focus --session` 只输出 `export PT_SNAP_DB_PATH=...`,以便被 shell 直接执行。该信息从数据库只读识别,不会写入 `.pt-snap/focus.json`。 ## 覆盖焦点 diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index c596d33..b601f2d 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -57,7 +57,8 @@ focus 中的设备。没有显式 `db_path` 时,已验证的 analyzer 设备 | `device_id` | analyzer 设备覆盖;未设置覆盖时为已解析 focus 所附带的设备 | | `source` | `explicit`、`env`、`project`、`global`、`none` 等解析来源 | | `available_devices` | 从 `trace_entry_` 表发现的设备 ID | -| `callstack_layout` | 内联调用栈文本为 `"v1"`,去重 `callstackId` 为 `"v2"`,无法识别时为 `None` | +| `callstack_layout` | 内联调用栈文本为 `"v1"`,去重 `callstackId` 为 `"v2"`,无法识别或冲突时为 `None` | +| `callstack_layout_error` | 布局无法使用时的冲突原因,否则为 `None` | 完整解析和持久化模型见 [Focus 管理](focus-management.md)。 diff --git a/src/pt_snap_cli/AGENTS.md b/src/pt_snap_cli/AGENTS.md index bb00d9b..63b627a 100644 --- a/src/pt_snap_cli/AGENTS.md +++ b/src/pt_snap_cli/AGENTS.md @@ -35,9 +35,10 @@ - Preserve focus precedence: explicit CLI/API path, `PT_SNAP_DB_PATH`, nearest project `.pt-snap/focus.json`, then legacy global config. - Keep SQLite access read-only for analysis paths. - When adding or changing a repository skill under `skills/`, copy the same `SKILL.md` into `bundled_skills//`. -- `Context.callstack_layout` is `v1`, `v2`, or `None`; conflicting layouts raise - `SchemaVersionError` during construction. Query SQL variants are selected by - `QueryExecutor`, not by adapters. +- `Context.callstack_layout` is `v1`, `v2`, or `None`. Conflicting layouts leave + it `None` and set `callstack_layout_error`; construction still only requires + the `dictionary` table. Query SQL variants are selected by `QueryExecutor`, + not by adapters. ### Testing Requirements - CLI changes should update/run `tests/test_cli.py` and any affected config/focus tests. diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index 03eb61e..887cad3 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -28,6 +28,7 @@ class FocusState: source: str available_devices: list[int] callstack_layout: str | None = None + callstack_layout_error: str | None = None class SnapshotAnalyzer: @@ -73,6 +74,7 @@ def get_focus(self) -> FocusState: source=state.source, available_devices=state.available_devices, callstack_layout=state.callstack_layout, + callstack_layout_error=state.callstack_layout_error, ) def set_focus(self, db_path: str | None = None, device_id: int | None = None) -> FocusState: diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 3854e62..04f24a9 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -151,7 +151,7 @@ def focus_database( typer.echo(f"Focus file: {state.focus_file}") if state.device_id is not None: typer.echo(f"Focused device: {state.device_id}") - _echo_callstack_layout(state.callstack_layout) + _echo_callstack_layout(state.callstack_layout, state.callstack_layout_error) if not state.db_path.exists(): typer.secho("Warning: Database file does not exist!", fg=typer.colors.YELLOW) else: @@ -180,6 +180,7 @@ def focus_database( typer.secho(f"Focused device ({state.source}): {device}", fg=typer.colors.GREEN) if state.source == "project" and state.focus_file: typer.echo(f"Focus file: {state.focus_file}") + _echo_callstack_layout(state.callstack_layout, state.callstack_layout_error) raise typer.Exit() try: @@ -204,7 +205,7 @@ def focus_database( typer.echo(f"Available devices: {', '.join(map(str, state.available_devices))}") else: typer.echo("No devices found in database.") - _echo_callstack_layout(state.callstack_layout) + _echo_callstack_layout(state.callstack_layout, state.callstack_layout_error) except ( DatabaseMissingError, DatabaseSchemaError, @@ -973,11 +974,13 @@ def show_config( typer.echo(f" {key}: {value}") -def _echo_callstack_layout(layout: str | None) -> None: +def _echo_callstack_layout(layout: str | None, error: str | None = None) -> None: if layout == "v1": typer.echo("Callstack layout: v1 (inline text)") elif layout == "v2": typer.echo("Callstack layout: v2 (deduplicated)") + elif error: + typer.secho(f"Warning: {error}", fg=typer.colors.YELLOW) def _error(message: str) -> NoReturn: diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index f47c194..5e334b5 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -65,6 +65,7 @@ def __init__( self._connect_depth = 0 self._device_ids: list[int] | None = None self._callstack_layout: CallstackLayout | None = None + self._callstack_layout_error: str | None = None if not self.db_path.exists(): raise DatabaseNotFoundError(f"Database not found: {self.db_path}") @@ -83,7 +84,9 @@ def _validate_schema(self) -> None: "Invalid database schema: 'dictionary' table not found. " "This may not be a valid PyTorch memory snapshot database." ) - self._callstack_layout = self._detect_callstack_layout(conn) + self._callstack_layout, self._callstack_layout_error = self._detect_callstack_layout( + conn + ) @property def callstack_layout(self) -> CallstackLayout | None: @@ -92,12 +95,19 @@ def callstack_layout(self) -> CallstackLayout | None: ``v1`` is inline ``callstack`` text on each ``trace_entry_`` table. ``v2`` stores ``callstackId`` and a shared ``callstack`` table. Detection is read-only and cached on this context; it is not written - to focus files. Conflicting or damaged layouts raise - :class:`SchemaVersionError` during construction. + to focus files. Conflicting or damaged layouts leave this ``None`` and + set :attr:`callstack_layout_error` instead of failing construction. """ return self._callstack_layout - def _detect_callstack_layout(self, conn: sqlite3.Connection) -> CallstackLayout | None: + @property + def callstack_layout_error(self) -> str | None: + """Return a layout conflict reason, or None when none was detected.""" + return self._callstack_layout_error + + def _detect_callstack_layout( + self, conn: sqlite3.Connection + ) -> tuple[CallstackLayout | None, str | None]: """Identify v1/v2 callstack layout from table columns and metadata.""" cursor = conn.cursor() layouts: dict[int, CallstackLayout] = {} @@ -107,7 +117,7 @@ def _detect_callstack_layout(self, conn: sqlite3.Connection) -> CallstackLayout has_id = "callstackid" in columns has_text = "callstack" in columns if has_id and has_text: - raise SchemaVersionError( + return None, ( "Incompatible callstack layout: " f"'{table_name}' has both callstack and callstackId columns." ) @@ -119,7 +129,7 @@ def _detect_callstack_layout(self, conn: sqlite3.Connection) -> CallstackLayout unrecognized.append(device_id) if layouts and unrecognized: - raise SchemaVersionError( + return None, ( "Incompatible callstack layout: some devices have a recognized " "callstack schema but others do not " f"(unrecognized devices: {unrecognized})." @@ -127,42 +137,48 @@ def _detect_callstack_layout(self, conn: sqlite3.Connection) -> CallstackLayout distinct = set(layouts.values()) if len(distinct) > 1: - raise SchemaVersionError( + return None, ( "Incompatible callstack layout: devices disagree " f"({', '.join(sorted(distinct))})." ) layout = next(iter(distinct), None) - has_callstack_table = _has_table(cursor, "callstack") - if layout == "v1" and has_callstack_table: - raise SchemaVersionError( + callstack_table = _find_table(cursor, "callstack") + if layout == "v1" and callstack_table is not None: + return None, ( "Incompatible callstack layout: inline callstack text coexists " "with a shared callstack table." ) if layout == "v2": - if not has_callstack_table: - raise SchemaVersionError( + if callstack_table is None: + return None, ( "Incompatible callstack layout: callstackId is present but " "the shared callstack table is missing." ) - callstack_columns = _table_columns(cursor, "callstack") + callstack_columns = _table_columns(cursor, callstack_table) if not {"id", "callstack"}.issubset(callstack_columns): - raise SchemaVersionError( + return None, ( "Incompatible callstack layout: shared callstack table is " "missing id or callstack columns." ) metadata_version = _import_format_version(cursor) - if metadata_version in (1, 2) and layout is not None: + if metadata_version in (1, 2): expected: CallstackLayout = "v1" if metadata_version == 1 else "v2" + if layout is None: + return None, ( + "Incompatible callstack layout: " + f"pt_snap_metadata.import_format_version is {metadata_version} " + "but the database structure is unrecognized." + ) if layout != expected: - raise SchemaVersionError( + return None, ( "Incompatible callstack layout: " f"pt_snap_metadata.import_format_version is {metadata_version} " f"but the database structure is {layout}." ) - return layout + return layout, None @property def device_ids(self) -> list[int]: @@ -251,9 +267,17 @@ def _quote_ident(name: str) -> str: return '"' + name.replace('"', '""') + '"' +def _find_table(cursor: sqlite3.Cursor, name: str) -> str | None: + cursor.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=? COLLATE NOCASE", + (name,), + ) + row = cursor.fetchone() + return str(row[0]) if row is not None else None + + def _has_table(cursor: sqlite3.Cursor, name: str) -> bool: - cursor.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)) - return cursor.fetchone() is not None + return _find_table(cursor, name) is not None def _table_columns(cursor: sqlite3.Cursor, table_name: str) -> set[str]: @@ -277,13 +301,16 @@ def _trace_entry_tables(cursor: sqlite3.Cursor) -> list[tuple[int, str]]: def _import_format_version(cursor: sqlite3.Cursor) -> int | None: - if not _has_table(cursor, _METADATA_TABLE): + metadata_table = _find_table(cursor, _METADATA_TABLE) + if metadata_table is None: return None - columns = _table_columns(cursor, _METADATA_TABLE) + columns = _table_columns(cursor, metadata_table) if "import_format_version" not in columns: return None try: - rows = cursor.execute(f"SELECT import_format_version FROM {_METADATA_TABLE}").fetchall() + rows = cursor.execute( + f"SELECT import_format_version FROM {_quote_ident(metadata_table)}" + ).fetchall() except sqlite3.DatabaseError: return None if len(rows) != 1: diff --git a/src/pt_snap_cli/core/focus_service.py b/src/pt_snap_cli/core/focus_service.py index 8eb3f5a..45e31df 100644 --- a/src/pt_snap_cli/core/focus_service.py +++ b/src/pt_snap_cli/core/focus_service.py @@ -51,12 +51,14 @@ def get_focus( resolved = self.resolve_focus(explicit_db_path, explicit_device_id, start_dir) available_devices: list[int] = [] callstack_layout: str | None = None + callstack_layout_error: str | None = None if resolved.db_path is not None and resolved.db_path.exists(): try: ctx = Context(resolved.db_path) available_devices = ctx.device_ids callstack_layout = ctx.callstack_layout + callstack_layout_error = ctx.callstack_layout_error except (DatabaseNotFoundError, SchemaVersionError): available_devices = [] @@ -67,6 +69,7 @@ def get_focus( source=resolved.source, focus_file=resolved.focus_file, callstack_layout=callstack_layout, + callstack_layout_error=callstack_layout_error, ) def set_project_focus( @@ -86,6 +89,7 @@ def set_project_focus( source="project", focus_file=self._config.project_focus_path(base_dir), callstack_layout=ctx.callstack_layout, + callstack_layout_error=ctx.callstack_layout_error, ) def set_global_focus(self, db_path: Path | str, device_id: int | None = None) -> FocusState: @@ -100,6 +104,7 @@ def set_global_focus(self, db_path: Path | str, device_id: int | None = None) -> source="global", focus_file=self._config.config_file, callstack_layout=ctx.callstack_layout, + callstack_layout_error=ctx.callstack_layout_error, ) def set_device( @@ -124,6 +129,7 @@ def set_device( source="global", focus_file=self._config.config_file, callstack_layout=ctx.callstack_layout, + callstack_layout_error=ctx.callstack_layout_error, ) focus_file = self._config.write_project_focus( @@ -138,6 +144,7 @@ def set_device( source="project", focus_file=focus_file, callstack_layout=ctx.callstack_layout, + callstack_layout_error=ctx.callstack_layout_error, ) def clear_global_focus(self) -> None: @@ -164,6 +171,7 @@ def validate_session_db( source="explicit", focus_file=None, callstack_layout=ctx.callstack_layout, + callstack_layout_error=ctx.callstack_layout_error, ) def _validated_context(self, db_path: Path) -> Context: diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index 577d2f0..bed40b3 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -62,6 +62,7 @@ class FocusState: source: FocusSource = "none" focus_file: Path | None = None callstack_layout: str | None = None + callstack_layout_error: str | None = None @dataclass(frozen=True) diff --git a/src/pt_snap_cli/mcp/server.py b/src/pt_snap_cli/mcp/server.py index 986c8ae..380b935 100644 --- a/src/pt_snap_cli/mcp/server.py +++ b/src/pt_snap_cli/mcp/server.py @@ -23,6 +23,7 @@ def get_focus() -> dict[str, Any]: "source": state.source, "available_devices": state.available_devices, "callstack_layout": state.callstack_layout, + "callstack_layout_error": state.callstack_layout_error, } @@ -34,6 +35,7 @@ def set_focus(db_path: str | None = None, device_id: int | None = None) -> dict[ "device_id": state.device_id, "available_devices": state.available_devices, "callstack_layout": state.callstack_layout, + "callstack_layout_error": state.callstack_layout_error, } @@ -74,6 +76,7 @@ def focus_resource() -> dict[str, Any]: "source": state.source, "available_devices": state.available_devices, "callstack_layout": state.callstack_layout, + "callstack_layout_error": state.callstack_layout_error, } diff --git a/src/pt_snap_cli/query/AGENTS.md b/src/pt_snap_cli/query/AGENTS.md index 47f4e36..b3d6494 100644 --- a/src/pt_snap_cli/query/AGENTS.md +++ b/src/pt_snap_cli/query/AGENTS.md @@ -29,9 +29,9 @@ - Keep SQL value filters parameterized where using builder/condition APIs. - Device-specific template SQL should use injected table names such as `device_trace_table` and `device_block_table`. - Keep direct `QueryParameter` validation errors local; `QueryExecutor` must normalize them to its `TemplateRenderError` before the core boundary. -- Variant SQL is selected from `Context.callstack_layout`; unknown layouts must - fail variant templates with a layout-specific error rather than a generic - re-import message. +- Variant SQL is selected from `Context.callstack_layout`; unknown or conflicting + layouts must fail variant templates with a layout-specific error rather than a + generic re-import message. Non-variant templates keep working. ### Testing Requirements - Run `pytest tests/query` for query subsystem changes; real SQLite template semantics live in `test_peak_memory_templates.py`, `test_callstack_schema_compat.py`, and `test_query_max_rows_pushdown.py`. diff --git a/src/pt_snap_cli/query/config.py b/src/pt_snap_cli/query/config.py index 99975a7..b2486b8 100644 --- a/src/pt_snap_cli/query/config.py +++ b/src/pt_snap_cli/query/config.py @@ -127,10 +127,17 @@ def from_dict(cls, data: dict[str, Any], default_category: str | None = None) -> description=param_data.get("description", ""), ) - query_variants = _parse_query_variants(data.get("name", ""), data.get("query_variants")) - query = data.get("query") or "" + template_name = data.get("name", "") + query_variants = _parse_query_variants(template_name, data.get("query_variants")) + explicit_query = data.get("query") if query_variants: - query = query_variants.get("v2") or query + if isinstance(explicit_query, str) and explicit_query.strip(): + raise ValueError( + f"template '{template_name}' cannot declare both query and query_variants" + ) + query = query_variants.get("v2") or "" + else: + query = explicit_query or "" return cls( name=data.get("name", ""), diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index 23a9d77..0ecb38f 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -87,6 +87,9 @@ def _sql_for_layout(self, template: QueryTemplate) -> str: sql = template.sql_for_layout(self._callstack_layout()) if sql is not None: return sql + error = self._callstack_layout_error() + if error: + raise QueryExecutionError(f"Query execution failed: {error}") layout = self._callstack_layout() if layout is None: raise QueryExecutionError( @@ -106,6 +109,13 @@ def _callstack_layout(self) -> str | None: layout = getattr(context, "callstack_layout", None) return layout if isinstance(layout, str) else None + def _callstack_layout_error(self) -> str | None: + context = getattr(self, "_context", None) + if context is None: + return None + error = getattr(context, "callstack_layout_error", None) + return error if isinstance(error, str) and error else None + def _apply_limit(self, sql: str, limit: int) -> str: """Tighten a trailing numeric LIMIT or append one when absent.""" match = _TRAILING_LIMIT_RE.search(sql) @@ -128,15 +138,25 @@ def _append_limit(self, sql: str, limit: int) -> str: trimmed = trimmed[:-1].rstrip() return f"{trimmed} LIMIT {int(limit)}" - @staticmethod - def _execution_error(error: sqlite3.OperationalError) -> QueryExecutionError: + def _execution_error(self, error: sqlite3.OperationalError) -> QueryExecutionError: detail = str(error) if "no such table: callstack" in detail or re.search( r"no such column: (?:[\w]+\.)?callstackId", detail ): + layout = self._callstack_layout() + if layout == "v1": + return QueryExecutionError( + "Query execution failed: this SQL requires callstackId, " + "but this database uses the v1 inline-callstack layout" + ) + if layout == "v2": + return QueryExecutionError( + "Query execution failed: this SQL expects the v1 inline-callstack " + "layout, but this database uses the v2 callstackId layout" + ) return QueryExecutionError( - "Query execution failed: database uses the legacy inline-callstack layout; " - "re-import the snapshot to use callstack queries" + "Query execution failed: this SQL requires a recognized v1 or v2 " + "callstack layout" ) return QueryExecutionError(f"Query execution failed: {error}") diff --git a/tests/core/test_focus_service.py b/tests/core/test_focus_service.py index 8e8871c..023db70 100644 --- a/tests/core/test_focus_service.py +++ b/tests/core/test_focus_service.py @@ -124,5 +124,27 @@ def test_set_project_focus_reports_v1_layout_without_persisting_it( focus_data = json.loads((Path.cwd() / ".pt-snap" / "focus.json").read_text()) assert state.callstack_layout == "v1" + assert state.callstack_layout_error is None assert "callstack_layout" not in focus_data assert set(focus_data) == {"db_path"} + + def test_get_focus_reports_layout_conflict_without_hiding_devices(self, tmp_path: Path) -> None: + db_path = tmp_path / "conflict.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute( + "CREATE TABLE trace_entry_0 " + "(id INTEGER PRIMARY KEY, callstack TEXT, callstackId INTEGER)" + ) + conn.commit() + conn.close() + config = Config() + config.write_project_focus(db_path) + + state = FocusService(config).get_focus() + assert state.available_devices == [0] + assert state.callstack_layout is None + assert state.callstack_layout_error is not None + assert "both callstack and callstackId" in state.callstack_layout_error diff --git a/tests/query/test_callstack_schema_compat.py b/tests/query/test_callstack_schema_compat.py index b41b990..86f3c58 100644 --- a/tests/query/test_callstack_schema_compat.py +++ b/tests/query/test_callstack_schema_compat.py @@ -8,8 +8,9 @@ import pytest -from pt_snap_cli.context import Context, SchemaVersionError +from pt_snap_cli.context import Context from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.query.config import QueryTemplate from pt_snap_cli.query.executor import QueryExecutionError, QueryExecutor from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates, get_query @@ -305,10 +306,19 @@ def test_multi_device_v1_layout_is_consistent(tmp_path: Path) -> None: assert rows[0]["callstack"] == "train.py:10" -def test_metadata_conflict_is_a_schema_error(tmp_path: Path) -> None: +def test_metadata_conflict_fails_only_variant_templates(tmp_path: Path) -> None: db_path = create_v1_db(tmp_path / "conflict.db", metadata_version=2) - with pytest.raises(SchemaVersionError, match="import_format_version is 2"): - Context(db_path) + ctx = Context(db_path) + assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is not None + assert "import_format_version is 2" in ctx.callstack_layout_error + executor = QueryExecutor(ctx) + with pytest.raises(QueryExecutionError, match="import_format_version is 2"): + executor.execute_template("event", device_id=0) + executor.register_template( + QueryTemplate(name="plain_size", query="SELECT size FROM {{ device_trace_table }}") + ) + assert executor.execute_template("plain_size", device_id=0)[0]["size"] == 1024 def test_unknown_metadata_version_does_not_override_structure(tmp_path: Path) -> None: @@ -316,7 +326,7 @@ def test_unknown_metadata_version_does_not_override_structure(tmp_path: Path) -> assert Context(db_path).callstack_layout == "v2" -def test_mixed_device_layouts_are_a_schema_error(tmp_path: Path) -> None: +def test_mixed_device_layouts_fail_only_variant_templates(tmp_path: Path) -> None: db_path = tmp_path / "mixed.db" conn = sqlite3.connect(str(db_path)) _create_dictionary(conn) @@ -325,8 +335,18 @@ def test_mixed_device_layouts_are_a_schema_error(tmp_path: Path) -> None: conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") conn.commit() conn.close() - with pytest.raises(SchemaVersionError, match="devices disagree"): - Context(db_path) + ctx = Context(db_path) + assert ctx.device_ids == [0, 1] + assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is not None + assert "devices disagree" in ctx.callstack_layout_error + executor = QueryExecutor(ctx) + with pytest.raises(QueryExecutionError, match="devices disagree"): + executor.execute_template("event", device_id=0) + executor.register_template( + QueryTemplate(name="plain_id", query="SELECT id FROM {{ device_trace_table }}") + ) + assert executor.execute_template("plain_id", device_id=0) == [] def test_unknown_layout_errors_only_for_variant_templates(tmp_path: Path) -> None: @@ -339,5 +359,54 @@ def test_unknown_layout_errors_only_for_variant_templates(tmp_path: Path) -> Non ctx = Context(db_path) assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is None + executor = QueryExecutor(ctx) with pytest.raises(QueryExecutionError, match="needs a v1 or v2 callstack layout"): + executor.execute_template("event", device_id=0) + executor.register_template( + QueryTemplate(name="plain_size", query="SELECT size FROM {{ device_trace_table }}") + ) + assert executor.execute_template("plain_size", device_id=0) == [] + + +def test_unrecognized_structure_conflicts_with_metadata_version(tmp_path: Path) -> None: + db_path = tmp_path / "unrecognized-meta.db" + conn = sqlite3.connect(str(db_path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, size INTEGER)") + _insert_metadata(conn, 2) + conn.commit() + conn.close() + + ctx = Context(db_path) + assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is not None + assert "import_format_version is 2" in ctx.callstack_layout_error + assert "unrecognized" in ctx.callstack_layout_error + with pytest.raises(QueryExecutionError, match="import_format_version is 2"): QueryExecutor(ctx).execute_template("event", device_id=0) + + +def test_v2_layout_accepts_case_insensitive_callstack_table(tmp_path: Path) -> None: + db_path = tmp_path / "Callstack.db" + conn = sqlite3.connect(str(db_path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE Callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstackId INTEGER)") + conn.commit() + conn.close() + assert Context(db_path).callstack_layout == "v2" + + +def test_v1_layout_detects_case_insensitive_shared_table(tmp_path: Path) -> None: + db_path = tmp_path / "v1-Callstack.db" + conn = sqlite3.connect(str(db_path)) + _create_dictionary(conn) + conn.execute("CREATE TABLE Callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + ctx = Context(db_path) + assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is not None + assert "coexists" in ctx.callstack_layout_error diff --git a/tests/query/test_config.py b/tests/query/test_config.py index 4af0edf..0a49148 100644 --- a/tests/query/test_config.py +++ b/tests/query/test_config.py @@ -138,6 +138,19 @@ def test_from_dict_rejects_incomplete_query_variants(self): } ) + def test_from_dict_rejects_query_and_query_variants(self): + with pytest.raises(ValueError, match="cannot declare both query and query_variants"): + QueryTemplate.from_dict( + { + "name": "event", + "query": "SELECT 1", + "query_variants": { + "v1": "SELECT callstack FROM t", + "v2": "SELECT callstackId FROM t", + }, + } + ) + class TestQueryConfig: def test_load_yaml_from_string(self): diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 981f8b0..977d26a 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -353,7 +353,7 @@ def test_legacy_callstack_schema_error_is_actionable(self, tmp_path): ) ) - with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + with pytest.raises(QueryExecutionError, match="v1 inline-callstack layout"): executor.execute_template("legacy_callstack", device_id=0) def test_qualified_legacy_callstack_schema_error_is_actionable(self, tmp_path): @@ -369,7 +369,7 @@ def test_qualified_legacy_callstack_schema_error_is_actionable(self, tmp_path): ) ) - with pytest.raises(QueryExecutionError, match="legacy inline-callstack layout"): + with pytest.raises(QueryExecutionError, match="recognized v1 or v2"): executor.execute_template("qualified_legacy_callstack", device_id=0) def test_variant_template_uses_inline_sql_on_v1_database(self, tmp_path): diff --git a/tests/test_api.py b/tests/test_api.py index e656e27..9185211 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -82,6 +82,26 @@ def test_get_focus_reports_v1_callstack_layout(self, valid_db: Path) -> None: analyzer = SnapshotAnalyzer(db_path=valid_db) state = analyzer.get_focus() assert state.callstack_layout == "v1" + assert state.callstack_layout_error is None + + def test_get_focus_reports_conflicting_callstack_layout(self, tmp_path: Path) -> None: + db_path = tmp_path / "conflict.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute( + "CREATE TABLE trace_entry_0 " + "(id INTEGER PRIMARY KEY, callstack TEXT, callstackId INTEGER)" + ) + conn.commit() + conn.close() + + state = SnapshotAnalyzer(db_path=db_path).get_focus() + assert state.available_devices == [0] + assert state.callstack_layout is None + assert state.callstack_layout_error is not None + assert "both callstack and callstackId" in state.callstack_layout_error class TestSnapshotAnalyzerWithDB: diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d6f445..c787929 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -513,6 +513,45 @@ def test_focus_database_reports_v1_callstack_layout(self, tmp_path: Path) -> Non focus_data = json.loads((Path.cwd() / ".pt-snap" / "focus.json").read_text()) assert "callstack_layout" not in focus_data + def test_focus_database_warns_on_conflicting_callstack_layout(self, tmp_path: Path) -> None: + db_path = tmp_path / "conflict.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute( + "CREATE TABLE trace_entry_0 " + "(id INTEGER PRIMARY KEY, callstack TEXT, callstackId INTEGER)" + ) + conn.commit() + conn.close() + + result = runner.invoke(app, ["focus", str(db_path)]) + assert result.exit_code == 0 + assert "Warning:" in result.stdout + assert "both callstack and callstackId" in result.stdout + focus_data = json.loads((Path.cwd() / ".pt-snap" / "focus.json").read_text()) + assert focus_data["db_path"] == str(db_path.resolve()) + assert "callstack_layout" not in focus_data + + def test_focus_device_only_reports_callstack_layout(self, tmp_path: Path) -> None: + db_path = tmp_path / "v1.db" + conn = sqlite3.connect(str(db_path)) + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.commit() + conn.close() + focus_dir = Path.cwd() / ".pt-snap" + focus_dir.mkdir() + (focus_dir / "focus.json").write_text(json.dumps({"db_path": str(db_path)})) + + result = runner.invoke(app, ["focus", "--device", "0"]) + assert result.exit_code == 0 + assert "Focused device" in result.stdout + assert "Callstack layout: v1 (inline text)" in result.stdout + def test_focus_database_with_device(self, sample_db: Path) -> None: """Test 'focus --device' sets both database and device.""" result = runner.invoke(app, ["focus", str(sample_db), "--device", "0"]) diff --git a/tests/test_context.py b/tests/test_context.py index 3e9d40e..9ff48e6 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -152,8 +152,10 @@ def test_both_callstack_columns_are_incompatible(self, tmp_path: Path) -> None: conn.commit() conn.close() - with pytest.raises(SchemaVersionError, match="both callstack and callstackId"): - Context(db_path) + ctx = Context(db_path) + assert ctx.callstack_layout is None + assert ctx.callstack_layout_error is not None + assert "both callstack and callstackId" in ctx.callstack_layout_error def test_close_connection(self, valid_db: Path) -> None: """Test closing connection.""" diff --git a/tests/test_contract_cli_mcp.py b/tests/test_contract_cli_mcp.py index 9ee14c6..3863f97 100644 --- a/tests/test_contract_cli_mcp.py +++ b/tests/test_contract_cli_mcp.py @@ -128,9 +128,12 @@ def _normalize_cli_focus(output: str) -> dict[str, object]: ] elif line.startswith("Callstack layout: "): focus["callstack_layout"] = line.removeprefix("Callstack layout: ").split(" ", 1)[0] + elif line.startswith("Warning: "): + focus["callstack_layout_error"] = line.removeprefix("Warning: ") focus.setdefault("device_id", None) focus.setdefault("available_devices", []) focus.setdefault("callstack_layout", None) + focus.setdefault("callstack_layout_error", None) return focus @@ -234,8 +237,10 @@ def test_focus_contract_matches_cli_and_mcp_semantics( "device_id": mcp_focus["device_id"], "available_devices": mcp_focus["available_devices"], "callstack_layout": mcp_focus["callstack_layout"], + "callstack_layout_error": mcp_focus["callstack_layout_error"], } assert cli_focus["callstack_layout"] == "v1" + assert cli_focus["callstack_layout_error"] is None def test_invalid_focus_device_contract_matches_cli_and_mcp_semantics(