diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01ce41b..220a439 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -86,6 +86,10 @@ jobs: FIXTURE=tests/fixtures/snapshots/snapshot_with_empty_cache.pkl rm -rf /tmp/pt-snap-wheel-smoke mkdir -p /tmp/pt-snap-wheel-smoke + DB=/tmp/pt-snap-wheel-smoke/snapshot_with_empty_cache.pkl.db pt-snap import "$FIXTURE" --no-focus --output-dir /tmp/pt-snap-wheel-smoke - test -f /tmp/pt-snap-wheel-smoke/snapshot_with_empty_cache.pkl.db + test -f "$DB" + SECOND_IMPORT=$(pt-snap import "$FIXTURE" --no-focus --output-dir /tmp/pt-snap-wheel-smoke) + echo "$SECOND_IMPORT" | grep -q '^Reused:' + pt-snap metadata "$DB" --json | python -c "import json, sys; data=json.load(sys.stdin); assert data['status'] == 'available'" rm -rf /tmp/pt-snap-wheel-smoke diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebcf5b4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/superpowers/ diff --git a/README.md b/README.md index 2c9f27d..8af5dcb 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ To start from a raw PyTorch memory snapshot, import the pickle into a SnapshotDB ```bash pt-snap import snapshot.pkl +pt-snap metadata snapshot.pkl.db pt-snap query --list ``` @@ -41,7 +42,9 @@ See the [full quick start guide](docs/en/quickstart.md) for a walkthrough. |---------|-------------| | `pt-snap focus` | Set and manage analysis focus (database + device) | | `pt-snap import ` | Import a PyTorch memory snapshot pickle into a SnapshotDB | +| `pt-snap metadata [database.db]` | Inspect SnapshotDB import provenance and compatibility metadata | | `pt-snap query` | Run memory analysis queries | +| `pt-snap report` | Generate higher-level memory analysis reports | | `pt-snap config` | Manage global configuration | | `pt-snap-mcp` | Start the MCP server for agent integration | @@ -63,6 +66,7 @@ The server exposes the following tools: | `list_templates` | List available query templates | | `get_template_info` | Get template details and parameters | | `execute_query` | Run a query template against the focused database | +| `get_database_metadata` | Inspect import metadata for the focused or specified database | See the [MCP guide](docs/en/mcp.md) for setup and usage details. @@ -79,11 +83,11 @@ See the [MCP guide](docs/en/mcp.md) for setup and usage details. ## Query Templates -6 built-in templates across 3 categories: +9 built-in templates across 3 categories: - **Basic**: `block`, `event`, `allocation` -- **Statistical**: `callstack_analysis`, `memory_peak` -- **Business**: `leak_detection` +- **Statistical**: `active_blocks_at_event`, `allocator_gap`, `callstack_analysis`, `memory_peak` +- **Business**: `active_memory_callstack_at_event`, `leak_detection` See [Querying](docs/en/querying.md) for details. diff --git a/README_zh.md b/README_zh.md index f8a1160..c2b34ce 100644 --- a/README_zh.md +++ b/README_zh.md @@ -13,6 +13,10 @@ pip install -e . ## 快速开始 ```bash +# 从原始 pickle 导入;相同内容和配置会自动复用已有 DB +pt-snap import snapshot.pkl +pt-snap metadata snapshot.pkl.db + # 设置快照数据库和设备 pt-snap focus examples/snapshot_expandable.pkl.db --device 0 @@ -33,7 +37,10 @@ pt-snap query --template-use leak_detection --params '{"min_size": 1024}' | 命令 | 说明 | |------|------| | `pt-snap focus` | 设置和管理分析焦点(数据库 + 设备) | +| `pt-snap import ` | 将 PyTorch 原始内存快照导入 SnapshotDB | +| `pt-snap metadata [database.db]` | 查看 SnapshotDB 的导入来源与兼容性 metadata | | `pt-snap query` | 运行内存分析查询 | +| `pt-snap report` | 生成高层内存分析报告 | | `pt-snap config` | 管理全局配置 | | `pt-snap-mcp` | 启动 MCP 服务器以支持 Agent 集成 | @@ -55,6 +62,7 @@ pt-snap-mcp | `list_templates` | 列出可用的查询模板 | | `get_template_info` | 获取模板详情和参数 | | `execute_query` | 对焦点数据库执行查询模板 | +| `get_database_metadata` | 查看焦点或指定数据库的导入 metadata | 详见 [MCP 指南](docs/zh/mcp.md)。 @@ -71,11 +79,11 @@ pt-snap-mcp ## 查询模板 -6 个内置模板,3 个分类: +9 个内置模板,3 个分类: - **Basic**: `block`, `event`, `allocation` -- **Statistical**: `callstack_analysis`, `memory_peak` -- **Business**: `leak_detection` +- **Statistical**: `active_blocks_at_event`, `allocator_gap`, `callstack_analysis`, `memory_peak` +- **Business**: `active_memory_callstack_at_event`, `leak_detection` 详见 [Querying](docs/zh/querying.md)。 diff --git a/docs/README.md b/docs/README.md index 38ba05d..3e60b2c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,7 @@ | Quick Start | [en/quickstart.md](en/quickstart.md) | | Context Management | [en/focus-management.md](en/focus-management.md) | | Querying | [en/querying.md](en/querying.md) | +| MCP server | [en/mcp.md](en/mcp.md) | | Database Schema | [en/database.md](en/database.md) | | ResultMapper API | [en/result-mapper-api.md](en/result-mapper-api.md) | @@ -17,5 +18,6 @@ | 快速开始 | [zh/quickstart.md](zh/quickstart.md) | | Focus 管理 | [zh/focus-management.md](zh/focus-management.md) | | 运行查询 | [zh/querying.md](zh/querying.md) | +| MCP 服务器 | [zh/mcp.md](zh/mcp.md) | | 数据库格式 | [zh/database.md](zh/database.md) | | ResultMapper API | [zh/result-mapper-api.md](zh/result-mapper-api.md) | diff --git a/docs/en/database.md b/docs/en/database.md index dcc6ee3..6bc1333 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -28,6 +28,25 @@ pt-snap query --list You can also generate the same SnapshotDB format with another compatible producer, then point `pt-snap focus` at the resulting `.db` file. +### Import cache and metadata + +Generated databases contain a `pt_snap_metadata` table with the source SHA-256, exact device +selection, import format version, producer version, and completion time. `pt-snap import` reuses an +existing target only when the full source SHA-256, import format version, and device selection match. +Package version changes alone do not invalidate the cache. + +```bash +# Inspect provenance and compatibility metadata +pt-snap metadata snapshot.pkl.db +pt-snap metadata snapshot.pkl.db --json + +# Bypass a matching cache +pt-snap import snapshot.pkl --force +``` + +Legacy or externally generated compatible databases without this table remain queryable. Their +metadata status is reported as unavailable, and a later import rebuilds them once before reuse. + --- ## Database Structure diff --git a/docs/en/mcp.md b/docs/en/mcp.md index a8fdd1d..73b5b2a 100644 --- a/docs/en/mcp.md +++ b/docs/en/mcp.md @@ -29,6 +29,7 @@ This starts a FastMCP server that exposes tools for analyzing PyTorch memory sna | `list_templates` | List available query templates, optionally filtered by category | | `get_template_info` | Get detailed information about a template including parameters | | `execute_query` | Execute a query template against the focused database | +| `get_database_metadata` | Inspect import provenance for the focused or specified database | ## Available Resources @@ -48,6 +49,7 @@ This starts a FastMCP server that exposes tools for analyzing PyTorch memory sna 2. Call `list_templates` to discover available queries 3. Call `get_template_info` with a template name to see its parameters 4. Call `execute_query` with the template name and parameters +5. Call `get_database_metadata` when import provenance or cache compatibility is needed ## Example Usage @@ -66,6 +68,10 @@ get_template_info("leak_detection") # Run a query execute_query("leak_detection", params={"min_size": 1024}) # Returns: {"total": 5, "returned": 5, "rows": [...]} + +# Inspect database import metadata +get_database_metadata() +# Returns: {"status": "available", "metadata": {"source_sha256": "...", ...}} ``` ## CLI Commands diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 8ce8443..d2d198b 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -18,9 +18,18 @@ If you have a raw `.pkl` memory snapshot, import it first with the built-in back ```bash pt-snap import snapshot.pkl +pt-snap metadata snapshot.pkl.db pt-snap query --list ``` +The import command stores a SHA-256 source fingerprint and import compatibility metadata inside the +generated database. Repeating the same import with the same device selection reuses the existing DB. +Use `--force` when you intentionally need to rebuild it: + +```bash +pt-snap import snapshot.pkl --force +``` + ### Step 1: Set the Snapshot Database and Device Point `pt-snap` to your SQLite snapshot database file: diff --git a/docs/zh/database.md b/docs/zh/database.md index 9eb5656..693cf63 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -11,6 +11,27 @@ SnapshotDB 是内存快照数据的 SQLite 数据库存储格式,用于持久 **数据库示例文件**: `snapshot_expandable.pkl.db` +## 生成数据库与导入缓存 + +如果输入是 PyTorch 原始 `.pkl` 内存快照,可以直接导入: + +```bash +pt-snap import snapshot.pkl +``` + +新生成的数据库包含 `pt_snap_metadata` 表,记录原始文件 SHA-256、精确设备选择、导入格式 +版本、生产者版本和完成时间。只有完整 SHA-256、导入格式版本和设备选择均匹配时,重复导入 +才会复用已有 DB;仅升级 `pt-snap-cli` 包版本不会让缓存失效。 + +```bash +pt-snap metadata snapshot.pkl.db +pt-snap metadata snapshot.pkl.db --json +pt-snap import snapshot.pkl --force +``` + +旧版或外部生成且结构兼容的 DB 仍可正常查询。若没有 metadata,查询会返回 unavailable, +下一次执行对应 pickle 导入时会重建一次。 + --- ## 数据库结构 diff --git a/docs/zh/mcp.md b/docs/zh/mcp.md index f488083..a0ea112 100644 --- a/docs/zh/mcp.md +++ b/docs/zh/mcp.md @@ -29,6 +29,7 @@ pt-snap-mcp | `list_templates` | 列出可用的查询模板,可按类别筛选 | | `get_template_info` | 获取模板详情,包括参数信息 | | `execute_query` | 对焦点数据库执行查询模板 | +| `get_database_metadata` | 查看焦点或指定数据库的导入来源 metadata | ## 可用资源 @@ -48,6 +49,7 @@ pt-snap-mcp 2. 调用 `list_templates` 发现可用查询 3. 调用 `get_template_info` 查看模板参数 4. 调用 `execute_query` 执行查询 +5. 需要确认导入来源或缓存兼容性时调用 `get_database_metadata` ## 使用示例 @@ -66,6 +68,10 @@ get_template_info("leak_detection") # 运行查询 execute_query("leak_detection", params={"min_size": 1024}) # 返回: {"total": 5, "returned": 5, "rows": [...]} + +# 查看数据库导入 metadata +get_database_metadata() +# 返回: {"status": "available", "metadata": {"source_sha256": "...", ...}} ``` ## CLI 命令 diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 46c78c1..5a82258 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -18,9 +18,17 @@ pip install -e . ```bash pt-snap import snapshot.pkl +pt-snap metadata snapshot.pkl.db pt-snap query --list ``` +导入命令会在数据库内保存原始文件的 SHA-256 和导入兼容性 metadata。再次使用相同设备配置 +导入相同内容时,会直接复用已有 DB。如需明确重建,可使用: + +```bash +pt-snap import snapshot.pkl --force +``` + ### 第一步:设置快照数据库和设备 将 `pt-snap` 指向你的 SQLite 快照数据库文件: diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index 88280d3..620cf50 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -12,6 +12,7 @@ DatabaseSchemaError, FocusNotConfiguredError, FocusService, + ImportMetadataService, QueryService, ) @@ -39,6 +40,7 @@ def __init__( self._device_id = device_id self._focus_service = FocusService(self._config) self._query_service = QueryService(self._focus_service) + self._metadata_service = ImportMetadataService() def get_focus(self) -> FocusState: state = self._focus_service.get_focus( @@ -121,3 +123,18 @@ def execute_query( "device_id": result.device_id, "rows": result.rows, } + + def get_database_metadata(self, db_path: str | None = None) -> dict[str, Any]: + resolved = self._focus_service.resolve_focus( + explicit_db_path=db_path if db_path is not None else self._db_path, + explicit_device_id=self._device_id, + ) + if resolved.db_path is None: + raise RuntimeError("No database configured. Call set_focus() first.") + try: + inspection = self._metadata_service.inspect(resolved.db_path) + except DatabaseMissingError as exc: + raise FileNotFoundError(str(exc)) from exc + except DatabaseSchemaError as exc: + raise ValueError(str(exc)) from exc + return self._metadata_service.inspection_to_dict(inspection) diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 3f8d649..049bdfc 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -21,6 +21,7 @@ FocusNotConfiguredError, FocusService, ImportExecutionError, + ImportMetadataService, ImportOptions, ImportService, ImportToolMissingError, @@ -164,6 +165,7 @@ def import_snapshot( output_dir: Annotated[Path | None, typer.Option("--output-dir", "-o")] = None, device: Annotated[int | None, typer.Option("--device", "-d")] = None, no_focus: Annotated[bool, typer.Option("--no-focus", help="Skip focus update")] = False, + force: Annotated[bool, typer.Option("--force", help="Rebuild even when cache matches")] = False, ) -> None: """Import a PyTorch memory snapshot into a SQLite database.""" try: @@ -173,6 +175,7 @@ def import_snapshot( output_dir=output_dir, device=device, set_focus=not no_focus, + force=force, ) ) except ( @@ -182,11 +185,51 @@ def import_snapshot( ) as e: _error(str(e)) - typer.echo(f"Imported: {result.db_path}") + action = "Reused" if result.reused else "Imported" + typer.echo(f"{action}: {result.db_path}") + if result.cache_miss_reason is not None: + typer.echo(f"Cache miss: {result.cache_miss_reason}") if result.focus_state is not None and result.focus_state.focus_file is not None: typer.echo(f"Focus: {result.focus_state.focus_file}") +@app.command("metadata") +def show_database_metadata( + db_path: Annotated[ + Path | None, typer.Argument(help="Path to database file (optional if configured)") + ] = None, + json_output: Annotated[bool, typer.Option("--json", help="Emit machine-readable JSON")] = False, +) -> None: + """Show import metadata for a SnapshotDB.""" + focus_service = _focus_service() + try: + resolved = focus_service.resolve_focus(explicit_db_path=db_path) + if resolved.db_path is None: + raise FocusNotConfiguredError("No database path specified and no database configured.") + service = ImportMetadataService() + inspection = service.inspect(resolved.db_path) + except ( + FocusFileInvalidError, + FocusNotConfiguredError, + DatabaseMissingError, + DatabaseSchemaError, + ) as e: + _error(str(e)) + + if json_output: + typer.echo(json.dumps(service.inspection_to_dict(inspection), indent=2)) + return + + typer.echo(f"Database: {inspection.db_path}") + typer.echo(f"Status: {inspection.status}") + if inspection.reason is not None: + typer.echo(f"Reason: {inspection.reason}") + if inspection.metadata is not None: + typer.echo("Metadata:") + for key, value in cast(Mapping[str, object], asdict(inspection.metadata)).items(): + typer.echo(f" {key}: {value}") + + @app.command("query") def query_database( db_path: Annotated[ diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index 11826e5..66beda6 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -6,6 +6,7 @@ from collections.abc import Generator from contextlib import contextmanager from pathlib import Path +from urllib.parse import quote class DatabaseNotFoundError(FileNotFoundError): @@ -116,7 +117,9 @@ def connect(self) -> Generator[sqlite3.Connection, None, None]: yield self._conn return - uri = f"file:{self.db_path}?mode=ro" + # Keep the colon in Windows drive prefixes while encoding SQLite URI metacharacters. + encoded_path = quote(self.db_path.as_posix(), safe="/:") + uri = f"file:{encoded_path}?mode=ro" self._conn = sqlite3.connect(uri, uri=True) self._conn.row_factory = sqlite3.Row try: diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index d69218a..32f25a0 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -4,21 +4,26 @@ FocusFileInvalidError, FocusNotConfiguredError, ImportExecutionError, + ImportMetadataError, ImportToolMissingError, InvalidCategoryError, InvalidDeviceError, PtSnapCoreError, QueryExecutionError, SnapshotFileInvalidError, + SourceChangedError, TemplateNotFoundError, TemplateRenderError, ) from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.import_metadata import ImportMetadataService from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.models import ( FocusState, + ImportMetadata, ImportOptions, ImportResult, + MetadataInspection, PeakMemoryReport, QueryResult, ResolvedFocus, @@ -43,8 +48,12 @@ "ImportToolMissingError", "SnapshotFileInvalidError", "ImportExecutionError", + "ImportMetadataError", + "SourceChangedError", "ResolvedFocus", "FocusState", + "ImportMetadata", + "MetadataInspection", "TemplateParameter", "TemplateSummary", "TemplateInfo", @@ -53,6 +62,7 @@ "ImportOptions", "ImportResult", "FocusService", + "ImportMetadataService", "QueryService", "ReportService", "ImportService", diff --git a/src/pt_snap_cli/core/errors.py b/src/pt_snap_cli/core/errors.py index 62e20e2..a09fd61 100644 --- a/src/pt_snap_cli/core/errors.py +++ b/src/pt_snap_cli/core/errors.py @@ -69,3 +69,15 @@ class ImportExecutionError(PtSnapCoreError): """ pass + + +class ImportMetadataError(PtSnapCoreError): + """Raised when import metadata cannot be read, written, or validated.""" + + pass + + +class SourceChangedError(ImportExecutionError): + """Raised when the source snapshot changes while an import is running.""" + + pass diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py new file mode 100644 index 0000000..c1d1584 --- /dev/null +++ b/src/pt_snap_cli/core/import_metadata.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import hashlib +import re +import sqlite3 +from contextlib import closing +from dataclasses import asdict +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from pt_snap_cli import __version__ +from pt_snap_cli.context import Context, DatabaseNotFoundError, SchemaVersionError +from pt_snap_cli.core.errors import DatabaseMissingError, DatabaseSchemaError, ImportMetadataError +from pt_snap_cli.core.models import ( + CacheDecision, + ImportMetadata, + MetadataInspection, +) + +METADATA_TABLE = "pt_snap_metadata" +METADATA_SCHEMA_VERSION = 1 +IMPORT_FORMAT_VERSION = 1 +IMPORTER_NAME = "pt-snap-cli" +HASH_CHUNK_SIZE = 1024 * 1024 + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +_REQUIRED_COLUMNS = { + "id", + "metadata_schema_version", + "import_format_version", + "source_sha256", + "source_size", + "source_name", + "requested_device", + "importer_name", + "importer_version", + "completed_at", +} + + +class ImportMetadataService: + def calculate_sha256(self, source_path: Path) -> str: + digest = hashlib.sha256() + try: + with source_path.open("rb") as source: + while chunk := source.read(HASH_CHUNK_SIZE): + digest.update(chunk) + except OSError as exc: + raise ImportMetadataError(f"Cannot hash snapshot file {source_path}: {exc}") from exc + return digest.hexdigest() + + def build_metadata( + self, + source_path: Path, + source_sha256: str, + requested_device: int | None, + ) -> ImportMetadata: + try: + source_size = source_path.stat().st_size + except OSError as exc: + raise ImportMetadataError(f"Cannot inspect snapshot file {source_path}: {exc}") from exc + + return ImportMetadata( + metadata_schema_version=METADATA_SCHEMA_VERSION, + import_format_version=IMPORT_FORMAT_VERSION, + source_sha256=source_sha256, + source_size=source_size, + source_name=source_path.name, + requested_device=requested_device, + importer_name=IMPORTER_NAME, + importer_version=__version__, + completed_at=datetime.now(timezone.utc).isoformat(), + ) + + def inspect(self, db_path: Path | str) -> MetadataInspection: + path = Path(db_path).expanduser().resolve() + try: + context = Context(path) + except DatabaseNotFoundError as exc: + raise DatabaseMissingError(str(exc)) from exc + except (SchemaVersionError, sqlite3.DatabaseError) as exc: + raise DatabaseSchemaError(str(exc)) from exc + + try: + with context.connect() as conn: + table = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (METADATA_TABLE,), + ).fetchone() + if table is None: + return MetadataInspection( + db_path=path, + status="unavailable", + reason="metadata_missing", + ) + + columns = { + row["name"] for row in conn.execute(f"PRAGMA table_info({METADATA_TABLE})") + } + if not _REQUIRED_COLUMNS.issubset(columns): + return MetadataInspection( + db_path=path, + status="invalid", + reason="metadata_invalid", + ) + + rows = conn.execute(f"SELECT * FROM {METADATA_TABLE}").fetchall() + except sqlite3.DatabaseError: + return MetadataInspection( + db_path=path, + status="invalid", + reason="metadata_invalid", + ) + + if len(rows) != 1 or rows[0]["id"] != 1: + return MetadataInspection( + db_path=path, + status="invalid", + reason="metadata_invalid", + ) + + try: + metadata = self._metadata_from_row(dict(rows[0])) + except (KeyError, TypeError, ValueError): + return MetadataInspection( + db_path=path, + status="invalid", + reason="metadata_invalid", + ) + + if metadata.metadata_schema_version != METADATA_SCHEMA_VERSION: + return MetadataInspection( + db_path=path, + status="invalid", + metadata=metadata, + reason="metadata_version_unsupported", + ) + + return MetadataInspection(db_path=path, status="available", metadata=metadata) + + def write(self, db_path: Path | str, metadata: ImportMetadata) -> None: + path = Path(db_path) + try: + self._validate_metadata(metadata, allow_unsupported_version=False) + except (TypeError, ValueError) as exc: + raise ImportMetadataError(f"Cannot write invalid import metadata: {exc}") from exc + try: + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute(f"DROP TABLE IF EXISTS {METADATA_TABLE}") + conn.execute(f""" + CREATE TABLE {METADATA_TABLE} ( + id INTEGER PRIMARY KEY CHECK (id = 1), + metadata_schema_version INTEGER NOT NULL, + import_format_version INTEGER NOT NULL, + source_sha256 TEXT NOT NULL, + source_size INTEGER NOT NULL, + source_name TEXT NOT NULL, + requested_device INTEGER, + importer_name TEXT NOT NULL, + importer_version TEXT NOT NULL, + completed_at TEXT NOT NULL + ) + """) + conn.execute( + f""" + INSERT INTO {METADATA_TABLE} ( + id, metadata_schema_version, import_format_version, + source_sha256, source_size, source_name, requested_device, + importer_name, importer_version, completed_at + ) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + metadata.metadata_schema_version, + metadata.import_format_version, + metadata.source_sha256, + metadata.source_size, + metadata.source_name, + metadata.requested_device, + metadata.importer_name, + metadata.importer_version, + metadata.completed_at, + ), + ) + except sqlite3.DatabaseError as exc: + raise ImportMetadataError(f"Cannot write import metadata to {path}: {exc}") from exc + + def evaluate_cache( + self, + db_path: Path | str, + source_sha256: str, + requested_device: int | None, + force: bool = False, + ) -> CacheDecision: + path = Path(db_path) + if not path.exists(): + return CacheDecision(reused=False, metadata=None, reason="database_missing") + if force: + return CacheDecision(reused=False, metadata=None, reason="forced") + + try: + inspection = self.inspect(path) + except (DatabaseMissingError, DatabaseSchemaError): + return CacheDecision(reused=False, metadata=None, reason="database_invalid") + + if inspection.status != "available" or inspection.metadata is None: + return CacheDecision( + reused=False, + metadata=inspection.metadata, + reason=inspection.reason or "metadata_invalid", + ) + + metadata = inspection.metadata + if metadata.source_sha256 != source_sha256: + return CacheDecision(reused=False, metadata=metadata, reason="source_changed") + if metadata.import_format_version != IMPORT_FORMAT_VERSION: + return CacheDecision(reused=False, metadata=metadata, reason="import_format_changed") + if metadata.requested_device != requested_device: + return CacheDecision(reused=False, metadata=metadata, reason="device_changed") + return CacheDecision(reused=True, metadata=metadata, reason=None) + + @staticmethod + def inspection_to_dict(inspection: MetadataInspection) -> dict[str, Any]: + return { + "db_path": str(inspection.db_path), + "status": inspection.status, + "reason": inspection.reason, + "metadata": asdict(inspection.metadata) if inspection.metadata is not None else None, + } + + def _metadata_from_row(self, row: dict[str, Any]) -> ImportMetadata: + metadata = ImportMetadata( + metadata_schema_version=self._required_int(row["metadata_schema_version"]), + import_format_version=self._required_int(row["import_format_version"]), + source_sha256=self._required_str(row["source_sha256"]), + source_size=self._required_int(row["source_size"]), + source_name=self._required_str(row["source_name"]), + requested_device=self._optional_int(row["requested_device"]), + importer_name=self._required_str(row["importer_name"]), + importer_version=self._required_str(row["importer_version"]), + completed_at=self._required_str(row["completed_at"]), + ) + self._validate_metadata(metadata, allow_unsupported_version=True) + return metadata + + @staticmethod + def _validate_metadata( + metadata: ImportMetadata, + *, + allow_unsupported_version: bool, + ) -> None: + if ( + not allow_unsupported_version + and metadata.metadata_schema_version != METADATA_SCHEMA_VERSION + ): + raise ValueError("Unsupported metadata schema version") + if metadata.metadata_schema_version < 1 or metadata.import_format_version < 1: + raise ValueError("Metadata versions must be positive integers") + if not _SHA256_RE.fullmatch(metadata.source_sha256): + raise ValueError("source_sha256 must be a lowercase SHA-256 digest") + if metadata.source_size < 0 or not metadata.source_name: + raise ValueError("Source metadata is invalid") + if metadata.requested_device is not None and metadata.requested_device < 0: + raise ValueError("requested_device must be non-negative") + if not metadata.importer_name or not metadata.importer_version: + raise ValueError("Importer metadata is invalid") + parsed_time = datetime.fromisoformat(metadata.completed_at) + if parsed_time.tzinfo is None or parsed_time.utcoffset() != timedelta(0): + raise ValueError("completed_at must use UTC") + + @staticmethod + def _required_int(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError("Expected integer metadata value") + return value + + @staticmethod + def _optional_int(value: Any) -> int | None: + if value is None: + return None + return ImportMetadataService._required_int(value) + + @staticmethod + def _required_str(value: Any) -> str: + if not isinstance(value, str) or not value: + raise TypeError("Expected non-empty metadata string") + return value diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py index 6bea179..36af999 100644 --- a/src/pt_snap_cli/core/import_service.py +++ b/src/pt_snap_cli/core/import_service.py @@ -7,10 +7,13 @@ DatabaseSchemaError, FocusFileInvalidError, ImportExecutionError, + ImportMetadataError, SnapshotFileInvalidError, + SourceChangedError, ) from pt_snap_cli.core.focus_service import FocusService -from pt_snap_cli.core.models import ImportOptions, ImportResult +from pt_snap_cli.core.import_metadata import ImportMetadataService +from pt_snap_cli.core.models import FocusState, ImportMetadata, ImportOptions, ImportResult from pt_snap_cli.core.snapshot_import_backend import SnapshotImportBackend logger = logging.getLogger(__name__) @@ -25,34 +28,106 @@ def __init__( self, focus_service: FocusService | None = None, backend: SnapshotImportBackend | None = None, + metadata_service: ImportMetadataService | None = None, ) -> None: - self._focus_service = focus_service - self._backend = backend or SnapshotImportBackend() + self._focus_service: FocusService | None = focus_service + self._backend: SnapshotImportBackend = backend or SnapshotImportBackend() + self._metadata_service: ImportMetadataService = metadata_service or ImportMetadataService() def import_snapshot(self, options: ImportOptions) -> ImportResult: self._validate_snapshot_file(options.snapshot_file) output_dir = self._resolve_output_dir(options.snapshot_file, options.output_dir) - db_path = self._backend.dump_to_db(options.snapshot_file, output_dir, options.device) + db_path = self._backend.target_db_path(options.snapshot_file, output_dir) + + try: + source_sha256 = self._metadata_service.calculate_sha256(options.snapshot_file) + except ImportMetadataError as exc: + raise ImportExecutionError(str(exc)) from exc + + decision = self._metadata_service.evaluate_cache( + db_path=db_path, + source_sha256=source_sha256, + requested_device=options.device, + force=options.force, + ) + if decision.reused: + if decision.metadata is None: + raise AssertionError("Cache hit must include import metadata") + focus_state = self._set_focus_if_requested(db_path, options) + return ImportResult( + db_path=db_path, + device_id=options.device, + focus_state=focus_state, + reused=True, + metadata=decision.metadata, + cache_miss_reason=None, + ) + + completed_metadata: ImportMetadata | None = None + + def finalize_temp_db(tmp_db_path: Path) -> None: + nonlocal completed_metadata + try: + final_sha256 = self._metadata_service.calculate_sha256(options.snapshot_file) + except ImportMetadataError as exc: + raise ImportExecutionError(str(exc)) from exc + if final_sha256 != source_sha256: + raise SourceChangedError( + "Snapshot source changed while import was running; existing database preserved." + ) - focus_state = None - if options.set_focus: - if self._focus_service is None: - self._focus_service = FocusService() try: - focus_state = self._focus_service.set_project_focus( - db_path=db_path, - device_id=options.device, + metadata = self._metadata_service.build_metadata( + source_path=options.snapshot_file, + source_sha256=final_sha256, + requested_device=options.device, ) - except (FocusFileInvalidError, DatabaseSchemaError) as exc: - # Focus write happens after the .db is on disk; if the new - # database fails validation (e.g. vendor produced a non-pt-snap - # schema), report it as an import execution failure rather than - # leaking the underlying domain error through the import API. - raise ImportExecutionError( - f"Imported database cannot be registered as focus: {exc}" - ) from exc - - return ImportResult(db_path=db_path, device_id=options.device, focus_state=focus_state) + self._metadata_service.write(tmp_db_path, metadata) + inspection = self._metadata_service.inspect(tmp_db_path) + except (ImportMetadataError, DatabaseSchemaError) as exc: + raise ImportExecutionError(f"Imported database metadata is invalid: {exc}") from exc + + if inspection.status != "available" or inspection.metadata != metadata: + raise ImportExecutionError("Imported database metadata validation failed.") + completed_metadata = metadata + + db_path = self._backend.dump_to_db( + options.snapshot_file, + output_dir, + options.device, + finalize_temp_db=finalize_temp_db, + ) + if completed_metadata is None: + raise ImportExecutionError("Imported database metadata was not produced.") + + focus_state = self._set_focus_if_requested(db_path, options) + return ImportResult( + db_path=db_path, + device_id=options.device, + focus_state=focus_state, + reused=False, + metadata=completed_metadata, + cache_miss_reason=decision.reason, + ) + + def _set_focus_if_requested( + self, + db_path: Path, + options: ImportOptions, + ) -> FocusState | None: + if not options.set_focus: + return None + if self._focus_service is None: + self._focus_service = FocusService() + try: + return self._focus_service.set_project_focus( + db_path=db_path, + device_id=options.device, + ) + except (FocusFileInvalidError, DatabaseSchemaError) as exc: + raise ImportExecutionError( + f"Imported database cannot be registered as focus: {exc}" + ) from exc @staticmethod def _validate_snapshot_file(path: Path) -> None: diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index d06ef98..8f93afe 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -5,6 +5,18 @@ from typing import Any, Literal FocusSource = Literal["explicit", "env", "project", "global", "none"] +CacheMissReason = Literal[ + "database_missing", + "database_invalid", + "metadata_missing", + "metadata_invalid", + "metadata_version_unsupported", + "source_changed", + "import_format_changed", + "device_changed", + "forced", +] +MetadataStatus = Literal["available", "unavailable", "invalid"] @dataclass(frozen=True) @@ -89,6 +101,35 @@ class ImportOptions: output_dir: Path | None = None device: int | None = None set_focus: bool = True + force: bool = False + + +@dataclass(frozen=True) +class ImportMetadata: + metadata_schema_version: int + import_format_version: int + source_sha256: str + source_size: int + source_name: str + requested_device: int | None + importer_name: str + importer_version: str + completed_at: str + + +@dataclass(frozen=True) +class MetadataInspection: + db_path: Path + status: MetadataStatus + metadata: ImportMetadata | None = None + reason: CacheMissReason | None = None + + +@dataclass(frozen=True) +class CacheDecision: + reused: bool + metadata: ImportMetadata | None + reason: CacheMissReason | None @dataclass(frozen=True) @@ -105,3 +146,6 @@ class ImportResult: db_path: Path device_id: int | None focus_state: FocusState | None + reused: bool + metadata: ImportMetadata + cache_miss_reason: CacheMissReason | None diff --git a/src/pt_snap_cli/core/snapshot_import_backend.py b/src/pt_snap_cli/core/snapshot_import_backend.py index c57dfbd..e4bb023 100644 --- a/src/pt_snap_cli/core/snapshot_import_backend.py +++ b/src/pt_snap_cli/core/snapshot_import_backend.py @@ -4,6 +4,7 @@ import pickle import sqlite3 import tempfile +from collections.abc import Callable from pathlib import Path from pt_snap_cli.core.errors import ImportExecutionError, ImportToolMissingError @@ -17,8 +18,9 @@ def dump_to_db( snapshot_file: Path, output_dir: Path, device: int | None = None, + finalize_temp_db: Callable[[Path], None] | None = None, ) -> Path: - db_path = output_dir / f"{snapshot_file.name}.db" + db_path = self.target_db_path(snapshot_file, output_dir) output_dir.mkdir(parents=True, exist_ok=True) run_dump_to_db = self._load_run_dump_to_db() @@ -41,6 +43,9 @@ def dump_to_db( if not tmp_db_path.is_file(): raise ImportExecutionError(f"Expected database not produced: {tmp_db_path}") + if finalize_temp_db is not None: + finalize_temp_db(tmp_db_path) + try: os.replace(tmp_db_path, db_path) except OSError as exc: @@ -50,6 +55,10 @@ def dump_to_db( return db_path + @staticmethod + def target_db_path(snapshot_file: Path, output_dir: Path) -> Path: + return output_dir / f"{snapshot_file.name}.db" + @staticmethod def _load_run_dump_to_db(): try: diff --git a/src/pt_snap_cli/mcp/server.py b/src/pt_snap_cli/mcp/server.py index 5dfe9e2..9784c8f 100644 --- a/src/pt_snap_cli/mcp/server.py +++ b/src/pt_snap_cli/mcp/server.py @@ -58,6 +58,11 @@ def execute_query( return _analyzer.execute_query(template, params, device_id, max_rows) +@mcp.tool() +def get_database_metadata(db_path: str | None = None) -> dict[str, Any]: + return _analyzer.get_database_metadata(db_path) + + @mcp.resource("focus://current") def focus_resource() -> dict[str, Any]: state = _analyzer.get_focus() diff --git a/tests/core/test_import_metadata.py b/tests/core/test_import_metadata.py new file mode 100644 index 0000000..a116017 --- /dev/null +++ b/tests/core/test_import_metadata.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import sqlite3 +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from pt_snap_cli.core.import_metadata import ( + IMPORT_FORMAT_VERSION, + METADATA_SCHEMA_VERSION, + ImportMetadataService, +) +from pt_snap_cli.core.models import ImportMetadata + + +def _create_snapshot_db(path: Path) -> Path: + with sqlite3.connect(path) as conn: + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE block_0 (id INTEGER PRIMARY KEY)") + return path + + +def _metadata(**overrides: object) -> ImportMetadata: + values: dict[str, object] = { + "metadata_schema_version": METADATA_SCHEMA_VERSION, + "import_format_version": IMPORT_FORMAT_VERSION, + "source_sha256": "a" * 64, + "source_size": 123, + "source_name": "snapshot.pkl", + "requested_device": None, + "importer_name": "pt-snap-cli", + "importer_version": "1.2.3", + "completed_at": "2026-07-18T10:00:00+00:00", + } + values.update(overrides) + return ImportMetadata(**values) # type: ignore[arg-type] + + +def test_import_metadata_is_frozen() -> None: + metadata = _metadata() + with pytest.raises(FrozenInstanceError): + metadata.source_name = "other.pkl" # type: ignore[misc] + + +def test_calculate_sha256_reads_complete_file(tmp_path: Path) -> None: + source = tmp_path / "snapshot.pkl" + source.write_bytes(b"abc") + + assert ImportMetadataService().calculate_sha256(source) == ( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ) + + +def test_write_and_inspect_metadata(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + metadata = _metadata() + + service.write(db_path, metadata) + inspection = service.inspect(db_path) + + assert inspection.status == "available" + assert inspection.reason is None + assert inspection.metadata == metadata + + +def test_write_closes_database_connection(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + from unittest.mock import MagicMock + + import pt_snap_cli.core.import_metadata as metadata_module + + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + connection = sqlite3.connect(db_path) + tracked_connection = MagicMock(wraps=connection) + tracked_connection.__enter__.return_value = tracked_connection + tracked_connection.__exit__.side_effect = connection.__exit__ + monkeypatch.setattr(metadata_module.sqlite3, "connect", lambda path: tracked_connection) + + ImportMetadataService().write(db_path, _metadata()) + + tracked_connection.close.assert_called_once_with() + + +def test_inspect_reports_missing_metadata(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "legacy.db") + + inspection = ImportMetadataService().inspect(db_path) + + assert inspection.status == "unavailable" + assert inspection.reason == "metadata_missing" + + +def test_inspect_reports_invalid_metadata_columns(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "invalid.db") + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE pt_snap_metadata (id INTEGER)") + conn.execute("INSERT INTO pt_snap_metadata VALUES (1)") + + inspection = ImportMetadataService().inspect(db_path) + + assert inspection.status == "invalid" + assert inspection.reason == "metadata_invalid" + + +def test_inspect_reports_unsupported_metadata_version(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "future.db") + service = ImportMetadataService() + service.write(db_path, _metadata()) + with sqlite3.connect(db_path) as conn: + conn.execute("UPDATE pt_snap_metadata SET metadata_schema_version = 999") + + inspection = service.inspect(db_path) + + assert inspection.status == "invalid" + assert inspection.reason == "metadata_version_unsupported" + + +def test_inspect_rejects_non_utc_completed_at(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "non-utc.db") + service = ImportMetadataService() + service.write(db_path, _metadata()) + with sqlite3.connect(db_path) as conn: + conn.execute("UPDATE pt_snap_metadata SET completed_at = '2026-07-18T18:00:00+08:00'") + + inspection = service.inspect(db_path) + + assert inspection.status == "invalid" + assert inspection.reason == "metadata_invalid" + + +def test_inspect_rejects_duplicate_metadata_rows(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "duplicate.db") + with sqlite3.connect(db_path) as conn: + conn.execute(""" + CREATE TABLE pt_snap_metadata ( + id INTEGER, + metadata_schema_version INTEGER, + import_format_version INTEGER, + source_sha256 TEXT, + source_size INTEGER, + source_name TEXT, + requested_device INTEGER, + importer_name TEXT, + importer_version TEXT, + completed_at TEXT + ) + """) + values = ( + METADATA_SCHEMA_VERSION, + IMPORT_FORMAT_VERSION, + "a" * 64, + 123, + "snapshot.pkl", + None, + "pt-snap-cli", + "1.2.3", + "2026-07-18T10:00:00+00:00", + ) + conn.execute("INSERT INTO pt_snap_metadata VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)", values) + conn.execute("INSERT INTO pt_snap_metadata VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?)", values) + + inspection = ImportMetadataService().inspect(db_path) + + assert inspection.status == "invalid" + assert inspection.reason == "metadata_invalid" + + +@pytest.mark.parametrize( + ("source_sha256", "device", "expected_reason"), + [ + ("b" * 64, None, "source_changed"), + ("a" * 64, 0, "device_changed"), + ], +) +def test_cache_decision_invalidations( + tmp_path: Path, + source_sha256: str, + device: int | None, + expected_reason: str, +) -> None: + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + service.write(db_path, _metadata()) + + decision = service.evaluate_cache(db_path, source_sha256, device) + + assert decision.reused is False + assert decision.reason == expected_reason + + +def test_cache_hit_ignores_importer_version(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + service.write(db_path, _metadata(importer_version="old-version")) + + decision = service.evaluate_cache(db_path, "a" * 64, None) + + assert decision.reused is True + assert decision.reason is None + + +def test_same_size_source_change_invalidates_cache(tmp_path: Path) -> None: + first_source = tmp_path / "first.pkl" + second_source = tmp_path / "second.pkl" + first_source.write_bytes(b"abc") + second_source.write_bytes(b"abd") + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + first_hash = service.calculate_sha256(first_source) + second_hash = service.calculate_sha256(second_source) + service.write(db_path, _metadata(source_sha256=first_hash, source_size=3)) + + decision = service.evaluate_cache(db_path, second_hash, None) + + assert first_source.stat().st_size == second_source.stat().st_size + assert decision.reused is False + assert decision.reason == "source_changed" + + +@pytest.mark.parametrize("requested_device", [None, 1]) +def test_exact_device_selection_invalidates_cache( + requested_device: int | None, tmp_path: Path +) -> None: + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + service.write(db_path, _metadata(requested_device=0)) + + decision = service.evaluate_cache(db_path, "a" * 64, requested_device) + + assert decision.reused is False + assert decision.reason == "device_changed" + + +def test_force_bypasses_compatible_cache(tmp_path: Path) -> None: + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + service.write(db_path, _metadata()) + + decision = service.evaluate_cache(db_path, "a" * 64, None, force=True) + + assert decision.reused is False + assert decision.reason == "forced" + + +def test_force_missing_database_reports_database_missing(tmp_path: Path) -> None: + decision = ImportMetadataService().evaluate_cache( + tmp_path / "missing.db", "a" * 64, None, force=True + ) + + assert decision.reused is False + assert decision.reason == "database_missing" + + +def test_import_format_change_invalidates_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import pt_snap_cli.core.import_metadata as metadata_module + + db_path = _create_snapshot_db(tmp_path / "snapshot.db") + service = ImportMetadataService() + service.write(db_path, _metadata()) + monkeypatch.setattr(metadata_module, "IMPORT_FORMAT_VERSION", IMPORT_FORMAT_VERSION + 1) + + decision = service.evaluate_cache(db_path, "a" * 64, None) + + assert decision.reused is False + assert decision.reason == "import_format_changed" diff --git a/tests/core/test_import_models.py b/tests/core/test_import_models.py index e1b90c6..6f1c7ef 100644 --- a/tests/core/test_import_models.py +++ b/tests/core/test_import_models.py @@ -5,7 +5,22 @@ import dataclasses from pathlib import Path -from pt_snap_cli.core.models import ImportOptions, ImportResult +from pt_snap_cli.core.import_metadata import IMPORT_FORMAT_VERSION, METADATA_SCHEMA_VERSION +from pt_snap_cli.core.models import ImportMetadata, ImportOptions, ImportResult + + +def _metadata() -> ImportMetadata: + return ImportMetadata( + metadata_schema_version=METADATA_SCHEMA_VERSION, + import_format_version=IMPORT_FORMAT_VERSION, + source_sha256="a" * 64, + source_size=123, + source_name="sample.pkl", + requested_device=0, + importer_name="pt-snap-cli", + importer_version="1.2.3", + completed_at="2026-07-18T10:00:00+00:00", + ) def test_import_options_defaults() -> None: @@ -13,6 +28,7 @@ def test_import_options_defaults() -> None: assert options.output_dir is None assert options.device is None assert options.set_focus is True + assert options.force is False def test_import_options_is_frozen() -> None: @@ -25,14 +41,29 @@ def test_import_options_is_frozen() -> None: def test_import_result_construction() -> None: - result = ImportResult(db_path=Path("sample.pkl.db"), device_id=0, focus_state=None) + result = ImportResult( + db_path=Path("sample.pkl.db"), + device_id=0, + focus_state=None, + reused=False, + metadata=_metadata(), + cache_miss_reason="database_missing", + ) assert result.db_path == Path("sample.pkl.db") assert result.device_id == 0 assert result.focus_state is None + assert result.reused is False def test_import_result_is_frozen() -> None: - result = ImportResult(db_path=Path("sample.pkl.db"), device_id=0, focus_state=None) + result = ImportResult( + db_path=Path("sample.pkl.db"), + device_id=0, + focus_state=None, + reused=False, + metadata=_metadata(), + cache_miss_reason="database_missing", + ) try: result.db_path = Path("other.db") # type: ignore[misc] except dataclasses.FrozenInstanceError: diff --git a/tests/core/test_import_service.py b/tests/core/test_import_service.py index 0c91206..15d3733 100644 --- a/tests/core/test_import_service.py +++ b/tests/core/test_import_service.py @@ -42,6 +42,8 @@ def test_import_writes_db_and_sets_focus(monkeypatch: pytest.MonkeyPatch, tmp_pa assert result.db_path.suffix == ".db" assert (project_dir / ".pt-snap" / "focus.json").exists() assert not (output_dir / ".pt-snap" / "focus.json").exists() + assert result.reused is False + assert result.cache_miss_reason == "database_missing" @pytest.mark.slow @@ -176,7 +178,12 @@ def test_import_raises_on_upstream_failure(monkeypatch: pytest.MonkeyPatch, tmp_ import_service_cls = _import_service_type() service = import_service_cls() - def fail_dump_to_db(snapshot_file: Path, dump_dir: Path, device: int | None) -> Path: + def fail_dump_to_db( + snapshot_file: Path, + dump_dir: Path, + device: int | None, + finalize_temp_db=None, + ) -> Path: raise ImportExecutionError("Vendored snapshot import backend failed: upstream failed") monkeypatch.setattr(service._backend, "dump_to_db", fail_dump_to_db) @@ -215,3 +222,281 @@ def test_import_output_path_uses_full_filename(tmp_path: Path) -> None: ) assert result.db_path.name == "snapshot.pickle.db" + + +@pytest.mark.parametrize("character", ["?", "#"]) +def test_import_supports_sqlite_uri_characters_in_filename(character: str, tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / f"snapshot{character}.pickle" + snapshot_file.write_bytes(EMPTY_CACHE_SNAPSHOT.read_bytes()) + + result = service.import_snapshot( + ImportOptions(snapshot_file=snapshot_file, output_dir=tmp_path, set_focus=False) + ) + + assert result.db_path.name == f"snapshot{character}.pickle.db" + assert service._metadata_service.inspect(result.db_path).status == "available" + + +def test_repeated_import_reuses_database_without_backend( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + first = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + def fail_if_called(*args: object, **kwargs: object) -> Path: + raise AssertionError("backend must not run on cache hit") + + monkeypatch.setattr(service._backend, "dump_to_db", fail_if_called) + second = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + assert second.db_path == first.db_path + assert second.reused is True + assert second.cache_miss_reason is None + assert second.metadata == first.metadata + + +def test_force_rebuilds_compatible_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + original_dump = service._backend.dump_to_db + calls = 0 + + def counting_dump(*args: object, **kwargs: object) -> Path: + nonlocal calls + calls += 1 + return original_dump(*args, **kwargs) + + monkeypatch.setattr(service._backend, "dump_to_db", counting_dump) + result = service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + force=True, + ) + ) + + assert calls == 1 + assert result.reused is False + assert result.cache_miss_reason == "forced" + + +def test_force_first_import_reports_database_missing(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + result = import_service_cls().import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + force=True, + ) + ) + + assert result.reused is False + assert result.cache_miss_reason == "database_missing" + + +def test_malformed_metadata_rebuilds_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import sqlite3 + + import_service_cls = _import_service_type() + service = import_service_cls() + first = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + with sqlite3.connect(first.db_path) as conn: + conn.execute("UPDATE pt_snap_metadata SET source_sha256 = 'invalid'") + + original_dump = service._backend.dump_to_db + calls = 0 + + def counting_dump(*args: object, **kwargs: object) -> Path: + nonlocal calls + calls += 1 + return original_dump(*args, **kwargs) + + monkeypatch.setattr(service._backend, "dump_to_db", counting_dump) + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + assert calls == 1 + assert result.reused is False + assert result.cache_miss_reason == "metadata_invalid" + assert service._metadata_service.inspect(result.db_path).status == "available" + + +def test_legacy_database_rebuilds_once_then_reuses(tmp_path: Path) -> None: + import sqlite3 + + import_service_cls = _import_service_type() + service = import_service_cls() + legacy_db = tmp_path / f"{EMPTY_CACHE_SNAPSHOT.name}.db" + with sqlite3.connect(legacy_db) as conn: + conn.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + conn.execute("CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY)") + conn.execute("CREATE TABLE block_0 (id INTEGER PRIMARY KEY)") + + rebuilt = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + reused = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + assert rebuilt.reused is False + assert rebuilt.cache_miss_reason == "metadata_missing" + assert reused.reused is True + + +def test_metadata_write_failure_preserves_existing_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from pt_snap_cli.core.errors import ImportMetadataError + + import_service_cls = _import_service_type() + service = import_service_cls() + first = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + original_metadata = service._metadata_service.inspect(first.db_path).metadata + + def fail_write(*args: object, **kwargs: object) -> None: + raise ImportMetadataError("metadata write failed") + + monkeypatch.setattr(service._metadata_service, "write", fail_write) + with pytest.raises(ImportExecutionError, match="metadata write failed"): + service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + force=True, + ) + ) + + assert service._metadata_service.inspect(first.db_path).metadata == original_metadata + + +def test_initial_hash_failure_preserves_existing_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from pt_snap_cli.core.errors import ImportMetadataError + + import_service_cls = _import_service_type() + service = import_service_cls() + first = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + original_metadata = service._metadata_service.inspect(first.db_path).metadata + monkeypatch.setattr( + service._metadata_service, + "calculate_sha256", + lambda path: (_ for _ in ()).throw(ImportMetadataError("hash failed")), + ) + + with pytest.raises(ImportExecutionError, match="hash failed"): + service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + force=True, + ) + ) + + assert service._metadata_service.inspect(first.db_path).metadata == original_metadata + + +def test_import_format_change_rebuilds_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import pt_snap_cli.core.import_metadata as metadata_module + + import_service_cls = _import_service_type() + service = import_service_cls() + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + monkeypatch.setattr( + metadata_module, + "IMPORT_FORMAT_VERSION", + metadata_module.IMPORT_FORMAT_VERSION + 1, + ) + + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + assert result.reused is False + assert result.cache_miss_reason == "import_format_changed" + + +def test_device_change_rebuilds_database(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + service.import_snapshot( + ImportOptions( + snapshot_file=MULTI_DEVICE_SNAPSHOT, + output_dir=tmp_path, + device=0, + set_focus=False, + ) + ) + + result = service.import_snapshot( + ImportOptions( + snapshot_file=MULTI_DEVICE_SNAPSHOT, + output_dir=tmp_path, + device=1, + set_focus=False, + ) + ) + + assert result.reused is False + assert result.cache_miss_reason == "device_changed" + assert result.metadata.requested_device == 1 + + +def test_source_change_during_import_preserves_existing_database( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from pt_snap_cli.core.errors import SourceChangedError + + import_service_cls = _import_service_type() + service = import_service_cls() + first = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + original_size = first.db_path.stat().st_size + hashes = iter(["a" * 64, "b" * 64]) + monkeypatch.setattr(service._metadata_service, "calculate_sha256", lambda path: next(hashes)) + + with pytest.raises(SourceChangedError, match="source changed"): + service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + force=True, + ) + ) + + assert first.db_path.stat().st_size == original_size + assert sorted(path.name for path in tmp_path.iterdir()) == [first.db_path.name] diff --git a/tests/test_api.py b/tests/test_api.py index 1eb3fc1..e0bfe53 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -160,3 +160,27 @@ def test_execute_query_max_rows_zero(self, valid_db: Path) -> None: # leak_detection on empty table returns 0 rows assert result["total"] == 0 assert result["returned"] == 0 + + def test_get_database_metadata_unavailable(self, valid_db: Path) -> None: + analyzer = SnapshotAnalyzer(db_path=valid_db) + + result = analyzer.get_database_metadata() + + assert result["status"] == "unavailable" + assert result["reason"] == "metadata_missing" + + def test_get_database_metadata_available(self, valid_db: Path) -> None: + from pt_snap_cli.core.import_metadata import ImportMetadataService + + source = valid_db.parent / "snapshot.pkl" + source.write_bytes(b"snapshot") + service = ImportMetadataService() + digest = service.calculate_sha256(source) + service.write(valid_db, service.build_metadata(source, digest, None)) + analyzer = SnapshotAnalyzer(db_path=valid_db) + + result = analyzer.get_database_metadata() + + assert result["status"] == "available" + assert result["metadata"]["source_sha256"] == digest + source.unlink() diff --git a/tests/test_cli.py b/tests/test_cli.py index e118fcf..46208de 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -825,6 +825,7 @@ def test_import_success_writes_focus(self, tmp_path: Path) -> None: assert result.exit_code == 0 assert "Imported:" in result.stdout + assert "Cache miss: database_missing" in result.stdout assert (tmp_path / ".pt-snap" / "focus.json").exists() def test_import_missing_tool_message( @@ -884,6 +885,67 @@ def test_import_no_focus(self, tmp_path: Path) -> None: assert not (tmp_path / ".pt-snap").exists() assert (tmp_path / "sample.pkl.db").exists() + def test_import_reuses_matching_database(self, tmp_path: Path) -> None: + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + + first = runner.invoke(app, ["import", str(snapshot), "--no-focus"]) + second = runner.invoke(app, ["import", str(snapshot), "--no-focus"]) + + assert first.exit_code == 0 + assert second.exit_code == 0 + assert "Imported:" in first.stdout + assert "Reused:" in second.stdout + assert "Cache miss:" not in second.stdout + + def test_import_force_rebuilds_matching_database(self, tmp_path: Path) -> None: + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + runner.invoke(app, ["import", str(snapshot), "--no-focus"]) + + result = runner.invoke(app, ["import", str(snapshot), "--no-focus", "--force"]) + + assert result.exit_code == 0 + assert "Imported:" in result.stdout + assert "Cache miss: forced" in result.stdout + + def test_metadata_json_reports_generated_database(self, tmp_path: Path) -> None: + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + runner.invoke(app, ["import", str(snapshot), "--no-focus"]) + + result = runner.invoke(app, ["metadata", str(tmp_path / "sample.pkl.db"), "--json"]) + payload = json.loads(result.stdout) + + assert result.exit_code == 0 + assert payload["status"] == "available" + assert payload["metadata"]["source_name"] == "sample.pkl" + assert "source_path" not in payload["metadata"] + + def test_metadata_uses_current_focus(self, tmp_path: Path) -> None: + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + imported = runner.invoke(app, ["import", str(snapshot)]) + assert imported.exit_code == 0 + + result = runner.invoke(app, ["metadata", "--json"]) + payload = json.loads(result.stdout) + + assert result.exit_code == 0 + assert payload["status"] == "available" + assert payload["db_path"] == str((tmp_path / "sample.pkl.db").resolve()) + + def test_metadata_reports_unavailable_for_legacy_database(self, sample_db: Path) -> None: + result = runner.invoke(app, ["metadata", str(sample_db)]) + + assert result.exit_code == 0 + assert "Status: unavailable" in result.stdout + assert "Reason: metadata_missing" in result.stdout + class TestReportCommand: """Test report subcommands.""" diff --git a/tests/test_contract_cli_mcp.py b/tests/test_contract_cli_mcp.py index 0c36c28..1e2b5da 100644 --- a/tests/test_contract_cli_mcp.py +++ b/tests/test_contract_cli_mcp.py @@ -316,3 +316,14 @@ def test_missing_template_error_contract_matches_cli_and_mcp_semantics( assert _normalize_cli_missing_template(cli_result.stdout) == mcp_server.get_template_info( template_name ) + + +def test_metadata_contract_matches_cli_and_mcp_semantics( + contract_db: Path, mcp_server: ModuleType +) -> None: + cli_result = runner.invoke(app, ["metadata", str(contract_db), "--json"]) + assert cli_result.exit_code == 0 + + server = _set_mcp_focus(mcp_server, contract_db) + + assert json.loads(cli_result.stdout) == server.get_database_metadata() diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 25b649c..c02595f 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -166,6 +166,17 @@ def test_execute_query_returns_dict(self, valid_db: Path) -> None: assert "device_id" in result assert "rows" in result + def test_get_database_metadata(self, valid_db: Path) -> None: + import pt_snap_cli.mcp.server as server_mod + from pt_snap_cli.api import SnapshotAnalyzer + + server_mod._analyzer = SnapshotAnalyzer(db_path=valid_db) + + result = server_mod.get_database_metadata() + + assert result["status"] == "unavailable" + assert result["reason"] == "metadata_missing" + class TestMCPEntryPoint: """Test the MCP server entry point."""