diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53549a3..170edf3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,7 @@ jobs: pip-${{ runner.os }}-${{ matrix.python-version }}- - name: Install dependencies - run: pip install -e .[dev] + run: pip install -e .[dev,memsnapdump] - name: Run tests with coverage run: pytest --cov --cov-report=xml --cov-report=term-missing diff --git a/README.md b/README.md index 24b7120..8cad5a8 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,14 @@ pip install -e . ## Quick Start +To start from a raw PyTorch memory snapshot, install the optional MemSnapDump integration from local source, import the pickle into a SnapshotDB, then list the available query templates: + +```bash +pip install -e ".[memsnapdump]" +pt-snap import snapshot.pkl +pt-snap query --list +``` + ```bash # Set the snapshot database and device pt-snap focus examples/snapshot_expandable.pkl.db --device 0 @@ -33,6 +41,7 @@ See the [full quick start guide](docs/en/quickstart.md) for a walkthrough. | Command | Description | |---------|-------------| | `pt-snap focus` | Set and manage analysis focus (database + device) | +| `pt-snap import ` | Import a PyTorch memory snapshot pickle into a SnapshotDB | | `pt-snap query` | Run memory analysis queries | | `pt-snap config` | Manage global configuration | | `pt-snap-mcp` | Start the MCP server for agent integration | diff --git a/docs/en/database.md b/docs/en/database.md index ed4430d..ac7e8de 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -12,6 +12,25 @@ SnapshotDB is the SQLite database format for persisting PyTorch memory profiling --- +## Producing a snapshot DB + +`pt-snap` analyzes SQLite SnapshotDB files. If you start with a raw PyTorch memory snapshot pickle, install the optional MemSnapDump integration from local source and import the snapshot: + +```bash +pip install -e ".[memsnapdump]" +pt-snap import snapshot.pkl +``` + +The import command uses [codeasier/MemSnapDump](https://github.com/codeasier/MemSnapDump) to produce `snapshot.pkl.db` next to the input file by default, then updates project focus so subsequent commands can use it directly: + +```bash +pt-snap query --list +``` + +You can also generate the same SnapshotDB format with the `memsnapdump` package directly, then point `pt-snap focus` at the resulting `.db` file. + +--- + ## Database Structure ### Tables diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 1685de9..a2de675 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -12,6 +12,16 @@ pip install -e . ## Your First Analysis +### Optional: Import a PyTorch Snapshot + +If you have a raw `.pkl` memory snapshot, install the optional integration and import it first: + +```bash +pip install -e ".[memsnapdump]" +pt-snap import snapshot.pkl +pt-snap query --list +``` + ### Step 1: Set the Snapshot Database and Device Point `pt-snap` to your SQLite snapshot database file: diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index e241dd6..afda1a3 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -12,6 +12,16 @@ pip install -e . ## 第一次分析 +### 可选:导入 PyTorch 快照 + +如果你手上是原始 `.pkl` 内存快照,请先安装可选集成并导入: + +```bash +pip install -e ".[memsnapdump]" +pt-snap import snapshot.pkl +pt-snap query --list +``` + ### 第一步:设置快照数据库和设备 将 `pt-snap` 指向你的 SQLite 快照数据库文件: diff --git a/pyproject.toml b/pyproject.toml index 4087312..0906f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ dev = [ "build>=1.0.0", "basedpyright>=1.39.3", ] +memsnapdump = [ + "memsnapdump @ git+https://github.com/codeasier/MemSnapDump.git@master", +] rag = [ "chromadb>=0.4.0", "langchain>=0.1.0", diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 828f9ec..37c7437 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -153,6 +153,49 @@ def focus_database( _error(str(e)) +@app.command("import") +def import_snapshot( + snapshot_file: Annotated[Path, typer.Argument(help="Path to .pkl snapshot")], + output_dir: Annotated[Path | None, typer.Option("--output-dir", "-o")] = None, + device: Annotated[int | None, typer.Option("--device", "-d")] = None, + no_focus: Annotated[bool, typer.Option("--no-focus", help="Skip focus update")] = False, +) -> None: + """Import a PyTorch memory snapshot into a SQLite database.""" + from pt_snap_cli.core.errors import ( + DatabaseSchemaError, + FocusFileInvalidError, + ImportExecutionError, + ImportToolMissingError, + SnapshotFileInvalidError, + ) + from pt_snap_cli.core.import_service import ImportService + from pt_snap_cli.core.models import ImportOptions + + try: + result = ImportService().import_snapshot( + ImportOptions( + snapshot_file=snapshot_file, + output_dir=output_dir, + device=device, + set_focus=not no_focus, + ) + ) + except ( + ImportToolMissingError, + ImportExecutionError, + SnapshotFileInvalidError, + FocusFileInvalidError, + DatabaseSchemaError, + ) as e: + _error(str(e)) + + typer.echo(f"Imported: {result.db_path}") + if result.focus_state is not None: + focus_file = getattr(result.focus_state, "focus_file", None) + if focus_file is not None: + typer.echo(f"Focus: {focus_file}") + + @app.command("query") def query_database( db_path: Annotated[ diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index 43ff514..1f1adb3 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -67,6 +67,10 @@ def device_ids(self) -> list[int]: self._device_ids = self._discover_device_ids() return self._device_ids + def discover_devices(self) -> list[int]: + """Discover device IDs available in the database.""" + return self.device_ids + def _discover_device_ids(self) -> list[int]: """Discover device IDs from database table names.""" device_ids = set() diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index 1dd1064..c74834d 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -3,14 +3,18 @@ DatabaseSchemaError, FocusFileInvalidError, FocusNotConfiguredError, + ImportExecutionError, + ImportToolMissingError, InvalidCategoryError, InvalidDeviceError, PtSnapCoreError, QueryExecutionError, + SnapshotFileInvalidError, TemplateNotFoundError, TemplateRenderError, ) from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.models import ( FocusState, PeakMemoryReport, @@ -34,6 +38,9 @@ "TemplateNotFoundError", "TemplateRenderError", "QueryExecutionError", + "ImportToolMissingError", + "SnapshotFileInvalidError", + "ImportExecutionError", "ResolvedFocus", "FocusState", "TemplateParameter", @@ -42,6 +49,7 @@ "QueryResult", "PeakMemoryReport", "FocusService", + "ImportService", "QueryService", "ReportService", ] diff --git a/src/pt_snap_cli/core/errors.py b/src/pt_snap_cli/core/errors.py index e977746..b572dc0 100644 --- a/src/pt_snap_cli/core/errors.py +++ b/src/pt_snap_cli/core/errors.py @@ -36,3 +36,15 @@ class TemplateRenderError(PtSnapCoreError): class QueryExecutionError(PtSnapCoreError): pass + + +class ImportToolMissingError(PtSnapCoreError): + pass + + +class SnapshotFileInvalidError(PtSnapCoreError): + pass + + +class ImportExecutionError(PtSnapCoreError): + pass diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py new file mode 100644 index 0000000..b85edd9 --- /dev/null +++ b/src/pt_snap_cli/core/import_service.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import pickle +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from pt_snap_cli.core.errors import ( + ImportExecutionError, + ImportToolMissingError, + SnapshotFileInvalidError, +) +from pt_snap_cli.core.models import ImportOptions, ImportResult + +if TYPE_CHECKING: + from pt_snap_cli.core.focus_service import FocusService + +VALID_SUFFIXES = {".pkl", ".pickle"} + +try: + from memsnapdump.tools.adaptors.snapshot2db import ( # pyright: ignore[reportMissingImports] + run_dump_to_db as _upstream_run_dump_to_db, + ) +except ImportError: + _upstream_run_dump_to_db: Any = None + + +class ImportService: + """Import PyTorch snapshot pickle files through the source-integrated MemSnapDump API. + + The upstream tool is imported lazily at module load so this package can still be + imported without the optional dependency. All upstream execution goes through a + single private call point for testability. There is intentionally no subprocess + fallback because this PR integrates against MemSnapDump source directly, not a + published PyPI command contract. + """ + + def __init__(self, focus_service: FocusService | None = None) -> None: + self._focus_service = focus_service + + def import_snapshot(self, options: ImportOptions) -> ImportResult: + self._validate_snapshot_file(options.snapshot_file) + output_dir = self._resolve_output_dir(options.snapshot_file, options.output_dir) + + if _upstream_run_dump_to_db is None: + raise ImportToolMissingError( + "memsnapdump package is required; install with pip install -e .[memsnapdump] " + "(or pip install /tmp/MemSnapDump)" + ) + + try: + self._run_dump_to_db( + snapshot_file=str(options.snapshot_file), + dump_dir=str(output_dir), + device=options.device, + ) + except TypeError as exc: + # Monkeypatched / misconfigured call point -> treat as missing tool. + if "NoneType" in str(exc) and "not callable" in str(exc): + raise ImportToolMissingError( + "memsnapdump call point is unavailable; install with " + "pip install -e .[memsnapdump]" + ) from exc + raise ImportExecutionError(f"memsnapdump run_dump_to_db failed: {exc}") from exc + except Exception as exc: + raise ImportExecutionError(f"memsnapdump run_dump_to_db failed: {exc}") from exc + + db_path = output_dir / (options.snapshot_file.name + ".db") + if not db_path.is_file(): + raise ImportExecutionError(f"Expected database not produced: {db_path}") + + focus_state = None + if options.set_focus: + if self._focus_service is None: + from pt_snap_cli.core.focus_service import FocusService + + self._focus_service = FocusService() + focus_state = self._focus_service.set_project_focus( + db_path=db_path, + device_id=options.device, + base_dir=output_dir, + ) + + return ImportResult(db_path=db_path, device_id=options.device, focus_state=focus_state) + + def _run_dump_to_db( + self, + snapshot_file: str, + dump_dir: str, + device: int | None, + ) -> bool: + kwargs: dict[str, Any] = {"snapshot_file": snapshot_file, "dump_dir": dump_dir} + if device is not None: + kwargs["device"] = device + assert _upstream_run_dump_to_db is not None # checked by import_snapshot + return _upstream_run_dump_to_db(**kwargs) + + @staticmethod + def _validate_snapshot_file(path: Path) -> None: + if not path.exists(): + raise SnapshotFileInvalidError(f"Snapshot file does not exist: {path}") + if not path.is_file(): + raise SnapshotFileInvalidError(f"Snapshot path is not a file: {path}") + if path.suffix.lower() not in VALID_SUFFIXES: + raise SnapshotFileInvalidError( + f"Snapshot suffix must be .pkl or .pickle, got {path.suffix!r}" + ) + try: + with path.open("rb") as file: + obj = pickle.load(file) + except Exception as exc: + raise SnapshotFileInvalidError(f"Cannot unpickle snapshot file: {exc}") from exc + if not isinstance(obj, dict): + raise SnapshotFileInvalidError( + f"Snapshot top-level must be a dict, got {type(obj).__name__}" + ) + + @staticmethod + def _resolve_output_dir(snapshot_file: Path, output_dir: Path | None) -> Path: + return output_dir.resolve() if output_dir is not None else snapshot_file.resolve().parent diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index e1768a5..ebac17d 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -69,3 +69,18 @@ class PeakMemoryReport: peak: dict[str, Any] allocator_gap: dict[str, Any] | None callstack_groups: list[dict[str, Any]] + + +@dataclass(frozen=True) +class ImportOptions: + snapshot_file: Path + output_dir: Path | None = None + device: int | None = None + set_focus: bool = True + + +@dataclass(frozen=True) +class ImportResult: + db_path: Path + device_id: int | None + focus_state: FocusState | None diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_import_errors.py b/tests/core/test_import_errors.py new file mode 100644 index 0000000..94eafb0 --- /dev/null +++ b/tests/core/test_import_errors.py @@ -0,0 +1,18 @@ +from pt_snap_cli.core import ( + ImportExecutionError, + ImportToolMissingError, + PtSnapCoreError, + SnapshotFileInvalidError, +) + + +def test_import_tool_missing_error_is_pt_snap_core(): + assert issubclass(ImportToolMissingError, PtSnapCoreError) + + +def test_snapshot_file_invalid_error_is_pt_snap_core(): + assert issubclass(SnapshotFileInvalidError, PtSnapCoreError) + + +def test_import_execution_error_is_pt_snap_core(): + assert issubclass(ImportExecutionError, PtSnapCoreError) diff --git a/tests/core/test_import_models.py b/tests/core/test_import_models.py new file mode 100644 index 0000000..4fce72d --- /dev/null +++ b/tests/core/test_import_models.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from pt_snap_cli.core.models import ImportOptions, ImportResult + + +def test_import_options_defaults() -> None: + options = ImportOptions(snapshot_file=Path("x.pkl")) + + assert options.snapshot_file == Path("x.pkl") + assert options.output_dir is None + assert options.device is None + assert options.set_focus is True + + +def test_import_options_is_frozen() -> None: + options = ImportOptions(snapshot_file=Path("x.pkl")) + + with pytest.raises(dataclasses.FrozenInstanceError): + options.device = 0 # type: ignore[misc] + + +def test_import_result_construction() -> None: + result = ImportResult(db_path=Path("x.db"), device_id=0, focus_state=None) + + assert result.db_path == Path("x.db") + assert result.device_id == 0 + assert result.focus_state is None + + +def test_import_result_is_frozen() -> None: + result = ImportResult(db_path=Path("x.db"), device_id=0, focus_state=None) + + with pytest.raises(dataclasses.FrozenInstanceError): + result.device_id = 1 # type: ignore[misc] diff --git a/tests/core/test_import_service.py b/tests/core/test_import_service.py new file mode 100644 index 0000000..39bc793 --- /dev/null +++ b/tests/core/test_import_service.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import pickle +from pathlib import Path + +import pytest + +from pt_snap_cli.context import Context +from pt_snap_cli.core.errors import ( + ImportExecutionError, + ImportToolMissingError, + SnapshotFileInvalidError, +) +from pt_snap_cli.core.models import ImportOptions + +FIXTURE_DIR = Path(__file__).parents[1] / "fixtures" / "snapshots" +EMPTY_CACHE_SNAPSHOT = FIXTURE_DIR / "snapshot_with_empty_cache.pkl" +MULTI_DEVICE_SNAPSHOT = FIXTURE_DIR / "snapshot_with_multi_devices.pkl" + + +def _import_service_type(): + from pt_snap_cli.core.import_service import ImportService + + return ImportService + + +def test_import_writes_db_and_sets_focus(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + assert result.db_path.exists() + assert result.db_path.suffix == ".db" + assert (tmp_path / ".pt-snap" / "focus.json").exists() + + +@pytest.mark.slow +def test_import_with_multi_device_fixture(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions(snapshot_file=MULTI_DEVICE_SNAPSHOT, output_dir=tmp_path) + ) + + assert len(Context(result.db_path).discover_devices()) >= 2 + + +def test_import_returns_focus_state(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + assert result.focus_state is not None + assert result.focus_state.source == "project" + + +def test_import_snapshot_file_invalid_missing(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + with pytest.raises(SnapshotFileInvalidError): + service.import_snapshot(ImportOptions(snapshot_file=tmp_path / "missing.pkl")) + + +def test_import_snapshot_file_invalid_suffix(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "foo.txt" + snapshot_file.write_bytes(b"not a pickle") + + with pytest.raises(SnapshotFileInvalidError): + service.import_snapshot(ImportOptions(snapshot_file=snapshot_file)) + + +def test_import_snapshot_file_invalid_corrupt_pickle(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "corrupt.pkl" + snapshot_file.write_bytes(b"not a pickle") + + with pytest.raises(SnapshotFileInvalidError): + service.import_snapshot(ImportOptions(snapshot_file=snapshot_file)) + + +def test_import_raises_on_missing_tool(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import_service_cls = _import_service_type() + monkeypatch.setattr(import_service_cls, "_run_dump_to_db", None) + service = import_service_cls() + + with pytest.raises(ImportToolMissingError, match="memsnapdump"): + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + +def test_import_raises_on_upstream_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import_service_cls = _import_service_type() + + def fail_dump_to_db(self, snapshot_file, dump_dir, device): + raise RuntimeError("upstream failed") + + monkeypatch.setattr(import_service_cls, "_run_dump_to_db", fail_dump_to_db) + service = import_service_cls() + + with pytest.raises(ImportExecutionError): + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + +def test_import_skip_focus(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + ) + ) + + assert result.db_path.exists() + assert not (tmp_path / ".pt-snap").exists() + assert result.focus_state is None + + +def test_import_output_path_uses_full_filename(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "snapshot.pickle" + snapshot_file.write_bytes(pickle.dumps({})) + + result = service.import_snapshot( + ImportOptions(snapshot_file=snapshot_file, output_dir=tmp_path, set_focus=False) + ) + + assert result.db_path.name == "snapshot.pickle.db" diff --git a/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl b/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl new file mode 100644 index 0000000..160e449 Binary files /dev/null and b/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl differ diff --git a/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl b/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl new file mode 100644 index 0000000..ed83d18 Binary files /dev/null and b/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl differ diff --git a/tests/test_cli.py b/tests/test_cli.py index b206f8e..b97d7fe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ """Tests for CLI.""" import json +import shutil import sqlite3 from pathlib import Path from unittest.mock import patch @@ -811,6 +812,75 @@ def test_safe_call_reraises_unrelated_keyerror(self) -> None: _safe_call() +class TestImportCommand: + """Test import command.""" + + def test_import_success_writes_focus(self, tmp_path: Path) -> None: + """Test import writes a database and project focus by default.""" + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + + result = runner.invoke(app, ["import", str(snapshot)], catch_exceptions=False) + + assert result.exit_code == 0 + assert "Imported:" in result.stdout + assert (tmp_path / ".pt-snap" / "focus.json").exists() + + def test_import_missing_tool_message( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test import reports memsnapdump installation guidance when the tool is missing.""" + from pt_snap_cli.core.import_service import ImportToolMissingError + + snapshot = tmp_path / "sample.pkl" + snapshot.write_bytes(b"not a real snapshot") + + def raise_missing_tool(*args: object, **kwargs: object) -> object: + raise ImportToolMissingError( + "memsnapdump missing; install with pip install -e .[memsnapdump]" + ) + + monkeypatch.setattr( + "pt_snap_cli.core.import_service.ImportService.import_snapshot", + raise_missing_tool, + ) + + result = runner.invoke(app, ["import", str(snapshot)]) + + assert result.exit_code != 0 + assert "memsnapdump" in result.output + + def test_import_invalid_path(self) -> None: + """Test import rejects missing snapshot paths.""" + result = runner.invoke(app, ["import", "/no/such/file.pkl"]) + + assert result.exit_code != 0 + assert "exist" in result.output.lower() + + def test_import_invalid_suffix(self, tmp_path: Path) -> None: + """Test import rejects non-pkl snapshot paths.""" + snapshot = tmp_path / "sample.txt" + snapshot.write_text("not a snapshot") + + result = runner.invoke(app, ["import", str(snapshot)]) + + assert result.exit_code != 0 + assert "suffix" in result.output.lower() or ".pkl" in result.output.lower() + + def test_import_no_focus(self, tmp_path: Path) -> None: + """Test import --no-focus creates a database without writing project focus.""" + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + + result = runner.invoke(app, ["import", str(snapshot), "--no-focus"], catch_exceptions=False) + + assert result.exit_code == 0 + assert not (tmp_path / ".pt-snap").exists() + assert (tmp_path / "sample.pkl.db").exists() + + class TestReportCommand: """Test report subcommands."""