diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53549a3..01ce41b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,3 +67,25 @@ jobs: name: coverage-report path: coverage.xml retention-days: 30 + + - name: Build wheel + if: matrix.python-version == '3.13' + run: | + python -m pip install --upgrade build + python -m build --wheel + + - name: Wheel smoke test (verify vendored backend is packaged) + if: matrix.python-version == '3.13' + run: | + set -euo pipefail + python -m pip install --force-reinstall dist/*.whl + # Confirm the vendored closure was actually packaged into the wheel. + python -c "import pt_snap_cli.vendor.memsnapdump.tools.adaptors.snapshot2db as m; assert hasattr(m, 'run_dump_to_db'), 'vendored run_dump_to_db missing in installed wheel'" + # End-to-end import against a real fixture — catches any packaging + # regression where setuptools drops src/pt_snap_cli/vendor/. + FIXTURE=tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + rm -rf /tmp/pt-snap-wheel-smoke + mkdir -p /tmp/pt-snap-wheel-smoke + pt-snap import "$FIXTURE" --no-focus --output-dir /tmp/pt-snap-wheel-smoke + test -f /tmp/pt-snap-wheel-smoke/snapshot_with_empty_cache.pkl.db + rm -rf /tmp/pt-snap-wheel-smoke diff --git a/README.md b/README.md index 24b7120..2c9f27d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ pip install -e . ## Quick Start +To start from a raw PyTorch memory snapshot, import the pickle into a SnapshotDB with the built-in backend, then list the available query templates: + +```bash +pt-snap import snapshot.pkl +pt-snap query --list +``` + ```bash # Set the snapshot database and device pt-snap focus examples/snapshot_expandable.pkl.db --device 0 @@ -33,6 +40,7 @@ See the [full quick start guide](docs/en/quickstart.md) for a walkthrough. | Command | Description | |---------|-------------| | `pt-snap focus` | Set and manage analysis focus (database + device) | +| `pt-snap import ` | Import a PyTorch memory snapshot pickle into a SnapshotDB | | `pt-snap query` | Run memory analysis queries | | `pt-snap config` | Manage global configuration | | `pt-snap-mcp` | Start the MCP server for agent integration | diff --git a/docs/en/database.md b/docs/en/database.md index ed4430d..dcc6ee3 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -12,6 +12,24 @@ SnapshotDB is the SQLite database format for persisting PyTorch memory profiling --- +## Producing a snapshot DB + +`pt-snap` analyzes SQLite SnapshotDB files. If you start with a raw PyTorch memory snapshot pickle, import the snapshot with the built-in backend: + +```bash +pt-snap import snapshot.pkl +``` + +The import command uses a vendored MemSnapDump backend to produce `snapshot.pkl.db` next to the input file by default, then updates project focus so subsequent commands can use it directly: + +```bash +pt-snap query --list +``` + +You can also generate the same SnapshotDB format with another compatible producer, then point `pt-snap focus` at the resulting `.db` file. + +--- + ## Database Structure ### Tables diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 1685de9..8ce8443 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -12,6 +12,15 @@ pip install -e . ## Your First Analysis +### Optional: Import a PyTorch Snapshot + +If you have a raw `.pkl` memory snapshot, import it first with the built-in backend: + +```bash +pt-snap import snapshot.pkl +pt-snap query --list +``` + ### Step 1: Set the Snapshot Database and Device Point `pt-snap` to your SQLite snapshot database file: diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index e241dd6..46c78c1 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -12,6 +12,15 @@ pip install -e . ## 第一次分析 +### 可选:导入 PyTorch 快照 + +如果你手上是原始 `.pkl` 内存快照,请先使用内建后端导入: + +```bash +pt-snap import snapshot.pkl +pt-snap query --list +``` + ### 第一步:设置快照数据库和设备 将 `pt-snap` 指向你的 SQLite 快照数据库文件: diff --git a/pyproject.toml b/pyproject.toml index 4087312..42a48e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,10 +67,16 @@ pt_snap_cli = [ [tool.black] line-length = 100 target-version = ["py310"] +extend-exclude = ''' +( + ^/src/pt_snap_cli/vendor/ +) +''' [tool.ruff] line-length = 100 target-version = "py310" +extend-exclude = ["src/pt_snap_cli/vendor"] [tool.ruff.lint] select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 828f9ec..3f8d649 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -20,12 +20,17 @@ FocusFileInvalidError, FocusNotConfiguredError, FocusService, + ImportExecutionError, + ImportOptions, + ImportService, + ImportToolMissingError, InvalidCategoryError, InvalidDeviceError, PeakMemoryReport, QueryExecutionError, QueryService, ReportService, + SnapshotFileInvalidError, TemplateNotFoundError, TemplateRenderError, ) @@ -153,6 +158,35 @@ def focus_database( _error(str(e)) +@app.command("import") +def import_snapshot( + snapshot_file: Annotated[Path, typer.Argument(help="Path to .pkl snapshot")], + output_dir: Annotated[Path | None, typer.Option("--output-dir", "-o")] = None, + device: Annotated[int | None, typer.Option("--device", "-d")] = None, + no_focus: Annotated[bool, typer.Option("--no-focus", help="Skip focus update")] = False, +) -> None: + """Import a PyTorch memory snapshot into a SQLite database.""" + try: + result = ImportService().import_snapshot( + ImportOptions( + snapshot_file=snapshot_file, + output_dir=output_dir, + device=device, + set_focus=not no_focus, + ) + ) + except ( + ImportToolMissingError, + ImportExecutionError, + SnapshotFileInvalidError, + ) as e: + _error(str(e)) + + typer.echo(f"Imported: {result.db_path}") + if result.focus_state is not None and result.focus_state.focus_file is not None: + typer.echo(f"Focus: {result.focus_state.focus_file}") + + @app.command("query") def query_database( db_path: Annotated[ diff --git a/src/pt_snap_cli/context.py b/src/pt_snap_cli/context.py index 43ff514..11826e5 100644 --- a/src/pt_snap_cli/context.py +++ b/src/pt_snap_cli/context.py @@ -67,6 +67,14 @@ def device_ids(self) -> list[int]: self._device_ids = self._discover_device_ids() return self._device_ids + def discover_devices(self) -> list[int]: + """Discover device IDs available in the database. + + Method-style alias of the ``device_ids`` property, exposed for + callers (e.g. import service) that prefer an explicit method. + """ + return self.device_ids + def _discover_device_ids(self) -> list[int]: """Discover device IDs from database table names.""" device_ids = set() diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index 1dd1064..d69218a 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -3,16 +3,22 @@ DatabaseSchemaError, FocusFileInvalidError, FocusNotConfiguredError, + ImportExecutionError, + ImportToolMissingError, InvalidCategoryError, InvalidDeviceError, PtSnapCoreError, QueryExecutionError, + SnapshotFileInvalidError, TemplateNotFoundError, TemplateRenderError, ) from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.models import ( FocusState, + ImportOptions, + ImportResult, PeakMemoryReport, QueryResult, ResolvedFocus, @@ -34,6 +40,9 @@ "TemplateNotFoundError", "TemplateRenderError", "QueryExecutionError", + "ImportToolMissingError", + "SnapshotFileInvalidError", + "ImportExecutionError", "ResolvedFocus", "FocusState", "TemplateParameter", @@ -41,7 +50,10 @@ "TemplateInfo", "QueryResult", "PeakMemoryReport", + "ImportOptions", + "ImportResult", "FocusService", "QueryService", "ReportService", + "ImportService", ] diff --git a/src/pt_snap_cli/core/errors.py b/src/pt_snap_cli/core/errors.py index e977746..62e20e2 100644 --- a/src/pt_snap_cli/core/errors.py +++ b/src/pt_snap_cli/core/errors.py @@ -36,3 +36,36 @@ class TemplateRenderError(PtSnapCoreError): class QueryExecutionError(PtSnapCoreError): pass + + +class ImportToolMissingError(PtSnapCoreError): + """Raised when the vendored snapshot import backend is unavailable. + + In the vendored model, this no longer means "the user forgot to install + memsnapdump" — the dump2db backend is built in. It means the vendored + backend could not be imported or initialized, which is a packaging + or installation problem. + """ + + pass + + +class SnapshotFileInvalidError(PtSnapCoreError): + """Raised when a candidate snapshot pickle file cannot be used as input. + + Triggers include: path does not exist, path is not a regular file, + suffix is not .pkl or .pickle, file cannot be unpickled, or the + unpickled top-level object is not a dict. + """ + + pass + + +class ImportExecutionError(PtSnapCoreError): + """Raised when the import backend itself fails to produce a database. + + Covers upstream dump2db returning False, raising an exception, + or producing no .db artifact where one was expected. + """ + + pass diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py new file mode 100644 index 0000000..6bea179 --- /dev/null +++ b/src/pt_snap_cli/core/import_service.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import logging +from pathlib import Path + +from pt_snap_cli.core.errors import ( + DatabaseSchemaError, + FocusFileInvalidError, + ImportExecutionError, + SnapshotFileInvalidError, +) +from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.models import ImportOptions, ImportResult +from pt_snap_cli.core.snapshot_import_backend import SnapshotImportBackend + +logger = logging.getLogger(__name__) + +VALID_SUFFIXES = {".pkl", ".pickle"} + + +class ImportService: + """Import PyTorch snapshot pickle files through the vendored backend.""" + + def __init__( + self, + focus_service: FocusService | None = None, + backend: SnapshotImportBackend | None = None, + ) -> None: + self._focus_service = focus_service + self._backend = backend or SnapshotImportBackend() + + def import_snapshot(self, options: ImportOptions) -> ImportResult: + self._validate_snapshot_file(options.snapshot_file) + output_dir = self._resolve_output_dir(options.snapshot_file, options.output_dir) + db_path = self._backend.dump_to_db(options.snapshot_file, output_dir, options.device) + + focus_state = None + if options.set_focus: + if self._focus_service is None: + self._focus_service = FocusService() + try: + focus_state = self._focus_service.set_project_focus( + db_path=db_path, + device_id=options.device, + ) + except (FocusFileInvalidError, DatabaseSchemaError) as exc: + # Focus write happens after the .db is on disk; if the new + # database fails validation (e.g. vendor produced a non-pt-snap + # schema), report it as an import execution failure rather than + # leaking the underlying domain error through the import API. + raise ImportExecutionError( + f"Imported database cannot be registered as focus: {exc}" + ) from exc + + return ImportResult(db_path=db_path, device_id=options.device, focus_state=focus_state) + + @staticmethod + def _validate_snapshot_file(path: Path) -> None: + if not path.exists(): + raise SnapshotFileInvalidError(f"Snapshot file does not exist: {path}") + if not path.is_file(): + raise SnapshotFileInvalidError(f"Snapshot path is not a file: {path}") + if path.suffix.lower() not in VALID_SUFFIXES: + raise SnapshotFileInvalidError( + f"Snapshot suffix must be .pkl or .pickle, got {path.suffix!r}" + ) + + @staticmethod + def _resolve_output_dir(snapshot_file: Path, output_dir: Path | None) -> Path: + return output_dir.resolve() if output_dir is not None else snapshot_file.resolve().parent diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index e1768a5..d06ef98 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -69,3 +69,39 @@ class PeakMemoryReport: peak: dict[str, Any] allocator_gap: dict[str, Any] | None callstack_groups: list[dict[str, Any]] + + +@dataclass(frozen=True) +class ImportOptions: + """Options for `pt-snap import`. + + Attributes: + snapshot_file: Path to the input `.pkl` or `.pickle` snapshot. + output_dir: Directory where the generated `.db` is written. + Defaults to the snapshot file's parent directory. + device: Optional device id to focus on. When None, all available + devices are imported. + set_focus: When True (default), also write project focus so subsequent + `pt-snap query` / `pt-snap report` commands reuse the new DB. + """ + + snapshot_file: Path + output_dir: Path | None = None + device: int | None = None + set_focus: bool = True + + +@dataclass(frozen=True) +class ImportResult: + """Result of `pt-snap import`. + + Attributes: + db_path: Path to the generated SQLite database. + device_id: Device id used during import, or None when not specified. + focus_state: The FocusState written to project focus when + options.set_focus is True, otherwise None. + """ + + db_path: Path + device_id: int | None + focus_state: FocusState | None diff --git a/src/pt_snap_cli/core/snapshot_import_backend.py b/src/pt_snap_cli/core/snapshot_import_backend.py new file mode 100644 index 0000000..c57dfbd --- /dev/null +++ b/src/pt_snap_cli/core/snapshot_import_backend.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import os +import pickle +import sqlite3 +import tempfile +from pathlib import Path + +from pt_snap_cli.core.errors import ImportExecutionError, ImportToolMissingError + + +class SnapshotImportBackend: + """Stable adapter over the vendored snapshot-to-database implementation.""" + + def dump_to_db( + self, + snapshot_file: Path, + output_dir: Path, + device: int | None = None, + ) -> Path: + db_path = output_dir / f"{snapshot_file.name}.db" + output_dir.mkdir(parents=True, exist_ok=True) + run_dump_to_db = self._load_run_dump_to_db() + + with tempfile.TemporaryDirectory(dir=output_dir) as tmp_dir_name: + tmp_dir = Path(tmp_dir_name) + tmp_db_path = tmp_dir / db_path.name + try: + ok = run_dump_to_db( + snapshot_file=str(snapshot_file), + dump_dir=str(tmp_dir), + device=device, + ) + except (OSError, sqlite3.DatabaseError, pickle.UnpicklingError, ValueError) as exc: + raise ImportExecutionError( + f"Vendored snapshot import backend failed: {exc}" + ) from exc + + if not ok: + raise ImportExecutionError("Vendored snapshot import backend reported failure.") + if not tmp_db_path.is_file(): + raise ImportExecutionError(f"Expected database not produced: {tmp_db_path}") + + try: + os.replace(tmp_db_path, db_path) + except OSError as exc: + raise ImportExecutionError( + f"Vendored snapshot import backend failed: {exc}" + ) from exc + + return db_path + + @staticmethod + def _load_run_dump_to_db(): + try: + from pt_snap_cli.vendor.memsnapdump.tools.adaptors.snapshot2db import run_dump_to_db + except ImportError as exc: + raise ImportToolMissingError( + "Vendored snapshot import backend is unavailable. Reinstall pt-snap-cli; " + "the package may be incomplete." + ) from exc + return run_dump_to_db diff --git a/src/pt_snap_cli/vendor/__init__.py b/src/pt_snap_cli/vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pt_snap_cli/vendor/memsnapdump/ATTRIBUTION.md b/src/pt_snap_cli/vendor/memsnapdump/ATTRIBUTION.md new file mode 100644 index 0000000..542820c --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/ATTRIBUTION.md @@ -0,0 +1,74 @@ +# Vendored MemSnapDump + +This directory contains a vendored copy of the `dump2db` toolchain from +[`codeasier/MemSnapDump`](https://github.com/codeasier/MemSnapDump). The vendored +code is the minimum closure required to invoke +`memsnapdump.tools.adaptors.snapshot2db.run_dump_to_db` from +`pt_snap_cli.core.snapshot_import_backend`. + +## Upstream metadata + +| Field | Value | +| ------------ | ------------------------------------------------------------------ | +| Repository | https://github.com/codeasier/MemSnapDump | +| Pinned SHA | `87ea207372e0985790e1a28dab499dccf3c3b9a4` (branch `master`, HEAD at vendoring time) | +| Vendored on | 2026-06-22 | +| Vendored by | pt-snap-cli maintainers | +| Closure size | 22 Python source files (5 `__init__.py` + 17 modules) | + +## License + +The upstream repository does not publish a top-level `LICENSE` file. Both +`pt-snap-cli` and `MemSnapDump` are developed by the same author and pt-snap-cli +inherits the upstream code under the same MIT license terms; see [`LICENSE`](./LICENSE) +in this directory for the full text. When upstream publishes a formal license +file in a future release, the pinned SHA in this file should be updated and +the upstream license text should replace this one. + +## Syncing strategy + +The closure is intentionally laid out to mirror the upstream module structure +one-to-one: + +``` +memsnapdump/ +├── __init__.py +├── base/ +├── simulate/ +├── tools/adaptors/database/ +└── util/ +``` + +This makes future re-syncs a matter of `diff -ru` between this tree and a fresh +upstream checkout, file by file. To resync: + +1. `git fetch https://github.com/codeasier/MemSnapDump.git master` +2. For each Python file under `src/pt_snap_cli/vendor/memsnapdump/`, copy the + upstream file from the same path, preserving the upstream license header + (if any). +3. Update the pinned SHA in the table above. +4. Re-run `pytest` and the wheel smoke test in CI. + +## Local modifications + +**None.** The vendored code is intended to be byte-identical to the pinned +upstream SHA, so that re-syncing is a clean replace operation. Any local fix +or workaround that the project needs should be applied in +`src/pt_snap_cli/core/snapshot_import_backend.py` (the semantic adapter) or +in `src/pt_snap_cli/core/import_service.py`, **not** in this directory. + +## Lint / format exclusion + +`pyproject.toml` excludes this entire tree from `black` and `ruff` so that +upstream code is not reformatted on every lint pass. When you upgrade the +pinned SHA, run `black` and `ruff` against the new tree first to surface any +drift between the upstream style and our pinned configuration. + +## Tooling + +- `black` and `ruff` are configured with `extend-exclude` covering + `src/pt_snap_cli/vendor/`. +- CI includes a wheel smoke test (`.github/workflows/test.yml`) that builds + the package, installs the wheel, and runs `pt-snap import` against a real + fixture. This guarantees that any packaging regression (e.g. setuptools + dropping the vendored closure) is caught before release. diff --git a/src/pt_snap_cli/vendor/memsnapdump/LICENSE b/src/pt_snap_cli/vendor/memsnapdump/LICENSE new file mode 100644 index 0000000..e37fe9a --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/LICENSE @@ -0,0 +1,29 @@ +MIT License + +Copyright (c) 2026 codeasier + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- + +The software in this directory is vendored from +https://github.com/codeasier/MemSnapDump, copyright (c) 2026 codeasier. The +upstream repository does not publish a top-level LICENSE file; the same +author publishes the same code under MIT terms in pt-snap-cli. See +ATTRIBUTION.md for the pinned upstream commit SHA and sync strategy. diff --git a/src/pt_snap_cli/vendor/memsnapdump/__init__.py b/src/pt_snap_cli/vendor/memsnapdump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/pt_snap_cli/vendor/memsnapdump/base/__init__.py b/src/pt_snap_cli/vendor/memsnapdump/base/__init__.py new file mode 100644 index 0000000..2f6be79 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/base/__init__.py @@ -0,0 +1,3 @@ +from .entities import Frame, TraceEntry, Block, BlockState, Segment, DeviceSnapshot + +__all__ = ["Frame", "TraceEntry", "Block", "BlockState", "Segment", "DeviceSnapshot"] diff --git a/src/pt_snap_cli/vendor/memsnapdump/base/entities.py b/src/pt_snap_cli/vendor/memsnapdump/base/entities.py new file mode 100644 index 0000000..7c7d329 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/base/entities.py @@ -0,0 +1,302 @@ +from typing import List, Literal, Any +from dataclasses import dataclass, field + + +class Frame: + filename: str = "" + line: int = -1 + name: str = "" + + _origin: dict = None # Readonly + + @classmethod + def from_dict(cls, frame_dict: dict): + frame = cls() + frame.filename = frame_dict["filename"] + frame.line = frame_dict["line"] + frame.name = frame_dict["name"] + frame._origin = frame_dict + return frame + + def to_dict(self): + return ( + self._origin + if self._origin + else {"filename": self.filename, "line": self.line, "name": self.name} + ) + + +@dataclass +class TraceEntry: + # When `torch.npu.memory._record_memory_history()` is enabled, + # the snapshot will contain TraceEntry objects that record each + # action the allocator took. + """ + action: Literal[ + 'alloc' # memory allocated + 'free_requested', # the allocated received a call to free memory + 'free_completed', # the memory that was requested to be freed is now + # able to be used in future allocation calls + 'segment_alloc', # the caching allocator ask aclrtMalloc for more memory + # and added it as a segment in its cache + 'segment_free', # the caching allocator called aclrtFree to return memory + # to npu possibly trying free up memory to + # allocate more segments or because empty_caches was called + 'oom', # the allocator threw an OOM exception. 'size' is + # the requested number of bytes that did not succeed + 'snapshot' # the allocator generated a memory snapshot + # useful to coorelate a previously taken + # snapshot with this trace + ] + """ + action: str = "" + addr: int = -1 # not present for OOM + frames: List[Frame] = field(default_factory=list) + size: int = 0 + stream: int = 0 + device_free: int = -1 # only present for OOM, the amount of + # memory npu still reports to be free + + _origin: dict = None # Readonly + idx: int = -1 # 索引,全局唯一 + + @classmethod + def from_dict(cls, trace_dict: dict): + trace_entry = cls( + action=trace_dict["action"], + addr=int(trace_dict["addr"]), + size=int(trace_dict["size"]), + stream=int(trace_dict["stream"]), + _origin=trace_dict, + frames=[ + Frame.from_dict(_frame_dict) + for _frame_dict in trace_dict.get("frames", []) + ], + ) + return trace_entry + + def get_callstack(self): + if not self.frames: + return "" + return "\n".join( + [ + f"{frame.filename}:{frame.line} {frame.name}" + for frame in self.frames[::-1] + ] + ) + + def to_dict(self): + return ( + self._origin + if self._origin + else dict( + action=self.action, + addr=self.addr, + size=self.size, + stream=self.stream, + frames=[frame.to_dict() for frame in self.frames], + ) + ) + + +class BlockState: + ACTIVE_PENDING_FREE = "active_pending_free" + ACTIVE_ALLOCATED = "active_allocated" + INACTIVE = "inactive" + + +@dataclass +class Block: + # A piece of memory returned from the allocator, or + # current cached but inactive. + size: int = 0 + requested_size: int = 0 # size requested during malloc, may be smaller than + # size due to rounding + address: int = -1 + state: Literal[ + "active_allocated", # used by a tensor + "active_pending_free", # waiting for another stream to finish using this, then it will become free + "inactive", + ] = BlockState.INACTIVE # free for reuse + frames: List[Frame] = field( + default_factory=list + ) # stack trace from where the allocation occurred + + # 指向持有该block的segment对象 + segment_ptr: Any = None + free_event_idx: int = None + alloc_event_idx: int = None + + @classmethod + def from_dict(cls, block_dict: dict): + block = cls( + size=block_dict["size"], + requested_size=block_dict["requested_size"], + address=block_dict["address"], + state=block_dict["state"], + frames=[Frame.from_dict(frame) for frame in block_dict.get("frames", [])], + ) + return block + + @classmethod + def build_from_event(cls, event: TraceEntry): + block = cls( + size=event.size, + requested_size=event.size, + address=event.addr, + frames=event.frames, + ) + return block + + def to_dict(self): + return dict( + size=self.size, + requested_size=self.requested_size, + address=self.address, + state=self.state, + frames=[frame.to_dict() for frame in self.frames], + ) + + +@dataclass +class Segment: + # Segments are memory returned from a aclrtMalloc call. + # The size of reserved memory is the sum of all Segments. + # Segments are cached and reused for future allocations. + # If the reuse is smaller than the segment, the segment + # is split into more then one Block. + # empty_cache() frees Segments that are entirely inactive. + address: int = -1 + total_size: int = 0 # aclrtMalloc'd size of segment + stream: int = 0 + segment_type: Literal["small", "large"] = "" # 'large' (>1MB) + allocated_size: int = 0 # size of memory in use + active_size: int = 0 # size of memory in use or in active_awaiting_free state + blocks: List[Block] = field(default_factory=list) + device: int = 0 + frames: List[Frame] = field(default_factory=list) + is_expandable: bool = False + _origin: dict = None # Readonly + free_or_unmap_event_idx: int = None + alloc_or_map_event_idx: int = None + + @classmethod + def from_dict(cls, segment_dict: dict, ignore_inactive_blocks: bool = False): + segment = cls( + address=segment_dict["address"], + total_size=segment_dict["total_size"], + stream=segment_dict["stream"], + segment_type=segment_dict["segment_type"], + allocated_size=segment_dict["allocated_size"], + active_size=segment_dict["active_size"], + frames=[ + Frame.from_dict(_frame) for _frame in segment_dict.get("frames", []) + ], + device=segment_dict.get("device", 0), + _origin=segment_dict, + is_expandable=segment_dict.get("is_expandable", False), + ) + for block in segment_dict["blocks"]: + if ignore_inactive_blocks and block["state"] == BlockState.INACTIVE: + continue + _block = Block.from_dict(block) + _block.segment_ptr = segment + segment.blocks.append(_block) + return segment + + @classmethod + def build_from_event(cls, event: TraceEntry, with_inactive_block: bool = False): + segment = cls( + address=event.addr, + total_size=event.size, + stream=event.stream, + frames=event.frames, + device=event.device if hasattr(event, "device") else 0, + allocated_size=0, + active_size=0, + is_expandable=event.action in ["segment_map", "segment_unmap"], + ) + segment.blocks = ( + [] + if not with_inactive_block + else [ + Block( + size=event.size, + requested_size=event.size, + address=event.addr, + state=BlockState.INACTIVE, + segment_ptr=segment, + ) + ] + ) + return segment + + def to_dict(self): + return dict( + address=self.address, + total_size=self.total_size, + stream=self.stream, + segment_type=self.segment_type, + allocated_size=self.allocated_size, + active_size=self.active_size, + device=self.device, + is_expandable=self.is_expandable, + frames=[frame.to_dict() for frame in self.frames], + blocks=[block.to_dict() for block in self.blocks], + ) + + +class DeviceSnapshot: + segments: List[Segment] + trace_entries: List[TraceEntry] + + total_allocated: int # 二次分配总量 + total_reserved: int # 内存池总量 + total_activated: int # 活跃内存总量 + + device: int + + @classmethod + def from_dict( + cls, snapshot_dict: dict, device: int, ignore_inactive_blocks: bool = False + ): + segments_dict = snapshot_dict.get("segments", []) + device_traces = snapshot_dict.get("device_traces", []) + device_trace_list = ( + device_traces[device] if 0 <= device <= len(device_traces) else [] + ) + snapshot = cls() + snapshot.segments = [] + snapshot.trace_entries = [] + snapshot.total_allocated = 0 + snapshot.total_reserved = 0 + snapshot.total_activated = 0 + # 读取dump_snapshot时内存状态 + for segment_dict in segments_dict: + # 当segment原始数据中缺少device字段明确指向其所属设备时,默认其归属为device0 + # 此时如果from_dict指定device为0或未指定而缺省为0,则未知归属的device也会纳入分析 + if segment_dict.get("device", 0) != device: + continue + _segment = Segment.from_dict( + segment_dict, ignore_inactive_blocks=ignore_inactive_blocks + ) + snapshot.segments.append(_segment) + snapshot.total_allocated += _segment.allocated_size + snapshot.total_reserved += _segment.total_size + snapshot.total_activated += _segment.active_size + snapshot.segments.sort(key=lambda segment: (segment.address, segment.stream)) + # 读取事件序列 + for idx, trace_entry_dict in enumerate(device_trace_list): + trace_entry = TraceEntry.from_dict(trace_entry_dict) + trace_entry.idx = idx + snapshot.trace_entries.append(trace_entry) + snapshot.device = device + return snapshot + + def to_dict(self): + return { + "segments": [segment.to_dict() for segment in self.segments], + # 需要根据deviceId,将事件列表填入,如果deviceId不为0,前序还要padding空事件列表 + "device_traces": [[] for _ in range(self.device)] + + [[trace.to_dict() for trace in self.trace_entries]], + } diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/__init__.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/__init__.py new file mode 100644 index 0000000..fa76d11 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/__init__.py @@ -0,0 +1,10 @@ +from .simulate import SimulateDeviceSnapshot +from .hooker_defs import SimulateHooker, AllocatorHooker +from .allocator_context import AllocatorContext + +__all__ = [ + "SimulateDeviceSnapshot", + "SimulateHooker", + "AllocatorHooker", + "AllocatorContext", +] diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_context.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_context.py new file mode 100644 index 0000000..0a911d5 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_context.py @@ -0,0 +1,11 @@ +from ..base import DeviceSnapshot, TraceEntry + + +class AllocatorContext: + def __init__(self, snapshot: DeviceSnapshot): + self.device_snapshot = snapshot + self.current_undo_event: TraceEntry = None + self.workspace_flag = False + + def set_current_undo_event(self, undo_event: TraceEntry): + self.current_undo_event = undo_event diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_hook_dispatcher.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_hook_dispatcher.py new file mode 100644 index 0000000..f80b29f --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/allocator_hook_dispatcher.py @@ -0,0 +1,63 @@ +import copy +from typing import Dict + +from ..base import Block, DeviceSnapshot, Segment + +from .hooker_defs import AllocatorHooker + + +class AllocatorHookDispatcher: + def __init__(self): + self.hookers: Dict[int, AllocatorHooker] = {} + + def register_hooker(self, hooker: AllocatorHooker) -> int: + idx = hash(hooker) + self.hookers[idx] = hooker + return idx + + def unregister_hooker(self, hooker_id: int): + if hooker_id in self.hookers: + del self.hookers[hooker_id] + + def pre_replay_alloc_block(self, block: Block, snapshot: DeviceSnapshot): + for hooker in self.hookers.values(): + hooker.pre_replay_alloc_block(block, snapshot) + + def post_replay_alloc_block(self, block: Block, snapshot: DeviceSnapshot): + for hooker in self.hookers.values(): + hooker.post_replay_alloc_block(block, snapshot) + + def pre_replay_free_block(self, block: Block, snapshot: DeviceSnapshot): + for hooker in self.hookers.values(): + hooker.pre_replay_free_block(block, snapshot) + + def post_replay_free_block( + self, block: Block, snapshot: DeviceSnapshot, use_copy: bool = False + ): + payload = copy.copy(block) if use_copy else block + for hooker in self.hookers.values(): + hooker.post_replay_free_block(payload, snapshot) + + def pre_replay_map_or_alloc_segment( + self, segment: Segment, snapshot: DeviceSnapshot + ): + for hooker in self.hookers.values(): + hooker.pre_replay_map_or_alloc_segment(segment, snapshot) + + def post_replay_map_or_alloc_segment( + self, segment: Segment, snapshot: DeviceSnapshot + ): + for hooker in self.hookers.values(): + hooker.post_replay_map_or_alloc_segment(segment, snapshot) + + def pre_replay_unmap_or_free_segment( + self, segment: Segment, snapshot: DeviceSnapshot + ): + for hooker in self.hookers.values(): + hooker.pre_replay_unmap_or_free_segment(segment, snapshot) + + def post_replay_unmap_or_free_segment( + self, segment: Segment, snapshot: DeviceSnapshot + ): + for hooker in self.hookers.values(): + hooker.post_replay_unmap_or_free_segment(segment, snapshot) diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/hooker_defs.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/hooker_defs.py new file mode 100644 index 0000000..d68e31e --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/hooker_defs.py @@ -0,0 +1,110 @@ +from abc import ABC, abstractmethod +from ..base import DeviceSnapshot, Segment, Block, TraceEntry + + +class SimulateHooker(ABC): + @abstractmethod + def pre_undo_event( + self, wait4undo_event: TraceEntry, current_snapshot: DeviceSnapshot + ) -> bool: + """ + 【READONLY】在回放事件前回调,此时事件列表**并未POP**出该事件 + :param wait4undo_event: 【只读】待回放的事件(仍在事件列表中) + :param current_snapshot: 【只读】当前的内存块快照 + :return 返回true继续执行,返回false将中断 + """ + ... + + @abstractmethod + def post_undo_event( + self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot + ) -> bool: + """ + 【READONLY】在回放事件后回调,此时snapshot已经将该事件从事件列表中丢弃,且segments已回放到事件发生前 + :param already_undo_event: 已回放的事件 + :param current_snapshot: 【只读】当前内存快照 + :return 返回true继续执行,返回false将中断 + """ + ... + + +class AllocatorHooker(ABC): + def pre_replay_alloc_block( + self, wait4alloc_block: Block, current_snapshot: DeviceSnapshot + ): + """ + 在**回放时**分配一个内存块**前**回调,对应一个内存块释放事件回滚前 + :param wait4alloc_block: 待分配的block + :param current_snapshot: 【只读】当前内存快照 + """ + ... + + def post_replay_alloc_block( + self, allocated_block: Block, current_snapshot: DeviceSnapshot + ): + """ + 在**回放时**分配一个内存块**后**回调,对应一个内存块释放事件回滚后 + :param allocated_block: 【只读】新分配的block + :param current_snapshot:【只读】分配block前内存快照 + """ + ... + + def pre_replay_free_block( + self, wait4free_block: Block, current_snapshot: DeviceSnapshot + ): + """ + 在**回放时**释放一个内存块**前**回调,对应一个内存块申请事件回滚前 + :param wait4free_block: 【只读】待释放的block + :param current_snapshot: 【只读】释放block前内存快照 + """ + ... + + def post_replay_free_block( + self, released_block: Block, current_snapshot: DeviceSnapshot + ): + """ + 在回放时,释放一个内存块**后**回调,对应一个内存块申请事件回滚后 + :param released_block: 【副本】已释放的block(副本) + :param current_snapshot: 【只读】释放block后内存快照 + """ + ... + + def pre_replay_map_or_alloc_segment( + self, wait4alloc_map_segment: Segment, current_snapshot: DeviceSnapshot + ): + """ + 在回放时,分配或映射一个内存段**前**回调,对应一个内存段释放或unmap事件回滚前 + :param wait4alloc_map_segment: 【只读】待分配或map的内存段 + :param current_snapshot: 分配或map内存段前的内存快照 + """ + ... + + def post_replay_map_or_alloc_segment( + self, allocated_mapped_segment: Segment, current_snapshot: DeviceSnapshot + ): + """ + 在回放时,分配或映射一个内存段**后**回调,对应一个内存段释放或unmap事件回滚后 + :param allocated_mapped_segment:【副本】已分配或map的内存段 + :param current_snapshot: 分配或map内存段后的内存快照 + """ + ... + + def pre_replay_unmap_or_free_segment( + self, wait4release_segment: Segment, current_snapshot: DeviceSnapshot + ): + """ + 在回放时,释放或unmap一个内存段**前**回调,对应一个内存段申请或map事件前 + :param wait4release_segment:【只读】待释放的内存段 + :param current_snapshot: 释放或unmap内存段前的内存快照 + """ + ... + + def post_replay_unmap_or_free_segment( + self, released_segment: Segment, current_snapshot: DeviceSnapshot + ): + """ + 在回放时,释放或unmap一个内存段**后**回调,对应一个内存段申请或map事件后 + :param released_segment:【副本】已释放的内存段 + :param current_snapshot: 释放或unmap内存段后的内存快照 + """ + ... diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/replay_executor.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/replay_executor.py new file mode 100644 index 0000000..803dc51 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/replay_executor.py @@ -0,0 +1,37 @@ +from logging import Logger + +from ..base import Block, BlockState, Segment, TraceEntry + +from .simulated_caching_allocator import SimulatedCachingAllocator + + +class ReplayExecutor: + def __init__(self, allocator: SimulatedCachingAllocator, replay_logger: Logger): + self.allocator = allocator + self.replay_logger = replay_logger + + def execute(self, event: TraceEntry) -> bool: + if event.action in ["free", "free_completed"]: + block = Block.build_from_event(event) + block.state = ( + BlockState.ACTIVE_ALLOCATED + if event.action == "free" + else BlockState.ACTIVE_PENDING_FREE + ) + return self.allocator.alloc_block(block) + if event.action == "free_requested": + return self.allocator.active_block(event) + if event.action == "alloc": + return self.allocator.free_block(event) + if event.action in ["segment_free", "segment_unmap"]: + segment = Segment.build_from_event(event) + segment.free_or_unmap_event_idx = event.idx + return self.allocator.alloc_or_map_segment( + segment, merge=event.action == "segment_unmap" + ) + if event.action == "segment_alloc": + return self.allocator.free_segment(event) + if event.action == "segment_map": + return self.allocator.unmap_segment(event) + self.replay_logger.warning(f"Skip event{event.to_dict()} during replay.") + return True diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/simulate.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/simulate.py new file mode 100644 index 0000000..cc1fb3b --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/simulate.py @@ -0,0 +1,112 @@ +from logging import Logger +from typing import Dict + +from ..base import DeviceSnapshot +from ..util import get_logger + +from .allocator_context import AllocatorContext +from .hooker_defs import AllocatorHooker, SimulateHooker +from .replay_executor import ReplayExecutor +from .simulated_caching_allocator import SimulatedCachingAllocator + +loading_logger = get_logger("LOAD") +replay_logger = get_logger("REPLAY") + + +class SimulateDeviceSnapshot: + device_snapshot: DeviceSnapshot + hookers: Dict[int, SimulateHooker] + + device: int + _loading_logger: Logger + _replay_logger: Logger + + def __init__(self, snapshot_dict: dict, device: int): + # 基于device初始化logger + self._loading_logger = loading_logger.getChild(f"{device}") + self._replay_logger = replay_logger.getChild(f"{device}") + if not snapshot_dict: + raise RuntimeError("Cannot init snapshot from empty data.") + self._loading_logger.info("Loading snapshot data...") + self.device_snapshot = DeviceSnapshot.from_dict( + snapshot_dict, device, ignore_inactive_blocks=True + ) + self._loading_logger.info( + f"Finished to load snapshot data: total of {len(self.device_snapshot.trace_entries)} " + f"entries and {len(self.device_snapshot.segments)} segments." + ) + self.hookers = dict[int, SimulateHooker]() + self.simulated_allocator_context = AllocatorContext( + snapshot=self.device_snapshot + ) + self.simulated_allocator = SimulatedCachingAllocator( + self.simulated_allocator_context + ) + self.replay_executor = ReplayExecutor( + self.simulated_allocator, self._replay_logger + ) + # 识别昇腾torch-npu采集的snapshot中的workspace事件 + if ( + self.device_snapshot.trace_entries + and self.device_snapshot.trace_entries[0].action == "workspace_snapshot" + ): + self.simulated_allocator_context.workspace_flag = True + + def register_hooker(self, hooker: SimulateHooker) -> int: + idx = hash(hooker) + self.hookers[idx] = hooker + return idx + + def unregister_hooker(self, hooker_id: int): + if hooker_id in self.hookers: + del self.hookers[hooker_id] + + def register_allocator_hooker(self, hooker: AllocatorHooker) -> int: + return self.simulated_allocator.register_hooker(hooker) + + def unregister_allocator_hooker(self, hooker_id: int): + self.simulated_allocator.unregister_hooker(hooker_id) + + def replay(self) -> bool: + """ + 开始仿真回放内存事件 + """ + # 倒序遍历 + total_size = len(self.device_snapshot.trace_entries) + self._replay_logger.info(f"Replaying {total_size} entries in snapshot...") + progress_update_point = [0.25, 0.5, 0.75] + while self.device_snapshot.trace_entries: + for hooker in self.hookers.values(): + if hooker and not hooker.pre_undo_event( + self.device_snapshot.trace_entries[-1], self.device_snapshot + ): + self._replay_logger.error( + "An interruption occurred during the replay of the single event pre hook." + ) + return False + event = self.device_snapshot.trace_entries[-1] + self.simulated_allocator_context.set_current_undo_event(event) + if not self.replay_executor.execute(event): + self._replay_logger.error( + "An interruption occurred during the replay of the single event." + ) + return False + self.device_snapshot.trace_entries.pop() + current_size = len(self.device_snapshot.trace_entries) + if ( + progress_update_point + and progress_update_point[-1] * total_size >= current_size + ): + self._replay_logger.info( + f"{(1 - progress_update_point[-1]) * 100}% of entries have been processed, " + f"{current_size} entries remain." + ) + progress_update_point.pop() + for hooker in self.hookers.values(): + if hooker and not hooker.post_undo_event(event, self.device_snapshot): + self._replay_logger.error( + "An interruption occurred during the replay of the single event post hook." + ) + return False + self._replay_logger.info("All events have been successfully replayed.") + return True diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/simulated_caching_allocator.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/simulated_caching_allocator.py new file mode 100644 index 0000000..7de5356 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/simulated_caching_allocator.py @@ -0,0 +1,279 @@ +import copy + +from ..base import BlockState, Block, Segment, TraceEntry +from ..util import get_logger + +from .allocator_context import AllocatorContext +from .allocator_hook_dispatcher import AllocatorHookDispatcher +from . import snapshot_lookup, snapshot_mutator + +allocator_logger = get_logger("ALLOCATOR") + + +class SimulatedCachingAllocator: + def __init__(self, ctx: AllocatorContext): + self.ctx = ctx + self.dispatcher = AllocatorHookDispatcher() + + def register_hooker(self, hooker) -> int: + return self.dispatcher.register_hooker(hooker) + + def unregister_hooker(self, hooker_id: int): + self.dispatcher.unregister_hooker(hooker_id) + + def alloc_block(self, new_block: Block) -> bool: + """ + 回放时模拟分配一个新的block + :param new_block: 待分配的block + """ + _error = "Failed to simulate alloc block" + gap_result = snapshot_lookup.find_gap_for_alloc_block( + self.ctx.device_snapshot, + new_block.address, + new_block.size, + self.ctx.current_undo_event.stream if self.ctx.current_undo_event else None, + ) + if gap_result is None: + allocator_logger.error( + f"{_error}: cannot find gap for block (addr={new_block.address}, size={new_block.size})" + ) + return False + segment, insert_idx = gap_result + if self.ctx.current_undo_event: + new_block.free_event_idx = self.ctx.current_undo_event.idx + if ( + self.ctx.current_undo_event + and self.ctx.current_undo_event.action == "free_completed" + ): + new_block.state = BlockState.ACTIVE_PENDING_FREE + else: + new_block.state = BlockState.ACTIVE_ALLOCATED + self.dispatcher.pre_replay_alloc_block(new_block, self.ctx.device_snapshot) + snapshot_mutator.attach_block( + self.ctx.device_snapshot, segment, new_block, insert_idx + ) + self.dispatcher.post_replay_alloc_block(new_block, self.ctx.device_snapshot) + return True + + def free_block(self, alloc_event: TraceEntry) -> bool: + """ + 回放时模拟释放一个block,可能涉及到拆分合并 + :param alloc_event: 待回滚的alloc事件 + """ + _error = "Failed to simulate free block" + seg_idx, segment = snapshot_lookup.find_overlapping_segment( + self.ctx.device_snapshot, alloc_event.addr, alloc_event.stream + ) + if seg_idx == -1 or segment is None: + allocator_logger.error( + f"{_error}: cannot find segment for block (addr={alloc_event.addr})" + ) + return False + block_idx, exist_block = snapshot_lookup.find_block(segment, alloc_event.addr) + if block_idx == -1 or exist_block is None: + # workspace场景容忍 + if self.ctx.workspace_flag: + allocator_logger.warning( + f"{_error}: cannot find block (addr={alloc_event.addr}), workspace scenario tolerance" + ) + return True + allocator_logger.error( + f"{_error}: cannot find block (addr={alloc_event.addr})" + ) + return False + if exist_block.size < alloc_event.size: + allocator_logger.error( + f"{_error}: block size ({exist_block.size}) < event size ({alloc_event.size})" + ) + return False + exist_block.alloc_event_idx = alloc_event.idx + self.dispatcher.pre_replay_free_block(exist_block, self.ctx.device_snapshot) + if not snapshot_mutator.detach_block(self.ctx.device_snapshot, exist_block): + allocator_logger.error(f"{_error}: block has no segment_ptr") + return False + self.dispatcher.post_replay_free_block( + exist_block, self.ctx.device_snapshot, use_copy=True + ) + return True + + def active_block(self, free_requested_event: TraceEntry) -> bool: + """ + 回放时模拟active一个block + :param free_requested_event: 待回放的free_requested请求 + """ + _error = "Failed to simulate active block" + seg_idx, segment = snapshot_lookup.find_overlapping_segment( + self.ctx.device_snapshot, + free_requested_event.addr, + free_requested_event.stream, + ) + if seg_idx == -1 or segment is None: + allocator_logger.error( + f"{_error}: cannot find segment for block (addr={free_requested_event.addr})" + ) + return False + block_idx, active_pending_free_block = snapshot_lookup.find_block( + segment, free_requested_event.addr + ) + if block_idx == -1 or active_pending_free_block is None: + allocator_logger.error( + f"{_error}: cannot find block (addr={free_requested_event.addr})" + ) + return False + if active_pending_free_block.state != BlockState.ACTIVE_PENDING_FREE: + # workspace场景容忍异常 + if self.ctx.workspace_flag: + allocator_logger.warning( + f"{_error}: block (addr={free_requested_event.addr}) is not in {BlockState.ACTIVE_PENDING_FREE} state, " + f"but workspace_flag is True, skipping" + ) + return True + allocator_logger.error( + f"{_error}: block (addr={free_requested_event.addr}) is not in {BlockState.ACTIVE_PENDING_FREE} state, " + f"current state: {active_pending_free_block.state}" + ) + return False + if not snapshot_mutator.promote_pending_free_block( + self.ctx.device_snapshot, active_pending_free_block + ): + allocator_logger.error( + f"{_error}: the found active pending block's segment is none." + ) + return False + return True + + def alloc_or_map_segment(self, new_segment: Segment, merge: bool = False) -> bool: + """ + 回放时模拟alloc或map一个新的内存段 + :param new_segment: 新内存段 + :param merge: 是否合并,map时对应虚拟内存场景,否则仅为alloc + """ + _error = "Failed to alloc or map segment" + segments = self.ctx.device_snapshot.segments + self.dispatcher.pre_replay_map_or_alloc_segment( + new_segment, self.ctx.device_snapshot + ) + if self.ctx.current_undo_event: + new_segment.free_or_unmap_event_idx = self.ctx.current_undo_event.idx + if not merge: + snapshot_mutator.insert_segment(self.ctx.device_snapshot, new_segment) + self.dispatcher.post_replay_map_or_alloc_segment( + new_segment, self.ctx.device_snapshot + ) + return True + new_seg_start = new_segment.address + new_seg_end = new_seg_start + new_segment.total_size + left_adjacent_idx = -1 + right_adjacent_idx = -1 + for i, seg in enumerate(segments): + if seg.stream != new_segment.stream: + continue + if seg.address + seg.total_size == new_seg_start: + left_adjacent_idx = i + elif new_seg_end == seg.address: + right_adjacent_idx = i + if left_adjacent_idx == -1 and right_adjacent_idx == -1: + snapshot_mutator.insert_segment(self.ctx.device_snapshot, new_segment) + self.dispatcher.post_replay_map_or_alloc_segment( + new_segment, self.ctx.device_snapshot + ) + return True + virtual_map_segment = copy.deepcopy(new_segment) + if not snapshot_mutator.merge_mapped_segment( + self.ctx.device_snapshot, + new_segment, + left_adjacent_idx, + right_adjacent_idx, + ): + allocator_logger.error(f"{_error}: failed to merge adjacent segments") + return False + snapshot_mutator.increase_reserved( + self.ctx.device_snapshot, virtual_map_segment.total_size + ) + self.dispatcher.post_replay_map_or_alloc_segment( + virtual_map_segment, self.ctx.device_snapshot + ) + return True + + def free_segment(self, alloc_seg_event: TraceEntry) -> bool: + """ + 回放时模拟free一个内存段(非虚拟内存场景) + :param alloc_seg_event: 待回滚的alloc事件 + """ + _error = "Free segment failed" + seg_addr = alloc_seg_event.addr + exist_seg_idx, exist_seg = snapshot_lookup.find_segment( + self.ctx.device_snapshot, seg_addr, alloc_seg_event.stream + ) + if exist_seg_idx == -1 or exist_seg is None: + allocator_logger.error( + f"{_error}: cannot found segment(addr={seg_addr}, stream={alloc_seg_event.stream})" + ) + return False + if exist_seg.total_size != alloc_seg_event.size: + allocator_logger.error( + f"{_error}: cannot free segment(addr={seg_addr}, size={alloc_seg_event.size}) in " + f"exist segment(addr={exist_seg.address}, size={exist_seg.total_size})" + ) + return False + if exist_seg.active_size > 0: + allocator_logger.error( + f"{_error}: cannot free a segment that still has active allocations." + ) + return False + + exist_seg.alloc_or_map_event_idx = alloc_seg_event.idx + self.dispatcher.pre_replay_unmap_or_free_segment( + exist_seg, self.ctx.device_snapshot + ) + snapshot_mutator.remove_segment(self.ctx.device_snapshot, exist_seg) + self.dispatcher.post_replay_unmap_or_free_segment( + exist_seg, self.ctx.device_snapshot + ) + return True + + def unmap_segment(self, map_event): + """ + 回放时模拟unmap一个已有的内存段(虚拟内存场景) + :param map_event: 待回滚的map事件 + """ + _error = "Unmap segment failed" + segments = self.ctx.device_snapshot.segments + virtual_free_segment = Segment.build_from_event(map_event) + seg_addr = virtual_free_segment.address + unmap_size = virtual_free_segment.total_size + exist_seg_idx, exist_seg = snapshot_lookup.find_overlapping_segment( + self.ctx.device_snapshot, seg_addr, map_event.stream + ) + if exist_seg_idx < 0 or exist_seg is None or exist_seg_idx >= len(segments): + allocator_logger.error(f"{_error}: cannot found segment(addr={seg_addr})") + return False + virtual_free_segment.free_or_unmap_event_idx = exist_seg.free_or_unmap_event_idx + virtual_free_segment.alloc_or_map_event_idx = map_event.idx + if not ( + seg_addr >= exist_seg.address + and seg_addr + unmap_size <= exist_seg.address + exist_seg.total_size + ): + allocator_logger.error( + f"{_error}: cannot unmap segment(addr={seg_addr}, unmap_size={unmap_size}) in exist segment(" + f"addr={exist_seg.address}, total_size={exist_seg.total_size})" + ) + return False + self.dispatcher.pre_replay_unmap_or_free_segment( + virtual_free_segment, self.ctx.device_snapshot + ) + if exist_seg.stream != map_event.stream: + allocator_logger.error( + f"{_error}: stream mismatch (segment: {exist_seg.stream}, event: {map_event.stream})" + ) + return False + if not snapshot_mutator.split_or_shrink_segment( + self.ctx.device_snapshot, exist_seg_idx, seg_addr, unmap_size + ): + allocator_logger.error(f"{_error}: failed to split or shrink segment") + return False + snapshot_mutator.decrease_reserved(self.ctx.device_snapshot, unmap_size) + self.dispatcher.post_replay_unmap_or_free_segment( + virtual_free_segment, self.ctx.device_snapshot + ) + return True diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_lookup.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_lookup.py new file mode 100644 index 0000000..58f5611 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_lookup.py @@ -0,0 +1,135 @@ +from typing import Optional, Tuple + +from ..base import Block, DeviceSnapshot, Segment + + +def is_valid_sub_block(block: Block, addr: int, size: int) -> bool: + """Return whether the given range is fully contained in a block.""" + return block.address <= addr and addr + size <= block.address + block.size + + +def find_block(segment: Segment, block_addr: int) -> tuple[int, Block | None]: + """Find the block containing an address inside a segment. + + Returns `(idx, block)` when matched, otherwise `(-1, None)`. + """ + left = 0 + right = len(segment.blocks) - 1 + while left <= right: + mid = (left + right) // 2 + if block_addr < segment.blocks[mid].address: + right = mid - 1 + elif block_addr >= segment.blocks[mid].address + segment.blocks[mid].size: + left = mid + 1 + else: + return mid, segment.blocks[mid] + return -1, None + + +def _scan_overlapping_segments_for_stream( + segments: list[Segment], mid: int, addr: int, stream: int +) -> tuple[int, Segment | None]: + """Scan neighboring overlapping segments to find a stream match.""" + for i in range(mid - 1, -1, -1): + if addr < segments[i].address: + break + if ( + addr < segments[i].address + segments[i].total_size + and segments[i].stream == stream + ): + return i, segments[i] + for i in range(mid + 1, len(segments)): + if addr < segments[i].address: + break + if ( + addr < segments[i].address + segments[i].total_size + and segments[i].stream == stream + ): + return i, segments[i] + return -1, None + + +def find_overlapping_segment( + snapshot: DeviceSnapshot, addr: int, stream: int | None = None +) -> tuple[int, Segment | None]: + """Find the segment whose range overlaps the given address. + + Returns `(idx, segment)` for a containing-range match, otherwise + `(-1, None)`. When `stream` is provided, the matched segment must also + share that stream. + """ + left = 0 + segments = snapshot.segments + right = len(segments) - 1 + while left <= right: + mid = (left + right) // 2 + if addr < segments[mid].address: + right = mid - 1 + elif addr >= segments[mid].address + segments[mid].total_size: + left = mid + 1 + else: + if stream is not None and segments[mid].stream != stream: + return _scan_overlapping_segments_for_stream( + segments, mid, addr, stream + ) + return mid, segments[mid] + return -1, None + + +def find_segment( + snapshot: DeviceSnapshot, addr: int, stream: int +) -> tuple[int, Segment | None]: + """Find the segment whose start address exactly matches the given address. + + Returns `(idx, segment)` for an exact start-address and stream match, + otherwise `(-1, None)`. + """ + seg_idx, seg = find_overlapping_segment(snapshot, addr, stream) + if seg is not None and seg.address == addr and seg.stream == stream: + return seg_idx, seg + return -1, None + + +def find_gap_for_alloc_block( + snapshot: DeviceSnapshot, event_addr: int, event_size: int, stream: int = None +) -> Optional[Tuple[Segment, int]]: + """Find the insertion gap for a block allocation inside a segment. + + Returns `(segment, insert_idx)` when a valid gap exists, otherwise `None`. + The returned index is the position where the new block should be inserted. + """ + seg_idx, segment = find_overlapping_segment(snapshot, event_addr, stream) + if seg_idx == -1 or segment is None: + return None + blocks = segment.blocks + event_end = event_addr + event_size + seg_start = segment.address + seg_end = seg_start + segment.total_size + if len(blocks) == 0: + if seg_start <= event_addr and event_end <= seg_end: + return segment, 0 + return None + + if blocks[0].address >= event_end: + if seg_start <= event_addr: + return segment, 0 + return None + + left, right = 0, len(blocks) - 1 + while left < right: + mid = (left + right + 1) // 2 + if blocks[mid].address <= event_addr: + left = mid + else: + right = mid - 1 + + gap_start = blocks[left].address + blocks[left].size + if left + 1 < len(blocks): + gap_end = blocks[left + 1].address + if gap_start <= event_addr and event_end <= gap_end: + return segment, left + 1 + else: + if gap_start <= event_addr and event_end <= seg_end: + return segment, len(blocks) + + return None diff --git a/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_mutator.py b/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_mutator.py new file mode 100644 index 0000000..4bc7126 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/simulate/snapshot_mutator.py @@ -0,0 +1,407 @@ +import bisect + +from ..base import Block, BlockState, DeviceSnapshot, Segment +from ..util import get_logger + +snapshot_mutator_logger = get_logger("ALLOCATOR") + + +# Public API: block mutation helpers + + +def attach_block( + snapshot: DeviceSnapshot, segment: Segment, block: Block, insert_idx: int +): + """Attach a block to a segment and update allocation totals. + + Mutates the block, the owning segment, and snapshot totals in place. + The block is inserted into `segment.blocks` at `insert_idx`. + """ + block.segment_ptr = segment + segment.blocks.insert(insert_idx, block) + segment.active_size += block.size + snapshot.total_activated += block.size + if block.state == BlockState.ACTIVE_ALLOCATED: + segment.allocated_size += block.size + snapshot.total_allocated += block.size + + +def detach_block(snapshot: DeviceSnapshot, block: Block) -> bool: + """Detach a block from its owning segment. + + Mutates the block, its segment, and snapshot totals in place. + Returns `True` on success, otherwise `False` when the block has no segment. + """ + segment = block.segment_ptr + if segment is None: + return False + segment.active_size -= block.size + snapshot.total_activated -= block.size + if block.state == BlockState.ACTIVE_ALLOCATED: + segment.allocated_size -= block.size + snapshot.total_allocated -= block.size + segment.blocks.remove(block) + block.segment_ptr = None + return True + + +def promote_pending_free_block(snapshot: DeviceSnapshot, block: Block) -> bool: + """Promote a pending-free block back to allocated state. + + Mutates the block state plus the owning segment and snapshot allocation + totals. Returns `False` when the block has no owning segment. + """ + segment = block.segment_ptr + if segment is None: + return False + block.state = BlockState.ACTIVE_ALLOCATED + segment.allocated_size += block.size + snapshot.total_allocated += block.size + return True + + +# Public API: segment collection mutation + + +def insert_segment(snapshot: DeviceSnapshot, segment: Segment): + """Insert a segment into the snapshot and increase reserved memory. + + Preserves segment ordering and mutates `snapshot.total_reserved` in place. + """ + _insert_segment_sorted(snapshot, segment) + snapshot.total_reserved += segment.total_size + + +def remove_segment(snapshot: DeviceSnapshot, segment: Segment): + """Remove a segment from the snapshot and decrease reserved memory. + + Clears every block's `segment_ptr` and mutates `snapshot.total_reserved` + in place. + """ + snapshot.total_reserved -= segment.total_size + snapshot.segments.remove(segment) + for block in segment.blocks: + block.segment_ptr = None + + +# Public API: segment shape mutation + + +def merge_mapped_segment( + snapshot: DeviceSnapshot, + new_segment: Segment, + left_adjacent_idx: int, + right_adjacent_idx: int, +) -> bool: + """Merge a newly mapped segment with adjacent segments. + + Supports left-only, right-only, or both-side adjacency. Mutates + `snapshot.segments` in place and returns `False` when neither adjacent + index is valid. + """ + _error = "Failed to merge mapped segment" + if left_adjacent_idx == -1 and right_adjacent_idx == -1: + snapshot_mutator_logger.error( + f"{_error}: both adjacent segment indices are missing" + ) + return False + + segments = snapshot.segments + if left_adjacent_idx != -1: + left_seg = segments[left_adjacent_idx] + left_seg.total_size += new_segment.total_size + left_seg.allocated_size += new_segment.allocated_size + left_seg.active_size += new_segment.active_size + for block in new_segment.blocks: + block.segment_ptr = left_seg + left_seg.blocks.append(block) + if right_adjacent_idx != -1: + return _merge_segments(snapshot, left_adjacent_idx, right_adjacent_idx) + return True + + _insert_segment_sorted(snapshot, new_segment) + new_idx = segments.index(new_segment) + corrected_right_idx = new_idx + 1 + if corrected_right_idx >= len(segments): + snapshot_mutator_logger.error( + f"{_error}: right adjacent segment missing after insert" + ) + segments.remove(new_segment) + return False + if corrected_right_idx != right_adjacent_idx + 1: + right_segment = segments[corrected_right_idx] + if right_segment.address != new_segment.address + new_segment.total_size: + snapshot_mutator_logger.error( + f"{_error}: inserted segment is not adjacent to the expected right segment" + ) + segments.remove(new_segment) + return False + if not _merge_segments(snapshot, new_idx, corrected_right_idx): + segments.remove(new_segment) + return False + return True + + +def split_or_shrink_segment( + snapshot: DeviceSnapshot, seg_idx: int, seg_addr: int, unmap_size: int +) -> bool: + """Shrink or split a segment to remove an unmapped range. + + Chooses the appropriate internal mutation strategy based on whether the + unmapped range touches the segment edges or lies in the middle. + Returns `True` on success and `False` when the mutation is invalid. + """ + exist_seg = snapshot.segments[seg_idx] + seg_start = exist_seg.address + unmap_end = seg_addr + unmap_size + if seg_addr == seg_start: + return _shrink_segment(snapshot, seg_idx, seg_addr, unmap_size, "left") + if unmap_end == seg_start + exist_seg.total_size: + return _shrink_segment(snapshot, seg_idx, seg_addr, unmap_size, "right") + return _split_segment_at(snapshot, seg_idx, seg_addr, unmap_size) + + +# Public API: snapshot total mutation + + +def increase_reserved(snapshot: DeviceSnapshot, size: int): + """Increase the snapshot reserved-memory total by `size`.""" + snapshot.total_reserved += size + + +def decrease_reserved(snapshot: DeviceSnapshot, size: int): + """Decrease the snapshot reserved-memory total by `size`.""" + snapshot.total_reserved -= size + + +# Private helpers + + +def _insert_segment_sorted(snapshot: DeviceSnapshot, segment: Segment): + """Insert a segment into `snapshot.segments` while preserving sort order.""" + segments = snapshot.segments + keys = [(seg.address, seg.stream) for seg in segments] + idx = bisect.bisect_left(keys, (segment.address, segment.stream)) + segments.insert(idx, segment) + + +def _merge_segments(snapshot: DeviceSnapshot, target_idx: int, source_idx: int) -> bool: + """Merge two adjacent segments with the same stream. + + Mutates `snapshot.segments` in place. Returns `True` on success, otherwise + `False` when indices, adjacency, or stream constraints are not satisfied. + """ + _error = "Failed to merge segments" + segments = snapshot.segments + if target_idx < 0 or target_idx >= len(segments): + snapshot_mutator_logger.error( + f"{_error}: invalid target segment index {target_idx}" + ) + return False + if source_idx < 0 or source_idx >= len(segments): + snapshot_mutator_logger.error( + f"{_error}: invalid source segment index {source_idx}" + ) + return False + if target_idx == source_idx: + snapshot_mutator_logger.error( + f"{_error}: target and source are the same segment" + ) + return False + target = segments[target_idx] + source = segments[source_idx] + if target.stream != source.stream: + snapshot_mutator_logger.error( + f"{_error}: segments have different streams (target: {target.stream}, source: {source.stream})" + ) + return False + are_adjacent = ( + target.address + target.total_size == source.address + or source.address + source.total_size == target.address + ) + if not are_adjacent: + snapshot_mutator_logger.error( + f"{_error}: segments are not adjacent (target: [{target.address}, {target.address + target.total_size}), source: [{source.address}, {source.address + source.total_size}))" + ) + return False + if target.address > source.address: + target, source = source, target + target_idx, source_idx = source_idx, target_idx + target.total_size += source.total_size + target.allocated_size += source.allocated_size + target.active_size += source.active_size + for block in source.blocks: + block.segment_ptr = target + target.blocks.append(block) + del segments[source_idx] + return True + + +def _split_segment_at( + snapshot: DeviceSnapshot, seg_idx: int, cut_addr: int, cut_size: int +) -> bool: + """Split a segment around a cut range and keep non-overlapping blocks. + + Mutates `snapshot.segments` in place. Returns `True` on success, otherwise + `False` when the cut range is invalid. + """ + _error = "Failed to split segment" + segments = snapshot.segments + if seg_idx < 0 or seg_idx >= len(segments): + snapshot_mutator_logger.error(f"{_error}: invalid segment index {seg_idx}") + return False + original_segment = segments[seg_idx] + seg_start = original_segment.address + seg_end = seg_start + original_segment.total_size + cut_end = cut_addr + cut_size + if cut_addr < seg_start or cut_end > seg_end: + snapshot_mutator_logger.error( + f"{_error}: cut range [{cut_addr}, {cut_end}) is outside segment [{seg_start}, {seg_end})" + ) + return False + if cut_addr == seg_start and cut_end == seg_end: + snapshot_mutator_logger.warning( + "Split Seg: cut range covers entire segment, nothing to split, just remove it" + ) + del snapshot.segments[seg_idx] + return True + left_segment = Segment( + address=seg_start, + total_size=cut_addr - seg_start, + stream=original_segment.stream, + segment_type=original_segment.segment_type, + allocated_size=0, + active_size=0, + blocks=[], + device=original_segment.device, + frames=original_segment.frames, + is_expandable=original_segment.is_expandable, + free_or_unmap_event_idx=original_segment.free_or_unmap_event_idx, + alloc_or_map_event_idx=original_segment.alloc_or_map_event_idx, + ) + right_segment = Segment( + address=cut_end, + total_size=seg_end - cut_end, + stream=original_segment.stream, + segment_type=original_segment.segment_type, + allocated_size=0, + active_size=0, + blocks=[], + device=original_segment.device, + frames=original_segment.frames, + is_expandable=original_segment.is_expandable, + free_or_unmap_event_idx=original_segment.free_or_unmap_event_idx, + alloc_or_map_event_idx=original_segment.alloc_or_map_event_idx, + ) + for block in original_segment.blocks: + block_start = block.address + block_end = block_start + block.size + if block_end <= cut_addr: + block.segment_ptr = left_segment + left_segment.blocks.append(block) + left_segment.active_size += block.size + if block.state == BlockState.ACTIVE_ALLOCATED: + left_segment.allocated_size += block.size + elif block_start >= cut_end: + block.segment_ptr = right_segment + right_segment.blocks.append(block) + right_segment.active_size += block.size + if block.state == BlockState.ACTIVE_ALLOCATED: + right_segment.allocated_size += block.size + else: + snapshot_mutator_logger.warning( + f"{_error}: active block [{block_start}, {block_end}) overlaps with cut range [{cut_addr}, {cut_end}), just drop it." + ) + del segments[seg_idx] + if left_segment.total_size > 0: + _insert_segment_sorted(snapshot, left_segment) + if right_segment.total_size > 0: + _insert_segment_sorted(snapshot, right_segment) + return True + + +def _shrink_segment( + snapshot: DeviceSnapshot, + seg_idx: int, + shrink_addr: int, + shrink_size: int, + direction: str, +) -> bool: + """Shrink a segment from the left or right side. + + Mutates the target segment in place and recomputes its size counters. + Returns `True` on success, otherwise `False` when the shrink is invalid. + """ + _error = "Failed to shrink segment" + segments = snapshot.segments + if seg_idx < 0 or seg_idx >= len(segments): + snapshot_mutator_logger.error(f"{_error}: invalid segment index {seg_idx}") + return False + if direction not in ["left", "right"]: + snapshot_mutator_logger.error( + f"{_error}: invalid direction '{direction}', must be 'left' or 'right'" + ) + return False + segment = segments[seg_idx] + seg_start = segment.address + seg_end = seg_start + segment.total_size + shrink_end = shrink_addr + shrink_size + if direction == "left": + if shrink_addr < seg_start or shrink_end > seg_end: + snapshot_mutator_logger.error( + f"{_error}: shrink range [{shrink_addr}, {shrink_end}) is outside segment [{seg_start}, {seg_end})" + ) + return False + new_start = shrink_end + new_size = seg_end - new_start + if new_size < 0: + snapshot_mutator_logger.error( + f"{_error}: shrink results in negative segment size" + ) + return False + for block in segment.blocks: + block_start = block.address + block_end = block_start + block.size + if block_end <= shrink_end: + snapshot_mutator_logger.error( + f"{_error}: active block [{block_start}, {block_end}) in shrink range [{shrink_addr}, {shrink_end})" + ) + return False + segment.address = new_start + segment.total_size = new_size + segment.blocks = [ + block for block in segment.blocks if block.address >= new_start + ] + else: + if shrink_addr < seg_start or shrink_end > seg_end: + snapshot_mutator_logger.error( + f"{_error}: shrink range [{shrink_addr}, {shrink_end}) is outside segment [{seg_start}, {seg_end})" + ) + return False + new_size = shrink_addr - seg_start + if new_size < 0: + snapshot_mutator_logger.error( + f"{_error}: shrink results in negative segment size" + ) + return False + for block in segment.blocks: + block_start = block.address + block_end = block_start + block.size + if block_start >= shrink_addr: + snapshot_mutator_logger.error( + f"{_error}: active block [{block_start}, {block_end}) in shrink range [{shrink_addr}, {shrink_end})" + ) + return False + segment.blocks = [ + block + for block in segment.blocks + if block.address + block.size <= shrink_addr + ] + segment.total_size = new_size + segment.allocated_size = sum( + b.size for b in segment.blocks if b.state == BlockState.ACTIVE_ALLOCATED + ) + segment.active_size = sum(b.size for b in segment.blocks) + if segment.total_size == 0: + del segments[seg_idx] + return True diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/__init__.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/__init__.py new file mode 100644 index 0000000..5e8aae8 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/__init__.py @@ -0,0 +1,4 @@ +from .snapshot_db import SnapshotDb, TRACE_ENTRY_ACTION_VALUE_MAP, BLOCK_STATE_VALUE_MAP +from .entity2record import event2record, block2record + +__all__ = ["SnapshotDb", "event2record", "block2record"] diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/defs.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/defs.py new file mode 100644 index 0000000..e6ad711 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/defs.py @@ -0,0 +1,20 @@ +class EventFieldDefs: + ID = "id" + ACTION = "action" + ADDR = "address" + SIZE = "size" + STREAM = "stream" + ALLOCATED = "allocated" + ACTIVE = "active" + RESERVED = "reserved" + CALLSTACK = "callstack" + + +class BlockFieldDefs: + ID = "id" + ADDR = "address" + SIZE = "size" + REQUESTED_SIZE = "requestedSize" + STATE = "state" + ALLOC_EVENT_ID = "allocEventId" + FREE_EVENT_ID = "freeEventId" diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/entity2record.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/entity2record.py new file mode 100644 index 0000000..52b9d1a --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/entity2record.py @@ -0,0 +1,61 @@ +from .defs import EventFieldDefs, BlockFieldDefs + +from ....base import TraceEntry, Block + + +def make_default_id_counter(start: int = -1): + count = start + + def next_id(): + nonlocal count + current = count + count -= 1 + return current + + return next_id + + +def get_timestamp_by_event_idx(event_idx: int) -> int: + return event_idx * 10 if event_idx is not None else -1 + + +next_default_block_id = make_default_id_counter() +next_default_event_id = make_default_id_counter() + + +def event2record( + event: TraceEntry, allocated: int = 0, active: int = 0, reserved: int = 0 +) -> dict: + return { + EventFieldDefs.ID: ( + event.idx if event.idx is not None else next_default_event_id() + ), + EventFieldDefs.ACTION: event.action, + EventFieldDefs.ADDR: event.addr, + EventFieldDefs.SIZE: event.size, + EventFieldDefs.STREAM: event.stream, + EventFieldDefs.ALLOCATED: allocated, + EventFieldDefs.ACTIVE: active, + EventFieldDefs.RESERVED: reserved, + EventFieldDefs.CALLSTACK: event.get_callstack(), + } + + +def block2record(block: Block) -> dict: + return { + BlockFieldDefs.ID: ( + block.alloc_event_idx + if block.alloc_event_idx is not None + else next_default_block_id() + ), + BlockFieldDefs.ADDR: block.address, + BlockFieldDefs.SIZE: block.size, + BlockFieldDefs.REQUESTED_SIZE: block.requested_size, + BlockFieldDefs.STATE: block.state, + BlockFieldDefs.ALLOC_EVENT_ID: ( + block.alloc_event_idx if block.alloc_event_idx is not None else -1 + ), + BlockFieldDefs.FREE_EVENT_ID: ( + block.free_event_idx if block.free_event_idx is not None else -1 + ), + } diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py new file mode 100644 index 0000000..8058fe5 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/database/snapshot_db.py @@ -0,0 +1,97 @@ +from ....util.sqlite_meta import SqliteColumn, SqliteTable, SqliteDB +from .defs import EventFieldDefs, BlockFieldDefs + +TRACE_ENTRY_ACTION_VALUE_MAP = { + "segment_map": 0, + "segment_unmap": 1, + "segment_alloc": 2, + "segment_free": 3, + "alloc": 4, + "free_requested": 5, + "free_completed": 6, + "workspace_snapshot": 7, +} + +BLOCK_STATE_VALUE_MAP = { + "inactive": -1, + "active_allocated": 1, + "active_pending_free": 0, +} + +_TRACE_ENTRY_TABLE_COLUMNS = [ + SqliteColumn(name=EventFieldDefs.ID, data_type=int, primary_key=True), + SqliteColumn( + name=EventFieldDefs.ACTION, + data_type=int, + value_map=TRACE_ENTRY_ACTION_VALUE_MAP.copy(), + ), + SqliteColumn(name=EventFieldDefs.ADDR, data_type=int), + SqliteColumn(name=EventFieldDefs.SIZE, data_type=int), + SqliteColumn(name=EventFieldDefs.STREAM, data_type=int), + SqliteColumn(name=EventFieldDefs.ALLOCATED, data_type=int), + SqliteColumn(name=EventFieldDefs.ACTIVE, data_type=int), + SqliteColumn(name=EventFieldDefs.RESERVED, data_type=int), + SqliteColumn(name=EventFieldDefs.CALLSTACK), +] + +_BLOCK_TABLE_COLUMNS = [ + SqliteColumn(name=BlockFieldDefs.ID, data_type=int, primary_key=True), + SqliteColumn(name=BlockFieldDefs.ADDR, data_type=int), + SqliteColumn(name=BlockFieldDefs.SIZE, data_type=int), + SqliteColumn(name=BlockFieldDefs.REQUESTED_SIZE, data_type=int), + SqliteColumn( + name=BlockFieldDefs.STATE, + default=99, + data_type=int, + value_map=BLOCK_STATE_VALUE_MAP.copy(), + ), + SqliteColumn(name=BlockFieldDefs.ALLOC_EVENT_ID, data_type=int), + SqliteColumn(name=BlockFieldDefs.FREE_EVENT_ID, data_type=int), +] + + +class SnapshotDb(SqliteDB): + TRACE_ENTRY_TABLE_NAME = "trace_entry" + BLOCK_TABLE_NAME = "block" + + def __init__(self, path: str): + super().__init__(path, auto_create=True, with_dictionary_table=True) + # 清理旧版本表格 + self._clear_old_tables() + + def create_trace_entry_table(self, device: int = 0): + self.create_table( + SqliteTable( + self.get_trace_table_name_by_device(device), _TRACE_ENTRY_TABLE_COLUMNS + ), + delete_if_exists=True, + ) + + def create_block_table(self, device: int = 0): + self.create_table( + SqliteTable( + self.get_block_table_name_by_device(device), _BLOCK_TABLE_COLUMNS + ), + delete_if_exists=True, + ) + + def get_trace_entry_table(self, device: int = 0): + return self.get_table_by_name(self.get_trace_table_name_by_device(device)) + + def get_block_table(self, device: int = 0): + return self.get_table_by_name(self.get_block_table_name_by_device(device)) + + def _clear_old_tables(self): + """ + 旧版本中创建的db不带device前缀,如果通过旧db打开,需要清理旧表 + """ + self.delete_table(self.TRACE_ENTRY_TABLE_NAME) + self.delete_table(self.BLOCK_TABLE_NAME) + + @staticmethod + def get_block_table_name_by_device(device: int = 0): + return f"{SnapshotDb.BLOCK_TABLE_NAME}_{device}" + + @staticmethod + def get_trace_table_name_by_device(device: int = 0): + return f"{SnapshotDb.TRACE_ENTRY_TABLE_NAME}_{device}" diff --git a/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/snapshot2db.py new file mode 100644 index 0000000..594c4e7 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/tools/adaptors/snapshot2db.py @@ -0,0 +1,277 @@ +import os +import sys +import argparse +from pathlib import Path +from ...simulate import SimulateHooker, SimulateDeviceSnapshot, AllocatorHooker +from ...base import DeviceSnapshot, TraceEntry, Block, BlockState +from ...util.file_util import check_dir_valid, load_pickle_to_dict + +from .database import SnapshotDb, block2record, event2record + +from ...util.logger import get_logger, set_global_log_file +from ...util.timer import timer + +dump_logger = get_logger("DatabaseDump") + + +class SnapshotDbHandler: + def __init__(self, db_path: str, devices: list[int], insert_cache_size: int = 1000): + self.db_path = db_path + self.db = SnapshotDb(db_path) + self._device_event_cache = dict() + self._device_block_cache = dict() + self._insert_cache_size = insert_cache_size + for device in devices: + self._device_block_cache[device] = list() + self._device_event_cache[device] = list() + self.db.create_trace_entry_table(device) + self.db.create_block_table(device) + + def insert_event(self, event_record: dict, device: int = 0): + if device not in self._device_event_cache: + self._device_event_cache[device] = list() + self._device_event_cache[device].append(event_record) + if len(self._device_event_cache[device]) >= self._insert_cache_size: + self._do_insert_events(device) + + def insert_block(self, block_record: dict, device: int = 0): + if device not in self._device_block_cache: + self._device_block_cache[device] = list() + self._device_block_cache[device].append(block_record) + if len(self._device_block_cache[device]) >= self._insert_cache_size: + self._do_insert_blocks(device) + + def flush(self, device: int = 0): + if self._device_event_cache.get(device, None): + self._do_insert_events(device) + if self._device_block_cache.get(device, None): + self._do_insert_blocks(device) + + def _do_insert_events(self, device: int = 0): + if device not in self._device_event_cache: + self._device_event_cache[device] = list() + return + self.db.get_trace_entry_table(device).insert_records( + self.db.conn, self._device_event_cache[device] + ) + self.db.conn.commit() + self._device_event_cache[device].clear() + + def _do_insert_blocks(self, device: int = 0): + if device not in self._device_block_cache: + self._device_block_cache[device] = list() + self.db.get_block_table(device).insert_records( + self.db.conn, self._device_block_cache[device] + ) + self.db.conn.commit() + self._device_block_cache[device].clear() + + def __del__(self): + self.db.conn.commit() + self.db.conn.close() + + +class DumpEventHooker(SimulateHooker, AllocatorHooker): + def __init__(self, db_path: str, devices: list[int], dump_cache_size: int = 1000): + self.db_handler = SnapshotDbHandler( + db_path, devices, insert_cache_size=dump_cache_size + ) + + def post_undo_event( + self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot + ) -> bool: + # 回放完毕,dump剩余Segment及block数据, 注意应该先插入blocks + if not current_snapshot.trace_entries: + for seg in current_snapshot.segments: + for block in seg.blocks: + if block.state != BlockState.INACTIVE: + self.db_handler.insert_block( + block2record(block), current_snapshot.device + ) + # segment不插入block表,而是以模拟事件插入事件表,便于后续重建segment + mock_segment_alloc_event = TraceEntry( + idx=None, + action="segment_map" if seg.is_expandable else "segment_alloc", + addr=seg.address, + frames=seg.frames, + size=seg.total_size, + stream=seg.stream, + ) + self.db_handler.insert_event( + event2record( + event=mock_segment_alloc_event, + allocated=current_snapshot.total_allocated, + active=current_snapshot.total_activated, + reserved=current_snapshot.total_reserved, + ), + current_snapshot.device, + ) + return True + + def pre_undo_event( + self, wait4undo_event: TraceEntry, current_snapshot: DeviceSnapshot + ) -> bool: + # 每个事件回放前dump一次event + self.db_handler.insert_event( + event2record( + event=wait4undo_event, + allocated=current_snapshot.total_allocated, + active=current_snapshot.total_activated, + reserved=current_snapshot.total_reserved, + ), + current_snapshot.device, + ) + return True + + def post_replay_free_block( + self, released_block: Block, current_snapshot: DeviceSnapshot + ): + self.db_handler.insert_block( + block2record(released_block), current_snapshot.device + ) + + def flush(self, device: int = 0): + self.db_handler.flush(device) + + +def dump(pickle_file: str, dump_file: str, device=None) -> bool: + try: + data = load_pickle_to_dict(Path(pickle_file)) + except Exception as e: + dump_logger.error("Failed to load pickle file: {}".format(e)) + return False + device_traces = data.get("device_traces", []) + # 当指定device为空时dump所有记录了跟踪事件的device,否则仅dump指定device + need_dump_devices = [ + device for device in range(len(device_traces)) if device_traces[device] + ] + dump_logger.info(f"Recognized have trace events devices {need_dump_devices}.") + if device is not None and device not in need_dump_devices: + dump_logger.error( + f"Specified device {device} is not found or has no trace events in the snapshot." + ) + return False + if device is not None: + need_dump_devices = [device] + dump_logger.info(f"Recognized need to dump devices {need_dump_devices}.") + hooker = DumpEventHooker(dump_file, need_dump_devices) + for device in need_dump_devices: + dump_logger.info(f"Start to dump the snapshot to database for device {device}.") + snapshot = SimulateDeviceSnapshot(data, device) + snapshot.register_hooker(hooker) + snapshot.register_allocator_hooker(hooker) + if not snapshot.replay(): + dump_logger.error( + f"Failed to dump the snapshot to database for device {device}." + ) + return False + dump_logger.info(f"Finished dump the snapshot to database for device {device}.") + hooker.flush(device) + dump_logger.info( + f"Successfully dump the snapshot to database for devices {need_dump_devices}." + ) + return True + + +def get_args(argv=None): + parser = argparse.ArgumentParser( + description="This script is used to parse and convert snapshot data into a database " + "format that is more convenient for visualization." + ) + arg_snapshot = parser.add_argument( + "snapshot_file", type=str, help="Memory snapshot file path." + ) + arg_dump_dir = parser.add_argument( + "--dump_dir", + "-o", + required=False, + type=str, + default="", + help="Specify the directory to store the parsed database file. If not provided, " + "the same directory as the specified snapshot file will be used by default.", + ) + arg_log_file = parser.add_argument( + "--log", + "-l", + required=False, + type=str, + default="", + help="Specify the log file path. If provided, all logs will be written to this file.", + ) + parser.add_argument( + "--device", + "-d", + required=False, + type=lambda x: ( + int(x) if int(x) >= 0 else parser.error("The device id must be at least 0") + ), + help="Specify the device id to dump. If not provided, " + "we will dump the data of all devices.", + ) + args = parser.parse_args(argv) + snapshot_path = Path(args.snapshot_file) + # 校验snapshot path + if not snapshot_path.is_file() or not os.access(args.snapshot_file, os.R_OK): + raise argparse.ArgumentError( + arg_snapshot, + "The specified snapshot file does not exist, or is not a file, or is not readable.", + ) + # 校验dump目标路径 + if not args.dump_dir: + args.dump_dir = snapshot_path.parent + if not Path(args.dump_dir).is_dir() or not check_dir_valid(args.dump_dir): + raise argparse.ArgumentError( + arg_dump_dir, + "The specified directory does not exist, or is not a directory, " + "or is not writable", + ) + if args.log: + try: + set_global_log_file(args.log) + except OSError as e: + raise argparse.ArgumentError(arg_log_file, str(e)) + return args + + +def run_dump_to_db( + snapshot_file: str, + dump_dir: str = "", + device: int | None = None, + log_file: str = "", +) -> bool: + resolved_dump_dir = dump_dir or str(Path(snapshot_file).parent) + if log_file: + set_global_log_file(log_file) + + return dump( + snapshot_file, + Path(resolved_dump_dir) / f"{Path(snapshot_file).name}.db", + device, + ) + + +class ExistCode: + SUCCESS = 0 + FAILED = -1 + + +@timer(name="Dump snapshot to database.", logger=dump_logger) +def main(argv=None): + try: + args = get_args(argv) + except argparse.ArgumentError as e: + dump_logger.error("Failed to parse arguments: {}".format(e)) + sys.exit(ExistCode.FAILED) + if not run_dump_to_db( + snapshot_file=args.snapshot_file, + dump_dir=str(args.dump_dir), + device=args.device, + log_file=args.log, + ): + dump_logger.error("Failed to dump the snapshot to database.") + sys.exit(ExistCode.FAILED) + sys.exit(ExistCode.SUCCESS) + + +if __name__ == "__main__": + main() diff --git a/src/pt_snap_cli/vendor/memsnapdump/util/__init__.py b/src/pt_snap_cli/vendor/memsnapdump/util/__init__.py new file mode 100644 index 0000000..823b21c --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/util/__init__.py @@ -0,0 +1,9 @@ +""" +工具模块 + +提供常用的工具函数和类,包括日志、文件操作、数据库操作、计时器等。 +""" + +from .logger import get_logger, set_global_log_file + +__all__ = ["get_logger", "set_global_log_file"] diff --git a/src/pt_snap_cli/vendor/memsnapdump/util/file_util.py b/src/pt_snap_cli/vendor/memsnapdump/util/file_util.py new file mode 100644 index 0000000..d2a15dd --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/util/file_util.py @@ -0,0 +1,132 @@ +""" +文件工具模块 + +提供文件和目录操作的实用函数,包括 pickle 文件的读写、文件和目录的校验等功能。 +""" + +import os +import pickle +from pathlib import Path +from typing import Dict, Any + + +def load_pickle_to_dict(pickle_file: Path) -> dict: + """ + 从指定路径加载 pickle 文件,并确保其内容为 dict 类型。 + + Args: + pickle_file (Path): pickle 文件路径 + + Returns: + dict: 加载的字典数据 + + Raises: + FileNotFoundError: 文件不存在 + ValueError: 文件内容不是 dict 类型 + pickle.UnpicklingError: 反序列化失败(如文件损坏或非 pickle 格式) + """ + if not pickle_file.is_file(): + raise FileNotFoundError(f"Cannot found pickle file: {pickle_file}") + + try: + with open(pickle_file, "rb") as f: + data = pickle.load(f) + except Exception: + raise pickle.UnpicklingError(f"Cannot load pickle file: {pickle_file}") + + if not isinstance(data, dict): + raise ValueError( + f"The content of the pickle file is not of type dict, actual type: {type(data).__name__}" + ) + + return data + + +def save_dict_to_pickle(data: Dict[Any, Any], path: Path, protocol: int = 4) -> None: + """ + 将字典保存为 pickle 文件。 + + Args: + data (dict): 要保存的字典 + path (Path): 保存路径(会自动创建父目录) + protocol (int): 保存版本 + + Raises: + TypeError: data 不是 dict 类型 + OSError: 文件写入失败(如权限不足、磁盘满等) + """ + if not isinstance(data, dict): + raise TypeError( + f"Only dict type is supported, but received: {type(data).__name__}" + ) + path.parent.mkdir(parents=True, exist_ok=True) # 自动创建父目录 + + try: + with open(path, "wb") as f: + pickle.dump(data, f, protocol=protocol) + except OSError as e: + raise OSError(f"Unable to write to file {path}: {e}") from e + + +def check_dir_valid( + path: str | Path, need_readable: bool = True, need_writable: bool = True +) -> bool: + """ + 校验目录是否合法,默认要求可读可写。 + + Args: + path: 路径字符串或 pathlib.Path 对象 + need_readable: 是否要求可读,默认为 True + need_writable: 是否要求可写,默认为 True + + Returns: + bool: 目录是否合法且满足权限要求 + + Examples: + >>> check_dir_valid('/tmp', need_readable=True, need_writable=True) + True + >>> check_dir_valid('/nonexistent') + False + """ + _path = path + if not isinstance(path, Path): + _path = Path(_path) + if not _path.is_dir(): + return False + if need_readable and not os.access(_path, os.R_OK): + return False + if need_writable and not os.access(_path, os.W_OK): + return False + return True + + +def check_file_valid( + path: str | Path, need_readable: bool = True, need_writable: bool = False +) -> bool: + """ + 校验文件是否合法,默认要求可读,不要求可写。 + + Args: + path: 路径字符串或 pathlib.Path 对象 + need_readable: 是否要求可读,默认为 True + need_writable: 是否要求可写,默认为 False + + Returns: + bool: 文件是否合法且满足权限要求 + + Examples: + >>> check_file_valid('/etc/passwd', need_readable=True, need_writable=False) + True + >>> check_file_valid('/nonexistent.txt') + False + """ + _path = path + if not isinstance(path, Path): + _path = Path(_path) + if not _path.is_file(): + return False + if need_readable and not os.access(_path, os.R_OK): + return False + if need_writable and not os.access(_path, os.W_OK): + return False + return True diff --git a/src/pt_snap_cli/vendor/memsnapdump/util/logger.py b/src/pt_snap_cli/vendor/memsnapdump/util/logger.py new file mode 100644 index 0000000..c132527 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/util/logger.py @@ -0,0 +1,137 @@ +""" +日志工具模块 + +提供日志记录器的创建和配置功能。 +""" + +import logging +from pathlib import Path +from typing import Optional +from .file_util import check_dir_valid + +_global_log_file: Optional[str] = None +_global_file_handler: Optional[logging.FileHandler] = None + + +def set_global_log_file(log_file: str) -> None: + """ + 设置全局日志文件路径,所有 logger 都会输出到该文件。 + + 包括已创建的 logger 和后续创建的 logger。 + + Args: + log_file: 日志文件路径 + + Raises: + OSError: 目录不存在或不可写 + """ + global _global_log_file, _global_file_handler + + log_path = Path(log_file) + parent_dir = log_path.parent + + if not parent_dir.exists(): + raise OSError(f"Log directory '{parent_dir}' does not exist.") + + if not parent_dir.is_dir() or not check_dir_valid(parent_dir): + raise OSError( + f"Log directory '{parent_dir}' is not writable or does not exist." + ) + + _global_log_file = str(log_path.absolute()) + + if _global_file_handler is not None: + _global_file_handler.close() + + formatter = logging.Formatter( + fmt="{asctime} [{levelname:^6}][ {name:^12} ]: {message}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{", + ) + + _global_file_handler = logging.FileHandler( + _global_log_file, mode="w", encoding="utf-8" + ) + _global_file_handler.setLevel(logging.DEBUG) + _global_file_handler.setFormatter(formatter) + + _attach_file_handler_to_existing_loggers() + + +_original_levels: dict[str, int] = {} + + +def suppress_logs(): + """Suppress INFO level logs during tests""" + for name in list(logging.Logger.manager.loggerDict.keys()): + logger = logging.getLogger(name) + _original_levels[name] = logger.level + logger.setLevel(logging.WARNING) + + +def restore_logs(): + """Restore original log levels after tests""" + for name, level in _original_levels.items(): + logging.getLogger(name).setLevel(level) + _original_levels.clear() + + +def _attach_file_handler_to_existing_loggers() -> None: + """为所有已存在的 logger 添加文件处理器""" + if _global_file_handler is None: + return + + manager = logging.Logger.manager + logger_dict = manager.loggerDict + + for name in list(logger_dict.keys()): + logger = logging.getLogger(name) + if _global_file_handler not in logger.handlers: + logger.addHandler(_global_file_handler) + + +def get_logger(name: str, level: int = logging.INFO) -> logging.Logger: + """ + 创建并返回一个配置好的 Logger 实例。 + + 该函数会创建一个带有控制台输出的日志记录器,使用统一的格式化样式。 + 如果 logger 已存在 handlers,会先清除再添加新的 handler。 + 如果设置了全局日志文件,同时会输出到文件。 + + Args: + name: logger 名称,通常使用 __name__ 或模块名 + level: 日志级别,默认为 logging.INFO + + Returns: + logging.Logger: 配置好的 Logger 对象 + + Examples: + >>> logger = get_logger(__name__) + >>> logger.info("This is an info message") + 2024-01-01 12:00:00 [ INFO ][ __main__ ]: This is an info message + + >>> logger = get_logger("my_module", level=logging.DEBUG) + >>> logger.debug("Debug message") + """ + logger = logging.getLogger(name) + logger.setLevel(level) + + # 避免重复添加handler + if logger.hasHandlers(): + logger.handlers.clear() + + formatter = logging.Formatter( + fmt="{asctime} [{levelname:^6}][ {name:^12} ]: {message}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{", + ) + + console_handler = logging.StreamHandler() + console_handler.setLevel(level) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + if _global_file_handler is not None: + logger.addHandler(_global_file_handler) + + return logger diff --git a/src/pt_snap_cli/vendor/memsnapdump/util/sqlite_meta.py b/src/pt_snap_cli/vendor/memsnapdump/util/sqlite_meta.py new file mode 100644 index 0000000..84604dd --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/util/sqlite_meta.py @@ -0,0 +1,520 @@ +""" +SQLite 元数据管理模块 + +提供 SQLite 数据库的表结构定义、类型映射、表创建和数据插入等功能。 +支持 Python 类型到 SQLite 类型的自动转换,以及表结构的元数据管理。 +""" + +import ast +import os.path +import sqlite3 +from typing import ( + Any, + Dict, + List, + Optional, + Type, + Union, + Iterable, + get_origin, + get_args, +) + +# 支持的 Python 类型映射 +_PY_TYPE_TO_SQLITE = { + int: "INTEGER", + float: "REAL", + str: "TEXT", + bool: "INTEGER", # SQLite 没有 BOOLEAN,用 0/1 + bytes: "BLOB", +} + + +def _map_py_type_to_sqlite(py_type: Type) -> str: + """将 Python 类型转换为 SQLite 类型""" + origin = get_origin(py_type) or py_type + if origin in _PY_TYPE_TO_SQLITE: + return _PY_TYPE_TO_SQLITE[origin] + # 处理 Optional[T] => T(Optional 是 Union[T, None]) + if origin is Union: + args = get_args(py_type) + non_none = [t for t in args if t is not type(None)] + if len(non_none) == 1: + return _map_py_type_to_sqlite(non_none[0]) + # 默认 fallback + return "TEXT" + + +def _sqlite_type_to_py_type(sqlite_type: str) -> type: + """将 SQLite 声明类型映射为最可能的 Python 类型""" + if not sqlite_type: + return str # 无类型默认为 TEXT + upper = sqlite_type.upper() + if "INT" in upper: + return int + if "CHAR" in upper or "CLOB" in upper or "TEXT" in upper: + return str + if "BLOB" in upper: + return bytes + if "REAL" in upper or "FLOA" in upper or "DOUB" in upper: + return float + # 兜底:可能是 NUMERIC 或自定义类型,按 TEXT 处理 + return str + + +def _parse_default_value(dflt_str: str) -> Any: + """将 PRAGMA 返回的默认值字符串转为 Python 对象""" + if dflt_str is None: + return None + + # 尝试解析为字面量(支持数字、字符串、布尔、None) + try: + # 处理 SQLite 中的布尔:'1'/'0' 或 'true'/'false'(但 SQLite 实际存整数) + if dflt_str == "1": + return True + elif dflt_str == "0": + return False + elif dflt_str.lower() in ("true", "false"): + return dflt_str.lower() == "true" + # 尝试用 ast.literal_eval 安全解析 + return ast.literal_eval(dflt_str) + except (ValueError, SyntaxError): + # 如果不是合法字面量,当作字符串处理(去掉外层引号) + if dflt_str.startswith("'") and dflt_str.endswith("'"): + return dflt_str[1:-1].replace("''", "'") + elif dflt_str.startswith('"') and dflt_str.endswith('"'): + return dflt_str[1:-1].replace('""', '"') + else: + return dflt_str + + +class SqliteColumn: + """ + SQLite 表的列定义类。 + + 用于定义表中的单个列,包括列名、数据类型、约束条件等属性。 + + Attributes: + name: 列名 + data_type: Python 类型,会自动映射为 SQLite 类型 + primary_key: 是否为主键 + autoincrement: 是否自增(仅适用于 INTEGER 主键) + not_null: 是否不允许 NULL + unique: 是否唯一 + default: 默认值 + value_map: 值映射字典,用于插入时的值转换 + + Examples: + >>> col = SqliteColumn('id', int, primary_key=True, autoincrement=True) + >>> col.to_sql_def() + '`id` INTEGER PRIMARY KEY AUTOINCREMENT' + + >>> col = SqliteColumn('name', str, not_null=True, default='unknown') + >>> col.to_sql_def() + "`name` TEXT NOT NULL DEFAULT 'unknown'" + """ + + def __init__( + self, + name: str, + data_type: Type = str, + primary_key: bool = False, # 是否主键 + autoincrement: bool = False, # 是否自增 + not_null: bool = False, # 是否不可为空 + unique: bool = False, # 是否唯一 + default: Optional[Any] = None, # 缺省值, + value_map: Dict[Any, Any] = None, + ): + if autoincrement and not primary_key: + raise ValueError("autoincrement requires primary_key=True") + if autoincrement and data_type is not int: + raise ValueError("autoincrement only supported for INTEGER type") + self.name = name + self.data_type = data_type + self.primary_key = primary_key + self.autoincrement = autoincrement + self.not_null = not_null + self.unique = unique + self.default = default + self.value_map = value_map + + def _format_default(self) -> str: + """格式化默认值为 SQL 字面量""" + val = self.default + if val is None: + return "NULL" + elif isinstance(val, bool): + return "1" if val else "0" + elif isinstance(val, str): + # 转义单引号(简单处理) + escaped = val.replace("'", "''") + return f"'{escaped}'" + elif isinstance(val, (int, float)): + return str(val) + else: + # 兜底:转为字符串并加引号 + return f"'{str(val)}'" + + def to_sql_def(self) -> str: + """ + 生成列的 SQL 定义语句。 + + Returns: + str: 完整的列定义 SQL,如 "`id` INTEGER PRIMARY KEY AUTOINCREMENT" + + Examples: + >>> col = SqliteColumn('id', int, primary_key=True) + >>> col.to_sql_def() + '`id` INTEGER PRIMARY KEY' + """ + parts = [f"`{self.name}`", _map_py_type_to_sqlite(self.data_type)] + + if self.primary_key: + parts.append("PRIMARY KEY") + if self.autoincrement: + parts.append("AUTOINCREMENT") + if self.not_null: + parts.append("NOT NULL") + if self.unique: + parts.append("UNIQUE") + if self.default is not None: + parts.append(f"DEFAULT {self._format_default()}") + + return " ".join(parts) + + +class SqliteTable: + """ + SQLite 表的定义和管理类。 + + 用于定义表结构、创建表、插入数据等操作。 + + Attributes: + name: 表名 + column_dict: 列定义字典,键为列名,值为 SqliteColumn 对象 + _column_value_map: 列值映射字典,用于插入时的值转换 + + Examples: + >>> columns = [ + ... SqliteColumn('id', int, primary_key=True, autoincrement=True), + ... SqliteColumn('name', str, not_null=True) + ... ] + >>> table = SqliteTable('users', columns) + >>> print(table.to_sql_def()) + CREATE TABLE IF NOT EXISTS users (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` TEXT NOT NULL); + """ + + name: str + column_dict: Dict[str, SqliteColumn] + _column_value_map: Dict[str, Dict[Any, Any]] + + def __init__(self, table_name: str, columns: Iterable[SqliteColumn] = None): + self.name = table_name + self.column_dict = {} + self._column_value_map = dict() + if columns: + for column in columns: + self.column_dict[column.name] = column + if column.value_map: + self._column_value_map[column.name] = column.value_map + + def to_sql_def(self, delete_if_exists: bool = False) -> str: + """ + 生成创建表的 SQL 语句。 + + Args: + delete_if_exists: 是否先 DROP TABLE IF EXISTS,默认为 False + + Returns: + str: 创建表的 SQL 语句 + + Examples: + >>> table.to_sql_def(delete_if_exists=True) + 'DROP TABLE IF EXISTS users;\\nCREATE TABLE users (...)' + """ + column_defs = [col.to_sql_def() for _, col in self.column_dict.items()] + create_sql = f"CREATE TABLE {self.name} ({', '.join(column_defs)});" + + if delete_if_exists: + drop_sql = f"DROP TABLE IF EXISTS {self.name};" + return f"{drop_sql}\n{create_sql}" + else: + # 使用 IF NOT EXISTS 更安全 + create_sql = create_sql.replace( + "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1 + ) + return create_sql + + def create_table(self, conn: sqlite3.Connection, delete_if_exists: bool = False): + """ + 在数据库中创建表 + :param conn: sqlite3.Connection 对象 + :param delete_if_exists: 是否先删除已存在的表 + """ + sql = self.to_sql_def(delete_if_exists=delete_if_exists) + conn.executescript(sql) # 支持多条 SQL(如 DROP + CREATE) + conn.commit() + + def create_index(self, conn: sqlite3.Connection, column_name: str): + """ + 创建索引 + :param conn: sqlite3.Connection 对象 + :param column_name: 列名 + :return: + """ + conn.execute( + f"CREATE INDEX IF NOT EXISTS idx_{self.name}_{column_name} ON {self.name} ({column_name});" + ) + conn.commit() + + def insert_record(self, conn: sqlite3.Connection, record: Dict[str, Any]): + """ + 插入单条记录。 + + Args: + conn: sqlite3.Connection 对象 + record: 记录字典,键为列名,值为列值 + + Examples: + >>> table.insert_record(conn, {'name': 'Alice', 'age': 30}) + """ + self.insert_records(conn, [record]) + + def insert_records(self, conn: sqlite3.Connection, records: List[Dict[str, Any]]): + """ + 批量插入多条记录。 + + Args: + conn: sqlite3.Connection 对象 + records: 记录字典列表,每个字典的键为列名,值为列值 + + Examples: + >>> records = [ + ... {'name': 'Alice', 'age': 30}, + ... {'name': 'Bob', 'age': 25} + ... ] + >>> table.insert_records(conn, records) + """ + if not records: + return + columns = SqliteTable.get_insert_columns_by_record(records[0]) + placeholders = SqliteTable.get_insert_placeholder_by_record(records[0]) + sql = f"INSERT INTO {self.name} ({', '.join(columns)}) VALUES ({placeholders})" + values = self.get_insert_values_by_records(records) + conn.executemany(sql, values) + conn.commit() + + @staticmethod + def get_insert_columns_by_record(record: Dict[str, Any]): + """ + 从记录字典中获取插入列名列表。 + + Args: + record: 记录字典 + + Returns: + List[str]: 列名列表(带反引号) + """ + return [f"`{key}`" for key in record.keys()] + + @staticmethod + def get_insert_placeholder_by_record(record: Dict[str, Any]): + """ + 生成插入语句的占位符字符串。 + + Args: + record: 记录字典 + + Returns: + str: 占位符字符串,如 "?, ?, ?" + """ + return ", ".join(["?" for _ in record.keys()]) + + def get_insert_values_by_records(self, records: List[Dict[str, Any]]): + """ + 从记录列表中提取插入值,应用值映射。 + + Args: + records: 记录字典列表 + + Returns: + List[Tuple]: 值元组列表,每个元组对应一条记录 + """ + if not records: + return [] + return [ + tuple( + self._column_value_map.get(k, {}).get(r[k], r[k]) + for k in records[0].keys() + ) + for r in records + ] + + +class SqliteDB: + """ + SQLite 数据库管理类。 + + 提供数据库连接、表创建、表结构查询等功能。 + + Attributes: + path: 数据库文件路径 + conn: sqlite3.Connection 对象 + table_cache: 表缓存字典,键为表名,值为 SqliteTable 对象 + with_dictionary_table: 是否创建字典表用于存储值映射 + + Examples: + >>> db = SqliteDB('/path/to/db.sqlite') + >>> table = SqliteTable('users', [ + ... SqliteColumn('id', int, primary_key=True), + ... SqliteColumn('name', str) + ... ]) + >>> db.create_table(table) + """ + + path: str + conn: sqlite3.Connection + table_cache: Dict[str, SqliteTable] + + def __init__( + self, path: str, auto_create: bool = True, with_dictionary_table: bool = False + ): + self.path = os.path.realpath(path) + if not os.path.exists(self.path): + if not auto_create: + raise FileNotFoundError(f"Db file not found: {self.path}.") + dir_path = os.path.dirname(self.path) + if not os.path.exists(dir_path): + os.makedirs(dir_path, exist_ok=True) + + self.conn = sqlite3.connect(self.path) + self.table_cache = {} + self.with_dictionary_table = with_dictionary_table + if self.with_dictionary_table: + self._create_dictionary_table() + + def create_table(self, table: SqliteTable, delete_if_exists: bool = True): + """ + 在数据库中创建表,并缓存表结构。 + + Args: + table: SqliteTable 对象 + delete_if_exists: 是否先删除已存在的表,默认为 True + + Note: + 如果启用了 with_dictionary_table,会自动将值映射存储到字典表中。 + """ + table.create_table(self.conn, delete_if_exists) + self.table_cache[table.name] = table + if not self.with_dictionary_table: + return + dictionary_table = self.get_table_by_name("dictionary") + for column_name, value_map in table._column_value_map.items(): + for key, value in value_map.items(): + dictionary_table.insert_record( + self.conn, + dict(table=table.name, column=column_name, key=value, value=key), + ) + self.conn.commit() + + def delete_table(self, table_name: str): + self.conn.execute(f"DROP TABLE IF EXISTS {table_name};") + self.conn.commit() + + def _create_dictionary_table(self): + _table_columns = [ + SqliteColumn(name="table"), + SqliteColumn(name="column"), + SqliteColumn(name="key"), + SqliteColumn(name="value"), + ] + _dictionary_table = SqliteTable("dictionary", _table_columns) + _dictionary_table.create_table(self.conn, delete_if_exists=True) + self.table_cache[_dictionary_table.name] = _dictionary_table + + def is_table_exists(self, table_name: str) -> bool: + """ + 检查表是否存在。 + + Args: + table_name: 表名 + + Returns: + bool: 表是否存在 + + Note: + 会优先检查缓存,如果缓存中不存在则查询数据库。 + """ + if table_name in self.table_cache: + return True + cursor = self.conn.cursor() + cursor.execute( + """ + SELECT name + FROM sqlite_master + WHERE type = 'table' + AND name = ? + """, + (table_name,), + ) + exist = cursor.fetchone() is not None + if exist: + self.table_cache[table_name] = self.get_table_by_name(table_name) + return exist + + def get_table_by_name(self, table_name: str) -> SqliteTable: + """ + 从数据库中读取表结构,还原为 SqliteTable 对象。 + + Args: + table_name: 表名 + + Returns: + SqliteTable: 表对象 + + Raises: + ValueError: 表不存在 + + Note: + 会优先从缓存中获取,缓存中不存在时才查询数据库。 + """ + if table_name in self.table_cache: + return self.table_cache[table_name] + # 获取列信息 + cur = self.conn.execute(f"PRAGMA table_info({table_name});") + rows = cur.fetchall() + + if not rows: + raise ValueError(f"Table '{table_name}' does not exist.") + table = SqliteTable(table_name) + for row in rows: + cid, name, type_affinity, notnull, dflt_value, pk = row + + py_type = _sqlite_type_to_py_type(type_affinity) + default_val = _parse_default_value(dflt_value) + + # 检测是否为自增(仅当 INTEGER 主键且有 sqlite_sequence 记录) + autoincrement = False + if pk and py_type is int: + # 检查是否存在 sqlite_sequence 表且包含该表 + seq_cur = self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='sqlite_sequence';" + ) + if seq_cur.fetchone(): + seq_cur = self.conn.execute( + "SELECT 1 FROM sqlite_sequence WHERE name = ?;", (table_name,) + ) + autoincrement = seq_cur.fetchone() is not None + + column = SqliteColumn( + name=name, + data_type=py_type, + primary_key=bool(pk), + autoincrement=autoincrement, + not_null=bool(notnull), + default=default_val, + # 注意:UNIQUE、COLLATE 无法从 table_info 获取,需解析 CREATE SQL + ) + table.column_dict[column.name] = column + self.table_cache[table_name] = table + return table diff --git a/src/pt_snap_cli/vendor/memsnapdump/util/timer.py b/src/pt_snap_cli/vendor/memsnapdump/util/timer.py new file mode 100644 index 0000000..30f4926 --- /dev/null +++ b/src/pt_snap_cli/vendor/memsnapdump/util/timer.py @@ -0,0 +1,58 @@ +""" +计时器工具模块 + +提供函数执行时间测量的装饰器。 +""" + +import time +import logging +from typing import Callable, Optional +from functools import wraps + + +def timer(name: Optional[str] = None, logger: Optional[logging.Logger] = None): + """ + 函数执行时间计时装饰器。 + + 用于测量函数执行时间并输出结果。可以选择输出到 logger 或控制台。 + + Args: + name: 计时器名称,默认使用函数名 + logger: 日志记录器,如果提供则使用 logger.info 输出,否则使用 print + + Returns: + Callable: 装饰器函数 + + Examples: + >>> from ..util import get_logger + >>> logger = get_logger(__name__) + + >>> @timer() + ... def slow_function(): + ... time.sleep(1) + slow_function took 1.0012 seconds + + >>> @timer(name="数据处理", logger=logger) + ... def process_data(): + ... time.sleep(0.5) + 2024-01-01 12:00:00 [ INFO ][ __main__ ]: 数据处理 took 0.5023 seconds + """ + + def decorator(func: Callable): + _name = func.__name__ if not name else name + + @wraps(func) + def wrapper(*args, **kwargs): + start = time.perf_counter() + result = func(*args, **kwargs) + end = time.perf_counter() + info = f"{_name} took {end - start:.4f} seconds" + if logger: + logger.info(info) + else: + print(info) + return result + + return wrapper + + return decorator diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/test_import_errors.py b/tests/core/test_import_errors.py new file mode 100644 index 0000000..240588e --- /dev/null +++ b/tests/core/test_import_errors.py @@ -0,0 +1,22 @@ +"""Tests for vendoring-era import error types.""" + +from __future__ import annotations + +from pt_snap_cli.core.errors import ( + ImportExecutionError, + ImportToolMissingError, + PtSnapCoreError, + SnapshotFileInvalidError, +) + + +def test_import_tool_missing_error_is_pt_snap_core() -> None: + assert issubclass(ImportToolMissingError, PtSnapCoreError) + + +def test_snapshot_file_invalid_error_is_pt_snap_core() -> None: + assert issubclass(SnapshotFileInvalidError, PtSnapCoreError) + + +def test_import_execution_error_is_pt_snap_core() -> None: + assert issubclass(ImportExecutionError, PtSnapCoreError) diff --git a/tests/core/test_import_models.py b/tests/core/test_import_models.py new file mode 100644 index 0000000..e1b90c6 --- /dev/null +++ b/tests/core/test_import_models.py @@ -0,0 +1,40 @@ +"""Tests for ImportOptions / ImportResult dataclasses.""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +from pt_snap_cli.core.models import ImportOptions, ImportResult + + +def test_import_options_defaults() -> None: + options = ImportOptions(snapshot_file=Path("sample.pkl")) + assert options.output_dir is None + assert options.device is None + assert options.set_focus is True + + +def test_import_options_is_frozen() -> None: + options = ImportOptions(snapshot_file=Path("sample.pkl")) + try: + options.set_focus = False # type: ignore[misc] + except dataclasses.FrozenInstanceError: + return + raise AssertionError("ImportOptions should be frozen") + + +def test_import_result_construction() -> None: + result = ImportResult(db_path=Path("sample.pkl.db"), device_id=0, focus_state=None) + assert result.db_path == Path("sample.pkl.db") + assert result.device_id == 0 + assert result.focus_state is None + + +def test_import_result_is_frozen() -> None: + result = ImportResult(db_path=Path("sample.pkl.db"), device_id=0, focus_state=None) + try: + result.db_path = Path("other.db") # type: ignore[misc] + except dataclasses.FrozenInstanceError: + return + raise AssertionError("ImportResult should be frozen") diff --git a/tests/core/test_import_service.py b/tests/core/test_import_service.py new file mode 100644 index 0000000..0c91206 --- /dev/null +++ b/tests/core/test_import_service.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import importlib +import pickle +import sys +from pathlib import Path + +import pytest + +from pt_snap_cli.context import Context +from pt_snap_cli.core.errors import ( + ImportExecutionError, + ImportToolMissingError, + SnapshotFileInvalidError, +) +from pt_snap_cli.core.models import ImportOptions + +FIXTURE_DIR = Path(__file__).parents[1] / "fixtures" / "snapshots" +EMPTY_CACHE_SNAPSHOT = FIXTURE_DIR / "snapshot_with_empty_cache.pkl" +MULTI_DEVICE_SNAPSHOT = FIXTURE_DIR / "snapshot_with_multi_devices.pkl" + + +def _import_service_type(): + from pt_snap_cli.core.import_service import ImportService + + return ImportService + + +def test_import_writes_db_and_sets_focus(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + project_dir = tmp_path / "project" + output_dir = tmp_path / "output" + project_dir.mkdir() + monkeypatch.chdir(project_dir) + + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=output_dir) + ) + + assert result.db_path.exists() + assert result.db_path.suffix == ".db" + assert (project_dir / ".pt-snap" / "focus.json").exists() + assert not (output_dir / ".pt-snap" / "focus.json").exists() + + +@pytest.mark.slow +def test_import_with_multi_device_fixture(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions(snapshot_file=MULTI_DEVICE_SNAPSHOT, output_dir=tmp_path) + ) + + assert len(Context(result.db_path).discover_devices()) >= 2 + + +def test_import_returns_focus_state(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + assert result.focus_state is not None + assert result.focus_state.source == "project" + + +def test_import_snapshot_file_invalid_missing(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + with pytest.raises(SnapshotFileInvalidError): + service.import_snapshot(ImportOptions(snapshot_file=tmp_path / "missing.pkl")) + + +def test_import_snapshot_file_invalid_suffix(tmp_path: Path) -> None: + """Wrong suffix is rejected before the vendor backend is invoked.""" + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "foo.txt" + snapshot_file.write_bytes(b"any content is fine here - suffix is the only check") + + with pytest.raises(SnapshotFileInvalidError, match=r"\.pkl|\.pickle|suffix"): + service.import_snapshot(ImportOptions(snapshot_file=snapshot_file)) + + +def test_import_snapshot_file_invalid_corrupt_pickle(tmp_path: Path) -> None: + """A .pkl file that cannot be unpickled surfaces as an import execution error. + + After removing the redundant upfront pickle.load, the suffix check passes + for ``.pkl`` and the failure is reported by the vendored backend, which + surfaces as :class:`ImportExecutionError`. + """ + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "corrupt.pkl" + # Real pickle protocol header followed by garbage: passes suffix check, + # fails inside the vendor's load_pickle_to_dict. + snapshot_file.write_bytes(b"\x80\x04not a valid pickle stream") + + with pytest.raises(ImportExecutionError, match=r"backend (failed|reported failure)"): + service.import_snapshot(ImportOptions(snapshot_file=snapshot_file)) + + +def test_failed_reimport_preserves_existing_database(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + result = service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + original_size = result.db_path.stat().st_size + assert Context(result.db_path).discover_devices() + + with pytest.raises(ImportExecutionError, match=r"backend (failed|reported failure)"): + service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, device=999, set_focus=False + ) + ) + + assert result.db_path.exists() + assert result.db_path.stat().st_size == original_size + assert Context(result.db_path).discover_devices() + + +def test_import_replaces_destination_errors_are_wrapped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + def fail_replace(*args: object) -> None: + raise OSError("cannot replace") + + monkeypatch.setattr("pt_snap_cli.core.snapshot_import_backend.os.replace", fail_replace) + + with pytest.raises(ImportExecutionError, match="cannot replace"): + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path, set_focus=False) + ) + + +def test_snapshot_import_backend_lazy_loads_vendor_module() -> None: + sys.modules.pop("pt_snap_cli.core.snapshot_import_backend", None) + vendor_module = "pt_snap_cli.vendor.memsnapdump.tools.adaptors.snapshot2db" + sys.modules.pop(vendor_module, None) + + importlib.import_module("pt_snap_cli.core.snapshot_import_backend") + + assert vendor_module not in sys.modules + + +def test_import_raises_on_missing_tool(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + def raise_missing_tool(*args: object, **kwargs: object) -> Path: + raise ImportToolMissingError( + "Vendored snapshot import backend is unavailable. Reinstall pt-snap-cli; " + "the package may be incomplete." + ) + + monkeypatch.setattr(service._backend, "dump_to_db", raise_missing_tool) + + with pytest.raises(ImportToolMissingError, match="Vendored snapshot import backend"): + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + +def test_import_raises_on_upstream_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """A backend that raises ImportExecutionError is propagated as-is.""" + import_service_cls = _import_service_type() + service = import_service_cls() + + def fail_dump_to_db(snapshot_file: Path, dump_dir: Path, device: int | None) -> Path: + raise ImportExecutionError("Vendored snapshot import backend failed: upstream failed") + + monkeypatch.setattr(service._backend, "dump_to_db", fail_dump_to_db) + + with pytest.raises(ImportExecutionError): + service.import_snapshot( + ImportOptions(snapshot_file=EMPTY_CACHE_SNAPSHOT, output_dir=tmp_path) + ) + + +def test_import_skip_focus(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + + result = service.import_snapshot( + ImportOptions( + snapshot_file=EMPTY_CACHE_SNAPSHOT, + output_dir=tmp_path, + set_focus=False, + ) + ) + + assert result.db_path.exists() + assert not (tmp_path / ".pt-snap").exists() + assert result.focus_state is None + + +def test_import_output_path_uses_full_filename(tmp_path: Path) -> None: + import_service_cls = _import_service_type() + service = import_service_cls() + snapshot_file = tmp_path / "snapshot.pickle" + snapshot_file.write_bytes(pickle.dumps({})) + + result = service.import_snapshot( + ImportOptions(snapshot_file=snapshot_file, output_dir=tmp_path, set_focus=False) + ) + + assert result.db_path.name == "snapshot.pickle.db" diff --git a/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl b/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl new file mode 100644 index 0000000..160e449 Binary files /dev/null and b/tests/fixtures/snapshots/snapshot_with_empty_cache.pkl differ diff --git a/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl b/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl new file mode 100644 index 0000000..ed83d18 Binary files /dev/null and b/tests/fixtures/snapshots/snapshot_with_multi_devices.pkl differ diff --git a/tests/test_cli.py b/tests/test_cli.py index b206f8e..e118fcf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ """Tests for CLI.""" import json +import shutil import sqlite3 from pathlib import Path from unittest.mock import patch @@ -811,6 +812,79 @@ def test_safe_call_reraises_unrelated_keyerror(self) -> None: _safe_call() +class TestImportCommand: + """Test import command.""" + + def test_import_success_writes_focus(self, tmp_path: Path) -> None: + """Test import writes a database and project focus by default.""" + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + + result = runner.invoke(app, ["import", str(snapshot)], catch_exceptions=False) + + assert result.exit_code == 0 + assert "Imported:" in result.stdout + assert (tmp_path / ".pt-snap" / "focus.json").exists() + + def test_import_missing_tool_message( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test import reports vendored backend packaging guidance when unavailable.""" + from pt_snap_cli.core.errors import ImportToolMissingError + + snapshot = tmp_path / "sample.pkl" + snapshot.write_bytes(b"not a real snapshot") + + def raise_missing_tool(*args: object, **kwargs: object) -> object: + raise ImportToolMissingError( + "Vendored snapshot import backend is unavailable. Reinstall pt-snap-cli; the package may be incomplete." + ) + + monkeypatch.setattr( + "pt_snap_cli.core.import_service.ImportService.import_snapshot", + raise_missing_tool, + ) + + result = runner.invoke(app, ["import", str(snapshot)]) + + assert result.exit_code != 0 + assert "vendored snapshot import backend" in result.output.lower() + assert "reinstall pt-snap-cli" in result.output.lower() + assert "memsnapdump" not in result.output.lower() + + def test_import_invalid_path(self) -> None: + """Test import rejects missing snapshot paths.""" + result = runner.invoke(app, ["import", "/no/such/file.pkl"]) + + assert result.exit_code != 0 + assert "exist" in result.output.lower() + + def test_import_invalid_suffix(self, tmp_path: Path) -> None: + """Test import rejects non-pkl snapshot paths.""" + snapshot = tmp_path / "sample.txt" + # Content is irrelevant: the .txt suffix is rejected before the + # backend ever opens the file. + snapshot.write_text("not a snapshot") + + result = runner.invoke(app, ["import", str(snapshot)]) + + assert result.exit_code != 0 + assert "suffix" in result.output.lower() or ".pkl" in result.output.lower() + + def test_import_no_focus(self, tmp_path: Path) -> None: + """Test import --no-focus creates a database without writing project focus.""" + source = Path(__file__).parent / "fixtures" / "snapshots" / "snapshot_with_empty_cache.pkl" + snapshot = tmp_path / "sample.pkl" + shutil.copy(source, snapshot) + + result = runner.invoke(app, ["import", str(snapshot), "--no-focus"], catch_exceptions=False) + + assert result.exit_code == 0 + assert not (tmp_path / ".pt-snap").exists() + assert (tmp_path / "sample.pkl.db").exists() + + class TestReportCommand: """Test report subcommands."""