diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index 3efb823..a9694d0 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -7,6 +7,12 @@ template discovery, capability listing, database overview, query execution, and SnapshotDB import metadata. Import it from `pt_snap_cli.api`; it is not re-exported from the package root. +`list_templates()` and `execute_query()` return ordinary dictionaries annotated +as `TemplateSummaryPayload` and `QueryResultPayload` (`TypedDict` definitions in +`pt_snap_cli.api`). Dynamic query cells and metadata values are typed as `object`; +narrow them using the template's output contract before performing typed +operations. The annotations do not convert values or change the result keys. + ## Create an Analyzer ```python diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index 9fa013f..4d5c5bf 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -6,6 +6,11 @@ 执行查询和检查 SnapshotDB 导入 metadata 的高层 Python 门面。应从 `pt_snap_cli.api` 导入;包根目录不会重新导出它。 +`list_templates()` 和 `execute_query()` 返回普通字典,分别使用 +`pt_snap_cli.api` 中的 `TemplateSummaryPayload` 和 `QueryResultPayload` +(`TypedDict`)进行类型标注。动态查询单元格和 metadata 值标注为 `object`; +进行有类型约束的操作前,应按模板输出契约收窄类型。这些标注不会转换值或改变结果键。 + ## 创建 Analyzer ```python diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index 6d6d6de..6ca2d90 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -1,11 +1,14 @@ """High-level API for PyTorch memory snapshot analysis.""" +# Keep dynamic values explicit at this public boundary; do not reintroduce Any. +# pyright: reportAny=error, reportExplicitAny=error, reportUnknownArgumentType=error, reportUnknownVariableType=error, reportUnknownMemberType=error + from __future__ import annotations from dataclasses import dataclass from pathlib import Path from types import TracebackType -from typing import Any +from typing import TypedDict from pt_snap_cli.config import Config from pt_snap_cli.core import ( @@ -23,6 +26,25 @@ from pt_snap_cli.core.context_cache import ContextCache +class TemplateSummaryPayload(TypedDict): + name: str + description: str + category: str | None + + +class QueryResultPayload(TypedDict): + total: int + returned: int + device_id: int | None + rows: list[dict[str, object]] + template: str | None + semantics_version: int | None + has_more: bool + truncated: bool + total_is_exact: bool + timeout_s: float | None + + @dataclass class FocusState: """Current focus state of the analyzer.""" @@ -45,24 +67,28 @@ def __init__( *, context_cache: ContextCache | None = None, ) -> None: - self._config = Config() - self._db_path = db_path - self._device_id = device_id - self._focus_service = FocusService(self._config) + self._config: Config = Config() + self._db_path: Path | None = db_path + self._device_id: int | None = device_id + self._focus_service: FocusService = FocusService(self._config) # Share one context cache across the analyzer so that long-lived # SnapshotAnalyzer instances reuse a single # SQLite connection and skip schema validation on every query. # Explicit ``is not None`` because an empty cache is falsy via # ``__len__`` and would be silently replaced otherwise. - self._context_cache = context_cache if context_cache is not None else ContextCache() + self._context_cache: ContextCache = ( + context_cache if context_cache is not None else ContextCache() + ) self._owns_context_cache: bool = context_cache is None self._closed: bool = False - self._query_service = QueryService(self._focus_service, context_cache=self._context_cache) - self._metadata_service = ImportMetadataService() - self._capability_service = CapabilityService( + self._query_service: QueryService = QueryService( + self._focus_service, context_cache=self._context_cache + ) + self._metadata_service: ImportMetadataService = ImportMetadataService() + self._capability_service: CapabilityService = CapabilityService( query_service=self._query_service, ) - self._overview_service = OverviewService( + self._overview_service: OverviewService = OverviewService( self._focus_service, metadata_service=self._metadata_service, context_cache=self._context_cache, @@ -141,7 +167,7 @@ def set_focus(self, db_path: str | None = None, device_id: int | None = None) -> if candidate_db is not None: try: - self._focus_service.validate_session_db(candidate_db, candidate_device) + _ = self._focus_service.validate_session_db(candidate_db, candidate_device) except DatabaseMissingError as exc: raise FileNotFoundError(str(exc)) from exc except DatabaseSchemaError as exc: @@ -154,7 +180,7 @@ def set_focus(self, db_path: str | None = None, device_id: int | None = None) -> self._device_id = device_id return self.get_focus() - def list_templates(self, category: str | None = None) -> list[dict[str, Any]]: + def list_templates(self, category: str | None = None) -> list[TemplateSummaryPayload]: self._ensure_open() return [ { @@ -165,7 +191,7 @@ def list_templates(self, category: str | None = None) -> list[dict[str, Any]]: for template in self._query_service.list_templates(category) ] - def get_template_info(self, name: str) -> dict[str, Any] | None: + def get_template_info(self, name: str) -> dict[str, object] | None: self._ensure_open() try: info = self._query_service.get_template_info(name) @@ -177,13 +203,13 @@ def get_template_info(self, name: str) -> dict[str, Any] | None: def execute_query( self, template: str, - params: dict[str, Any] | None = None, + params: dict[str, object] | None = None, device_id: int | None = None, max_rows: int | None = None, *, exact_total: bool = False, timeout_s: float | None = None, - ) -> dict[str, Any]: + ) -> QueryResultPayload: self._ensure_open() try: result = self._query_service.execute_query( @@ -210,11 +236,11 @@ def execute_query( "timeout_s": result.timeout_s, } - def list_capabilities(self) -> dict[str, Any]: + def list_capabilities(self) -> dict[str, object]: self._ensure_open() return self._capability_service.catalog_to_dict(self._capability_service.catalog()) - def get_database_overview(self, db_path: str | None = None) -> dict[str, Any]: + def get_database_overview(self, db_path: str | None = None) -> dict[str, object]: self._ensure_open() resolved_path = db_path if db_path is not None else self._db_path try: @@ -229,7 +255,7 @@ def get_database_overview(self, db_path: str | None = None) -> dict[str, Any]: raise ValueError(str(exc)) from exc return self._overview_service.overview_to_dict(overview) - def get_database_metadata(self, db_path: str | None = None) -> dict[str, Any]: + def get_database_metadata(self, db_path: str | None = None) -> dict[str, object]: self._ensure_open() resolved = self._focus_service.resolve_focus( explicit_db_path=db_path if db_path is not None else self._db_path, diff --git a/src/pt_snap_cli/core/models.py b/src/pt_snap_cli/core/models.py index 0bc5cb4..71b6253 100644 --- a/src/pt_snap_cli/core/models.py +++ b/src/pt_snap_cli/core/models.py @@ -1,8 +1,9 @@ from __future__ import annotations +# pyright: reportAny=error, reportExplicitAny=error, reportUnknownArgumentType=error, reportUnknownVariableType=error, reportUnknownMemberType=error from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Literal +from typing import Literal from pt_snap_cli.config import FocusSource @@ -89,7 +90,7 @@ class TemplateInfo: category: str | None devices: str | None parameters: dict[str, TemplateParameter] - output_schema: list[dict[str, Any]] | None + output_schema: list[dict[str, object]] | None semantics_version: int | None = None interpretation_limits: list[str] = field(default_factory=list) @@ -99,7 +100,7 @@ class QueryResult: total: int returned: int device_id: int | None - rows: list[dict[str, Any]] + rows: list[dict[str, object]] template: str | None = None semantics_version: int | None = None has_more: bool = False @@ -113,9 +114,9 @@ class PeakMemoryReport: device_id: int | None metric: str event_id: int | None - peak: dict[str, Any] - allocator_gap: dict[str, Any] | None - callstack_groups: list[dict[str, Any]] + peak: dict[str, object] + allocator_gap: dict[str, object] | None + callstack_groups: list[dict[str, object]] @dataclass(frozen=True) diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index 4b38973..e4a7fbf 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -1,7 +1,7 @@ from __future__ import annotations from pathlib import Path -from typing import Literal +from typing import Literal, cast from pt_snap_cli.core.focus_service import FocusService from pt_snap_cli.core.models import PeakMemoryReport @@ -46,7 +46,8 @@ def peak_memory_report( start_dir=start_dir, ) peak = peak_result.rows[0] if peak_result.rows else {} - event_id = peak.get(_EVENT_ID_BY_METRIC[metric]) + # memory_peak selects an INTEGER event ID (or NULL for an empty trace). + event_id = cast(int | None, peak.get(_EVENT_ID_BY_METRIC[metric])) if event_id is None: return PeakMemoryReport( diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 4b36413..75f3fd7 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -219,3 +219,12 @@ Unions with multiple non-None members and unsupported types retain the TEXT fallback. Regression tests cover both optional spellings, reversed union order, generated schema, numeric ordering, and nullable value round trips. Audited upstream source mappings and license terms are unchanged. + +## 2026-10-03 — Explicit snapshot value boundary (issue #165) + +Local first-party typing change in `snapshot/representation.py`: replace `Any` +values with `object`, narrow canonicalization containers explicitly, and make +discarded side-effect return values explicit. Database/slice adaptor consumers +annotate the existing per-device trace-list and segment-list format assumptions. +No serialization, validation, replay, or database layout behavior changes. +Audited upstream source mappings and license terms remain unchanged. diff --git a/src/pt_snap_cli/snapshot/representation.py b/src/pt_snap_cli/snapshot/representation.py index 1edd986..e021f4a 100644 --- a/src/pt_snap_cli/snapshot/representation.py +++ b/src/pt_snap_cli/snapshot/representation.py @@ -1,9 +1,11 @@ +# pyright: reportAny=error, reportExplicitAny=error, reportUnknownArgumentType=error, reportUnknownVariableType=error, reportUnknownMemberType=error + import hashlib import json import math import pickle from pathlib import Path -from typing import Any, Literal, cast +from typing import Literal, cast from .simulate import AllocatorHooker, SimulateDeviceSnapshot, SimulateHooker from .util.file_util import UnsafePickleError as UnsafePickleError @@ -12,25 +14,25 @@ SnapshotFormat = Literal["pickle", "json"] -def load_pickle_representation(snapshot_file: str | Path) -> dict[str, Any]: +def load_pickle_representation(snapshot_file: str | Path) -> dict[str, object]: """Load a trusted pickle snapshot representation.""" - return cast(dict[str, Any], load_pickle_to_dict(Path(snapshot_file))) + return cast(dict[str, object], load_pickle_to_dict(Path(snapshot_file))) -def load_json_representation(snapshot_file: str | Path) -> dict[str, Any]: +def load_json_representation(snapshot_file: str | Path) -> dict[str, object]: """Load a JSON snapshot representation.""" with Path(snapshot_file).open(encoding="utf-8") as stream: - value: object = json.load(stream) + value = cast(object, json.load(stream)) if not isinstance(value, dict): raise ValueError( f"The content of the JSON file is not of type dict, actual type: {type(value).__name__}" ) - return cast(dict[str, Any], value) + return cast(dict[str, object], value) def load_snapshot_representation( snapshot_file: str | Path, snapshot_format: SnapshotFormat = "pickle" -) -> dict[str, Any]: +) -> dict[str, object]: """Load either supported representation through one dispatch point.""" if snapshot_format == "pickle": return load_pickle_representation(snapshot_file) @@ -40,7 +42,7 @@ def load_snapshot_representation( def serialize_snapshot_representation( - representation: dict[str, Any], snapshot_format: SnapshotFormat = "pickle" + representation: dict[str, object], snapshot_format: SnapshotFormat = "pickle" ) -> bytes: """Serialize a snapshot in its supported pickle or normalized JSON form.""" if snapshot_format == "pickle": @@ -50,28 +52,30 @@ def serialize_snapshot_representation( raise ValueError(f"Unsupported snapshot format: {snapshot_format}") -def serialize_pickle_representation(representation: dict[str, Any]) -> bytes: +def serialize_pickle_representation(representation: dict[str, object]) -> bytes: """Serialize the complete runtime-coupled pickle representation.""" return pickle.dumps(representation, protocol=4) -def serialize_json_representation(representation: dict[str, Any]) -> bytes: +def serialize_json_representation(representation: dict[str, object]) -> bytes: """Serialize the normalized inspectable JSON representation.""" return canonical_snapshot_bytes(representation) -def save_pickle_representation(representation: dict[str, Any], snapshot_file: str | Path) -> None: +def save_pickle_representation( + representation: dict[str, object], snapshot_file: str | Path +) -> None: """Save a trusted pickle snapshot using the retained protocol.""" save_dict_to_pickle(representation, Path(snapshot_file), protocol=4) -def save_json_representation(representation: dict[str, Any], snapshot_file: str | Path) -> None: +def save_json_representation(representation: dict[str, object], snapshot_file: str | Path) -> None: """Save normalized compact UTF-8 JSON.""" - Path(snapshot_file).write_bytes(serialize_json_representation(representation)) + _ = Path(snapshot_file).write_bytes(serialize_json_representation(representation)) def save_snapshot_representation( - representation: dict[str, Any], + representation: dict[str, object], snapshot_file: str | Path, snapshot_format: SnapshotFormat = "pickle", ) -> None: @@ -96,13 +100,16 @@ def canonicalize_snapshot(value: object) -> object: return value if isinstance(value, dict): normalized: dict[str, object] = {} - if not all(isinstance(key, str) for key in value): + mapping = cast(dict[object, object], value) + if not all(isinstance(key, str) for key in mapping): raise ValueError("Canonical snapshots require string dictionary keys.") - for key in sorted(cast(dict[str, object], value)): - normalized[key] = canonicalize_snapshot(value[key]) + for key in sorted(cast(dict[str, object], mapping)): + normalized[key] = canonicalize_snapshot(mapping[key]) return normalized if isinstance(value, (list, tuple)): - return [canonicalize_snapshot(item) for item in value] + return [ + canonicalize_snapshot(item) for item in cast(list[object] | tuple[object, ...], value) + ] raise ValueError(f"Canonical snapshots do not support {type(value).__name__} values.") @@ -123,7 +130,7 @@ def canonical_snapshot_sha256(representation: object) -> str: def replay_snapshot( - representation: dict[str, Any], + representation: dict[str, object], device: int, *, hooker: SimulateHooker | None = None, @@ -133,9 +140,9 @@ def replay_snapshot( """Construct and replay one device through the shared snapshot runtime.""" snapshot = SimulateDeviceSnapshot(representation, device, _raw_frames=_raw_frames) if hooker is not None: - snapshot.register_hooker(hooker) + _ = snapshot.register_hooker(hooker) if allocator_hooker is not None: - snapshot.register_allocator_hooker(allocator_hooker) + _ = snapshot.register_allocator_hooker(allocator_hooker) return snapshot, snapshot.replay() @@ -143,7 +150,7 @@ def load_and_replay_snapshot( snapshot_file: str | Path, snapshot_format: SnapshotFormat, device: int, -) -> tuple[dict[str, Any], SimulateDeviceSnapshot, bool]: +) -> tuple[dict[str, object], SimulateDeviceSnapshot, bool]: """Load and replay either format through the shared validation entrypoint.""" representation = load_snapshot_representation(snapshot_file, snapshot_format) snapshot, replayed = replay_snapshot(representation, device) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index 67a757f..f9b84c5 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -1,7 +1,7 @@ import os import tempfile from pathlib import Path -from typing import Any +from typing import Any, cast from ...base import Block, BlockState, DeviceSnapshot, TraceEntry from ...representation import UnsafePickleError, load_snapshot_representation, replay_snapshot @@ -168,7 +168,8 @@ def dump(pickle_file: str | Path, dump_file: str | Path, device: int | None = No except Exception as e: dump_logger.error(f"Failed to load pickle file: {e}") return False - device_traces = data.get("device_traces", []) + # Snapshot format: one list of trace dictionaries per device. + device_traces = cast(list[list[object]], data.get("device_traces", [])) # 当指定device为空时dump所有记录了跟踪事件的device,否则仅dump指定device need_dump_devices = [device for device in range(len(device_traces)) if device_traces[device]] dump_logger.info(f"Recognized have trace events devices {need_dump_devices}.") diff --git a/src/pt_snap_cli/snapshot/tools/slice_dump/dump.py b/src/pt_snap_cli/snapshot/tools/slice_dump/dump.py index b30a425..f9fc53c 100644 --- a/src/pt_snap_cli/snapshot/tools/slice_dump/dump.py +++ b/src/pt_snap_cli/snapshot/tools/slice_dump/dump.py @@ -1,6 +1,6 @@ import os.path from pathlib import Path -from typing import Literal +from typing import Literal, cast from ...representation import load_snapshot_representation, replay_snapshot from ...util import get_logger @@ -26,14 +26,16 @@ def run_slice_dump( "history event recoding during collection." ) return - if device < 0 or len(df["device_traces"]) <= device or not df["device_traces"][device]: + # These are the sequence fields of the snapshot format; no values are coerced. + traces = cast(list[list[object]], df["device_traces"]) + segments = cast(list[object], df["segments"]) + if device < 0 or len(traces) <= device or not traces[device]: dump_logger.warning( f"The snapshot file did not record any event data for the specified device {device}." ) return dump_logger.info( - f"Start loading snapshot with {len(df['segments'])} segments, " - f"{len(df['device_traces'][device])} events" + f"Start loading snapshot with {len(segments)} segments, " f"{len(traces[device])} events" ) dump_logger.info("Successfully loaded snapshot, starting to replay and dump.") slice_dump_hooker = SliceDumpHooker(