From b9456c25755b1a9ec8bebb5df25f09baf9a71d2f Mon Sep 17 00:00:00 2001 From: codeasier Date: Mon, 5 Oct 2026 10:37:08 +0800 Subject: [PATCH] feat(snapshot): finalize continuous native shard replay (#199) --- README.md | 2 +- README_zh.md | 2 +- docs/en/sharded-snapshotdb.md | 47 ++ docs/zh/sharded-snapshotdb.md | 36 ++ .../core/sharded_replay_service.py | 57 +++ src/pt_snap_cli/snapshot/PROVENANCE.md | 25 + src/pt_snap_cli/snapshot/simulate/simulate.py | 25 +- .../tools/adaptors/database/snapshot_db.py | 19 +- .../snapshot/tools/adaptors/sharded_replay.py | 278 +++++++++++ .../snapshot/tools/adaptors/snapshot2db.py | 33 +- tests/core/test_sharded_replay_service.py | 99 ++++ tests/snapshot/test_sharded_replay.py | 440 ++++++++++++++++++ 12 files changed, 1037 insertions(+), 26 deletions(-) create mode 100644 src/pt_snap_cli/core/sharded_replay_service.py create mode 100644 src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py create mode 100644 tests/core/test_sharded_replay_service.py create mode 100644 tests/snapshot/test_sharded_replay.py diff --git a/README.md b/README.md index 5917953..fda331b 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ See the [documentation index](docs/README.md) for all English and Chinese guides | Splitting snapshots | [Splitting Snapshots](docs/en/splitting.md) | | Agent skills | [Agent Skills](docs/en/skills.md) | | Database format | [SnapshotDB Schema](docs/en/database.md) | -| Sharded artifact contract (P0; not an exporter) | [Sharded SnapshotDB protocol](docs/en/sharded-snapshotdb.md) | +| Sharded contract and internal replay (P0/P1; no CLI exporter) | [Sharded SnapshotDB protocol](docs/en/sharded-snapshotdb.md) | | Python API | [SnapshotAnalyzer API](docs/en/snapshot-analyzer-api.md) | | Result mapping utility | [ResultMapper API](docs/en/result-mapper-api.md) | diff --git a/README_zh.md b/README_zh.md index a37c593..31e5cec 100644 --- a/README_zh.md +++ b/README_zh.md @@ -91,7 +91,7 @@ helper 会先路由采集,无需先安装 CLI。分析仍需要 CLI 和 Snapsh | 拆分快照 | [拆分快照](docs/zh/splitting.md) | | Agent skill | [Agent Skills](docs/zh/skills.md) | | 数据库格式 | [SnapshotDB Schema](docs/zh/database.md) | -| 分片产物合同(P0,非导出器) | [分片 SnapshotDB 协议](docs/zh/sharded-snapshotdb.md) | +| 分片合同与内部回放(P0/P1,无 CLI 导出器) | [分片 SnapshotDB 协议](docs/zh/sharded-snapshotdb.md) | | Python API | [SnapshotAnalyzer API](docs/zh/snapshot-analyzer-api.md) | | 结果映射工具 | [ResultMapper API](docs/zh/result-mapper-api.md) | diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 6af500b..a5e396a 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -180,6 +180,53 @@ QueryScope("event_range", device_id=0, start_event_id=1, end_event_id=3).validat assert not inspection.structured_frames ``` +## Internal continuous replay (P1) + +`core.sharded_replay_service.ShardedReplayService.stage(source, directory, +events_per_slice=..., device=None)` is an **internal producer boundary**, not a +new CLI/API import command. It loads trusted pickle once, constructs one persistent +simulator per selected event-bearing device, and writes windows from last to first +into a new, caller-owned **private staging directory**. `omitted_devices` discloses +all unselected/empty device positions. Existing `import`, `split`, `SnapshotAnalyzer` +and single-DB v1/v2 reading are unchanged. + +The returned immutable result identifies **native-v2** slices, not compatibility-v1 +artifacts: trace rows use `callstackId` plus a local `callstack` table and retain +native OOM action 8 and workspace events. Sparse/nonzero-start native real IDs are +preserved if nonnegative, unique and chronological; windows count list positions, +not `max(id)+1`. These native slices are **not** accepted by `validate_dataset`. +Compatibility export, a compatible manifest, cache/publication and dataset queries +remain separate work. No `manifest.json`, `readySlices`, or incremental ready +notification is emitted. Only a successful return after all devices finalize and +validate may be handed to later publication. Failure retains private partial +files for their owner; none is advertised as a complete dataset. + +`SimulateDeviceSnapshot.replay_until(remaining_events)` undoes until that many +chronological list entries remain. The endpoint is a **position**, not an event ID; +repeated pauses/resumes share the full replay hook/error path. A real event row +records the state **after** its event (before undo). After undoing a window's first +event, active block observations and negative-ID `segment_alloc`/`segment_map` +rows reconstruct the state **before** that first event. Boundary rows do not count +against capacity or real-event peaks. + +A device-local registry retains original block objects across writer switches, +not copied hook payloads or an address-only key. Allocation replay resolves every +appearance to the original alloc ID; unknown/preexisting lifetimes retain stable +negative identities. All writers close before per-DB transactional batch backfill +and validation. `freeEventId` stays `free_completed`, never `free_requested`; +boundary `active_pending_free` observations remain state 0. Address reuse and +same-address different-stream blocks remain distinct. Each native slice adds +`pt_snap_block_reference(blockId, stream, allocCallstack, freeCallstack)` so finalized +out-of-slice lifecycle references do not discard their text stack sources. NULL +means no observed source event; empty text means the observed event had no frames. +Legacy slice-local query JOINs do not automatically consume this internal table; +this is not a cross-slice query engine or structured-frame extension. + +Pickle may still load entirely. Sharding bounds **real events per DB**, not block +rows, total staged bytes, registry size, or peak RSS. Synthetic regressions and +reviewed expandable/multi-device fixtures cover replay equivalence and resource +failure paths; no live upstream GUI or performance acceptance is claimed. + ## Evidence and limits Interoperability facts were read from fixed upstream diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index a07ce11..bd687b7 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -148,6 +148,42 @@ QueryScope("event_range", device_id=0, start_event_id=1, end_event_id=3).validat assert not inspection.structured_frames ``` +## 内部持续回放(P1) + +`core.sharded_replay_service.ShardedReplayService.stage(source, directory, +events_per_slice=..., device=None)` 是**内部生产边界**,不是新 CLI/API 导入命令。 +它只加载一次可信 pickle,每个所选非空设备只创建一个持续模拟器,从末片向前写入 +新的、调用者所有的**私有 staging 目录**。`omitted_devices` 披露全部未选/空设备位置。 +现有 `import`、`split`、`SnapshotAnalyzer` 与单库 v1/v2 读取保持不变。 + +返回的不可变结果标识 **native-v2** 分片,不是兼容 v1 产物:trace 使用 +`callstackId` 和片内 `callstack` 表,保留原生 OOM action 8 与 workspace 事件。 +原生真实 ID 可以稀疏或非零起点,但必须非负、唯一且按时间递增;窗口按列表位置计数, +不是 `max(id)+1`。这些原生片**不能**通过 `validate_dataset`。兼容导出、兼容 manifest、 +缓存/发布及数据集查询属于后续独立工作。不生成 `manifest.json`、`readySlices` 或增量 +ready 通知;仅所有设备完成回填及校验后的成功返回结果可交给后续发布层。 +失败保留调用者所有的私有部分文件,不将其宣称为完整数据集。 + +`SimulateDeviceSnapshot.replay_until(remaining_events)` 倒序回放到剩余指定数量的 +时间顺序列表事件。端点是**位置**而非 event ID;重复暂停/继续与完整 replay 共用 +hooks 和错误路径。真实事件行记录事件**之后**的状态(undo 之前)。undo 窗口首事件后, +活跃 block 观察和负 ID `segment_alloc`/`segment_map` 行重建首事件**之前**的状态; +边界行不计入容量或真实事件峰值。 + +设备内 registry 跨 writer 保留原始 block 对象,不用 hook 副本或仅地址作为身份。 +遇到真实 alloc 后,每次出现都最终指向原 alloc ID;未知/预存生命周期保持稳定负身份。 +所有 writer 关闭后才逐 DB 事务批量回填并校验。`freeEventId` 始终是 `free_completed`, +不是 `free_requested`;边界 `active_pending_free` 观察保留 state 0。同地址复用及不同 +stream 的同地址块不会合并。每个原生片加入 +`pt_snap_block_reference(blockId, stream, allocCallstack, freeCallstack)`,保留最终化的 +片外生命周期引用的文本调用栈来源。NULL 表示未观察到来源事件;空文本表示已观察事件 +没有 frames。旧有片内查询 JOIN 不自动消费这个内部表;它不是跨片查询引擎或结构化 +frame 扩展。 + +pickle 仍可能整体加载。分库只限制**单库真实事件数**,不限制 block 行数、总 staging +字节、registry 大小或峰值 RSS。合成回归及已审 expandable/多设备 fixture 覆盖回放 +等价和资源失败路径;不宣称真实上游 GUI 或性能验收。 + ## 来源证据与限制 接口事实已从固定版本真实源码读取: diff --git a/src/pt_snap_cli/core/sharded_replay_service.py b/src/pt_snap_cli/core/sharded_replay_service.py new file mode 100644 index 0000000..bb210c6 --- /dev/null +++ b/src/pt_snap_cli/core/sharded_replay_service.py @@ -0,0 +1,57 @@ +"""Internal P1 shared producer boundary, not a CLI import/export facade.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from ..snapshot.representation import load_snapshot_representation +from ..snapshot.tools.adaptors.sharded_replay import ReplaySlice, build_shards + + +@dataclass(frozen=True) +class ShardedReplayResult: + directory: Path + slices: tuple[ReplaySlice, ...] + omitted_devices: tuple[int, ...] + layout: str = "native-v2" + + +class ShardedReplayService: + def stage( + self, + source: Path | str, + directory: Path | str, + *, + events_per_slice: int, + device: int | None = None, + ) -> ShardedReplayResult: + """Load once and finalize native shards in a NEW private staging directory. + + The caller owns partial files on failure. No manifest, ready slice, focus, + cache reuse or final publication is exposed. Only a successful result can + be handed to a subsequent exporter/publication service. Pickle is trusted + executable input; this does not implement streaming or an RSS bound. + """ + if type(events_per_slice) is not int or events_per_slice <= 0: + raise ValueError("events_per_slice must be a positive integer") + if device is not None and (type(device) is not int or device < 0): + raise ValueError("device must be a nonnegative integer") + target = Path(directory).absolute() + if target.resolve() != target or target.is_symlink(): + raise ValueError("Staging directory must be canonical and non-symlink") + if target.exists(): + raise FileExistsError(target) + data = load_snapshot_representation(Path(source)) + traces = cast(list[list[object]], data.get("device_traces", [])) + available = tuple(index for index, entries in enumerate(traces) if entries) + if not available or (device is not None and device not in available): + raise ValueError("Requested device is absent or has no trace events") + selected = available if device is None else (device,) + # Exclusive allocation: never replace or adopt an existing directory. + target.mkdir() + slices = build_shards(data, target, events_per_slice, selected) + return ShardedReplayResult( + target, slices, tuple(index for index in range(len(traces)) if index not in selected) + ) diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 75f3fd7..f7d5edd 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -228,3 +228,28 @@ discarded side-effect return values explicit. Database/slice adaptor consumers annotate the existing per-device trace-list and segment-list format assumptions. No serialization, validation, replay, or database layout behavior changes. Audited upstream source mappings and license terms remain unchanged. + +## 2026-10-05 — Continuous native shard replay (issue #199) + +Interface facts were independently read through real GitHub GETs at +Ascend/msinsight revision `101f65b877a267ffd5f66ea3834706057ba243e5`: +[resumable replay](https://api.github.com/repos/Ascend/msinsight/git/blobs/1956490d46ccf11309d10853ea76175d0713934e) +and [boundary/registry interfaces](https://api.github.com/repos/Ascend/msinsight/git/blobs/95612509ef990ac6b7034919dae4938cc6acf3ad). +The fixed tree confirms both blob identities. These sources carry Copyright (c) +2026 Huawei Technologies Co.,Ltd. and Mulan PSL v2; the docs' CC BY 4.0 license +is separate. No upstream implementation is copied into the package. + +Local original implementation changes `simulate/simulate.py` to reuse the existing +full replay loop for position-based pauses (deliberately not an ID threshold), +adds `tools/adaptors/sharded_replay.py` for native-v2 multi-writer staging, +original-object lifecycle tracking, negative left-boundary rows, transactional +batch ID/free_completed/stack-source backfill and final validation, and adds the +shared `core/sharded_replay_service.py` producer boundary. Unknown allocations +keep device-local negative identities without upstream compaction. All writers +close before finalization; no incremental readiness, compatibility manifest, +cache/publication, CLI changes, or structured-frame extension is introduced. +`tools/adaptors/snapshot2db.py` and `database/snapshot_db.py` close connections on +constructor/commit failure while preserving the original single-DB data contract. +Synthetic failure/equivalence tests and bilingual docs are locally authored; +reviewed fixtures are unchanged. Existing source mappings and MIT relicensing +records are retained. GUI interoperability and streaming/RSS claims are not made. diff --git a/src/pt_snap_cli/snapshot/simulate/simulate.py b/src/pt_snap_cli/snapshot/simulate/simulate.py index 7e8d942..524bfba 100644 --- a/src/pt_snap_cli/snapshot/simulate/simulate.py +++ b/src/pt_snap_cli/snapshot/simulate/simulate.py @@ -154,14 +154,27 @@ def unregister_allocator_hooker(self, hooker_id: int): self.simulated_allocator.unregister_hooker(hooker_id) def replay(self) -> bool: + """Undo the entire remaining trace through the resumable replay path.""" + return self.replay_until(0) + + def replay_until(self, remaining_events: int) -> bool: + """Undo until exactly ``remaining_events`` chronological entries remain. + + The endpoint is a list position, not an event ID (native IDs may be sparse). + Undoing positions [start, end] leaves the state immediately BEFORE start. + Pre hooks observe the state AFTER the event; post hooks observe BEFORE it. + A pre/allocator failure leaves the event pending, while a post-hook failure + occurs after it was consumed, exactly as for full replay. """ - 开始仿真回放内存事件 - """ + if type(remaining_events) is not int or not ( + 0 <= remaining_events <= len(self.device_snapshot.trace_entries) + ): + raise ValueError("remaining_events must be an integer within the remaining trace") # 倒序遍历 total_size = len(self.device_snapshot.trace_entries) self._replay_logger.info(f"Replaying {total_size} entries in snapshot...") progress_update_point = [0.25, 0.5, 0.75] - while self.device_snapshot.trace_entries: + while len(self.device_snapshot.trace_entries) > remaining_events: for hooker in self.hookers.values(): if hooker and not hooker.pre_undo_event( self.device_snapshot.trace_entries[-1], self.device_snapshot @@ -191,5 +204,9 @@ def replay(self) -> bool: "An interruption occurred during the replay of the single event post hook." ) return False - self._replay_logger.info("All events have been successfully replayed.") + self._replay_logger.info( + "All events have been successfully replayed." + if not remaining_events + else f"Replay paused with {remaining_events} entries remaining." + ) return True diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py index 6fd43b2..ef71010 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/database/snapshot_db.py @@ -64,13 +64,18 @@ class SnapshotDb(SqliteDB): CALLSTACK_TABLE_NAME = "callstack" def __init__(self, path: str): - super().__init__(path, auto_create=True, with_dictionary_table=True) - # Import builds a replaceable database, so favor write throughput over crash recovery. - self.conn.execute("PRAGMA journal_mode = MEMORY") - self.conn.execute("PRAGMA synchronous = OFF") - self.conn.execute("PRAGMA cache_size = -65536") - # 清理旧版本表格 - self._clear_old_tables() + try: + super().__init__(path, auto_create=True, with_dictionary_table=True) + # Import builds a replaceable database, so favor write throughput over crash recovery. + self.conn.execute("PRAGMA journal_mode = MEMORY") + self.conn.execute("PRAGMA synchronous = OFF") + self.conn.execute("PRAGMA cache_size = -65536") + # 清理旧版本表格 + self._clear_old_tables() + except BaseException: + if hasattr(self, "conn"): + self.conn.close() + raise def create_trace_entry_table(self, device: int = 0): self.create_table( diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py new file mode 100644 index 0000000..1d2d130 --- /dev/null +++ b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py @@ -0,0 +1,278 @@ +"""Internal native-v2 shard construction; no compatibility export or publication. + +All writers close before finalization. Only a successful return makes identities +stable. Partial files in the caller's private staging directory are NOT ready. +""" + +from __future__ import annotations + +import sqlite3 +from contextlib import closing +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ...base import Block, BlockState, DeviceSnapshot, TraceEntry +from ...simulate import AllocatorHooker, SimulateDeviceSnapshot, SimulateHooker +from .database import CallstackInterner, event2record +from .snapshot2db import SnapshotDbHandler + + +@dataclass(frozen=True) +class ReplaySlice: + device: int + index: int + start_position: int + end_position: int + start_event_id: int + end_event_id: int + file: Path + + +@dataclass +class _Lifetime: + # Retain the ORIGINAL object: copied allocator hook payloads and address reuse + # cannot alias it, and Python cannot recycle its object identity during replay. + block: Block + token: int + stream: int + alloc_stack: str | None = None + free_stack: str | None = None + + @property + def final_id(self) -> int: + return self.token if self.block.alloc_event_idx is None else self.block.alloc_event_idx + + def signature(self) -> tuple[int, int, int, int, int]: + block = self.block + return ( + block.address, + block.size, + block.requested_size, + -1 if block.alloc_event_idx is None else block.alloc_event_idx, + -1 if block.free_event_idx is None else block.free_event_idx, + ) + + +@dataclass +class BlockRegistry: + """One snapshot/device-local registry; provisional IDs never escape staging.""" + + lifetimes: dict[int, _Lifetime] = field(default_factory=dict) + members: dict[Path, dict[int, _Lifetime]] = field(default_factory=dict) + + def observe(self, block: Block, stream: int) -> _Lifetime: + key = id(block) + lifetime = self.lifetimes.get(key) + if lifetime is None: + lifetime = _Lifetime(block, -len(self.lifetimes) - 1, stream) + self.lifetimes[key] = lifetime + elif lifetime.stream != stream: + raise RuntimeError("Block changed stream during its lifetime") + return lifetime + + def record(self, path: Path, block: Block, stream: int) -> dict[str, Any] | None: + lifetime = self.observe(block, stream) + members = self.members.setdefault(path, {}) + if lifetime.token in members: + return None + members[lifetime.token] = lifetime + return { + "id": lifetime.token, + "address": block.address, + "size": block.size, + "requestedSize": block.requested_size, + "state": block.state, + "allocEventId": -1, + "freeEventId": -1 if block.free_event_idx is None else block.free_event_idx, + } + + def finalize(self) -> None: + """Batch patches by closed DB, including free_completed and stack sources. + + Unknown/preexisting allocations keep their unique negative token; no + compaction is necessary or allowed to merge unrelated objects. + """ + for path, members in self.members.items(): + device = int(path.parent.name.removeprefix("device_")) + with closing(sqlite3.connect(path.as_uri() + "?mode=rw", uri=True)) as conn, conn: + changes = conn.total_changes + conn.executemany( + f'UPDATE "block_{device}" SET id=?, allocEventId=?, freeEventId=? WHERE id=?', + [ + (item.final_id, *item.signature()[-2:], token) + for token, item in members.items() + ], + ) + if conn.total_changes - changes != len(members): + raise RuntimeError("Missing provisional block during finalization") + conn.execute( + "CREATE TABLE pt_snap_block_reference (blockId INTEGER PRIMARY KEY, " + "stream INTEGER NOT NULL, allocCallstack TEXT, freeCallstack TEXT)" + ) + conn.executemany( + "INSERT INTO pt_snap_block_reference VALUES (?, ?, ?, ?)", + [ + (item.final_id, item.stream, item.alloc_stack, item.free_stack) + for item in members.values() + ], + ) + + def validate(self, item: ReplaySlice, expected_ids: list[int]) -> None: + with closing(sqlite3.connect(item.file.as_uri() + "?mode=ro", uri=True)) as conn: + if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]: + raise RuntimeError("Invalid finalized SQLite shard") + ids = [ + row[0] + for row in conn.execute( + f"SELECT id FROM trace_entry_{item.device} WHERE id>=0 ORDER BY id" + ) + ] + if ids != expected_ids: + raise RuntimeError("Shard lost or changed real events") + rows = conn.execute( + f"SELECT id,address,size,requestedSize,allocEventId,freeEventId FROM block_{item.device}" + ).fetchall() + expected = { + (lifetime.final_id, *lifetime.signature()) + for lifetime in self.members[item.file].values() + } + if len(rows) != len(expected) or set(rows) != expected: + raise RuntimeError("Incomplete block identity/lifetime finalization") + references = conn.execute("SELECT * FROM pt_snap_block_reference").fetchall() + if set(references) != { + (lifetime.final_id, lifetime.stream, lifetime.alloc_stack, lifetime.free_stack) + for lifetime in self.members[item.file].values() + }: + raise RuntimeError("Incomplete cross-slice callstack references") + + +class _SliceHook(SimulateHooker, AllocatorHooker): + def __init__(self, item: ReplaySlice, registry: BlockRegistry): + self.item = item + self.registry = registry + self.writer = SnapshotDbHandler(str(item.file), [item.device]) + self.stacks = CallstackInterner() + self.current_event: TraceEntry | None = None + + def pre_undo_event(self, wait4undo_event: TraceEntry, current_snapshot: DeviceSnapshot) -> bool: + event, snapshot = wait4undo_event, current_snapshot + self.current_event = event + self.writer.insert_event( + event2record( + event, + snapshot.total_allocated, + snapshot.total_activated, + snapshot.total_reserved, + callstacks=self.stacks, + ), + snapshot.device, + ) + return True + + def post_undo_event( + self, already_undo_event: TraceEntry, current_snapshot: DeviceSnapshot + ) -> bool: + return True + + def post_replay_alloc_block(self, allocated_block: Block, current_snapshot: DeviceSnapshot): + assert self.current_event is not None + lifetime = self.registry.observe(allocated_block, self.current_event.stream) + lifetime.free_stack = self.current_event.get_callstack() + + def pre_replay_free_block(self, wait4free_block: Block, current_snapshot: DeviceSnapshot): + # This hook receives the original block before detach; the post hook is + # a shallow copy and MUST NOT be used as an object-identity registry key. + assert self.current_event is not None + lifetime = self.registry.observe(wait4free_block, self.current_event.stream) + lifetime.alloc_stack = self.current_event.get_callstack() + self._write_block(wait4free_block, lifetime.stream, current_snapshot.device) + + def _write_block(self, block: Block, stream: int, device: int): + record = self.registry.record(self.item.file, block, stream) + if record is not None: + self.writer.insert_block(record, device) + + def dump_left_boundary(self, snapshot: DeviceSnapshot): + # Called AFTER undoing the first event in this window. Segment rows and + # block states describe the instant BEFORE that first event, not after. + for index, segment in enumerate(snapshot.segments, start=1): + for block in segment.blocks: + if block.state != BlockState.INACTIVE: + self._write_block(block, segment.stream, snapshot.device) + boundary = TraceEntry( + idx=-index, + action="segment_map" if segment.is_expandable else "segment_alloc", + addr=segment.address, + size=segment.total_size, + stream=segment.stream, + frames=segment.frames, + _raw_frames=segment._raw_frames, + ) + self.writer.insert_event( + event2record( + boundary, + snapshot.total_allocated, + snapshot.total_activated, + snapshot.total_reserved, + callstacks=self.stacks, + ), + snapshot.device, + ) + + +def build_shards( + data: dict[str, Any], directory: Path, events_per_slice: int, devices: tuple[int, ...] +) -> tuple[ReplaySlice, ...]: + """Build inside a caller-owned private EMPTY directory; never publish readiness. + + Core validates options and owns staging. Native v2 preserves OOM/workspace; + this is NOT the compatibility-v1 dataset validated by dataset_contract. + """ + results: list[ReplaySlice] = [] + for device in devices: + simulator = SimulateDeviceSnapshot(data, device, _raw_frames=True) + events = simulator.device_snapshot.trace_entries + event_ids = [event.idx for event in events] + if any(type(value) is not int or value < 0 for value in event_ids): + raise ValueError("Real event IDs must be nonnegative integers") + ids = [int(value) for value in event_ids if value is not None] + if ids != sorted(set(ids)): + raise ValueError("Real event IDs must be unique and chronological") + registry = BlockRegistry() + device_dir = directory / f"device_{device}" + device_dir.mkdir() + slices: list[ReplaySlice] = [] + for index in reversed(range((len(events) + events_per_slice - 1) // events_per_slice)): + start = index * events_per_slice + end = min(start + events_per_slice, len(ids)) - 1 + item = ReplaySlice( + device, + index, + start, + end, + ids[start], + ids[end], + device_dir / f"slice_{index:05d}.db", + ) + registry.members[item.file] = {} + hook = _SliceHook(item, registry) + hook_id = simulator.register_hooker(hook) + allocator_id = simulator.register_allocator_hooker(hook) + try: + if not simulator.replay_until(start): + raise RuntimeError(f"Replay failed for device {device}, slice {index}") + hook.dump_left_boundary(simulator.device_snapshot) + hook.writer.flush(device) + hook.writer.insert_callstacks(hook.stacks.records()) + hook.writer.close() + finally: + simulator.unregister_hooker(hook_id) + simulator.unregister_allocator_hooker(allocator_id) + hook.writer.close(commit=False) + slices.append(item) + registry.finalize() + for item in slices: + registry.validate(item, ids[item.start_position : item.end_position + 1]) + results.extend(reversed(slices)) + return tuple(results) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py index f9b84c5..282280a 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/snapshot2db.py @@ -21,18 +21,23 @@ class SnapshotDbHandler: def __init__( self, db_path: str, devices: list[int], insert_cache_size: int = DEFAULT_INSERT_CACHE_SIZE ): - self._closed = False + self._closed = True self.db_path = db_path self.db = SnapshotDb(db_path) + self._closed = False self._device_event_cache = {} self._device_block_cache = {} self._insert_cache_size = insert_cache_size - self.db.create_callstack_table() - for device in devices: - self._device_block_cache[device] = [] - self._device_event_cache[device] = [] - self.db.create_trace_entry_table(device) - self.db.create_block_table(device) + try: + self.db.create_callstack_table() + for device in devices: + self._device_block_cache[device] = [] + self._device_event_cache[device] = [] + self.db.create_trace_entry_table(device) + self.db.create_block_table(device) + except BaseException: + self.close(commit=False) + raise def insert_event(self, event_record: dict[str, Any], device: int = 0): if device not in self._device_event_cache: @@ -83,12 +88,14 @@ def _do_insert_blocks(self, device: int = 0): def close(self, *, commit: bool = True): if getattr(self, "_closed", True): return - if commit: - self.db.conn.commit() - else: - self.db.conn.rollback() - self.db.conn.close() - self._closed = True + try: + if commit: + self.db.conn.commit() + else: + self.db.conn.rollback() + finally: + self.db.conn.close() + self._closed = True def __del__(self): self.close() diff --git a/tests/core/test_sharded_replay_service.py b/tests/core/test_sharded_replay_service.py new file mode 100644 index 0000000..3965f9b --- /dev/null +++ b/tests/core/test_sharded_replay_service.py @@ -0,0 +1,99 @@ +import copy + +import pytest + +from pt_snap_cli.core import sharded_replay_service +from pt_snap_cli.core.sharded_replay_service import ShardedReplayService +from pt_snap_cli.snapshot.tools.adaptors import sharded_replay + +from ..snapshot.test_sharded_replay import lifecycle_data, write_source + + +def test_source_loaded_once_and_one_simulator_per_selected_device(tmp_path, monkeypatch): + data = lifecycle_data() + data["device_traces"].append(copy.deepcopy(data["device_traces"][0])) + data["segments"] += [{**copy.deepcopy(segment), "device": 1} for segment in data["segments"]] + source = write_source(tmp_path, data) + loads, devices = [], [] + load = sharded_replay_service.load_snapshot_representation + simulator = sharded_replay.SimulateDeviceSnapshot + + def counted_load(path): + loads.append(path) + return load(path) + + def counted_simulator(data, device, **kwargs): + devices.append(device) + return simulator(data, device, **kwargs) + + monkeypatch.setattr(sharded_replay_service, "load_snapshot_representation", counted_load) + monkeypatch.setattr(sharded_replay, "SimulateDeviceSnapshot", counted_simulator) + result = ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=1) + assert loads == [source] + assert devices == [0, 1] + assert result.layout == "native-v2" and result.omitted_devices == () + assert len(result.slices) == 18 + + +@pytest.mark.parametrize("capacity", [0, -1, True, 1.5]) +def test_invalid_capacity_before_load_or_staging(tmp_path, capacity): + with pytest.raises(ValueError, match="events_per_slice"): + ShardedReplayService().stage( + tmp_path / "missing", tmp_path / "stage", events_per_slice=capacity + ) + assert not (tmp_path / "stage").exists() + + +@pytest.mark.parametrize("device", [-1, True, 1.5, 5]) +def test_invalid_or_missing_device_never_creates_stage(tmp_path, device): + source = write_source(tmp_path, lifecycle_data()) + with pytest.raises(ValueError): + ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=2, device=device) + assert not (tmp_path / "stage").exists() + + +def test_omitted_empty_devices_disclosed(tmp_path): + data = lifecycle_data() + data["device_traces"].append([]) + result = ShardedReplayService().stage( + write_source(tmp_path, data), tmp_path / "stage", events_per_slice=2, device=0 + ) + assert result.omitted_devices == (1,) + + +def test_empty_static_only_and_existing_stage_rejected(tmp_path): + source = write_source(tmp_path, {"segments": [], "device_traces": [[]]}) + with pytest.raises(ValueError, match="trace events"): + ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=2) + existing = tmp_path / "existing" + existing.mkdir() + marker = existing / "owner.txt" + marker.write_text("preserve") + with pytest.raises(FileExistsError): + ShardedReplayService().stage(source, existing, events_per_slice=2) + assert marker.read_text() == "preserve" + + +def test_symlink_and_dangling_stage_rejected(tmp_path): + (tmp_path / "actual").mkdir() + for name, target in [("link", "actual"), ("dangling", "missing")]: + (tmp_path / name).symlink_to(target) + with pytest.raises(ValueError, match="non-symlink"): + ShardedReplayService().stage(tmp_path / "input", tmp_path / name, events_per_slice=1) + with pytest.raises(ValueError, match="non-symlink"): + ShardedReplayService().stage( + tmp_path / "input", tmp_path / "link" / "child", events_per_slice=1 + ) + + +@pytest.mark.parametrize("ids", [[0, 0], [1, 0], [-1, 0]]) +def test_invalid_native_ids_rejected_without_ready_result(tmp_path, ids): + data = { + "segments": [], + "device_traces": [[{"id": value, "action": "oom", "frames": []} for value in ids]], + } + with pytest.raises(ValueError, match="event IDs"): + ShardedReplayService().stage( + write_source(tmp_path, data), tmp_path / "stage", events_per_slice=1 + ) + assert not list((tmp_path / "stage").iterdir()) diff --git a/tests/snapshot/test_sharded_replay.py b/tests/snapshot/test_sharded_replay.py new file mode 100644 index 0000000..f836f43 --- /dev/null +++ b/tests/snapshot/test_sharded_replay.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import copy +import sqlite3 +from contextlib import closing + +import pytest + +from pt_snap_cli.core.sharded_replay_service import ShardedReplayService +from pt_snap_cli.snapshot.base import Block, BlockState +from pt_snap_cli.snapshot.representation import save_snapshot_representation +from pt_snap_cli.snapshot.simulate import SimulateDeviceSnapshot +from pt_snap_cli.snapshot.tools.adaptors import sharded_replay, snapshot2db +from pt_snap_cli.snapshot.tools.adaptors.database.snapshot_db import SnapshotDb + +from .helpers import FIXTURE_DIR +from .test_simulate import Hooker, make_torch_npu_workspace_snapshot + + +def event(action, addr=1000, size=16, stream=0): + return { + "action": action, + "addr": addr, + "size": size, + "stream": stream, + "frames": [{"filename": "source.py", "line": 4, "name": action}], + } + + +def block(addr, size, requested): + return { + "address": addr, + "size": size, + "requested_size": requested, + "state": "active_allocated", + "frames": [], + } + + +def segment(stream, total, blocks): + return { + "address": 1000, + "total_size": total, + "stream": stream, + "segment_type": "small", + "allocated_size": sum(item["size"] for item in blocks), + "active_size": sum(item["size"] for item in blocks), + "device": 0, + "is_expandable": False, + "frames": [], + "blocks": blocks, + } + + +def lifecycle_data(): + return { + "segments": [ + segment(0, 64, [block(1000, 16, 14), block(1040, 8, 6)]), + segment(1, 32, [block(1000, 8, 5)]), + ], + "device_traces": [ + [ + event("alloc"), + event("alloc", size=8, stream=1), + event("oom"), + event("free_requested"), + event("oom"), + event("free_completed"), + event("alloc"), + event("oom"), + event("oom"), + ] + ], + } + + +def write_source(tmp_path, data): + path = tmp_path / "input.pkl" + save_snapshot_representation(data, path, "pickle") + return path + + +def rows(path, sql): + with closing(sqlite3.connect(path)) as conn: + return conn.execute(sql).fetchall() + + +@pytest.mark.parametrize("capacity", [1, 2, 3, 100]) +def test_shards_equal_single_db_events_blocks_boundaries_and_references(tmp_path, capacity): + data = lifecycle_data() + original = copy.deepcopy(data) + source = write_source(tmp_path, data) + full = tmp_path / "full.db" + assert snapshot2db.dump(source, full) + result = ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=capacity) + full_events = rows( + full, + "SELECT t.id,t.action,t.address,t.size,t.stream,t.allocated,t.active,t.reserved,c.callstack FROM trace_entry_0 t JOIN callstack c ON c.id=t.callstackId WHERE t.id>=0 ORDER BY t.id", + ) + collected = [] + identities = {} + for item in result.slices: + collected += rows( + item.file, + "SELECT t.id,t.action,t.address,t.size,t.stream,t.allocated,t.active,t.reserved,c.callstack FROM trace_entry_0 t JOIN callstack c ON c.id=t.callstackId WHERE t.id>=0 ORDER BY t.id", + ) + assert len(rows(item.file, "SELECT id FROM trace_entry_0 WHERE id>=0")) <= capacity + references = { + row[0]: row[1:] for row in rows(item.file, "SELECT * FROM pt_snap_block_reference") + } + shard_blocks = rows(item.file, "SELECT * FROM block_0") + for record in shard_blocks: + signature = record[1:4] + record[5:] + assert record[0] not in identities or identities[record[0]] == signature + identities[record[0]] = signature + if record[5] >= 0: + assert record[0] == record[5] + assert references[record[0]][1] == "source.py:4 alloc" + else: + assert record[0] < 0 and references[record[0]][1] is None + if record[6] >= 0: + assert record[6] == 5 # Never free_requested=3. + assert references[record[0]][2] == "source.py:4 free_completed" + # Replay the full simulator to the identical left boundary as an oracle. + oracle = SimulateDeviceSnapshot(data, 0) + assert oracle.replay_until(item.start_position) + state = oracle.device_snapshot + expected_blocks = sorted( + (seg.stream, b.address, b.size, b.requested_size, b.state) + for seg in state.segments + for b in seg.blocks + ) + observed_blocks = sorted( + ( + references[r[0]][0], + r[1], + r[2], + r[3], + {-1: "inactive", 0: "active_pending_free", 1: "active_allocated"}[r[4]], + ) + for r in shard_blocks + if (r[5] == -1 or r[5] < item.start_event_id) + and (r[6] == -1 or r[6] >= item.start_event_id) + ) + assert observed_blocks == expected_blocks + boundaries = rows( + item.file, + "SELECT action,address,size,stream,allocated,active,reserved FROM trace_entry_0 WHERE id<0", + ) + assert sorted(row[1:4] for row in boundaries) == sorted( + (seg.address, seg.total_size, seg.stream) for seg in state.segments + ) + assert all( + row[4:] == (state.total_allocated, state.total_activated, state.total_reserved) + for row in boundaries + ) + # Active geometry at EVERY real event, not just shard edges. + for position in range(item.start_position, item.end_position + 1): + current = SimulateDeviceSnapshot(data, 0) + assert current.replay_until(position + 1) + expected = sorted( + (seg.stream, b.address, b.size, b.requested_size) + for seg in current.device_snapshot.segments + for b in seg.blocks + ) + active = sorted( + (references[r[0]][0], r[1], r[2], r[3]) + for r in shard_blocks + if (r[5] == -1 or r[5] <= position) and (r[6] == -1 or r[6] > position) + ) + assert active == expected + assert collected == full_events + expected_signatures = {row[1:4] + row[5:] for row in rows(full, "SELECT * FROM block_0")} + assert set(identities.values()) == expected_signatures + assert {key for key in identities if key >= 0} == {0, 1, 6} + assert data == original + assert not (result.directory / "manifest.json").exists() + + +def test_pending_free_left_boundary_has_state_zero(tmp_path): + result = ShardedReplayService().stage( + write_source(tmp_path, lifecycle_data()), tmp_path / "stage", events_per_slice=2 + ) + assert rows(result.slices[2].file, "SELECT id,state,freeEventId FROM block_0 WHERE id=0") == [ + (0, 0, 5) + ] + + +@pytest.mark.parametrize( + "fixture", + [ + "snapshot_expandable.pkl", + "snapshot_with_empty_cache_expandable.pkl", + "snapshot_with_multi_devices.pkl", + ], +) +def test_reviewed_fixtures_real_events_match_single_db(tmp_path, fixture): + source = FIXTURE_DIR / fixture + full = tmp_path / "full.db" + assert snapshot2db.dump(source, full) + result = ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=1000) + for device in {item.device for item in result.slices}: + observed = [] + for item in result.slices: + if item.device == device: + observed += rows( + item.file, + f"SELECT id,action,address,size,stream,allocated,active,reserved FROM trace_entry_{device} WHERE id>=0 ORDER BY id", + ) + assert observed == rows( + full, + f"SELECT id,action,address,size,stream,allocated,active,reserved FROM trace_entry_{device} WHERE id>=0 ORDER BY id", + ) + + +def test_workspace_oom_and_sparse_native_ids_are_not_relabelled(tmp_path): + data = make_torch_npu_workspace_snapshot() + data["device_traces"][0].append( + {"action": "oom", "size": 2048, "device_free": 512, "frames": []} + ) + for index, entry in enumerate(data["device_traces"][0]): + entry["id"] = 10 + index * 2 + source = write_source(tmp_path, data) + result = ShardedReplayService().stage(source, tmp_path / "stage", events_per_slice=1) + actions = [ + rows( + item.file, "SELECT id,action,allocated,active,reserved FROM trace_entry_0 WHERE id>=0" + )[0] + for item in result.slices + ] + assert [row[:2] for row in actions] == [(10, 7), (12, 2), (14, 4), (16, 8)] + assert actions[-1][2:] == (4096, 4096, 4096) + assert rows(result.slices[-1].file, "SELECT id,allocEventId FROM block_0") == [(14, 14)] + + +def test_pause_resume_matches_full_hooks_and_state(): + data = lifecycle_data() + complete = SimulateDeviceSnapshot(data, 0) + paused = SimulateDeviceSnapshot(data, 0) + hooks = [Hooker(), Hooker()] + complete.register_hooker(hooks[0]) + paused.register_hooker(hooks[1]) + assert complete.replay() + for endpoint in [9, 7, 7, 5, 2, 0, 0]: + assert paused.replay_until(endpoint) + assert hooks[0].calls == hooks[1].calls + assert complete.device_snapshot.to_dict() == paused.device_snapshot.to_dict() + + +@pytest.mark.parametrize("endpoint", [-1, True, 1.5, 10]) +def test_invalid_pause_endpoint_preserves_state(endpoint): + simulator = SimulateDeviceSnapshot(lifecycle_data(), 0) + before = simulator.device_snapshot.to_dict() + with pytest.raises(ValueError, match="remaining_events"): + simulator.replay_until(endpoint) + assert simulator.device_snapshot.to_dict() == before + + +@pytest.mark.parametrize("phase", ["pre", "post", "allocator"]) +def test_pause_failure_semantics_remain_original(phase, monkeypatch): + simulator = SimulateDeviceSnapshot(lifecycle_data(), 0) + hook = Hooker(pre=phase != "pre", post=phase != "post") + simulator.register_hooker(hook) + if phase == "allocator": + monkeypatch.setattr(simulator.replay_executor, "execute", lambda _: False) + assert not simulator.replay_until(8) + assert len(simulator.device_snapshot.trace_entries) == (8 if phase == "post" else 9) + + +@pytest.mark.parametrize( + "failure", ["replay", "boundary", "flush", "backfill", "validation", "constructor"] +) +def test_failures_close_every_writer_and_return_no_ready_result(tmp_path, monkeypatch, failure): + connections = [] + original = SnapshotDb.__init__ + + def track(self, *args, **kwargs): + original(self, *args, **kwargs) + connections.append(self.conn) + + monkeypatch.setattr(SnapshotDb, "__init__", track) + + def fail(*args, **kwargs): + raise RuntimeError("injected failure") + + if failure == "replay": + monkeypatch.setattr(SimulateDeviceSnapshot, "replay_until", lambda *_: False) + elif failure == "boundary": + monkeypatch.setattr(sharded_replay._SliceHook, "dump_left_boundary", fail) + elif failure == "flush": + monkeypatch.setattr(snapshot2db.SnapshotDbHandler, "flush", fail) + elif failure == "constructor": + monkeypatch.setattr(SnapshotDb, "create_block_table", fail) + elif failure == "backfill": + monkeypatch.setattr(sharded_replay.BlockRegistry, "finalize", fail) + else: + monkeypatch.setattr(sharded_replay.BlockRegistry, "validate", fail) + with pytest.raises(RuntimeError): + ShardedReplayService().stage( + write_source(tmp_path, lifecycle_data()), tmp_path / "stage", events_per_slice=2 + ) + assert connections + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + assert not (tmp_path / "stage" / "manifest.json").exists() + + +def test_registry_does_not_merge_distinct_same_address_objects(): + registry = sharded_replay.BlockRegistry() + a = Block(address=1, size=2, requested_size=1, state=BlockState.ACTIVE_ALLOCATED) + b = copy.copy(a) + assert registry.observe(a, 0).token != registry.observe(b, 0).token + assert registry.observe(a, 0).token == -1 + with pytest.raises(RuntimeError, match="stream"): + registry.observe(a, 1) + + +def test_finalization_sql_failure_closes_rw_connection(tmp_path, monkeypatch): + original = sharded_replay.BlockRegistry.finalize + captured = [] + + def broken(self): + path = next(iter(self.members)) + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("DELETE FROM block_0") + real_connect = sqlite3.connect + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + captured.append(conn) + return conn + + monkeypatch.setattr(sharded_replay.sqlite3, "connect", connect) + return original(self) + + monkeypatch.setattr(sharded_replay.BlockRegistry, "finalize", broken) + with pytest.raises(RuntimeError, match="Missing provisional"): + ShardedReplayService().stage( + write_source(tmp_path, lifecycle_data()), tmp_path / "stage", events_per_slice=2 + ) + assert captured + for conn in captured: + with pytest.raises(sqlite3.ProgrammingError): + conn.execute("SELECT 1") + + +def test_snapshot_db_constructor_failure_closes_connection(tmp_path, monkeypatch): + captured = [] + + def fail(self): + captured.append(self.conn) + raise RuntimeError("constructor failure") + + monkeypatch.setattr(SnapshotDb, "_clear_old_tables", fail) + with pytest.raises(RuntimeError): + SnapshotDb(str(tmp_path / "failure.db")) + with pytest.raises(sqlite3.ProgrammingError): + captured[0].execute("SELECT 1") + + +def test_writer_commit_failure_closes_connection(tmp_path): + handler = snapshot2db.SnapshotDbHandler(str(tmp_path / "db"), [0]) + conn = handler.db.conn + + class BrokenCommit: + def commit(self): + raise RuntimeError("commit failure") + + def close(self): + conn.close() + + handler.db.conn = BrokenCommit() + with pytest.raises(RuntimeError): + handler.close() + handler.close() + with pytest.raises(sqlite3.ProgrammingError): + conn.execute("SELECT 1") + + +def test_actual_sql_backfill_abort_rolls_back_and_closes(tmp_path, monkeypatch): + original = sharded_replay.BlockRegistry.finalize + real_connect = sqlite3.connect + captured, paths = [], [] + + def rejected(self): + path = next(iter(self.members)) + paths.append(path) + with closing(real_connect(path)) as conn, conn: + conn.execute( + "CREATE TRIGGER reject_patch BEFORE UPDATE ON block_0 " + "BEGIN SELECT RAISE(ABORT, 'backfill rejected'); END" + ) + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + captured.append(conn) + return conn + + monkeypatch.setattr(sharded_replay.sqlite3, "connect", connect) + original(self) + + monkeypatch.setattr(sharded_replay.BlockRegistry, "finalize", rejected) + with pytest.raises(sqlite3.IntegrityError, match="backfill rejected"): + ShardedReplayService().stage( + write_source(tmp_path, lifecycle_data()), tmp_path / "stage", events_per_slice=2 + ) + for conn in captured: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + with closing(real_connect(paths[0])) as conn: + assert all(row[0] < 0 for row in conn.execute("SELECT id FROM block_0")) + assert not conn.execute( + "SELECT name FROM sqlite_master WHERE name='pt_snap_block_reference'" + ).fetchall() + + +@pytest.mark.parametrize( + ("sql", "message"), + [ + ("DELETE FROM trace_entry_0 WHERE id>=0", "lost or changed real events"), + ("UPDATE block_0 SET requestedSize=0", "identity/lifetime"), + ("DELETE FROM pt_snap_block_reference", "callstack references"), + ], +) +def test_final_validation_rejects_corrupt_finalized_shard(tmp_path, monkeypatch, sql, message): + original = sharded_replay.BlockRegistry.finalize + + def corrupt(self): + original(self) + path = next(iter(self.members)) + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute(sql) + + monkeypatch.setattr(sharded_replay.BlockRegistry, "finalize", corrupt) + with pytest.raises(RuntimeError, match=message): + ShardedReplayService().stage( + write_source(tmp_path, lifecycle_data()), tmp_path / "stage", events_per_slice=2 + ) + assert not (tmp_path / "stage" / "manifest.json").exists()