From 48073031ac0cd6fb034557c226141f0ad3dfdef5 Mon Sep 17 00:00:00 2001 From: codeasier Date: Wed, 19 Aug 2026 22:32:20 +0800 Subject: [PATCH 1/3] fix: restore dump-time torch-npu workspace snapshot correction (#98) torch-npu dumps live workspace pools as inactive/zero occupancy, so import was recording empty dump-time allocated/active state. Co-authored-by: Cursor --- src/pt_snap_cli/snapshot/PROVENANCE.md | 10 ++ src/pt_snap_cli/snapshot/simulate/simulate.py | 69 +++++++- tests/snapshot/test_simulate.py | 148 +++++++++++++++++- 3 files changed, 224 insertions(+), 3 deletions(-) diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 8afbec5..6a98edc 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -123,6 +123,16 @@ The upstream standalone split frontend is deliberately excluded, not deferred: objects and preserves source frame dictionaries through synthetic segments. This is a local performance change; audited upstream mappings and licensing evidence are unchanged. +- On 2026-08-19, issue #98 added a first-party dump-time torch-npu workspace + pool correction after inactive-block filtering. `_adapt_workspace_snapshot` + is not present in audited MemSnapDump @87ea207; this is a local + reimplementation of the grouping behavior specified in #98. Consecutive + `workspace_snapshot` + `segment_alloc` + `alloc` groups rewrite the + matching dump-time segment to a single `active_allocated` block and + update device allocated/active totals. Size mismatches and missing + segments warn and stop later groups. `workspace_flag` remains a + fallback for unmatched workspace traces. Audited upstream source + mappings and license terms are unchanged. ## Migration toolchain diff --git a/src/pt_snap_cli/snapshot/simulate/simulate.py b/src/pt_snap_cli/snapshot/simulate/simulate.py index 1cb591e..96e0612 100644 --- a/src/pt_snap_cli/snapshot/simulate/simulate.py +++ b/src/pt_snap_cli/snapshot/simulate/simulate.py @@ -1,7 +1,8 @@ from logging import Logger -from ..base import DeviceSnapshot +from ..base import Block, BlockState, DeviceSnapshot from ..util import get_logger +from . import snapshot_lookup, snapshot_mutator from .allocator_context import AllocatorContext from .hooker_defs import AllocatorHooker, SimulateHooker from .replay_executor import ReplayExecutor @@ -46,6 +47,72 @@ def __init__(self, snapshot_dict: dict, device: int, *, _raw_frames: bool = Fals and self.device_snapshot.trace_entries[0].action == "workspace_snapshot" ): self.simulated_allocator_context.workspace_flag = True + # 校正 dump 时刻 workspace 池占用,必须在 ignore_inactive_blocks 之后 + self._adapt_workspace_snapshot() + + def _adapt_workspace_snapshot(self): + """Fix dump-time occupancy for torch-npu workspace pools. + + torch-npu prepends consecutive ``workspace_snapshot`` + ``segment_alloc`` + + ``alloc`` groups for each live workspace pool, but dumps the matching + segment as one inactive block with allocated/active size 0. After + ``ignore_inactive_blocks=True`` that block is gone, so replay cannot + find it and dump-time curves stay at zero. + + Matched groups become a single ``active_allocated`` block covering the + segment, with allocated/active/device totals equal to the workspace + size. Missing segments or size mismatches warn and stop later groups; + ``workspace_flag`` remains the fallback for those cases. + """ + self._loading_logger.info("Recognized workspace events in snapshot, start adapting...") + snapshot = self.device_snapshot + events = snapshot.trace_entries + group_start = 0 + while group_start + 2 < len(events): + group = events[group_start : group_start + 3] + if [event.action for event in group] != [ + "workspace_snapshot", + "segment_alloc", + "alloc", + ]: + break + workspace_snapshot = group[0] + _, existed_seg = snapshot_lookup.find_segment( + snapshot, workspace_snapshot.addr, workspace_snapshot.stream + ) + if existed_seg is None: + self._loading_logger.warning( + f"Workspace snapshot at addr {workspace_snapshot.addr} " + f"(stream {workspace_snapshot.stream}) not found in device snapshot" + ) + break + if existed_seg.total_size != workspace_snapshot.size: + self._loading_logger.warning( + f"Workspace snapshot at addr {workspace_snapshot.addr} " + f"(stream {workspace_snapshot.stream}) size {workspace_snapshot.size} " + f"does not match segment total_size {existed_seg.total_size}" + ) + break + snapshot.total_allocated -= existed_seg.allocated_size + snapshot.total_activated -= existed_seg.active_size + existed_seg.allocated_size = 0 + existed_seg.active_size = 0 + existed_seg.blocks = [] + snapshot_mutator.attach_block( + snapshot, + existed_seg, + Block( + size=existed_seg.total_size, + requested_size=existed_seg.total_size, + address=existed_seg.address, + state=BlockState.ACTIVE_ALLOCATED, + frames=existed_seg.frames, + segment_ptr=existed_seg, + ), + 0, + ) + group_start += 3 + self._loading_logger.info("Finished to adapt workspace snapshot.") def register_hooker(self, hooker: SimulateHooker) -> int: idx = hash(hooker) diff --git a/tests/snapshot/test_simulate.py b/tests/snapshot/test_simulate.py index 4514729..b9e7bfa 100644 --- a/tests/snapshot/test_simulate.py +++ b/tests/snapshot/test_simulate.py @@ -1,3 +1,4 @@ +import logging import os import unittest from collections import Counter @@ -169,15 +170,99 @@ def make_snapshot_dict(action="alloc", with_segment=False): return payload +def _inactive_workspace_segment(addr, size, stream): + return { + "address": addr, + "total_size": size, + "stream": stream, + "segment_type": "small", + "allocated_size": 0, + "active_size": 0, + "device": 0, + "is_expandable": False, + "frames": [], + "blocks": [ + { + "size": size, + "requested_size": size, + "state": "inactive", + "address": addr, + "frames": [], + } + ], + } + + +def _workspace_triplet(addr, size, stream): + return [ + { + "action": "workspace_snapshot", + "addr": addr, + "size": size, + "stream": stream, + "frames": [], + }, + { + "action": "segment_alloc", + "addr": addr, + "size": size, + "stream": stream, + "frames": [], + }, + {"action": "alloc", "addr": addr, "size": size, "stream": stream, "frames": []}, + ] + + +def make_torch_npu_workspace_snapshot( + *, addr=1000, size=4096, stream=1, segment_size=None, extra_segments=None, extra_events=None +): + segment_size = size if segment_size is None else segment_size + segments = [_inactive_workspace_segment(addr, segment_size, stream)] + if extra_segments: + segments.extend(extra_segments) + events = _workspace_triplet(addr, size, stream) + if extra_events: + events.extend(extra_events) + return {"segments": segments, "device_traces": [events]} + + +def _segment_by_addr_stream(snapshot, addr, stream): + matches = [ + segment + for segment in snapshot.device_snapshot.segments + if segment.address == addr and segment.stream == stream + ] + assert len(matches) == 1 + return matches[0] + + +def assert_dump_time_workspace_pool(snapshot, *, addr=1000, size=4096, stream=1): + assert snapshot.simulated_allocator_context.workspace_flag is True + segment = _segment_by_addr_stream(snapshot, addr, stream) + assert segment.allocated_size == size + assert segment.active_size == size + assert len(segment.blocks) == 1 + block = segment.blocks[0] + assert block.state == BlockState.ACTIVE_ALLOCATED + assert block.address == addr + assert block.size == size + assert_valid_snapshot(snapshot.device_snapshot) + + def test_simulate_device_snapshot_raises_on_empty_snapshot(): with pytest.raises(RuntimeError): SimulateDeviceSnapshot({}, 0) def test_simulate_device_snapshot_sets_workspace_flag_from_first_event(): - snapshot = SimulateDeviceSnapshot(make_snapshot_dict("workspace_snapshot"), 0) + snapshot = SimulateDeviceSnapshot(make_torch_npu_workspace_snapshot(), 0) + device = snapshot.device_snapshot assert snapshot.simulated_allocator_context.workspace_flag is True + assert_dump_time_workspace_pool(snapshot) + assert device.total_allocated == 4096 + assert device.total_activated == 4096 + assert device.total_reserved == 4096 def test_simulate_register_unregister_hookers_and_allocator_hookers(): @@ -253,9 +338,68 @@ def test_multi_device_fixture_keeps_device_state_isolated(): assert snapshot_observation(device_zero) != snapshot_observation(device_one) -def test_minimal_workspace_input_only_establishes_tolerance_flag_and_skip_behavior(): +def test_minimal_workspace_input_corrects_dump_time_state_before_replay(caplog): + snapshot = SimulateDeviceSnapshot(make_torch_npu_workspace_snapshot(), 0) + + assert_dump_time_workspace_pool(snapshot) + with caplog.at_level(logging.WARNING, logger="ALLOCATOR"): + assert snapshot.replay() is True + assert "workspace scenario tolerance" not in caplog.text + assert snapshot.device_snapshot.trace_entries == [] + assert snapshot.device_snapshot.segments == [] + assert snapshot.device_snapshot.total_allocated == 0 + assert snapshot.device_snapshot.total_activated == 0 + assert snapshot.device_snapshot.total_reserved == 0 + + +def test_incomplete_workspace_marker_keeps_tolerance_flag_and_skip_behavior(): snapshot = SimulateDeviceSnapshot(make_snapshot_dict("workspace_snapshot"), 0) assert snapshot.simulated_allocator_context.workspace_flag is True assert snapshot.replay() is True assert snapshot.device_snapshot.trace_entries == [] + + +def test_workspace_adapt_multiple_stream_groups_before_replay(): + data = { + "segments": [ + _inactive_workspace_segment(1000, 4096, 1), + _inactive_workspace_segment(10000, 8192, 2), + ], + "device_traces": [_workspace_triplet(1000, 4096, 1) + _workspace_triplet(10000, 8192, 2)], + } + snapshot = SimulateDeviceSnapshot(data, 0) + + assert_dump_time_workspace_pool(snapshot, addr=1000, size=4096, stream=1) + assert_dump_time_workspace_pool(snapshot, addr=10000, size=8192, stream=2) + assert snapshot.device_snapshot.total_allocated == 12288 + assert snapshot.device_snapshot.total_activated == 12288 + + +def test_workspace_adapt_warns_and_stops_when_segment_missing(caplog): + data = { + "segments": [_inactive_workspace_segment(2000, 8192, 2)], + "device_traces": [_workspace_triplet(1000, 4096, 1) + _workspace_triplet(2000, 8192, 2)], + } + with caplog.at_level(logging.WARNING, logger="LOAD"): + snapshot = SimulateDeviceSnapshot(data, 0) + + assert "Workspace snapshot at addr 1000 (stream 1) not found" in caplog.text + later = _segment_by_addr_stream(snapshot, 2000, 2) + assert later.allocated_size == 0 + assert later.active_size == 0 + assert later.blocks == [] + + +def test_workspace_adapt_warns_and_skips_when_size_mismatches(caplog): + data = make_torch_npu_workspace_snapshot(size=4096, segment_size=8192) + with caplog.at_level(logging.WARNING, logger="LOAD"): + snapshot = SimulateDeviceSnapshot(data, 0) + + assert "does not match segment total_size" in caplog.text + segment = _segment_by_addr_stream(snapshot, 1000, 1) + assert segment.allocated_size == 0 + assert segment.active_size == 0 + assert segment.blocks == [] + assert snapshot.device_snapshot.total_allocated == 0 + assert snapshot.simulated_allocator_context.workspace_flag is True From 2e81f8c034830c5aad2de426cd2a8264811bd27c Mon Sep 17 00:00:00 2001 From: codeasier Date: Wed, 19 Aug 2026 23:50:27 +0800 Subject: [PATCH 2/3] test: lock workspace adapt when a later group fails Earlier tests only failed the first group, so partial application of later-stop was untested. Co-authored-by: Cursor --- tests/snapshot/test_simulate.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/snapshot/test_simulate.py b/tests/snapshot/test_simulate.py index b9e7bfa..7dd787d 100644 --- a/tests/snapshot/test_simulate.py +++ b/tests/snapshot/test_simulate.py @@ -391,6 +391,30 @@ def test_workspace_adapt_warns_and_stops_when_segment_missing(caplog): assert later.blocks == [] +def test_workspace_adapt_keeps_earlier_group_when_later_group_is_missing(caplog): + data = { + "segments": [ + _inactive_workspace_segment(1000, 4096, 1), + _inactive_workspace_segment(10000, 8192, 2), + ], + "device_traces": [_workspace_triplet(1000, 4096, 1) + _workspace_triplet(2000, 8192, 2)], + } + with caplog.at_level(logging.WARNING, logger="LOAD"): + snapshot = SimulateDeviceSnapshot(data, 0) + + assert_dump_time_workspace_pool(snapshot, addr=1000, size=4096, stream=1) + later = _segment_by_addr_stream(snapshot, 10000, 2) + assert later.allocated_size == 0 + assert later.active_size == 0 + assert later.blocks == [] + assert snapshot.device_snapshot.total_allocated == 4096 + assert snapshot.device_snapshot.total_activated == 4096 + assert snapshot.device_snapshot.total_reserved == 12288 + warnings = [record for record in caplog.records if record.levelno == logging.WARNING] + assert len(warnings) == 1 + assert "Workspace snapshot at addr 2000 (stream 2) not found" in caplog.text + + def test_workspace_adapt_warns_and_skips_when_size_mismatches(caplog): data = make_torch_npu_workspace_snapshot(size=4096, segment_size=8192) with caplog.at_level(logging.WARNING, logger="LOAD"): From 8e6cbf5d68850cf1db268a3501cc84cfdb70fe8a Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 20 Aug 2026 00:27:04 +0800 Subject: [PATCH 3/3] fix: skip workspace adapt on inconsistent triplets and live blocks Adapting those dumps replaced a tolerable missing-block replay path with hard size/segment failures, and silently discarded leftover allocations. Co-authored-by: Cursor --- src/pt_snap_cli/snapshot/PROVENANCE.md | 5 +- src/pt_snap_cli/snapshot/simulate/simulate.py | 26 +++++++- tests/snapshot/test_simulate.py | 62 +++++++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 6a98edc..8707b8a 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -129,8 +129,9 @@ The upstream standalone split frontend is deliberately excluded, not deferred: reimplementation of the grouping behavior specified in #98. Consecutive `workspace_snapshot` + `segment_alloc` + `alloc` groups rewrite the matching dump-time segment to a single `active_allocated` block and - update device allocated/active totals. Size mismatches and missing - segments warn and stop later groups. `workspace_flag` remains a + update device allocated/active totals. Size mismatches, missing + segments, internally inconsistent triplets, and leftover live blocks + warn and stop later groups. `workspace_flag` remains a fallback for unmatched workspace traces. Audited upstream source mappings and license terms are unchanged. diff --git a/src/pt_snap_cli/snapshot/simulate/simulate.py b/src/pt_snap_cli/snapshot/simulate/simulate.py index 96e0612..6968d50 100644 --- a/src/pt_snap_cli/snapshot/simulate/simulate.py +++ b/src/pt_snap_cli/snapshot/simulate/simulate.py @@ -61,8 +61,9 @@ def _adapt_workspace_snapshot(self): Matched groups become a single ``active_allocated`` block covering the segment, with allocated/active/device totals equal to the workspace - size. Missing segments or size mismatches warn and stop later groups; - ``workspace_flag`` remains the fallback for those cases. + size. Missing segments, size mismatches, internally inconsistent + triplets, or leftover live blocks warn and stop later groups. + ``workspace_flag`` remains the fallback for those skipped cases. """ self._loading_logger.info("Recognized workspace events in snapshot, start adapting...") snapshot = self.device_snapshot @@ -76,7 +77,18 @@ def _adapt_workspace_snapshot(self): "alloc", ]: break - workspace_snapshot = group[0] + workspace_snapshot, segment_alloc, alloc = group + if ( + segment_alloc.addr != workspace_snapshot.addr + or segment_alloc.size != workspace_snapshot.size + or alloc.addr != workspace_snapshot.addr + or alloc.size != workspace_snapshot.size + ): + self._loading_logger.warning( + f"Workspace snapshot triplet at addr {workspace_snapshot.addr} " + f"(stream {workspace_snapshot.stream}) is internally inconsistent" + ) + break _, existed_seg = snapshot_lookup.find_segment( snapshot, workspace_snapshot.addr, workspace_snapshot.stream ) @@ -93,6 +105,14 @@ def _adapt_workspace_snapshot(self): f"does not match segment total_size {existed_seg.total_size}" ) break + if existed_seg.blocks: + n_live = len(existed_seg.blocks) + self._loading_logger.warning( + f"Workspace snapshot at addr {workspace_snapshot.addr} " + f"(stream {workspace_snapshot.stream}) skipped because the matching " + f"segment still has {n_live} live block{'s' if n_live != 1 else ''}" + ) + break snapshot.total_allocated -= existed_seg.allocated_size snapshot.total_activated -= existed_seg.active_size existed_seg.allocated_size = 0 diff --git a/tests/snapshot/test_simulate.py b/tests/snapshot/test_simulate.py index 7dd787d..e26135a 100644 --- a/tests/snapshot/test_simulate.py +++ b/tests/snapshot/test_simulate.py @@ -427,3 +427,65 @@ def test_workspace_adapt_warns_and_skips_when_size_mismatches(caplog): assert segment.blocks == [] assert snapshot.device_snapshot.total_allocated == 0 assert snapshot.simulated_allocator_context.workspace_flag is True + + +def test_workspace_adapt_skips_inconsistent_triplet_so_replay_keeps_tolerance(caplog): + data = make_torch_npu_workspace_snapshot() + data["device_traces"][0][2]["size"] = 8192 + with caplog.at_level(logging.WARNING): + snapshot = SimulateDeviceSnapshot(data, 0) + + segment = _segment_by_addr_stream(snapshot, 1000, 1) + assert segment.allocated_size == 0 + assert segment.blocks == [] + assert "internally inconsistent" in caplog.text + assert snapshot.replay() is True + assert "workspace scenario tolerance" in caplog.text + + +def test_workspace_adapt_skips_when_segment_still_has_live_blocks(caplog): + data = { + "segments": [ + { + "address": 1000, + "total_size": 4096, + "stream": 1, + "segment_type": "small", + "allocated_size": 1024, + "active_size": 1024, + "device": 0, + "is_expandable": False, + "frames": [], + "blocks": [ + { + "size": 1024, + "requested_size": 1024, + "state": "active_allocated", + "address": 1000, + "frames": [], + }, + { + "size": 3072, + "requested_size": 3072, + "state": "inactive", + "address": 2048, + "frames": [], + }, + ], + } + ], + "device_traces": [ + _workspace_triplet(1000, 4096, 1) + + [{"action": "alloc", "addr": 1000, "size": 1024, "stream": 1, "frames": []}] + ], + } + with caplog.at_level(logging.WARNING, logger="LOAD"): + snapshot = SimulateDeviceSnapshot(data, 0) + + segment = _segment_by_addr_stream(snapshot, 1000, 1) + assert len(segment.blocks) == 1 + assert segment.blocks[0].size == 1024 + assert segment.blocks[0].state == BlockState.ACTIVE_ALLOCATED + assert segment.allocated_size == 1024 + assert snapshot.device_snapshot.total_allocated == 1024 + assert "still has 1 live block" in caplog.text