From c98b66b3ac0fb668267398431edd5f81d296292b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:38:04 +0800 Subject: [PATCH] fix: bound exporter event-id validation memory --- docs/en/sharded-snapshotdb.md | 6 + docs/zh/sharded-snapshotdb.md | 5 + src/pt_snap_cli/core/msinsight_export.py | 69 +++++-- .../core/test_msinsight_export_validation.py | 193 ++++++++++++++++++ 4 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 tests/core/test_msinsight_export_validation.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53bf44a..afa5867 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -457,6 +457,12 @@ Those queries and global aggregation follow the P3 query matrix. Text cannot rec original ordered frame arrays; original workspace raw-frame loss and string-action OOM coercion are known upstream limitations, not compatible features to imitate. +Before converting any shard, the exporter checks every selected device's real IDs +and all actions. ID continuity uses each shard's SQL `COUNT/MIN/MAX` and a per-device +next expected ID, after verifying a unique integer primary key; negative boundary +rows are excluded. It does not collect the device's event IDs in Python. This bounds +additional ID-validation memory, not snapshot loading, replay, or total import RSS. + `cacheHash = SHA256(b"mem_snapshot_parser_v2" + raw_pickle_bytes)`: the salt is **prepended**. Independent `ptSnap.identity` fields are `sourceSha256`, `requestedDevice`, `eventsPerSlice`, `outputFormat=compatibility-v1`, diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index f89cfad..7e889ae 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -365,6 +365,11 @@ native-v2 输出仍独立且不变。 跨来源查询/全局聚合遵循 P3 查询矩阵。文本不能恢复原始有序 frame 数组;原版 workspace raw-frame 丢失与字符串 OOM action 强制转换属于已知局限,不应仿制为兼容能力。 +转换任何分片前,导出器先校验所有选中设备的真实 ID 和全部 action。确认 ID 为唯一整数 +主键后,用逐片 SQL `COUNT/MIN/MAX` 和每设备下一个预期 ID 校验连续性,排除负 ID +边界行,不再在 Python 中收集整个设备的事件 ID。这只限制额外的 ID 校验内存, +不代表快照加载、回放或完整导入的 RSS 有界。 + `cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`,salt **前置**。 独立 `ptSnap.identity` 字段为 `sourceSha256`、`requestedDevice`、`eventsPerSlice`、 `outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=1`、 diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py index 834efeb..e1764af 100644 --- a/src/pt_snap_cli/core/msinsight_export.py +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -22,10 +22,10 @@ MANIFEST_SCHEMA_VERSION, MSINSIGHT_REVISION, TRACE_COLUMNS, + DatasetContractError, DatasetManifest, _unique_object, validate_dataset, - validate_event_ids, ) from .dataset_files import hash_file, require_readonly_member from .dataset_import_backend import DatasetImportBackend @@ -81,33 +81,74 @@ def inspect_target(target: Path) -> None: return None -def convert_shards(result: ShardedReplayResult) -> None: - """Preserve event/stream/lifecycle values, materialize inline ninth column. +def _require_integer_primary_key(conn: sqlite3.Connection, trace: str) -> None: + """Prove unique integer IDs before relying on count and range endpoints. - First validate all selected IDs/actions, without inventing event zero or - coercing unsupported OOM. Block/reference tables are left byte-semantically - intact; no foreign real events are copied into the slice window. + The native writer uses a rowid alias, which enforces integer storage. SQLite's + inline PRIMARY KEY DESC exception and WITHOUT ROWID tables instead have a + primary-key index: retain support for these only after checking storage types. """ + kind = conn.execute("SELECT type FROM sqlite_master WHERE name=?", (trace,)).fetchone() + primary_key = [row for row in conn.execute(f'PRAGMA table_info("{trace}")') if row[5]] + if ( + kind != ("table",) + or len(primary_key) != 1 + or primary_key[0][1] != "id" + or primary_key[0][2].upper() != "INTEGER" + or primary_key[0][5] != 1 + ): + raise ValueError(f"{trace}.id must be the sole INTEGER PRIMARY KEY") + if any(row[3] == "pk" for row in conn.execute(f'PRAGMA index_list("{trace}")')) and ( + conn.execute(f"SELECT 1 FROM {trace} WHERE typeof(id)!='integer' LIMIT 1").fetchone() + is not None + ): + raise ValueError(f"{trace}.id INTEGER PRIMARY KEY must contain only integers") + + +def _validate_shards(result: ShardedReplayResult) -> None: + """Validate every selected device before conversion without collecting IDs.""" for device in sorted({s.device for s in result.slices}): - ids: list[int] = [] + next_expected = 0 for item in (s for s in result.slices if s.device == device): require_readonly_member(result.directory, item.file) with closing(sqlite3.connect(item.file.as_uri() + "?mode=ro", uri=True)) as conn: - ids.extend( - row[0] - for row in conn.execute( - f"SELECT id FROM trace_entry_{device} WHERE id>=0 ORDER BY id" + trace = f"trace_entry_{device}" + _require_integer_primary_key(conn, trace) + count, low, high = conn.execute( + f'SELECT COUNT(*), MIN(id), MAX(id) FROM "{trace}" WHERE id>=0' + ).fetchone() + # With unique integer IDs, count and endpoints prove there are + # no holes inside this shard. Checking each shard in order also + # rejects overlaps/gaps that a device-wide aggregate can hide. + if count and (low != next_expected or high != next_expected + count - 1): + raise DatasetContractError( + f"events[{next_expected}]", + "event_identity", + "compatibility requires original real IDs 0..N-1; do not silently renumber", ) - ) + next_expected += count if ( conn.execute( - f"SELECT id FROM trace_entry_{device} WHERE typeof(action)!='integer' " + f"SELECT id FROM {trace} WHERE typeof(action)!='integer' " "OR action NOT BETWEEN 0 AND 7 OR action IS NULL LIMIT 1" ).fetchone() is not None ): raise ValueError("msinsight compatibility does not support OOM/unknown actions") - validate_event_ids(ids) + if not next_expected: + raise DatasetContractError( + "events", "empty", "empty/static-only devices cannot use compatibility v1" + ) + + +def convert_shards(result: ShardedReplayResult) -> None: + """Preserve event/stream/lifecycle values, materialize inline ninth column. + + First validate all selected IDs/actions, without inventing event zero or + coercing unsupported OOM. Block/reference tables are left byte-semantically + intact; no foreign real events are copied into the slice window. + """ + _validate_shards(result) for item in result.slices: device = item.device trace = f"trace_entry_{device}" diff --git a/tests/core/test_msinsight_export_validation.py b/tests/core/test_msinsight_export_validation.py new file mode 100644 index 0000000..b60b3e7 --- /dev/null +++ b/tests/core/test_msinsight_export_validation.py @@ -0,0 +1,193 @@ +"""Compatibility preflight stays per-shard and avoids a device-wide ID list.""" + +import sqlite3 +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_contract import DatasetContractError +from pt_snap_cli.core.msinsight_export import _validate_shards, convert_shards +from pt_snap_cli.core.sharded_replay_service import ShardedReplayResult +from pt_snap_cli.snapshot.tools.adaptors.sharded_replay import ReplaySlice +from tests.core.test_dataset_import import hashes + + +def shard( + root, + device, + index, + ids, + *, + id_schema="id INTEGER PRIMARY KEY", + suffix="", + table_suffix="", + action=7, +): + """Construct native-layout SQL only; no replay, pickle or source session.""" + path = root / f"device_{device}" / f"slice_{index:05d}.db" + path.parent.mkdir(parents=True, exist_ok=True) + trace = f"trace_entry_{device}" + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute( + f"CREATE TABLE {trace} ({id_schema}, action INTEGER, address INTEGER, size INTEGER, " + "stream INTEGER, allocated INTEGER, active INTEGER, reserved INTEGER, " + f"callstackId INTEGER{suffix}) {table_suffix}" + ) + conn.executemany( + f"INSERT INTO {trace} VALUES (?, ?, 0, 0, ?, 0, 0, 0, 0)", + ((value, action, position) for position, value in enumerate(ids)), + ) + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)") + conn.execute("INSERT INTO callstack VALUES (0, 'synthetic stack')") + conn.execute('CREATE TABLE dictionary ("table" TEXT, "column" TEXT, key TEXT, value TEXT)') + real = [value for value in ids if value is not None and value >= 0] + return ReplaySlice( + device, index, 0, len(real) - 1, min(real, default=0), max(real, default=-1), path + ) + + +def result(root, *items): + return ShardedReplayResult(root, items, ()) + + +def unchanged_failure(staged, error=DatasetContractError, match="event_identity"): + before = hashes(staged.directory) + with pytest.raises(error, match=match): + convert_shards(staged) + assert hashes(staged.directory) == before + + +@pytest.mark.parametrize( + "ids", + [ + ((1, 2),), # Nonzero start. + ((0, 2),), # Within-shard hole. + ((0, 1), (3, 4)), # Cross-shard hole. + ((0, 1), (1, 2)), # Cross-shard overlap. + ((2, 3), (0, 1)), # Reversed shard order. + ((0, 1), (3, 4), (4, 5)), # Global COUNT/MIN/MAX hides the hole and duplicate. + ((0, 2), (1, 3)), # Globally continuous IDs cannot repair per-shard holes/order. + ], +) +def test_rejects_invalid_continuity_without_converting_any_shard(tmp_path, ids): + items = tuple(shard(tmp_path, 0, index, values) for index, values in enumerate(ids)) + unchanged_failure(result(tmp_path, *items)) + + +@pytest.mark.parametrize("ids", [(), (-10, -1)]) +def test_empty_or_boundary_only_device_still_requires_a_real_event(tmp_path, ids): + first = shard(tmp_path, 0, 0, (0, 1)) + empty_device = shard(tmp_path, 3, 0, ids) + unchanged_failure(result(tmp_path, first, empty_device), match="empty/static-only") + + +def test_boundary_and_empty_shards_do_not_advance_real_ids(tmp_path): + items = ( + shard(tmp_path, 0, 0, (-2, -1)), + shard(tmp_path, 0, 1, (-100, 0, 1)), + shard(tmp_path, 0, 2, ()), + shard(tmp_path, 0, 3, (-20, 2, 3)), + ) + convert_shards(result(tmp_path, *items)) + for item in items: + with closing(sqlite3.connect(item.file)) as conn: + assert conn.execute("PRAGMA table_info(trace_entry_0)").fetchall()[-1][1] == "callstack" + + +def test_each_device_restarts_at_zero_even_when_shards_are_interleaved(tmp_path): + items = ( + shard(tmp_path, 0, 0, (0, 1)), + shard(tmp_path, 4, 0, (0,)), + shard(tmp_path, 0, 1, (2, 3)), + shard(tmp_path, 4, 1, (1, 2)), + ) + convert_shards(result(tmp_path, *items)) + + +def test_invalid_second_device_does_not_convert_first_device(tmp_path): + items = (shard(tmp_path, 0, 0, (0, 1)), shard(tmp_path, 4, 0, (2, 3))) + unchanged_failure(result(tmp_path, *items)) + + +@pytest.mark.parametrize("action", [8, -1, 1.5, "unknown", None]) +@pytest.mark.parametrize("boundary", [False, True]) +def test_actions_are_checked_on_real_and_boundary_rows_before_conversion( + tmp_path, action, boundary +): + items = ( + shard(tmp_path, 0, 0, (0, 1)), + shard(tmp_path, 0, 1, (-1,) if boundary else (2,), action=action), + ) + unchanged_failure(result(tmp_path, *items), ValueError, "OOM/unknown actions") + + +@pytest.mark.parametrize( + ("id_schema", "suffix", "ids"), + [ + ("id INTEGER", "", (0, 0, 2)), # Duplicate offsets a hole in aggregate checks. + ("id INTEGER", ", PRIMARY KEY (id, stream)", (0, 0, 2)), + ("id INT PRIMARY KEY", "", (0, 1, 2)), + ("id REAL PRIMARY KEY", "", (0, 1.5, 2)), + ("id TEXT PRIMARY KEY", "", (0, 1, 2)), + ("id INTEGER PRIMARY KEY DESC", "", (0, 1.5, 2)), + ("id INTEGER PRIMARY KEY DESC", "", (None, 0, 1)), + ], +) +def test_aggregate_requires_a_true_integer_primary_key(tmp_path, id_schema, suffix, ids): + first = shard(tmp_path, 0, 0, (0, 1)) + invalid = shard(tmp_path, 4, 0, ids, id_schema=id_schema, suffix=suffix) + unchanged_failure(result(tmp_path, first, invalid), ValueError, "INTEGER PRIMARY KEY") + + +@pytest.mark.parametrize("kind", ["view", "wrong-device"]) +def test_rejects_non_native_trace_before_converting_prior_shards(tmp_path, kind): + first = shard(tmp_path, 0, 0, (0, 1)) + invalid = shard(tmp_path, 4, 0, (0, 1)) + with closing(sqlite3.connect(invalid.file)) as conn, conn: + conn.execute("ALTER TABLE trace_entry_4 RENAME TO other") + if kind == "view": + conn.execute("CREATE VIEW trace_entry_4 AS SELECT * FROM other") + unchanged_failure(result(tmp_path, first, invalid), ValueError, "INTEGER PRIMARY KEY") + + +@pytest.mark.parametrize( + "id_schema,suffix", [("id integer primary key", ""), ("id INTEGER", ", PRIMARY KEY (id DESC)")] +) +def test_accepts_rowid_alias_spelling_variants(tmp_path, id_schema, suffix): + item = shard(tmp_path, 0, 0, (0, 1, 2), id_schema=id_schema, suffix=suffix) + convert_shards(result(tmp_path, item)) + + +def test_many_small_shards_fetch_only_one_aggregate_per_shard(tmp_path, monkeypatch): + items = tuple(shard(tmp_path, 0, index, (-1, index)) for index in range(128)) + statements = [] + connect = sqlite3.connect + + def traced_connect(*args, **kwargs): + conn = connect(*args, **kwargs) + conn.set_trace_callback(statements.append) + return conn + + monkeypatch.setattr(sqlite3, "connect", traced_connect) + _validate_shards(result(tmp_path, *items)) + aggregates = [sql for sql in statements if "SELECT COUNT(*), MIN(id), MAX(id)" in sql] + assert len(aggregates) == len(items) + assert not any("ORDER BY id" in sql for sql in statements) + + +@pytest.mark.parametrize("without_rowid", [False, True]) +def test_accepts_integer_values_with_a_non_rowid_primary_key(tmp_path, without_rowid): + item = shard( + tmp_path, + 0, + 0, + (-1, 0, 1, 2), + id_schema="id INTEGER PRIMARY KEY DESC", + table_suffix="WITHOUT ROWID" if without_rowid else "", + ) + convert_shards(result(tmp_path, item)) + + +def test_without_rowid_integer_affinity_does_not_prove_integer_storage(tmp_path): + item = shard(tmp_path, 0, 0, (0, 1.5, 2), table_suffix="WITHOUT ROWID") + unchanged_failure(result(tmp_path, item), ValueError, "only integers")