Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/en/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,12 @@ Those queries and global aggregation follow the P3 query matrix. Text cannot rec
original ordered frame arrays; original workspace raw-frame loss and string-action
OOM coercion are known upstream limitations, not compatible features to imitate.

Before converting any shard, the exporter checks every selected device's real IDs
and all actions. ID continuity uses each shard's SQL `COUNT/MIN/MAX` and a per-device
next expected ID, after verifying a unique integer primary key; negative boundary
rows are excluded. It does not collect the device's event IDs in Python. This bounds
additional ID-validation memory, not snapshot loading, replay, or total import RSS.

`cacheHash = SHA256(b"mem_snapshot_parser_v2" + raw_pickle_bytes)`: the salt is
**prepended**. Independent `ptSnap.identity` fields are `sourceSha256`,
`requestedDevice`, `eventsPerSlice`, `outputFormat=compatibility-v1`,
Expand Down
5 changes: 5 additions & 0 deletions docs/zh/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,11 @@ native-v2 输出仍独立且不变。
跨来源查询/全局聚合遵循 P3 查询矩阵。文本不能恢复原始有序 frame 数组;原版 workspace
raw-frame 丢失与字符串 OOM action 强制转换属于已知局限,不应仿制为兼容能力。

转换任何分片前,导出器先校验所有选中设备的真实 ID 和全部 action。确认 ID 为唯一整数
主键后,用逐片 SQL `COUNT/MIN/MAX` 和每设备下一个预期 ID 校验连续性,排除负 ID
边界行,不再在 Python 中收集整个设备的事件 ID。这只限制额外的 ID 校验内存,
不代表快照加载、回放或完整导入的 RSS 有界。

`cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`,salt **前置**。
独立 `ptSnap.identity` 字段为 `sourceSha256`、`requestedDevice`、`eventsPerSlice`、
`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=1`、
Expand Down
69 changes: 55 additions & 14 deletions src/pt_snap_cli/core/msinsight_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
MANIFEST_SCHEMA_VERSION,
MSINSIGHT_REVISION,
TRACE_COLUMNS,
DatasetContractError,
DatasetManifest,
_unique_object,
validate_dataset,
validate_event_ids,
)
from .dataset_files import hash_file, require_readonly_member
from .dataset_import_backend import DatasetImportBackend
Expand Down Expand Up @@ -81,33 +81,74 @@ def inspect_target(target: Path) -> None:
return None


def convert_shards(result: ShardedReplayResult) -> None:
"""Preserve event/stream/lifecycle values, materialize inline ninth column.
def _require_integer_primary_key(conn: sqlite3.Connection, trace: str) -> None:
"""Prove unique integer IDs before relying on count and range endpoints.

First validate all selected IDs/actions, without inventing event zero or
coercing unsupported OOM. Block/reference tables are left byte-semantically
intact; no foreign real events are copied into the slice window.
The native writer uses a rowid alias, which enforces integer storage. SQLite's
inline PRIMARY KEY DESC exception and WITHOUT ROWID tables instead have a
primary-key index: retain support for these only after checking storage types.
"""
kind = conn.execute("SELECT type FROM sqlite_master WHERE name=?", (trace,)).fetchone()
primary_key = [row for row in conn.execute(f'PRAGMA table_info("{trace}")') if row[5]]
if (
kind != ("table",)
or len(primary_key) != 1
or primary_key[0][1] != "id"
or primary_key[0][2].upper() != "INTEGER"
or primary_key[0][5] != 1
):
raise ValueError(f"{trace}.id must be the sole INTEGER PRIMARY KEY")
if any(row[3] == "pk" for row in conn.execute(f'PRAGMA index_list("{trace}")')) and (
conn.execute(f"SELECT 1 FROM {trace} WHERE typeof(id)!='integer' LIMIT 1").fetchone()
is not None
):
raise ValueError(f"{trace}.id INTEGER PRIMARY KEY must contain only integers")


def _validate_shards(result: ShardedReplayResult) -> None:
"""Validate every selected device before conversion without collecting IDs."""
for device in sorted({s.device for s in result.slices}):
ids: list[int] = []
next_expected = 0
for item in (s for s in result.slices if s.device == device):
require_readonly_member(result.directory, item.file)
with closing(sqlite3.connect(item.file.as_uri() + "?mode=ro", uri=True)) as conn:
ids.extend(
row[0]
for row in conn.execute(
f"SELECT id FROM trace_entry_{device} WHERE id>=0 ORDER BY id"
trace = f"trace_entry_{device}"
_require_integer_primary_key(conn, trace)
count, low, high = conn.execute(
f'SELECT COUNT(*), MIN(id), MAX(id) FROM "{trace}" WHERE id>=0'
).fetchone()
# With unique integer IDs, count and endpoints prove there are
# no holes inside this shard. Checking each shard in order also
# rejects overlaps/gaps that a device-wide aggregate can hide.
if count and (low != next_expected or high != next_expected + count - 1):
raise DatasetContractError(
f"events[{next_expected}]",
"event_identity",
"compatibility requires original real IDs 0..N-1; do not silently renumber",
)
)
next_expected += count
if (
conn.execute(
f"SELECT id FROM trace_entry_{device} WHERE typeof(action)!='integer' "
f"SELECT id FROM {trace} WHERE typeof(action)!='integer' "
"OR action NOT BETWEEN 0 AND 7 OR action IS NULL LIMIT 1"
).fetchone()
is not None
):
raise ValueError("msinsight compatibility does not support OOM/unknown actions")
validate_event_ids(ids)
if not next_expected:
raise DatasetContractError(
"events", "empty", "empty/static-only devices cannot use compatibility v1"
)


def convert_shards(result: ShardedReplayResult) -> None:
"""Preserve event/stream/lifecycle values, materialize inline ninth column.

First validate all selected IDs/actions, without inventing event zero or
coercing unsupported OOM. Block/reference tables are left byte-semantically
intact; no foreign real events are copied into the slice window.
"""
_validate_shards(result)
for item in result.slices:
device = item.device
trace = f"trace_entry_{device}"
Expand Down
193 changes: 193 additions & 0 deletions tests/core/test_msinsight_export_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
"""Compatibility preflight stays per-shard and avoids a device-wide ID list."""

import sqlite3
from contextlib import closing

import pytest

from pt_snap_cli.core.dataset_contract import DatasetContractError
from pt_snap_cli.core.msinsight_export import _validate_shards, convert_shards
from pt_snap_cli.core.sharded_replay_service import ShardedReplayResult
from pt_snap_cli.snapshot.tools.adaptors.sharded_replay import ReplaySlice
from tests.core.test_dataset_import import hashes


def shard(
root,
device,
index,
ids,
*,
id_schema="id INTEGER PRIMARY KEY",
suffix="",
table_suffix="",
action=7,
):
"""Construct native-layout SQL only; no replay, pickle or source session."""
path = root / f"device_{device}" / f"slice_{index:05d}.db"
path.parent.mkdir(parents=True, exist_ok=True)
trace = f"trace_entry_{device}"
with closing(sqlite3.connect(path)) as conn, conn:
conn.execute(
f"CREATE TABLE {trace} ({id_schema}, action INTEGER, address INTEGER, size INTEGER, "
"stream INTEGER, allocated INTEGER, active INTEGER, reserved INTEGER, "
f"callstackId INTEGER{suffix}) {table_suffix}"
)
conn.executemany(
f"INSERT INTO {trace} VALUES (?, ?, 0, 0, ?, 0, 0, 0, 0)",
((value, action, position) for position, value in enumerate(ids)),
)
conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY, callstack TEXT)")
conn.execute("INSERT INTO callstack VALUES (0, 'synthetic stack')")
conn.execute('CREATE TABLE dictionary ("table" TEXT, "column" TEXT, key TEXT, value TEXT)')
real = [value for value in ids if value is not None and value >= 0]
return ReplaySlice(
device, index, 0, len(real) - 1, min(real, default=0), max(real, default=-1), path
)


def result(root, *items):
return ShardedReplayResult(root, items, ())


def unchanged_failure(staged, error=DatasetContractError, match="event_identity"):
before = hashes(staged.directory)
with pytest.raises(error, match=match):
convert_shards(staged)
assert hashes(staged.directory) == before


@pytest.mark.parametrize(
"ids",
[
((1, 2),), # Nonzero start.
((0, 2),), # Within-shard hole.
((0, 1), (3, 4)), # Cross-shard hole.
((0, 1), (1, 2)), # Cross-shard overlap.
((2, 3), (0, 1)), # Reversed shard order.
((0, 1), (3, 4), (4, 5)), # Global COUNT/MIN/MAX hides the hole and duplicate.
((0, 2), (1, 3)), # Globally continuous IDs cannot repair per-shard holes/order.
],
)
def test_rejects_invalid_continuity_without_converting_any_shard(tmp_path, ids):
items = tuple(shard(tmp_path, 0, index, values) for index, values in enumerate(ids))
unchanged_failure(result(tmp_path, *items))


@pytest.mark.parametrize("ids", [(), (-10, -1)])
def test_empty_or_boundary_only_device_still_requires_a_real_event(tmp_path, ids):
first = shard(tmp_path, 0, 0, (0, 1))
empty_device = shard(tmp_path, 3, 0, ids)
unchanged_failure(result(tmp_path, first, empty_device), match="empty/static-only")


def test_boundary_and_empty_shards_do_not_advance_real_ids(tmp_path):
items = (
shard(tmp_path, 0, 0, (-2, -1)),
shard(tmp_path, 0, 1, (-100, 0, 1)),
shard(tmp_path, 0, 2, ()),
shard(tmp_path, 0, 3, (-20, 2, 3)),
)
convert_shards(result(tmp_path, *items))
for item in items:
with closing(sqlite3.connect(item.file)) as conn:
assert conn.execute("PRAGMA table_info(trace_entry_0)").fetchall()[-1][1] == "callstack"


def test_each_device_restarts_at_zero_even_when_shards_are_interleaved(tmp_path):
items = (
shard(tmp_path, 0, 0, (0, 1)),
shard(tmp_path, 4, 0, (0,)),
shard(tmp_path, 0, 1, (2, 3)),
shard(tmp_path, 4, 1, (1, 2)),
)
convert_shards(result(tmp_path, *items))


def test_invalid_second_device_does_not_convert_first_device(tmp_path):
items = (shard(tmp_path, 0, 0, (0, 1)), shard(tmp_path, 4, 0, (2, 3)))
unchanged_failure(result(tmp_path, *items))


@pytest.mark.parametrize("action", [8, -1, 1.5, "unknown", None])
@pytest.mark.parametrize("boundary", [False, True])
def test_actions_are_checked_on_real_and_boundary_rows_before_conversion(
tmp_path, action, boundary
):
items = (
shard(tmp_path, 0, 0, (0, 1)),
shard(tmp_path, 0, 1, (-1,) if boundary else (2,), action=action),
)
unchanged_failure(result(tmp_path, *items), ValueError, "OOM/unknown actions")


@pytest.mark.parametrize(
("id_schema", "suffix", "ids"),
[
("id INTEGER", "", (0, 0, 2)), # Duplicate offsets a hole in aggregate checks.
("id INTEGER", ", PRIMARY KEY (id, stream)", (0, 0, 2)),
("id INT PRIMARY KEY", "", (0, 1, 2)),
("id REAL PRIMARY KEY", "", (0, 1.5, 2)),
("id TEXT PRIMARY KEY", "", (0, 1, 2)),
("id INTEGER PRIMARY KEY DESC", "", (0, 1.5, 2)),
("id INTEGER PRIMARY KEY DESC", "", (None, 0, 1)),
],
)
def test_aggregate_requires_a_true_integer_primary_key(tmp_path, id_schema, suffix, ids):
first = shard(tmp_path, 0, 0, (0, 1))
invalid = shard(tmp_path, 4, 0, ids, id_schema=id_schema, suffix=suffix)
unchanged_failure(result(tmp_path, first, invalid), ValueError, "INTEGER PRIMARY KEY")


@pytest.mark.parametrize("kind", ["view", "wrong-device"])
def test_rejects_non_native_trace_before_converting_prior_shards(tmp_path, kind):
first = shard(tmp_path, 0, 0, (0, 1))
invalid = shard(tmp_path, 4, 0, (0, 1))
with closing(sqlite3.connect(invalid.file)) as conn, conn:
conn.execute("ALTER TABLE trace_entry_4 RENAME TO other")
if kind == "view":
conn.execute("CREATE VIEW trace_entry_4 AS SELECT * FROM other")
unchanged_failure(result(tmp_path, first, invalid), ValueError, "INTEGER PRIMARY KEY")


@pytest.mark.parametrize(
"id_schema,suffix", [("id integer primary key", ""), ("id INTEGER", ", PRIMARY KEY (id DESC)")]
)
def test_accepts_rowid_alias_spelling_variants(tmp_path, id_schema, suffix):
item = shard(tmp_path, 0, 0, (0, 1, 2), id_schema=id_schema, suffix=suffix)
convert_shards(result(tmp_path, item))


def test_many_small_shards_fetch_only_one_aggregate_per_shard(tmp_path, monkeypatch):
items = tuple(shard(tmp_path, 0, index, (-1, index)) for index in range(128))
statements = []
connect = sqlite3.connect

def traced_connect(*args, **kwargs):
conn = connect(*args, **kwargs)
conn.set_trace_callback(statements.append)
return conn

monkeypatch.setattr(sqlite3, "connect", traced_connect)
_validate_shards(result(tmp_path, *items))
aggregates = [sql for sql in statements if "SELECT COUNT(*), MIN(id), MAX(id)" in sql]
assert len(aggregates) == len(items)
assert not any("ORDER BY id" in sql for sql in statements)


@pytest.mark.parametrize("without_rowid", [False, True])
def test_accepts_integer_values_with_a_non_rowid_primary_key(tmp_path, without_rowid):
item = shard(
tmp_path,
0,
0,
(-1, 0, 1, 2),
id_schema="id INTEGER PRIMARY KEY DESC",
table_suffix="WITHOUT ROWID" if without_rowid else "",
)
convert_shards(result(tmp_path, item))


def test_without_rowid_integer_affinity_does_not_prove_integer_storage(tmp_path):
item = shard(tmp_path, 0, 0, (0, 1.5, 2), table_suffix="WITHOUT ROWID")
unchanged_failure(result(tmp_path, item), ValueError, "only integers")
Loading