From 4b93721f98037afcd861b30797d3c7bbd7346741 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:38:12 +0800 Subject: [PATCH 1/2] fix: attest compatible base content across Insight cache writes Refs #266. Keep legacy byte proofs and no-replace publication; allow only versioned device-local derived cache additions with unchanged base content. --- README.md | 6 +- README_zh.md | 4 +- docs/en/sharded-snapshotdb.md | 45 ++- docs/zh/sharded-snapshotdb.md | 28 +- src/pt_snap_cli/core/import_metadata.py | 9 +- .../core/msinsight_content_proof.py | 156 +++++++++ src/pt_snap_cli/core/msinsight_export.py | 101 +++++- tests/core/test_msinsight_content_proof.py | 306 ++++++++++++++++++ tests/core/test_msinsight_content_races.py | 133 ++++++++ 9 files changed, 750 insertions(+), 38 deletions(-) create mode 100644 src/pt_snap_cli/core/msinsight_content_proof.py create mode 100644 tests/core/test_msinsight_content_proof.py create mode 100644 tests/core/test_msinsight_content_races.py diff --git a/README.md b/README.md index 2c9b1b7..dd11e78 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,10 @@ Explicit `pt-snap import snapshot.pkl --format msinsight --json` produces `snapshot.pkl.msinsight/` for the fixed msinsight revision's **same original pickle + adjacent cache** entrance. Inline stacks cost more space than native interning. Existing compatible targets are never replaced, even with `--force`; -only fully validated identical pt-snap caches are reused. GUI acceptance remains -**pending/not run**; hash equality alone is not acceptance. See +only fully validated pt-snap caches are reused. New exports attest base content so +known device-local Insight cache tables can change without losing reuse; changed +legacy caches without that proof still fail closed. Hash equality alone is not GUI +acceptance; dated fixed-version results and their limits are recorded below. See [compatible export and limits](docs/en/sharded-snapshotdb.md#explicit-msinsight-compatible-export-p2). See [offline interoperability acceptance and bounded performance](docs/en/interop-acceptance.md) diff --git a/README_zh.md b/README_zh.md index b7b65c3..16c2a2b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -68,7 +68,9 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices 显式 `pt-snap import snapshot.pkl --format msinsight --json` 生成 `snapshot.pkl.msinsight/`,对齐固定 msinsight 版本的**同一原始 pickle + 邻接缓存**入口。 内联栈比原生去重占更多空间;即使 `--force` 也不替换已有兼容目标,只复用完整校验且 -身份相同的 pt-snap 缓存。GUI **待验收/未运行**,hash 相等不代表验收通过。详见 +身份相同的 pt-snap 缓存。新产物的基础内容证明允许已知的本设备 Insight 派生缓存表变化; +缺少该证明的旧产物字节变化仍保守拒绝。hash 相等不代表 GUI 验收通过; +带日期的固定版本实测及限制见下方记录。详见 [兼容导出及限制](docs/zh/sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。 [离线双向验收与有界性能](docs/zh/interop-acceptance.md) 提供固定版本两条链路、 diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 3dc0480..6fd46d2 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -510,17 +510,40 @@ additional ID-validation memory, not snapshot loading, replay, or total import R `manifestContractVersion=1`, `exportContractVersion=1`, `referenceContractVersion=1`, `metadataSchemaVersion=1`, `importFormatVersion=1`, and full `targetRevision`. The `ptSnap` object also contains `metadata`, `omittedDevices`, and `members` -(canonical relative path → finalized SHA256). Whole validated ownership/identity -requires every member/hash and identical per-slice metadata; unknown/external -caches and unrelated files are not adopted. CLI version alone does not invalidate. - -Compatible publication is **no-replace even with force**. Identical recognized -pt-snap caches reuse without loading pickle (force does not bypass this rule). -Changed source path/content, options/contracts, corrupted members or GUI-modified -DBs require a new `--output-dir`; no repair, overwrite or deletion of the old -artifact occurs. Exclusive publication and focus compensation use the existing -checked transaction; source SHA256 is rechecked immediately before publish/reuse. -Native force replacement and recovery semantics are untouched. +(canonical relative path → finalized SHA256). New exports additionally store +`memberContent = {"version": 1, "members": {relative_path: proof}}`. Each proof +records every original SQLite schema object's type/name/table/SQL (not physical +rootpage) and every original table's row count and SHA256. This includes dictionary, +metadata, lifecycle references and internal tables. Rows use type-tagged, +length-framed NULL/integer/IEEE-754 real/UTF-8 text/blob encoding, sorted as BLOBs; +duplicates remain significant. Declared IDs are included; incidental hidden rowid +and physical row order are not. SQLite performs the sort; Python streams rows, +but sorting can require temporary storage. This is not a constant-work import. + +Byte-identical members retain the fast path without logical scans. On byte drift, +a prior version-1 proof must match every base schema object and row. Only a newly +added ordinary table exactly named `memory_allocation_cache_v1_` +is permitted; other tables, versions, devices, indexes, views and triggers are +rejected. An originally attested cache-named table is still protected. Stable +header/layout-only changes can also pass when the entire base proof matches. +Unknown/malformed proofs fail closed, including on the byte fast path. Old exports +without a proof remain reusable only with exact member hashes: changed bytes mean +lost integrity evidence, not a diagnosis of corrupted base data. No proof is +backfilled from already modified content. Attestations are integrity baselines, +not authentication of untrusted manifests or origins. + +Compatible publication is **no-replace even with force**. Recognized caches with +matching source/options/identity and byte or base-content proof reuse without +loading pickle; force cannot bypass these checks. Source/options changes, base +changes, unrecognized objects and unproved legacy drift require a new +`--output-dir`. No repair, overwrite, deletion or manifest rewrite occurs. +Per-member metadata still must agree with manifest identity. Manifest/member bytes, +full file generation, directory identities, exact inventory and alias/sidecar checks +bracket inspection; writers must already be stopped and members finalized. This +is change detection, not a filesystem lock or live-writer support. Ordinary +read-only compatibility queries retain their separate validation contract. +Exclusive publication/focus compensation and the final source SHA256 recheck are +unchanged, as are native force replacement and recovery semantics. Inline text costs `sum(len(callstack.encode("utf-8")) for each event)` rather than one copy per distinct stack. Existing native interning remains available. Conversion diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index d608cc7..35e5578 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -407,13 +407,27 @@ raw-frame 丢失与字符串 OOM action 强制转换属于已知局限,不应 `outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=1`、 `referenceContractVersion=1`、`metadataSchemaVersion=1`、`importFormatVersion=1`、完整 `targetRevision`。`ptSnap` 另含 `metadata`、`omittedDevices` 和 `members`(规范相对路径 -→ 最终化 SHA256)。整体已校验所有权/身份要求每个成员/哈希及逐片 metadata 相同; -不接管未知/外部缓存或无关文件。CLI 版本变化本身不使缓存失效。 - -兼容发布 **force 也 no-replace**。已识别、身份相同的 pt-snap 缓存不加载 pickle 即复用, -force 不绕过该规则。源路径/内容、选项/合同变化、成员损坏或 GUI 修改 DB 后须换新 -`--output-dir`,不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务, -发布/复用前再次核对源 SHA256;原生 force 替换与恢复语义不变。 +→ 最终化 SHA256)。新导出还保存 `memberContent = {"version": 1, "members": {相对路径: proof}}`。 +proof 记录全部原始 SQLite schema 对象的 type/name/table/SQL(不含物理 rootpage), +以及每张原表的行数和 SHA256,包括 dictionary、metadata、生命周期引用和内部表。 +行按带类型标签和长度分隔的 NULL/整数/IEEE-754 浮点/UTF-8 文本/blob 编码,再按 BLOB +排序;重复行仍计入。声明的 ID 被保护,隐式 rowid 和物理行顺序不作为语义。SQLite +负责排序,Python 流式消费;排序可能使用临时存储,不是常数工作量导入。 + +成员字节相同时保留快速路径,不扫描逻辑内容。字节改变时,必须有事前 version-1 proof +且全部基础 schema/行匹配;只允许新增普通表 `memory_allocation_cache_v1_<该片 deviceId>`, +其他表、版本、设备名、索引、视图和 trigger 均拒绝。原本已纳入证明的同名缓存表仍受保护。 +基础证明全量相同时,稳定的文件头/物理布局变化也可通过。未知或非法 proof 即使字节未变 +也拒绝。旧产物无 proof 时仍须整文件 hash 相同;字节变化表示完整性证据失效,不等于基础 +数据损坏,不从已修改现状补造历史证明。证明是完整性基线,不认证不可信 manifest 或来源。 + +兼容发布 **force 也 no-replace**。已识别、源/选项/身份相同、字节或基础内容证明匹配的 +pt-snap 缓存不加载 pickle 即复用;force 不绕过检查。源/选项变化、基础内容变化、未知对象、 +旧产物未被证明的字节变化须换新 `--output-dir`。不修复、不覆盖、不删除、不改写 manifest。 +逐片 metadata 仍须与 manifest 身份相符。检查前后核对 manifest/成员字节、完整文件 generation、 +目录身份、精确成员清单及 alias/sidecar;writer 必须已停止、成员已最终化。这是变更检测, +不是文件系统锁或 live-writer 支持。普通兼容数据集只读查询保留独立校验合同。 +排他发布、focus 补偿、发布/复用前源 SHA256 复查及原生 force 替换/恢复语义不变。 内联文本成本按每个事件的 UTF-8 调用栈字节求和,不按不同栈各存一份;原生去重仍可用。 转换暂时并存两种布局,可能留下 SQLite free pages;容量不是 RSS/registry/block/磁盘 diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index e713803..422afb7 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -96,7 +96,14 @@ def inspect(self, db_path: Path | str) -> MetadataInspection: if "ptSnap" in raw: try: owned = inspect_owned_cache(dataset.root) - except (ValueError, KeyError, TypeError, OSError, sqlite3.DatabaseError): + except ( + ValueError, + KeyError, + TypeError, + OSError, + sqlite3.DatabaseError, + DatabaseSchemaError, + ): return MetadataInspection(path, "invalid", reason="metadata_invalid") return MetadataInspection(path, "available", owned.metadata) return MetadataInspection(path, "unavailable", reason="metadata_missing") diff --git a/src/pt_snap_cli/core/msinsight_content_proof.py b/src/pt_snap_cli/core/msinsight_content_proof.py new file mode 100644 index 0000000..ff1c7a0 --- /dev/null +++ b/src/pt_snap_cli/core/msinsight_content_proof.py @@ -0,0 +1,156 @@ +"""Versioned logical attestation for finalized, caller-owned compatibility members. + +This is an integrity baseline, not authentication of an untrusted manifest. It +covers every original schema object and table. Only a new, ordinary allocation +cache table for the member's own device can be ignored during comparison. +""" + +from __future__ import annotations + +import hashlib +import re +import sqlite3 +import struct +from contextlib import closing +from pathlib import Path + +CONTENT_VERSION = 1 + + +def _quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _row_bytes(*values: object) -> bytes: + """Injective storage-class encoding; length framing retains empty/NULL/rows.""" + result = bytearray() + for value in values: + if value is None: + tag, payload = b"n", b"" + elif type(value) is int: + tag, payload = b"i", str(value).encode("ascii") + elif type(value) is float: + tag, payload = b"f", struct.pack(">d", value) + elif isinstance(value, str): + tag, payload = b"t", value.encode("utf-8") + elif isinstance(value, bytes): + tag, payload = b"b", value + else: + raise ValueError("Unsupported SQLite storage class") + result.extend(tag + len(payload).to_bytes(8, "big") + payload) + return bytes(result) + + +def _schema(conn: sqlite3.Connection) -> list[list[str | None]]: + # rootpage is physical allocation, not semantic schema. Include internal + # tables and indexes (whose SQL can be NULL), views and triggers in baseline. + return [ + list(row) + for row in conn.execute( + "SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name COLLATE BINARY" + ) + ] + + +def _table_content(conn: sqlite3.Connection, name: str) -> dict[str, object]: + # SQLite sorts canonical BLOB row encodings, not collated/coerced SQL values. + # Stream the sorted result, preserving duplicate multiplicity. No whole-table + # Python materialization, executable dump, or modification of the member. + columns = [row[1] for row in conn.execute(f"PRAGMA table_xinfo({_quote(name)})")] + arguments = ",".join(_quote(column) for column in columns) + digest = hashlib.sha256(b"pt-snap-member-table-v1\0") + count = 0 + for (encoded,) in conn.execute( + f"SELECT pt_snap_row({arguments}) AS encoded FROM {_quote(name)} ORDER BY encoded" + ): + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + count += 1 + return {"rows": count, "sha256": digest.hexdigest()} + + +def member_content(path: Path) -> dict[str, object]: + """Caller must preflight and bracket finalized members with file guards.""" + with closing(sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: + conn.execute("PRAGMA query_only=ON") + conn.create_function("pt_snap_row", -1, _row_bytes, deterministic=True) + schema = _schema(conn) + tables = {} + for kind, name, _, sql in schema: + if kind == "table": + if not isinstance(name, str) or ( + sql and re.match(r"CREATE\s+VIRTUAL\s+TABLE\s", sql, re.IGNORECASE) + ): + raise ValueError("Unsupported virtual table in compatible content proof") + tables[name] = _table_content(conn, name) + return {"schema": schema, "tables": tables} + + +def validate_member_content(proof: object) -> None: + """Fail closed for malformed evidence even on the byte-identical fast path.""" + if not isinstance(proof, dict) or set(proof) != {"schema", "tables"}: + raise ValueError("Malformed compatible base-content proof") + schema, tables = proof["schema"], proof["tables"] + if not isinstance(schema, list) or not isinstance(tables, dict): + raise ValueError("Malformed compatible base-content proof") + names = set() + table_names = set() + for item in schema: + if ( + not isinstance(item, list) + or len(item) != 4 + or item[0] not in ("table", "index", "view", "trigger") + or not all(isinstance(value, str) and value for value in item[:3]) + or (item[3] is not None and not isinstance(item[3], str)) + or (item[0], item[1]) in names + ): + raise ValueError("Malformed compatible base schema proof") + names.add((item[0], item[1])) + if item[0] == "table": + table_names.add(item[1]) + if set(tables) != table_names or not table_names: + raise ValueError("Incomplete compatible base table proof") + for entry in tables.values(): + if ( + not isinstance(entry, dict) + or set(entry) != {"rows", "sha256"} + or type(entry["rows"]) is not int + or entry["rows"] < 0 + or not isinstance(entry["sha256"], str) + or re.fullmatch("[0-9a-f]{64}", entry["sha256"]) is None + ): + raise ValueError("Malformed compatible base table proof") + + +def compare_member_content(path: Path, proof: object, device: int) -> None: + validate_member_content(proof) + assert isinstance(proof, dict) + expected_schema = proof["schema"] + assert isinstance(expected_schema, list) + with closing(sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: + conn.execute("PRAGMA query_only=ON") + conn.create_function("pt_snap_row", -1, _row_bytes, deterministic=True) + schema = _schema(conn) + added = [item for item in schema if item not in expected_schema] + allowed = f"memory_allocation_cache_v1_{device}" + if added: + if ( + len(added) != 1 + or added[0][:3] != ["table", allowed, allowed] + or not isinstance(added[0][3], str) + or not re.match(r"CREATE\s+TABLE\s", added[0][3], re.IGNORECASE) + or any(item[1] == allowed for item in expected_schema) + ): + raise ValueError("Unrecognized derived schema in compatible member") + schema.remove(added[0]) + if schema != expected_schema: + raise ValueError("Compatible base schema or data changed") + # Validate schema before scanning rows; never scan the newly permitted + # derived table or unknown objects. Every original table remains covered. + tables = { + item[1]: _table_content(conn, item[1]) + for item in schema + if item[0] == "table" and isinstance(item[1], str) + } + if tables != proof["tables"]: + raise ValueError("Compatible base schema or data changed") diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py index e1764af..2d8d958 100644 --- a/src/pt_snap_cli/core/msinsight_export.py +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -25,6 +25,7 @@ DatasetContractError, DatasetManifest, _unique_object, + parse_manifest, validate_dataset, ) from .dataset_files import hash_file, require_readonly_member @@ -32,6 +33,12 @@ from .errors import ImportExecutionError from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata +from .msinsight_content_proof import ( + CONTENT_VERSION, + compare_member_content, + member_content, + validate_member_content, +) from .sharded_replay_service import ShardedReplayResult COMPATIBILITY_FORMAT = "compatibility-v1" @@ -193,6 +200,7 @@ def build_compatible_manifest( ) -> dict[str, object]: devices: dict[str, object] = {} files: dict[str, str] = {} + contents: dict[str, object] = {} for device in sorted({s.device for s in result.slices}): slices = [s for s in result.slices if s.device == device] devices[str(device)] = { @@ -211,7 +219,14 @@ def build_compatible_manifest( ], } for item in slices: - files[item.file.relative_to(result.directory).as_posix()] = hash_file(item.file) + relative = item.file.relative_to(result.directory).as_posix() + require_readonly_member(result.directory, item.file, immutable=True) + before = hash_file(item.file) + contents[relative] = member_content(item.file) + require_readonly_member(result.directory, item.file, immutable=True) + if hash_file(item.file) != before: + raise ValueError("Compatible member changed while building proof") + files[relative] = before return { "schemaVersion": MANIFEST_SCHEMA_VERSION, "status": "complete", @@ -226,6 +241,7 @@ def build_compatible_manifest( "metadata": asdict(metadata), "omittedDevices": list(result.omitted_devices), "members": files, + "memberContent": {"version": CONTENT_VERSION, "members": contents}, }, } @@ -238,16 +254,48 @@ class CompatibleCache: omitted_devices: tuple[int, ...] +def _cache_inventory(target: Path) -> dict[str, tuple[int, ...]]: + """Snapshot identity of the entire owned tree without following aliases.""" + if target.resolve() != target or target.is_symlink() or not target.is_dir(): + raise ValueError("Compatible cache must be a canonical directory") + result = {} + for path in (target, *target.rglob("*")): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise ValueError("Unrecognized files or aliases in compatible cache") + stat = path.stat() + result[path.relative_to(target).as_posix()] = ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + stat.st_mode, + stat.st_nlink, + ) + return result + + def inspect_owned_cache(target: Path) -> CompatibleCache: - """Whole valid recognized identity only; salted hash alone is never ownership. + """Recognized identity plus byte or versioned base-content attestation. - Reject unexpected files/aliases rather than adopt them. GUI may change - derived tables: that invalidates pt-snap's immutable cache attestation, and - requires a new output, never repair or force-replacement. + Stable cache-only changes can reuse new proofs. Legacy byte drift cannot be + classified retrospectively. Bracketing detects changes, not a writer lock. """ - validation = validate_dataset(target) - with (target / "manifest.json").open(encoding="utf-8") as file: + generation = _cache_inventory(target) + manifest_path = target / "manifest.json" + manifest_hash = hash_file(manifest_path) + with manifest_path.open(encoding="utf-8") as file: data = json.load(file, object_pairs_hook=_unique_object) + planned = parse_manifest(data) + paths = {s.file: d.device_id for d in planned.devices for s in d.slices} + expected = {".", "manifest.json", *paths, *(str(Path(p).parent) for p in paths)} + if set(generation) != expected: + raise ValueError("Unrecognized files or aliases in compatible cache") + before = {} + for relative in paths: + require_readonly_member(target, target / relative, immutable=True) + before[relative] = hash_file(target / relative) + validation = validate_dataset(target) extension = data["ptSnap"] metadata = metadata_from_mapping(extension["metadata"]) identity = extension["identity"] @@ -264,17 +312,32 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: or metadata.metadata_schema_version != 1 ): raise ValueError("Metadata must match physical v1 layout") - paths = {s.file for d in validation.manifest.devices for s in d.slices} - if set(extension["members"]) != paths: + if set(extension["members"]) != set(paths): raise ValueError("Incomplete member attestation") - expected = {"manifest.json", *paths, *(str(Path(p).parent) for p in paths)} - actual = {p.relative_to(target).as_posix() for p in target.rglob("*")} - if actual != expected or any(p.is_symlink() for p in target.rglob("*")): - raise ValueError("Unrecognized files or aliases in compatible cache") - for relative in paths: + proofs = None + if "memberContent" in extension: + content = extension["memberContent"] + if ( + not isinstance(content, dict) + or set(content) != {"version", "members"} + or type(content["version"]) is not int + or content["version"] != CONTENT_VERSION + or not isinstance(content["members"], dict) + or set(content["members"]) != set(paths) + ): + raise ValueError("Unsupported or incomplete compatible base-content proof") + proofs = content["members"] + for proof in proofs.values(): + validate_member_content(proof) + for relative, device in paths.items(): member = target / relative - if hash_file(member) != extension["members"][relative]: - raise ValueError("Compatible member hash changed") + if before[relative] != extension["members"][relative]: + if proofs is None: + raise ValueError( + "Compatible member hash changed; legacy integrity proof invalidated. " + "No prior base-content proof is available; base-data corruption is not established." + ) + compare_member_content(member, proofs[relative], device) with closing(sqlite3.connect(member.as_uri() + "?mode=ro&immutable=1", uri=True)) as conn: conn.row_factory = sqlite3.Row rows = conn.execute("SELECT * FROM pt_snap_metadata").fetchall() @@ -296,4 +359,10 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError("Device selection disagrees with identity") if Path(validation.manifest.source_file).name != metadata.source_name: raise ValueError("Source path disagrees with metadata") + for relative in paths: + require_readonly_member(target, target / relative, immutable=True) + if hash_file(target / relative) != before[relative]: + raise ValueError("Compatible member changed during inspection") + if hash_file(manifest_path) != manifest_hash or _cache_inventory(target) != generation: + raise ValueError("Compatible cache changed during inspection") return CompatibleCache(validation.manifest, identity, metadata, tuple(omitted)) diff --git a/tests/core/test_msinsight_content_proof.py b/tests/core/test_msinsight_content_proof.py new file mode 100644 index 0000000..a13d757 --- /dev/null +++ b/tests/core/test_msinsight_content_proof.py @@ -0,0 +1,306 @@ +"""Cache-only SQLite drift must not weaken compatibility member ownership.""" + +import dataclasses +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from tests.core.test_dataset_import import hashes +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +def execute(path, sql, parameters=()): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute(sql, parameters) + + +def read_manifest(root): + return json.loads((root / "manifest.json").read_text()) + + +def write_manifest(root, manifest): + (root / "manifest.json").write_text(json.dumps(manifest)) + + +def add_cache(path, device=0): + execute( + path, f"CREATE TABLE memory_allocation_cache_v1_{device} (event_id INTEGER, payload BLOB)" + ) + execute(path, f"INSERT INTO memory_allocation_cache_v1_{device} VALUES (?, ?)", (0, b"cache")) + + +def assert_refused_without_changes(service, request, root): + before = hashes(root) + focus = request.snapshot_file.parent / ".pt-snap/focus.json" + old_focus = focus.read_bytes() if focus.exists() else None + for force in (False, True): + with pytest.raises(ImportExecutionError, match="preserved even under --force"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert hashes(root) == before + assert (focus.read_bytes() if focus.exists() else None) == old_focus + + +def test_fresh_export_records_complete_content_proofs_and_keeps_export_version(tmp_path): + from pt_snap_cli.core.msinsight_content_proof import member_content + + result = ImportService().import_snapshot(options(tmp_path, multiple=True)) + extension = read_manifest(result.db_path)["ptSnap"] + assert extension["identity"]["exportContractVersion"] == 1 + content = extension["memberContent"] + assert content["version"] == 1 + assert set(content["members"]) == set(extension["members"]) + for relative, proof in content["members"].items(): + path = result.db_path / relative + assert proof == member_content(path) + assert extension["members"][relative] == hashlib.sha256(path.read_bytes()).hexdigest() + assert set(proof) == {"schema", "tables"} + assert {"dictionary", "pt_snap_metadata", "pt_snap_block_reference"} <= set(proof["tables"]) + assert all(len(row) == 4 for row in proof["schema"]) + + +@pytest.mark.parametrize("multiple", [False, True]) +def test_cache_addition_reuses_every_device_without_replay_or_writes( + tmp_path, monkeypatch, multiple +): + service = ImportService() + request = options(tmp_path, multiple=multiple) + first = service.import_snapshot(request) + manifest = read_manifest(first.db_path) + with SnapshotAnalyzer(first.db_path) as analyzer: + expected = { + device: analyzer.execute_query("event", {"id": 0}, device_id=device)["rows"] + for device in first.devices + } + for device, inventory in manifest["devices"].items(): + for shard in inventory["slices"]: + add_cache(first.db_path / shard["file"], int(device)) + before = hashes(first.db_path) + monkeypatch.setattr( + service._msinsight_backend.replay, + "stage", + lambda *a, **k: pytest.fail("cache-only reuse replayed input"), + ) + for force in (False, True): + reused = service.import_snapshot(dataclasses.replace(request, force=force)) + assert reused.reused and reused.metadata == first.metadata + with SnapshotAnalyzer(first.db_path) as analyzer: + for device in first.devices: + assert ( + analyzer.execute_query("event", {"id": 0}, device_id=device)["rows"] + == expected[device] + ) + assert hashes(first.db_path) == before + + +@pytest.mark.parametrize( + "sql", + [ + "UPDATE trace_entry_0 SET size=size+1 WHERE id=0", + "UPDATE block_0 SET size=size+1", + "ALTER TABLE trace_entry_0 ADD COLUMN unexpected TEXT", + "UPDATE pt_snap_metadata SET importer_version='changed'", + "DELETE FROM pt_snap_block_reference", + "UPDATE dictionary SET value='changed' WHERE key='0'", + "DELETE FROM dictionary WHERE rowid=(SELECT MIN(rowid) FROM dictionary)", + "INSERT INTO dictionary SELECT * FROM dictionary LIMIT 1", + "DROP INDEX trace_entry_0_allocated", + ], +) +def test_cache_does_not_mask_changes_to_base_content(tmp_path, sql): + service = ImportService() + request = options(tmp_path) + result = service.import_snapshot(request) + path = result.db_path / "device_0/slice_00000.db" + add_cache(path) + execute(path, sql) + assert_refused_without_changes(service, request, result.db_path) + + +@pytest.mark.parametrize( + "sql", + [ + "CREATE TABLE memory_allocation_cache_v2_0 (id INTEGER)", + "CREATE TABLE memory_allocation_cache_v1_1 (id INTEGER)", + "CREATE TABLE memory_allocation_cache_v1_00 (id INTEGER)", + "CREATE TABLE unrelated (id INTEGER)", + "CREATE VIEW memory_allocation_cache_v1_0 AS SELECT * FROM trace_entry_0", + "CREATE VIRTUAL TABLE memory_allocation_cache_v1_0 USING fts5(payload)", + "CREATE INDEX memory_allocation_cache_v1_0 ON trace_entry_0(size)", + "CREATE TRIGGER memory_allocation_cache_v1_0 AFTER INSERT ON dictionary BEGIN SELECT 1; END", + ], +) +def test_only_exact_ordinary_device_cache_table_is_permitted(tmp_path, sql): + service = ImportService() + request = options(tmp_path) + result = service.import_snapshot(request) + execute(result.db_path / "device_0/slice_00000.db", sql) + assert_refused_without_changes(service, request, result.db_path) + + +@pytest.mark.parametrize( + "object_sql", + [ + "CREATE INDEX cache_index ON memory_allocation_cache_v1_0(event_id)", + "CREATE TRIGGER cache_trigger AFTER INSERT ON memory_allocation_cache_v1_0 BEGIN SELECT 1; END", + ], +) +def test_allowed_cache_table_does_not_allow_extra_schema_objects(tmp_path, object_sql): + service = ImportService() + request = options(tmp_path) + result = service.import_snapshot(request) + path = result.db_path / "device_0/slice_00000.db" + add_cache(path) + execute(path, object_sql) + assert_refused_without_changes(service, request, result.db_path) + + +@pytest.mark.parametrize( + "kind", + [ + "null", + "version", + "boolean-version", + "members", + "missing-member", + "extra-member", + "schema", + "tables", + "rows", + "digest", + ], +) +def test_present_malformed_proof_is_rejected_even_when_raw_hash_matches(tmp_path, kind): + service = ImportService() + request = options(tmp_path) + result = service.import_snapshot(request) + manifest = read_manifest(result.db_path) + content = manifest["ptSnap"]["memberContent"] + first = next(iter(content["members"])) + if kind == "null": + manifest["ptSnap"]["memberContent"] = None + elif kind == "version": + content["version"] = 999 + elif kind == "boolean-version": + content["version"] = True + elif kind == "members": + content["members"] = [] + elif kind == "missing-member": + del content["members"][first] + elif kind == "extra-member": + content["members"]["extra.db"] = content["members"][first] + elif kind == "schema": + content["members"][first]["schema"] = "invalid" + elif kind == "tables": + content["members"][first]["tables"] = [] + elif kind == "rows": + content["members"][first]["tables"]["dictionary"]["rows"] = True + else: + content["members"][first]["tables"]["dictionary"]["sha256"] = "not-a-digest" + write_manifest(result.db_path, manifest) + assert_refused_without_changes(service, request, result.db_path) + + +@pytest.mark.parametrize("changed", [False, True]) +def test_legacy_no_proof_requires_exact_raw_bytes(tmp_path, changed): + service = ImportService() + request = options(tmp_path, focus=False) + result = service.import_snapshot(request) + manifest = read_manifest(result.db_path) + del manifest["ptSnap"]["memberContent"] + write_manifest(result.db_path, manifest) + if changed: + add_cache(result.db_path / "device_0/slice_00000.db") + assert_refused_without_changes(service, request, result.db_path) + else: + before = hashes(result.db_path) + assert service.import_snapshot(request).reused + assert hashes(result.db_path) == before + + +def test_content_proof_hashes_sqlite_types_and_duplicate_multiplicity(tmp_path): + from pt_snap_cli.core.msinsight_content_proof import member_content + + path = tmp_path / "types.db" + execute(path, "CREATE TABLE sample (value)") + proofs = [] + for value in (None, 1, 1.0, "1", b"1", b"\x00\xff"): + execute(path, "DELETE FROM sample") + execute(path, "INSERT INTO sample VALUES (?)", (value,)) + proofs.append(member_content(path)["tables"]["sample"]) + assert all(proof["rows"] == 1 for proof in proofs) + assert len({proof["sha256"] for proof in proofs}) == len(proofs) + execute(path, "INSERT INTO sample SELECT * FROM sample") + duplicate = member_content(path)["tables"]["sample"] + assert duplicate["rows"] == 2 + assert duplicate["sha256"] != proofs[-1]["sha256"] + + +def test_content_proof_ignores_row_order_and_schema_rootpage(tmp_path): + from pt_snap_cli.core.msinsight_content_proof import member_content + + path = tmp_path / "layout.db" + execute(path, "CREATE TABLE discarded (value)") + execute(path, "CREATE TABLE sample (value)") + execute(path, "DROP TABLE discarded") + for value in ("first", "second", "first"): + execute(path, "INSERT INTO sample VALUES (?)", (value,)) + before = member_content(path) + with closing(sqlite3.connect(path)) as conn: + old_rootpage = conn.execute( + "SELECT rootpage FROM sqlite_master WHERE name='sample'" + ).fetchone() + conn.execute("VACUUM") + new_rootpage = conn.execute( + "SELECT rootpage FROM sqlite_master WHERE name='sample'" + ).fetchone() + assert old_rootpage != new_rootpage + assert member_content(path) == before + execute(path, "DELETE FROM sample") + for value in ("first", "first", "second"): + execute(path, "INSERT INTO sample VALUES (?)", (value,)) + assert member_content(path) == before + + +def test_compare_proves_base_even_when_allowed_cache_exists(tmp_path): + from pt_snap_cli.core.msinsight_content_proof import ( + compare_member_content, + member_content, + ) + + path = tmp_path / "base.db" + execute(path, "CREATE TABLE sample (value)") + execute(path, "INSERT INTO sample VALUES (?)", (b"base",)) + proof = member_content(path) + compare_member_content(path, proof, 1) + add_cache(path, 1) + compare_member_content(path, proof, 1) + execute(path, "UPDATE memory_allocation_cache_v1_1 SET payload=?", (b"recomputed",)) + compare_member_content(path, proof, 1) + with pytest.raises(ValueError): + compare_member_content(path, proof, 0) + execute(path, "UPDATE sample SET value=?", (b"modified",)) + with pytest.raises(ValueError): + compare_member_content(path, proof, 1) + + +def test_cache_for_another_device_cannot_hide_in_multi_device_dataset(tmp_path): + service = ImportService() + request = options(tmp_path, multiple=True) + result = service.import_snapshot(request) + add_cache(result.db_path / "device_1/slice_00000.db", 0) + assert_refused_without_changes(service, request, result.db_path) diff --git a/tests/core/test_msinsight_content_races.py b/tests/core/test_msinsight_content_races.py new file mode 100644 index 0000000..6427fa4 --- /dev/null +++ b/tests/core/test_msinsight_content_races.py @@ -0,0 +1,133 @@ +"""Deterministic mutations across owned-cache inspection guard boundaries.""" + +import dataclasses +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import msinsight_export as export +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from tests.core.test_dataset_import import hashes +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +def add_cache(path): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("CREATE TABLE memory_allocation_cache_v1_0 (value INTEGER)") + conn.execute("INSERT INTO memory_allocation_cache_v1_0 VALUES (1)") + + +@pytest.mark.parametrize("force", [False, True]) +def test_manifest_swap_after_validation_preserves_cache_and_focus(tmp_path, monkeypatch, force): + request = options(tmp_path) + service = ImportService() + root = service.import_snapshot(request).db_path + manifest = root / "manifest.json" + focus = tmp_path / ".pt-snap/focus.json" + old_focus = focus.read_bytes() + original = export.validate_dataset + changed = {} + + def swap(*args, **kwargs): + result = original(*args, **kwargs) + data = json.loads(manifest.read_text()) + data["sourceFile"] = str(tmp_path / "other" / request.snapshot_file.name) + replacement = tmp_path / "replacement.json" + replacement.write_text(json.dumps(data)) + replacement.replace(manifest) + changed.update(hashes(root)) + return result + + monkeypatch.setattr(export, "validate_dataset", swap) + with pytest.raises(ImportExecutionError, match="preserved even under --force"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert changed and hashes(root) == changed + assert focus.read_bytes() == old_focus + + +@pytest.mark.parametrize("force", [False, True]) +def test_earlier_member_mutation_during_later_proof_is_rejected(tmp_path, monkeypatch, force): + request = options(tmp_path) + service = ImportService() + root = service.import_snapshot(request).db_path + members = sorted(root.glob("device_0/*.db")) + assert len(members) > 1 + for member in members: + add_cache(member) + focus = tmp_path / ".pt-snap/focus.json" + old_focus = focus.read_bytes() + original = export.compare_member_content + changed = {} + visited = [] + + def change_earlier(path, *args, **kwargs): + original(path, *args, **kwargs) + visited.append(path) + if len(visited) == 2: + with closing(sqlite3.connect(visited[0])) as conn, conn: + conn.execute("UPDATE memory_allocation_cache_v1_0 SET value=2") + changed.update(hashes(root)) + + monkeypatch.setattr(export, "compare_member_content", change_earlier) + with pytest.raises(ImportExecutionError, match="preserved even under --force"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert changed and hashes(root) == changed + assert focus.read_bytes() == old_focus + + +@pytest.mark.parametrize("kind", ["wal", "journal", "shm", "symlink", "unknown"]) +@pytest.mark.parametrize("force", [False, True]) +def test_late_transport_or_inventory_change_is_rejected(tmp_path, monkeypatch, kind, force): + request = options(tmp_path) + service = ImportService() + root = service.import_snapshot(request).db_path + members = sorted(root.glob("device_0/*.db")) + for member in members: + add_cache(member) + focus = tmp_path / ".pt-snap/focus.json" + old_focus = focus.read_bytes() + original = export.compare_member_content + changed = {} + changed_path = None + visited = [] + + def inject_after_last_proof(path, *args, **kwargs): + nonlocal changed_path + original(path, *args, **kwargs) + visited.append(path) + if len(visited) != len(members): + return + if kind in ("wal", "journal", "shm"): + changed_path = Path(str(visited[0]) + "-" + kind) + changed_path.write_bytes(b"test-owned concurrent sidecar") + elif kind == "symlink": + saved = tmp_path / "saved-member.db" + saved.write_bytes(visited[0].read_bytes()) + visited[0].unlink() + visited[0].symlink_to(saved) + changed_path = visited[0] + else: + changed_path = root / "user-notes" + changed_path.write_text("concurrent user data") + changed.update(hashes(root)) + + monkeypatch.setattr(export, "compare_member_content", inject_after_last_proof) + with pytest.raises(ImportExecutionError, match="preserved even under --force"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert changed and hashes(root) == changed + assert focus.read_bytes() == old_focus + assert changed_path is not None and changed_path.exists() + if kind == "symlink": + assert changed_path.is_symlink() From a23794034d4c7447530c0f0d7e2983b0fdb1b009 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:23:19 +0800 Subject: [PATCH 2/2] perf: reuse byte-bound compatible ownership validation Avoid duplicate P0 validation within metadata/overview and retain a bounded in-process ownership proof cache guarded by complete content hashes. Keep cold-process proof costs and all mutation/transport checks explicit. Refs #266 --- docs/en/sharded-snapshotdb.md | 20 ++ docs/zh/sharded-snapshotdb.md | 14 + src/pt_snap_cli/core/import_metadata.py | 15 +- src/pt_snap_cli/core/msinsight_export.py | 86 ++++++- src/pt_snap_cli/core/overview_service.py | 2 +- tests/core/test_msinsight_proof_cache.py | 314 +++++++++++++++++++++++ 6 files changed, 437 insertions(+), 14 deletions(-) create mode 100644 tests/core/test_msinsight_proof_cache.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 6fd46d2..478f929 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -532,6 +532,26 @@ lost integrity evidence, not a diagnosis of corrupted base data. No proof is backfilled from already modified content. Attestations are integrity baselines, not authentication of untrusted manifests or origins. +Owned-proof reuse also has a bounded, process-local 16-root LRU. It stores only +completed ownership results, keyed by the canonical root plus complete manifest +and member-byte fingerprint. Warm hits still read/hash every member and manifest +before and after the operation, recheck transport, inventory and full generation, +and return defensive copies; failures evict the entry. `immutable` resolver mode +never makes this ownership cache stat-only. Nothing is written to the dataset, +manifest, sidecars or a persistent receipt. Overview passes its validated dataset +to metadata/ownership checks; the base validation can be reused only after binding +it to the current full content fingerprint. A resolver result alone never proves +ownership or supplies a missing logical proof. Hard-linked files are rejected. + +**Cold-process cost remains:** Insight byte changes do not update the original +manifest hash. A new CLI process must therefore scan/sort the base content again +for import reuse and recognized dataset `metadata`/`overview`; the ordinary query +resolver does not perform this ownership proof. Repeated API calls in one process +can avoid logical rescans while retaining byte verification, but that benefit must +not be presented as a speedup for separate CLI invocations. Deep stacks, table size +and SQLite temporary sorting storage affect cold cost; no large-scale latency +bound is promised. Source proof generation likewise scans every original table. + Compatible publication is **no-replace even with force**. Recognized caches with matching source/options/identity and byte or base-content proof reuse without loading pickle; force cannot bypass these checks. Source/options changes, base diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 35e5578..5122d77 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -421,6 +421,20 @@ proof 记录全部原始 SQLite schema 对象的 type/name/table/SQL(不含物 也拒绝。旧产物无 proof 时仍须整文件 hash 相同;字节变化表示完整性证据失效,不等于基础 数据损坏,不从已修改现状补造历史证明。证明是完整性基线,不认证不可信 manifest 或来源。 +所有权证明另有进程内最多 16 个 root 的有界 LRU,只缓存完整成功结果;key 绑定规范 root、 +完整 manifest 和全部成员字节 fingerprint。warm 命中仍在操作前后读取并 hash 全部文件, +复查 transport、精确清单和完整 generation,返回深拷贝;失败驱逐缓存。resolver 的 +`immutable` 模式也不把此所有权缓存降级成仅 stat 信任。不写数据集、manifest、sidecar 或 +持久凭据。overview 把已校验数据集传给 metadata/ownership;重新绑定当前完整内容 +fingerprint 后才复用基础校验。普通 resolver 结果不代表所有权,也不能补缺失的逻辑证明。 +有硬链接别名的文件拒绝复用。 + +**冷进程成本仍存在:** Insight 修改字节后,原 manifest hash 不会刷新;每个新的 CLI +进程在 import 复用以及可信数据集 `metadata`/`overview` 时仍须重新扫描/排序基础内容。 +普通查询 resolver 不做这项所有权证明。同进程重复 API 调用可省逻辑扫描,但仍全量校验 +文件字节;不能把 warm API 收益声称为多个独立 CLI 调用的提速。深栈、表大小、SQLite +排序临时存储都会影响冷成本,不承诺大型数据延迟上界。新导出的 proof 生成也扫描全部原表。 + 兼容发布 **force 也 no-replace**。已识别、源/选项/身份相同、字节或基础内容证明匹配的 pt-snap 缓存不加载 pickle 即复用;force 不绕过检查。源/选项变化、基础内容变化、未知对象、 旧产物未被证明的字节变化须换新 `--output-dir`。不修复、不覆盖、不删除、不改写 manifest。 diff --git a/src/pt_snap_cli/core/import_metadata.py b/src/pt_snap_cli/core/import_metadata.py index 422afb7..ec742a8 100644 --- a/src/pt_snap_cli/core/import_metadata.py +++ b/src/pt_snap_cli/core/import_metadata.py @@ -11,7 +11,7 @@ from pt_snap_cli import __version__ from pt_snap_cli.context import Context, DatabaseNotFoundError, SchemaVersionError -from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_resolver import DatasetResolver, ResolvedDataset from pt_snap_cli.core.errors import DatabaseMissingError, DatabaseSchemaError, ImportMetadataError from pt_snap_cli.core.import_metadata_contract import ( metadata_from_mapping, @@ -83,11 +83,16 @@ def build_metadata( completed_at=datetime.now(timezone.utc).isoformat(), ) - def inspect(self, db_path: Path | str) -> MetadataInspection: - dataset = DatasetResolver().inspect(db_path) + def inspect( + self, db_path: Path | str, *, _resolved: ResolvedDataset | None = None + ) -> MetadataInspection: + dataset = _resolved if _resolved is not None else DatasetResolver().inspect(db_path) path = Path(db_path).expanduser().resolve() if dataset is not None: + if path not in (dataset.root, dataset.root / "manifest.json"): + raise DatabaseSchemaError("Dataset resolution does not match metadata path") if isinstance(dataset.validation.manifest, NativeManifest): + dataset.require_unchanged() return MetadataInspection(path, "available", dataset.validation.manifest.metadata) # External P0 metadata remains optional. Only a recognized complete # pt-snap manifest attestation can provide dataset-wide metadata. @@ -95,7 +100,7 @@ def inspect(self, db_path: Path | str) -> MetadataInspection: raw = json.load(source) if "ptSnap" in raw: try: - owned = inspect_owned_cache(dataset.root) + owned = inspect_owned_cache(dataset.root, _resolved=dataset) except ( ValueError, KeyError, @@ -104,8 +109,10 @@ def inspect(self, db_path: Path | str) -> MetadataInspection: sqlite3.DatabaseError, DatabaseSchemaError, ): + dataset.require_unchanged() return MetadataInspection(path, "invalid", reason="metadata_invalid") return MetadataInspection(path, "available", owned.metadata) + dataset.require_unchanged() return MetadataInspection(path, "unavailable", reason="metadata_missing") try: context = Context(path) diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py index 2d8d958..46d909b 100644 --- a/src/pt_snap_cli/core/msinsight_export.py +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -10,12 +10,16 @@ from __future__ import annotations +import copy import hashlib import json import sqlite3 +from collections import OrderedDict from contextlib import closing from dataclasses import asdict, dataclass from pathlib import Path +from threading import RLock +from typing import TYPE_CHECKING from .dataset_contract import ( ACTION_NAMES, @@ -41,6 +45,10 @@ ) from .sharded_replay_service import ShardedReplayResult +if TYPE_CHECKING: + from .dataset_resolver import ResolvedDataset + + COMPATIBILITY_FORMAT = "compatibility-v1" DEFAULT_CAPACITY = 500000 SALT = b"mem_snapshot_parser_v2" @@ -254,6 +262,13 @@ class CompatibleCache: omitted_devices: tuple[int, ...] +# No persisted receipt, live connection or stat-only reuse. Every hit is bracketed +# by current manifest/member byte hashes and transport/generation checks. +_OWNED_CACHE_LIMIT = 16 +_OWNED_CACHE: OrderedDict[Path, tuple[str, CompatibleCache]] = OrderedDict() +_OWNED_CACHE_LOCK = RLock() + + def _cache_inventory(target: Path) -> dict[str, tuple[int, ...]]: """Snapshot identity of the entire owned tree without following aliases.""" if target.resolve() != target or target.is_symlink() or not target.is_dir(): @@ -263,6 +278,8 @@ def _cache_inventory(target: Path) -> dict[str, tuple[int, ...]]: if path.is_symlink() or not (path.is_file() or path.is_dir()): raise ValueError("Unrecognized files or aliases in compatible cache") stat = path.stat() + if path.is_file() and stat.st_nlink != 1: + raise ValueError("Compatible cache files must not have hard-link aliases") result[path.relative_to(target).as_posix()] = ( stat.st_dev, stat.st_ino, @@ -275,7 +292,21 @@ def _cache_inventory(target: Path) -> dict[str, tuple[int, ...]]: return result -def inspect_owned_cache(target: Path) -> CompatibleCache: +def inspect_owned_cache( + target: Path, *, _resolved: ResolvedDataset | None = None +) -> CompatibleCache: + """Inspect complete ownership evidence, evicting any failed admission.""" + try: + return _inspect_owned_cache(target, _resolved=_resolved) + except BaseException: + with _OWNED_CACHE_LOCK: + _OWNED_CACHE.pop(target, None) + raise + + +def _inspect_owned_cache( + target: Path, *, _resolved: ResolvedDataset | None = None +) -> CompatibleCache: """Recognized identity plus byte or versioned base-content attestation. Stable cache-only changes can reuse new proofs. Legacy byte drift cannot be @@ -295,7 +326,42 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: for relative in paths: require_readonly_member(target, target / relative, immutable=True) before[relative] = hash_file(target / relative) - validation = validate_dataset(target) + digest = hashlib.sha256(manifest_hash.encode("ascii")) + for relative, content_hash in before.items(): + digest.update(relative.encode("utf-8")) + digest.update(content_hash.encode("ascii")) + fingerprint = digest.hexdigest() + + def require_unchanged() -> None: + for relative in paths: + require_readonly_member(target, target / relative, immutable=True) + if hash_file(target / relative) != before[relative]: + raise ValueError("Compatible member changed during inspection") + if hash_file(manifest_path) != manifest_hash or _cache_inventory(target) != generation: + raise ValueError("Compatible cache changed during inspection") + + # A resolver result proves only the P0 contract, never ownership. Bind it to + # these current bytes before reusing that base validation; the remaining + # identity, logical proof and metadata checks still run on a cold admission. + if _resolved is not None and ( + _resolved.root != target + or _resolved.fingerprint != fingerprint + or _resolved.validation.manifest != planned + or dict(_resolved.generation or ()) != generation + ): + raise ValueError("Compatible cache changed since dataset resolution") + with _OWNED_CACHE_LOCK: + cached = _OWNED_CACHE.get(target) + if cached is not None and cached[0] == fingerprint: + result = copy.deepcopy(cached[1]) + _OWNED_CACHE.move_to_end(target) + else: + result = None + _OWNED_CACHE.pop(target, None) + if result is not None: + require_unchanged() + return result + validation = _resolved.validation if _resolved is not None else validate_dataset(target) extension = data["ptSnap"] metadata = metadata_from_mapping(extension["metadata"]) identity = extension["identity"] @@ -359,10 +425,12 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError("Device selection disagrees with identity") if Path(validation.manifest.source_file).name != metadata.source_name: raise ValueError("Source path disagrees with metadata") - for relative in paths: - require_readonly_member(target, target / relative, immutable=True) - if hash_file(target / relative) != before[relative]: - raise ValueError("Compatible member changed during inspection") - if hash_file(manifest_path) != manifest_hash or _cache_inventory(target) != generation: - raise ValueError("Compatible cache changed during inspection") - return CompatibleCache(validation.manifest, identity, metadata, tuple(omitted)) + require_unchanged() + result = CompatibleCache(validation.manifest, identity, metadata, tuple(omitted)) + admitted = copy.deepcopy(result) + with _OWNED_CACHE_LOCK: + _OWNED_CACHE[target] = (fingerprint, admitted) + _OWNED_CACHE.move_to_end(target) + while len(_OWNED_CACHE) > _OWNED_CACHE_LIMIT: + _OWNED_CACHE.popitem(last=False) + return result diff --git a/src/pt_snap_cli/core/overview_service.py b/src/pt_snap_cli/core/overview_service.py index c8be173..20569df 100644 --- a/src/pt_snap_cli/core/overview_service.py +++ b/src/pt_snap_cli/core/overview_service.py @@ -65,7 +65,7 @@ def inspect( dataset.validation.manifest.devices, key=lambda d: d.device_id ) ], - metadata=self._metadata_service.inspect(db_path), + metadata=self._metadata_service.inspect(db_path, _resolved=dataset), dataset=dataset.to_dict(), ) ctx = self._validated_context(db_path) diff --git a/tests/core/test_msinsight_proof_cache.py b/tests/core/test_msinsight_proof_cache.py new file mode 100644 index 0000000..c15dd10 --- /dev/null +++ b/tests/core/test_msinsight_proof_cache.py @@ -0,0 +1,314 @@ +"""Process-local proof reuse keeps strong byte and publication guards.""" + +import os +import shutil +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from pt_snap_cli.core import dataset_resolver +from pt_snap_cli.core import msinsight_export as export +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.overview_service import OverviewService +from tests.core.test_dataset_import import hashes +from tests.core.test_msinsight_content_proof import ( + add_cache, + execute, + read_manifest, + write_manifest, +) +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + with export._OWNED_CACHE_LOCK: + export._OWNED_CACHE.clear() + with dataset_resolver._CACHE_LOCK: + dataset_resolver._VALIDATED.clear() + yield + with export._OWNED_CACHE_LOCK: + export._OWNED_CACHE.clear() + with dataset_resolver._CACHE_LOCK: + dataset_resolver._VALIDATED.clear() + + +@pytest.fixture +def root(tmp_path): + result = ImportService().import_snapshot(options(tmp_path, focus=False)) + for path in result.db_path.glob("device_*/*.db"): + add_cache(path) + with export._OWNED_CACHE_LOCK: + export._OWNED_CACHE.clear() + return result.db_path + + +def spies(monkeypatch): + validation = Mock(wraps=export.validate_dataset) + proof = Mock(wraps=export.compare_member_content) + monkeypatch.setattr(export, "validate_dataset", validation) + monkeypatch.setattr(export, "compare_member_content", proof) + return validation, proof + + +def test_cold_validates_and_warm_hashes_twice_without_sql(root, monkeypatch): + validation, proof = spies(monkeypatch) + before = hashes(root) + first = export.inspect_owned_cache(root) + members = sorted(root.glob("device_*/*.db")) + assert validation.call_count == 1 + assert proof.call_count == len(members) + digest = Mock(wraps=export.hash_file) + inventory = Mock(wraps=export._cache_inventory) + monkeypatch.setattr(export, "hash_file", digest) + monkeypatch.setattr(export, "_cache_inventory", inventory) + monkeypatch.setattr( + export.sqlite3, "connect", lambda *a, **kw: pytest.fail("warm proof reuse opened SQLite") + ) + assert export.inspect_owned_cache(root) == first + assert validation.call_count == 1 + assert proof.call_count == len(members) + hashed = [call.args[0] for call in digest.call_args_list] + for path in [root / "manifest.json", *members]: + assert hashed.count(path) >= 2 + assert inventory.call_count >= 2 + assert hashes(root) == before + + +@pytest.mark.parametrize("immutable", [False, True]) +def test_same_size_restored_mtime_base_tamper_is_rejected_and_evicts(root, immutable, monkeypatch): + if immutable: + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + export.inspect_owned_cache(root) + member = root / "device_0/slice_00000.db" + stamp = member.stat() + execute(member, "UPDATE trace_entry_0 SET size=size+1 WHERE id=0") + assert member.stat().st_size == stamp.st_size + os.utime(member, ns=(stamp.st_atime_ns, stamp.st_mtime_ns)) + assert member.stat().st_mtime_ns == stamp.st_mtime_ns + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert root not in export._OWNED_CACHE + + +@pytest.mark.parametrize("change", ["manifest", "proof", "cache-rows"]) +def test_changed_fingerprint_revalidates_instead_of_using_warm_proof(root, monkeypatch, change): + validation, proof = spies(monkeypatch) + export.inspect_owned_cache(root) + validation.reset_mock() + proof.reset_mock() + manifest = read_manifest(root) + if change == "manifest": + (root / "manifest.json").write_text((root / "manifest.json").read_text() + "\n") + elif change == "proof": + first = next(iter(manifest["ptSnap"]["memberContent"]["members"].values())) + first["tables"]["dictionary"]["sha256"] = "0" * 64 + write_manifest(root, manifest) + else: + execute( + root / "device_0/slice_00000.db", + "UPDATE memory_allocation_cache_v1_0 SET payload=X'1234'", + ) + if change == "proof": + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert root not in export._OWNED_CACHE + else: + export.inspect_owned_cache(root) + assert validation.call_count == 1 + assert proof.call_count >= 1 + + +@pytest.mark.parametrize("kind", ["wal", "journal", "extra", "alias"]) +def test_warm_cache_rejects_sidecars_extra_files_and_member_aliases(root, kind): + export.inspect_owned_cache(root) + member = root / "device_0/slice_00000.db" + if kind == "alias": + destination = root.parent / "member-copy.db" + shutil.copyfile(member, destination) + member.unlink() + member.symlink_to(destination) + else: + path = Path(str(member) + {"wal": "-wal", "journal": "-journal", "extra": ".notes"}[kind]) + path.write_bytes(b"unexpected") + before = hashes(root) + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert root not in export._OWNED_CACHE + assert hashes(root) == before + + +@pytest.mark.parametrize("warm", [False, True]) +def test_final_guard_failure_cannot_admit_or_retain_a_proof(root, monkeypatch, warm): + if warm: + export.inspect_owned_cache(root) + original = export.hash_file + manifest_reads = 0 + + def fail_final_hash(path): + nonlocal manifest_reads + digest = original(path) + if path == root / "manifest.json": + manifest_reads += 1 + if manifest_reads == 2: + return "0" * 64 + return digest + + with monkeypatch.context() as patch: + patch.setattr(export, "hash_file", fail_final_hash) + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert root not in export._OWNED_CACHE + validation, _ = spies(monkeypatch) + export.inspect_owned_cache(root) + assert validation.call_count == 1 + + +def test_owned_cache_is_bounded_and_evicts_least_recently_used(root, monkeypatch): + roots = [root] + for index in range(16): + destination = root.parent / f"copy-{index}" + shutil.copytree(root, destination) + roots.append(destination) + for path in roots[:16]: + export.inspect_owned_cache(path) + assert len(export._OWNED_CACHE) == 16 + export.inspect_owned_cache(roots[0]) + export.inspect_owned_cache(roots[16]) + assert len(export._OWNED_CACHE) == 16 + assert roots[0] in export._OWNED_CACHE + assert roots[1] not in export._OWNED_CACHE + validation, _ = spies(monkeypatch) + export.inspect_owned_cache(roots[1]) + assert validation.call_count == 1 + + +def test_mutating_returned_identity_cannot_poison_cached_result(root): + first = export.inspect_owned_cache(root) + expected = first.identity.copy() + first.identity["exportContractVersion"] = 999 + second = export.inspect_owned_cache(root) + assert second.identity == expected + second.identity.clear() + assert export.inspect_owned_cache(root).identity == expected + + +def test_overview_reuses_its_resolution_for_one_base_validation(root, monkeypatch): + with dataset_resolver._CACHE_LOCK: + dataset_resolver._VALIDATED.clear() + resolver_validation = Mock(wraps=dataset_resolver.validate_dataset) + monkeypatch.setattr(dataset_resolver, "validate_dataset", resolver_validation) + own_validation, proof = spies(monkeypatch) + resolver_inspect = Mock(wraps=dataset_resolver.DatasetResolver().inspect) + monkeypatch.setattr(dataset_resolver.DatasetResolver, "inspect", resolver_inspect) + before = hashes(root) + service = OverviewService() + try: + result = service.inspect(root) + finally: + service.close() + assert result.metadata.status == "available" + assert resolver_inspect.call_count == 1 + assert resolver_validation.call_count == 1 + assert own_validation.call_count == 0 + assert proof.call_count == len(list(root.glob("device_*/*.db"))) + assert hashes(root) == before + + +@pytest.mark.parametrize("change", ["bytes", "manifest", "different-root"]) +@pytest.mark.parametrize("immutable", [False, True]) +def test_resolved_dataset_cannot_authorize_changed_or_other_content( + root, change, immutable, monkeypatch +): + if immutable: + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + resolved = dataset_resolver.DatasetResolver().inspect(root) + assert resolved is not None + export.inspect_owned_cache(root, _resolved=resolved) + target = root + if change == "bytes": + execute( + root / "device_0/slice_00000.db", + "UPDATE memory_allocation_cache_v1_0 SET payload=X'9988'", + ) + elif change == "manifest": + (root / "manifest.json").write_text((root / "manifest.json").read_text() + "\n") + else: + target = root.parent / "another-dataset" + shutil.copytree(root, target) + before = hashes(target) + with pytest.raises(ValueError): + export.inspect_owned_cache(target, _resolved=resolved) + assert target not in export._OWNED_CACHE + assert hashes(target) == before + + +def test_matching_base_resolution_does_not_prove_owned_content(root, monkeypatch): + manifest = read_manifest(root) + first = next(iter(manifest["ptSnap"]["memberContent"]["members"].values())) + first["tables"]["dictionary"]["sha256"] = "0" * 64 + write_manifest(root, manifest) + resolved = dataset_resolver.DatasetResolver().inspect(root) + assert resolved is not None + validation, proof = spies(monkeypatch) + with pytest.raises(ValueError): + export.inspect_owned_cache(root, _resolved=resolved) + assert validation.call_count == 0 + assert proof.call_count >= 1 + assert root not in export._OWNED_CACHE + + +@pytest.mark.parametrize("warm", [False, True]) +def test_hardlinked_member_is_rejected_without_admission(root, warm): + if warm: + export.inspect_owned_cache(root) + member = root / "device_0/slice_00000.db" + alias = root.parent / "outside-hardlink.db" + os.link(member, alias) + before = hashes(root) + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert root not in export._OWNED_CACHE + assert hashes(root) == before + + +def test_concurrent_callers_receive_independent_results(root): + before = hashes(root) + expected_version = read_manifest(root)["ptSnap"]["identity"]["exportContractVersion"] + + def inspect_and_mutate(index): + result = export.inspect_owned_cache(root) + version = result.identity["exportContractVersion"] + result.identity["exportContractVersion"] = 100 + index + return version + + with ThreadPoolExecutor(max_workers=4) as executor: + versions = list(executor.map(inspect_and_mutate, range(8))) + assert versions == [expected_version] * 8 + assert export.inspect_owned_cache(root).identity["exportContractVersion"] == expected_version + assert hashes(root) == before + + +def test_concurrent_failed_inspections_never_seed_a_success(root): + manifest = read_manifest(root) + manifest["ptSnap"]["memberContent"]["version"] = 999 + write_manifest(root, manifest) + before = hashes(root) + + def inspect_invalid(_): + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(inspect_invalid, range(8))) + assert root not in export._OWNED_CACHE + with pytest.raises(ValueError): + export.inspect_owned_cache(root) + assert hashes(root) == before