From 56eb261b93fbdcf3e8dfd6418d8235afd9a8c30b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:03:40 +0800 Subject: [PATCH] perf: reuse content-verified dataset validation in process --- docs/en/querying.md | 8 +- docs/en/sharded-snapshotdb.md | 66 +++-- docs/zh/querying.md | 7 +- docs/zh/sharded-snapshotdb.md | 46 ++-- src/pt_snap_cli/core/dataset_resolver.py | 105 +++++-- tests/core/test_dataset_content_cache.py | 286 ++++++++++++++++++++ tests/core/test_dataset_validation_cache.py | 11 +- 7 files changed, 453 insertions(+), 76 deletions(-) create mode 100644 tests/core/test_dataset_content_cache.py diff --git a/docs/en/querying.md b/docs/en/querying.md index 1730893..3d68d68 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -153,10 +153,12 @@ fetched/output work is limited to **100000 rows / 64 MiB serialized values**, in repeated source reads. Exceeding either fails the entire operation, never an alleged complete partial result. Python materialization/sort are bounded by this work ceiling, **not** by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual -cells have overhead. Strong manifest/member validation and hashing remain the default -and share that deadline: hash chunks, Python row batches and SQLite progress handlers +cells have overhead. Strong manifest/member admission remains the default; each +inspection hashes all content, reusing prior in-process row validation only when +content and generation match. Warm work remains O(bytes + shards), and new CLI +processes are cold. Admission shares that deadline: hash chunks, Python row batches and SQLite progress handlers check cancellation. Individual filesystem calls are not OS-level hard-preempted. -`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into process-local validation reuse +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only validation reuse only when the caller guarantees immutable finalized files and reliable filesystem change times; see [dataset validation and cache contracts](sharded-snapshotdb.md). No temporary merge DB, repair or artifact write. diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53f7ae4..0edd799 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -242,32 +242,46 @@ full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. -By default **every independent inspection** fully validates the finalized contract -and hashes the manifest/members. File metadata alone is not content-integrity proof: -unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. -Pinned report-operation guards also rehash content by default, without repeating -row validation. - -`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most -16 validated generations, without open connections. Set it only when you control -finalized, immutable publication and trust your filesystem's kernel change-time -semantics. **Do not enable it for files that may be modified in place**, live writers, -unknown/coarse/network filesystems, or filesystem snapshots that may be rolled back. -This is a caller responsibility contract, not a filesystem capability detected by -the tool. Unset it to retain strong validation; unknown values also use strong mode. -Windows and platforms without a usable POSIX change token remain in strong mode. - -With that explicit opt-in, new resolver/service instances can reuse validated -results. Every hit and pinned-operation guard still rechecks canonical paths, -sidecars, native membership and each file's device, inode, size, mtime, **ctime**, -mode and link count. These conservative checks catch ordinary replacements and -in-place edits with restored mtime, but do not replace the immutable-file contract. -For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: -work is O(shards), independent of event/block counts. Default inspections and cache -misses remain proportional to content. Explicit `validate_dataset` / -`validate_native_dataset` calls always validate fully. Native admission verifies -hashes before and after validation and reuses the verified hashes for its fingerprint, -avoiding a third member-hash pass. +By default, the first inspection fully validates the finalized contract and hashes +its content before/after validation. A process-wide LRU holds at most 16 validated +generations, with no open connections or row-level validation collections. Each +subsequent inspection still hashes the manifest and **every member** and checks +canonical paths, sidecars, native membership and file metadata before/after hashing. +Only an unchanged generation with the same content fingerprint reuses its validated +result, avoiding repeated SQLite/schema/row validation. Changed metadata or content +falls back to complete validation; failures and cancellation evict the entry. + +Default warm work is **O(bytes + shards)**, not O(1) or independent of dataset size. +It saves repeated row-validation CPU and temporary memory, not full-file reads. +Every new CLI process is cold; this is not a persistent or cross-process cache. +Compatibility-v1 cold admission adds a pre-validation member hash to its existing +post-validation hash; native admission retains two passes, reusing verified hashes +for the fingerprint rather than adding a third pass. Pinned report-operation guards +also rehash content by default, without repeating row validation. Explicit +`validate_dataset` / `validate_native_dataset` calls always validate fully. + +File metadata alone is not content-integrity proof: unknown or coarse filesystems +may expose unchanged `ctime_ns` after a write. Unsetting `PT_SNAP_DATASET_CACHE` +selects the default content-verified cache. Setting it to `off` or an unknown value +fully revalidates on every inspection without caching; strong hashing remains active. + +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only reuse in the +same bounded process cache. Set it only when you control finalized, immutable +publication and trust your filesystem's kernel change-time semantics. **Do not +enable it for files that may be modified in place**, live writers, unknown/coarse/ +network filesystems, or filesystem snapshots that may be rolled back. This is a +caller responsibility contract, not a filesystem capability detected by the tool. +Windows and platforms without a usable POSIX change token fully revalidate in this +mode. An immutable-mode compatibility admission cannot seed default reuse until +it has passed the stronger content-bracketed admission. + +With that explicit opt-in, every hit and pinned-operation guard still rechecks +canonical paths, sidecars, native membership and each file's device, inode, size, +mtime, **ctime**, mode and link count. These conservative checks catch ordinary +replacements and in-place edits with restored mtime, but do not replace the +immutable-file contract. An opted-in hit performs no row scans or whole-file hashes: +work is O(shards), independent of event/block counts. None of these checks provides +a concurrent-writer lock or an atomic filesystem snapshot. The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, Python validation loops check every 256 rows, and SQLite validation (including diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 104cf01..52616c8 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -131,10 +131,11 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 精确总数共用递减 deadline;报告的峰值选择与归因明确共用**一个**预算。累计取回/输出工作上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ -SQLite、一个批次及单个 cell 有额外开销。默认仍对 manifest/成员执行完整校验和 hash, -并共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 +SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入:每次对 manifest/全部成员 +取哈希,仅当内容及 generation 一致时复用进程内逐行校验结果。热路径仍为 O(字节数 + +分片数),新 CLI 进程仍冷启动。准入共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 单个文件系统调用不提供 OS 级硬抢占。仅当调用方保证最终产物不可变且文件系统变更时间 -可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用进程内校验复用;参见 +可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用纯元数据校验复用;参见 [数据集校验与缓存合同](sharded-snapshotdb.md)。不生成合并临时 DB、不修复或写产物。 兼容导出器默认**每设备每片 500000 个真实事件**。该存储容量与 **100000 行 / 64 MiB diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index c2ee244..c606653 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -198,23 +198,35 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 -默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 -本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 -已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 - -`PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, -不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 -**可能被原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照 -不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, -未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 - -显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 -规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode -和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 -合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), -不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` -/ `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 -复用已验证摘要,不再做第三遍成员哈希。 +默认首次 inspect 完整校验最终化合同,并在校验前后对内容取哈希。进程级 LRU 最多保留 +16 个已校验 generation,不保留连接或逐行校验集合。后续 inspect 仍对 manifest 和 +**全部成员**取内容哈希,并在哈希前后检查规范路径、sidecar、原生成员集合和文件元数据。 +只有 generation 未变且完整 fingerprint 相同才复用结果,跳过重复 SQLite/schema/逐行 +校验。元数据或内容变化回退完整校验;失败或取消会清除对应缓存条目。 + +默认热路径仍是 **O(内容字节数 + 分片数)**,不是 O(1),也不独立于数据规模。节省的是 +重复逐行校验 CPU 和临时内存,不是整文件读取。每个新 CLI 进程仍为冷启动;没有持久化 +或跨进程缓存。兼容 v1 冷准入在已有校验后哈希外增加一遍校验前成员哈希;原生准入仍为 +两遍,fingerprint 复用已验证摘要,不增加第三遍。已固定 generation 的报告操作默认仍 +重新哈希内容,不重复逐行校验。显式 `validate_dataset` / `validate_native_dataset` +始终完整校验。 + +文件元数据本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 +`ctime_ns`。不设置 `PT_SNAP_DATASET_CACHE` 使用默认内容验证缓存;设为 `off` 或未知 +取值时每次完整重验且不缓存,仍执行强内容哈希。 + +`PT_SNAP_DATASET_CACHE=immutable` 显式选择同一有界进程缓存的纯元数据复用路径。 +仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。**可能被 +原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照不得 +启用**。这是调用方责任合同,不代表工具已检测文件系统能力。Windows 或没有可用 +POSIX change token 的平台在此模式下每次完整重验。immutable 模式准入的兼容数据必须 +先通过更强的前后内容哈希准入,才能用于默认缓存命中。 + +显式启用后,每次命中及报告 generation 检查仍验证规范路径、sidecar、原生成员集合, +以及文件的设备号、inode、size、mtime、**ctime**、mode 和硬链接数。这些保守检查可检测 +常规替换及恢复 mtime 的原地修改,但不能代替不可变文件合同。显式启用后的命中不扫描 +行、不做整文件哈希,工作量为 O(分片数),不随 event/block 数增长。这些检查均不提供 +并发 writer 锁或原子文件系统快照。 查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite 校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index b1d9da3..1f7ea7c 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -29,7 +29,12 @@ from pt_snap_cli.core.dataset_files import hash_file as _hash_file from pt_snap_cli.core.dataset_files import require_readonly_member as _require_readonly_member from pt_snap_cli.core.dataset_support import DATASET_SUPPORT -from pt_snap_cli.core.errors import DatabaseSchemaError, InvalidDeviceError, InvalidParameterError +from pt_snap_cli.core.errors import ( + DatabaseSchemaError, + InvalidDeviceError, + InvalidParameterError, + QueryTimeoutError, +) from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, NativeManifest, @@ -147,11 +152,20 @@ def to_dict(self) -> dict[str, object]: # This is deliberately process-local, bounded, and contains no open connections. -# Cross-call reuse is opt-in: unknown/coarse filesystems cannot supply a -# trustworthy change token merely because Python exposes st_ctime_ns. The caller -# must attest immutable publication and suitable change-time semantics explicitly. +# Default reuse verifies all content bytes; metadata alone never proves integrity. +# Skipping byte scans requires the explicit immutable-publication contract. _CACHE_LIMIT = 16 -_VALIDATED: OrderedDict[Path, tuple[_Generation, ResolvedDataset]] = OrderedDict() + + +@dataclass(frozen=True) +class _ValidatedEntry: + generation: _Generation + resolved: ResolvedDataset + # Only admissions bracketed by content hashes can seed default reuse. + content_verified: bool + + +_VALIDATED: OrderedDict[Path, _ValidatedEntry] = OrderedDict() _CACHE_LOCK = RLock() @@ -216,11 +230,17 @@ def _cacheable(generation: _Generation) -> bool: def _content_fingerprint( - root: Path, manifest: DatasetManifest, budget: ValidationBudget | None + root: Path, + manifest: DatasetManifest, + budget: ValidationBudget | None, + *, + manifest_hash: str | None = None, ) -> str: """Strong verification for pinned operations without repeating row validation.""" digest = hashlib.sha256() - digest.update(_hash_file(root / "manifest.json", budget=budget).encode("ascii")) + digest.update( + (manifest_hash or _hash_file(root / "manifest.json", budget=budget)).encode("ascii") + ) for device in checked_rows(manifest.devices, budget): for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) @@ -231,7 +251,9 @@ def _content_fingerprint( class DatasetResolver: """Resolve and validate finalized datasets, reusing unchanged generations. - By default every call validates and hashes content. Explicitly setting + By default every call hashes content; a matching, fully validated in-process + result skips repeated row validation. Warm work remains O(bytes + shards), + and new CLI processes are cold. Explicitly setting PT_SNAP_DATASET_CACHE=immutable enables bounded process-local reuse under the caller's immutable-publication/change-time contract. Even then paths, members, sidecars and kernel change times are checked on every call. @@ -241,15 +263,26 @@ class DatasetResolver: def inspect( self, path: Path | str, *, budget: ValidationBudget | None = None ) -> ResolvedDataset | None: - check_budget(budget) - root = _root(path) - if root is None: - return None + candidate = Path(path).expanduser().absolute() + with _CACHE_LOCK: + cache_key = ( + candidate.parent + if candidate.name == "manifest.json" and candidate not in _VALIDATED + else candidate + ) try: + check_budget(budget) + root = _root(path) + if root is None: + with _CACHE_LOCK: + _ = _VALIDATED.pop(cache_key, None) + return None + cache_key = root + strong_cache = os.environ.get("PT_SNAP_DATASET_CACHE") is None with _CACHE_LOCK: cached = _VALIDATED.get(root) if cached is not None: - previous, resolved = cached + previous, resolved = cached.generation, cached.resolved manifest_unchanged = ( _stamp(root / "manifest.json") == dict(previous)["manifest.json"] ) @@ -258,7 +291,18 @@ def inspect( if manifest_unchanged else None ) - if generation is not None and _cacheable(generation) and generation == previous: + reuse = False + if generation is not None and generation == previous: + if _cacheable(generation): + reuse = True + elif strong_cache and cached.content_verified: + fingerprint = _content_fingerprint( + root, resolved.validation.manifest, budget + ) + if _generation(root, resolved.validation.manifest, budget) != generation: + raise DatabaseSchemaError("Dataset changed during inspection; retry.") + reuse = fingerprint == resolved.fingerprint + if reuse: with _CACHE_LOCK: if root in _VALIDATED: _VALIDATED.move_to_end(root) @@ -266,7 +310,7 @@ def inspect( check_budget(budget) return result with _CACHE_LOCK: - _VALIDATED.pop(root, None) + _ = _VALIDATED.pop(root, None) manifest_path = root / "manifest.json" manifest_stamp = _stamp(manifest_path) before = _hash_file(manifest_path, budget=budget) @@ -283,6 +327,14 @@ def inspect( generation = _generation(root, planned, budget) if manifest_stamp != _stamp(manifest_path): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + # Compatibility manifests contain no shard digests. Bracket row + # validation with byte hashes before allowing its result to be reused. + # Native validation already checks both ends against declared hashes. + compatibility_before = ( + _content_fingerprint(root, planned, budget, manifest_hash=before) + if strong_cache and not isinstance(planned, NativeManifest) + else None + ) validation = ( validate_native_dataset(root, planned, budget=budget) if isinstance(planned, NativeManifest) @@ -301,6 +353,9 @@ def inspect( else _hash_file(root / item.file, budget=budget) ) digest.update(content_hash.encode("ascii")) + fingerprint = digest.hexdigest() + if compatibility_before is not None and compatibility_before != fingerprint: + raise DatabaseSchemaError("Dataset content changed during inspection; retry.") if before != _hash_file(manifest_path, budget=budget): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") if generation != _generation(root, planned, budget): @@ -311,21 +366,31 @@ def inspect( result = ResolvedDataset( root, validation, - digest.hexdigest(), + fingerprint, version if type(version) is int else None, generation, ) - if _cacheable(generation): + if strong_cache or _cacheable(generation): + entry = _ValidatedEntry( + generation, + copy.deepcopy(result), + isinstance(planned, NativeManifest) or compatibility_before is not None, + ) + check_budget(budget) with _CACHE_LOCK: - _VALIDATED[root] = (generation, copy.deepcopy(result)) + _VALIDATED[root] = entry _VALIDATED.move_to_end(root) while len(_VALIDATED) > _CACHE_LIMIT: - _VALIDATED.popitem(last=False) + _ = _VALIDATED.popitem(last=False) return result except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: with _CACHE_LOCK: - _VALIDATED.pop(root, None) + _ = _VALIDATED.pop(cache_key, None) raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc + except (DatabaseSchemaError, QueryTimeoutError): + with _CACHE_LOCK: + _ = _VALIDATED.pop(cache_key, None) + raise def completion_device_ids(self, path: Path | str) -> list[int] | None: """Bounded manifest-only hints, never evidence of dataset validity. diff --git a/tests/core/test_dataset_content_cache.py b/tests/core/test_dataset_content_cache.py new file mode 100644 index 0000000..ff5e7ad --- /dev/null +++ b/tests/core/test_dataset_content_cache.py @@ -0,0 +1,286 @@ +"""Default reuse must prove content equality, even with untrustworthy stat tokens.""" + +import os +import sqlite3 +from collections import Counter +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_resolver as resolver +from pt_snap_cli.core import native_dataset_contract as native +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from tests.core.test_dataset_validation_cache import CancellableBudget +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(autouse=True) +def default_cache(monkeypatch): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + resolver._VALIDATED.clear() + yield + resolver._VALIDATED.clear() + + +@pytest.fixture(params=[False, True], ids=["compatibility", "native"]) +def artifact(tmp_path, request): + return native_dataset(tmp_path) if request.param else make_dataset(tmp_path / "dataset") + + +def test_default_cold_and_warm_work_counts(artifact, monkeypatch): + hashes = Counter() + + def tracked(path, **kwargs): + hashes[path] += 1 + return hash_file(path, **kwargs) + + monkeypatch.setattr(resolver, "_hash_file", tracked) + monkeypatch.setattr(native, "_hash_file", tracked) + first = resolver.DatasetResolver().inspect(artifact) + for member in artifact.glob("*/*.db"): + assert hashes[member] == 2 + assert hashes[artifact / "manifest.json"] == 2 + hashes.clear() + + def forbidden(*args, **kwargs): + pytest.fail("content-verified warm hit must not repeat SQLite/row validation") + + monkeypatch.setattr(resolver, "validate_dataset", forbidden) + monkeypatch.setattr(resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + second = resolver.DatasetResolver().inspect(artifact / "manifest.json") + assert first == second and first is not second + for member in artifact.glob("*/*.db"): + assert hashes[member] == 1 + assert hashes[artifact / "manifest.json"] == 1 + # Report guards remain independently strong, even following a cache hit. + second.require_unchanged() + for member in artifact.glob("*/*.db"): + assert hashes[member] == 2 + + +def test_default_detects_tampering_with_all_change_tokens_hidden(artifact, monkeypatch): + resolved = resolver.DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + assert (member.stat().st_size, member.stat().st_mtime_ns) == ( + before.st_size, + before.st_mtime_ns, + ) + monkeypatch.setattr(resolver, "_generation", lambda *args: resolved.generation) + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +@pytest.mark.parametrize("change", ["touch", "replace", "manifest"]) +def test_default_metadata_changes_fully_revalidate(artifact, monkeypatch, change): + first = resolver.DatasetResolver().inspect(artifact) + member = artifact / "manifest.json" if change == "manifest" else next(artifact.glob("*/*.db")) + if change == "replace": + replacement = member.with_suffix(".replacement") + replacement.write_bytes(member.read_bytes()) + replacement.replace(member) + else: + before = member.stat() + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns + 1000000)) + calls = [] + for name in ("validate_dataset", "validate_native_dataset"): + original = getattr(resolver, name) + + def tracked(*args, _original=original, **kwargs): + calls.append(1) + return _original(*args, **kwargs) + + monkeypatch.setattr(resolver, name, tracked) + second = resolver.DatasetResolver().inspect(artifact) + assert first.fingerprint == second.fingerprint + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "change", ["wal", "journal", "shm", "member-alias", "root-alias", "missing"] +) +def test_default_preflight_failure_evicts_cached_entry(artifact, tmp_path, change): + resolver.DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + if change == "missing": + member.unlink() + elif change in ("member-alias", "root-alias"): + source = member if change == "member-alias" else artifact + saved = tmp_path / "renamed" + source.rename(saved) + source.symlink_to(saved, target_is_directory=change == "root-alias") + else: + Path(str(member) + "-" + change).touch() + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_native_unlisted_member_invalidates(tmp_path): + root = native_dataset(tmp_path) + resolver.DatasetResolver().inspect(root) + (root / "device_0" / "slice_99999.db").write_bytes(b"extra") + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(root) + assert root not in resolver._VALIDATED + + +def test_warm_hash_cancellation_evicts_and_next_call_recovers(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + budget = CancellableBudget() + original = resolver._hash_file + + def cancelled(path, **kwargs): + if path.suffix == ".db": + budget.cancelled = True + return original(path, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(resolver, "_hash_file", cancelled) + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(artifact, budget=budget) + assert artifact not in resolver._VALIDATED + assert resolver.DatasetResolver().inspect(artifact) is not None + + +def test_expired_budget_evicts_without_any_hash(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + budget = CancellableBudget() + budget.cancelled = True + + def forbidden(*args, **kwargs): + pytest.fail("expired deadline must precede file hashing") + + monkeypatch.setattr(resolver, "_hash_file", forbidden) + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(artifact, budget=budget) + assert artifact not in resolver._VALIDATED + + +def test_warm_concurrent_change_refuses_and_evicts(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + original = resolver._hash_file + + def changed(path, **kwargs): + value = original(path, **kwargs) + if path.suffix == ".db": + before = path.stat() + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 1000000)) + return value + + monkeypatch.setattr(resolver, "_hash_file", changed) + with pytest.raises(DatabaseSchemaError, match="changed during inspection"): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_compatibility_cold_hashes_bracket_validation(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + original = resolver.validate_dataset + + def changed(*args, **kwargs): + result = original(*args, **kwargs) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='valid changed text'") + return result + + monkeypatch.setattr(resolver, "validate_dataset", changed) + with pytest.raises(DatabaseSchemaError, match="content changed during inspection"): + resolver.DatasetResolver().inspect(root) + assert root not in resolver._VALIDATED + + +def test_default_cache_is_bounded_deep_copied_and_owns_no_connections(tmp_path, monkeypatch): + monkeypatch.setattr(resolver, "_CACHE_LIMIT", 2) + roots = [make_dataset(tmp_path / str(i), devices=(0,), slices=1) for i in range(3)] + for root in roots: + resolver.DatasetResolver().inspect(root) + assert list(resolver._VALIDATED) == roots[1:] + first = resolver.DatasetResolver().inspect(roots[1]) + object.__setattr__(first.validation, "real_event_count", -1) + assert resolver.DatasetResolver().inspect(roots[1]).validation.real_event_count == 2 + assert list(resolver._VALIDATED) == [roots[2], roots[1]] + # Cached entries only contain result data; file removal needs no cache close. + for root in roots: + for path in root.glob("*/*.db"): + path.unlink() + assert all(not tuple(root.glob("*/*.db")) for root in roots) + + +def test_immutable_v1_entry_cannot_seed_default_reuse(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + resolver.DatasetResolver().inspect(root) + assert not resolver._VALIDATED[root].content_verified + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + calls = [] + original = resolver.validate_dataset + + def tracked(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(resolver, "validate_dataset", tracked) + resolver.DatasetResolver().inspect(root) + resolver.DatasetResolver().inspect(root) + assert calls == [1] + assert resolver._VALIDATED[root].content_verified + + +def test_default_reuse_does_not_require_trusted_posix_tokens(artifact, monkeypatch): + monkeypatch.setattr(resolver, "_cacheable", lambda generation: False) + resolver.DatasetResolver().inspect(artifact) + assert resolver.DatasetResolver().inspect(artifact) is not None + assert artifact in resolver._VALIDATED + + +def test_valid_compatibility_content_change_revalidates_with_hidden_tokens(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + first = resolver.DatasetResolver().inspect(root) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='valid new content'") + monkeypatch.setattr(resolver, "_generation", lambda *args: first.generation) + calls = [] + original = resolver.validate_dataset + + def tracked(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(resolver, "validate_dataset", tracked) + second = resolver.DatasetResolver().inspect(root) + assert second.fingerprint != first.fingerprint + assert calls == [1] + assert resolver.DatasetResolver().inspect(root) == second + assert calls == [1] + + +def test_hash_io_failure_evicts_entry(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + + def failed(*args, **kwargs): + raise PermissionError("unreadable member") + + monkeypatch.setattr(resolver, "_hash_file", failed) + with pytest.raises(DatabaseSchemaError, match="unreadable member"): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_dataset_directory_named_manifest_cancellation_evicts_exact_entry(tmp_path): + root = make_dataset(tmp_path / "manifest.json", devices=(0,), slices=1) + resolver.DatasetResolver().inspect(root) + budget = CancellableBudget() + budget.cancelled = True + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(root, budget=budget) + assert root not in resolver._VALIDATED diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py index af0287a..094ecc2 100644 --- a/tests/core/test_dataset_validation_cache.py +++ b/tests/core/test_dataset_validation_cache.py @@ -347,12 +347,9 @@ def test_native_lock_wait_respects_deadline(tmp_path): assert DatasetResolver().inspect(root) is not None -@pytest.mark.parametrize("mode", [None, "off", "unknown"]) -def test_default_and_unknown_cache_mode_repeat_strong_validation(tmp_path, monkeypatch, mode): - if mode is None: - monkeypatch.delenv("PT_SNAP_DATASET_CACHE") - else: - monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) +@pytest.mark.parametrize("mode", ["off", "unknown"]) +def test_disabled_and_unknown_cache_mode_repeat_full_validation(tmp_path, monkeypatch, mode): + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) calls = [] original = dataset_resolver._hash_file @@ -387,7 +384,7 @@ def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch) root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) DatasetResolver().inspect(root) assert root in dataset_resolver._VALIDATED - monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "off") with pytest.raises(QueryTimeoutError): budget = CancellableBudget() budget.cancelled = True