From 32639e130413e10e7016a024be828c2e0ab9d296 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:27:08 +0800 Subject: [PATCH 1/2] fix: make dataset validation cancellable and cache reuse explicit (#219) --- docs/en/sharded-snapshotdb.md | 44 +- docs/zh/sharded-snapshotdb.md | 30 +- src/pt_snap_cli/completion.py | 6 +- src/pt_snap_cli/core/dataset_contract.py | 49 ++- src/pt_snap_cli/core/dataset_files.py | 6 +- src/pt_snap_cli/core/dataset_resolver.py | 243 +++++++++-- .../core/native_dataset_contract.py | 125 ++++-- src/pt_snap_cli/core/query_service.py | 2 +- src/pt_snap_cli/core/validation_budget.py | 79 ++++ tests/core/test_dataset_validation_cache.py | 409 ++++++++++++++++++ tests/test_dataset_focus.py | 4 +- 11 files changed, 896 insertions(+), 101 deletions(-) create mode 100644 src/pt_snap_cli/core/validation_budget.py create mode 100644 tests/core/test_dataset_validation_cache.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 3948d19..53f7ae4 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -242,12 +242,44 @@ full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. -It revalidates the finalized P0 contract on each call and hashes the manifest and -members; no unbounded manifest/connection cache is retained. Hashing uses bounded -chunks and is proportional to artifact size, not a performance claim. Context LRU -size defaults to four; dataset generation changes invalidate contexts on next -lookup even when size/mtime are unchanged. Analyzer-owned caches close on exit; -injected caches remain caller-owned. Validation opens/closes slices sequentially. +By default **every independent inspection** fully validates the finalized contract +and hashes the manifest/members. File metadata alone is not content-integrity proof: +unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. +Pinned report-operation guards also rehash content by default, without repeating +row validation. + +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most +16 validated generations, without open connections. Set it only when you control +finalized, immutable publication and trust your filesystem's kernel change-time +semantics. **Do not enable it for files that may be modified in place**, live writers, +unknown/coarse/network filesystems, or filesystem snapshots that may be rolled back. +This is a caller responsibility contract, not a filesystem capability detected by +the tool. Unset it to retain strong validation; unknown values also use strong mode. +Windows and platforms without a usable POSIX change token remain in strong mode. + +With that explicit opt-in, new resolver/service instances can reuse validated +results. Every hit and pinned-operation guard still rechecks canonical paths, +sidecars, native membership and each file's device, inode, size, mtime, **ctime**, +mode and link count. These conservative checks catch ordinary replacements and +in-place edits with restored mtime, but do not replace the immutable-file contract. +For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: +work is O(shards), independent of event/block counts. Default inspections and cache +misses remain proportional to content. Explicit `validate_dataset` / +`validate_native_dataset` calls always validate fully. Native admission verifies +hashes before and after validation and reuses the verified hashes for its fingerprint, +avoiding a third member-hash pass. + +The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, +Python validation loops check every 256 rows, and SQLite validation (including +`quick_check`) uses an interruptible progress handler, cleared on every exit. +Cancellation raises `QUERY_TIMEOUT`; no partial validation is cached. Device tab +completion reads only a size-bounded manifest (up to 1 MiB), never member SQLite +contents or hashes. Its candidates are hints, not proof the dataset is valid. + +Context LRU size defaults to four; dataset content-generation changes invalidate +contexts on next lookup even when size/mtime are unchanged. Analyzer-owned caches +close on exit; injected caches remain caller-owned. Validation opens/closes slices +sequentially. Building, missing/inconsistent slices, unknown base versions and unsafe paths are rejected before focus writes or query execution. Before **any** SQLite open, diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 137d3fd..c2ee244 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -198,10 +198,32 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 -每次调用重新校验最终化 P0 合同并对 manifest/所有成员取内容哈希,不保存无界 manifest/ -连接缓存。哈希按有界块读取,工作量与产物大小成正比,不宣称性能收益。Context LRU 默认 -最多四个;数据集 generation 变化在下次 lookup 使旧 Context 失效,即使 size/mtime 相同。 -analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有;校验逐片打开并关闭连接。 +默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 +本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 +已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 + +`PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, +不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 +**可能被原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照 +不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, +未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 + +显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 +规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode +和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 +合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), +不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` +/ `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 +复用已验证摘要,不再做第三遍成员哈希。 + +查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite +校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 +取消保留 `QUERY_TIMEOUT`,不会缓存不完整校验。设备 tab 补全只读取有界 manifest +(最多 1 MiB),不读取成员 SQLite 内容或整文件哈希;候选仅为提示,不证明数据集有效。 + +Context LRU 默认最多四个;数据集内容 generation 变化在下次 lookup 使旧 Context 失效, +即使 size/mtime 相同。analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有; +校验逐片打开并关闭连接。 building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写入或查询前拒绝。 **任何** SQLite 打开之前,裸 P0 validator 与 resolver 均先检查**全部**成员/父目录别名及 diff --git a/src/pt_snap_cli/completion.py b/src/pt_snap_cli/completion.py index 797dfff..749104c 100644 --- a/src/pt_snap_cli/completion.py +++ b/src/pt_snap_cli/completion.py @@ -77,9 +77,9 @@ def complete_device_ids() -> list[str]: try: from pt_snap_cli.core.dataset_resolver import DatasetResolver - dataset = DatasetResolver().inspect(db_path) - if dataset is not None: - return [str(device) for device in dataset.device_ids] + devices = DatasetResolver().completion_device_ids(db_path) + if devices is not None: + return [str(device) for device in devices] uri = f"file:{db_path}?mode=ro" with closing(sqlite3.connect(uri, uri=True)) as conn: cursor = conn.cursor() diff --git a/src/pt_snap_cli/core/dataset_contract.py b/src/pt_snap_cli/core/dataset_contract.py index 9d8be4c..a8b03fd 100644 --- a/src/pt_snap_cli/core/dataset_contract.py +++ b/src/pt_snap_cli/core/dataset_contract.py @@ -18,6 +18,13 @@ from .dataset_files import require_readonly_member from .errors import DatabaseSchemaError +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) MANIFEST_SCHEMA_VERSION = 1 MSINSIGHT_REVISION = "101f65b877a267ffd5f66ea3834706057ba243e5" @@ -118,12 +125,15 @@ def require_complete(self) -> None: ) -def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetManifest: +def parse_manifest( + value: object, *, require_complete: bool = True, budget: ValidationBudget | None = None +) -> DatasetManifest: """Validate JSON-shaped data; inspection may explicitly accept building. Unknown extension declarations are ignored, never promoted to capabilities. This does not check files; use validate_dataset for finalized artifacts. """ + check_budget(budget) data = _object(value, "manifest") version = _integer(data.get("schemaVersion"), "schemaVersion", maximum=2**31 - 1) _need( @@ -137,7 +147,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa devices_data = _object(data.get("devices"), "devices") _need(bool(devices_data), "devices", "empty", "at least one event-bearing device required") devices: list[DeviceRecord] = [] - for key, value in devices_data.items(): + for key, value in checked_rows(devices_data.items(), budget): location = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -153,7 +163,9 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) ready = tuple( _integer(item, location + ".readySlices", maximum=size - 1) - for item in _array(device.get("readySlices"), location + ".readySlices") + for item in checked_rows( + _array(device.get("readySlices"), location + ".readySlices"), budget + ) ) _need( len(set(ready)) == len(ready), @@ -170,7 +182,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) slices: list[SliceRecord] = [] expected_start = 0 - for index, raw in enumerate(records): + for index, raw in enumerate(checked_rows(records, budget)): loc = f"{location}.slices[{index}]" item = _object(raw, loc) number = _integer(item.get("index"), loc + ".index", maximum=2**31 - 1) @@ -357,6 +369,7 @@ def _validate_slice( device: DeviceRecord, item: SliceRecord, identities: dict[tuple[int, int], tuple[object, ...]], + budget: ValidationBudget | None = None, ) -> int: loc = item.file trace, block = f"trace_entry_{device.device_id}", f"block_{device.device_id}" @@ -415,7 +428,7 @@ def _validate_slice( ).fetchone() _need(bad is None, loc, "event_values", "invalid scalar type or negative byte metric") boundary_count = conn.execute(f'SELECT COUNT(*) FROM "{trace}" WHERE id<0').fetchone()[0] - for row in conn.execute(f'SELECT * FROM "{block}"'): + for row in checked_rows(conn.execute(f'SELECT * FROM "{block}"'), budget): block_id, address, size, requested, state, alloc, free = row for name, value in zip( ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId"), @@ -455,12 +468,15 @@ def _validate_slice( return cast(int, boundary_count) -def validate_dataset(directory: Path | str) -> DatasetValidation: +def validate_dataset( + directory: Path | str, *, budget: ValidationBudget | None = None +) -> DatasetValidation: """Read a complete compatibility-v1 dataset without migration or pickle. Checks are validation, not a filesystem sandbox or concurrent publication lock. Missing pt_snap_metadata/extension tables are valid external base artifacts. """ + check_budget(budget) root = Path(directory).absolute() _need( root.resolve() == root and not root.is_symlink(), @@ -472,11 +488,11 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: path = _safe_file(root, "manifest.json") with path.open(encoding="utf-8") as source: data: object = json.load(source, object_pairs_hook=_unique_object) - manifest = parse_manifest(data) + manifest = parse_manifest(data, budget=budget) # Preflight ALL members before the first SQLite open, also for callers # using this bare P0 validator rather than DatasetResolver. for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): member = _safe_file(root, item.file) try: require_readonly_member(root, member, immutable=True) @@ -485,14 +501,21 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: identities: dict[tuple[int, int], tuple[object, ...]] = {} boundaries = 0 for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): db_path = _safe_file(root, item.file) try: - with closing( - sqlite3.connect(db_path.as_uri() + "?mode=ro&immutable=1", uri=True) - ) as conn: + with ( + closing( + sqlite3.connect( + db_path.as_uri() + "?mode=ro&immutable=1", + uri=True, + timeout=sqlite_timeout(budget), + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") - boundaries += _validate_slice(conn, device, item, identities) + boundaries += _validate_slice(conn, device, item, identities, budget) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc return DatasetValidation( diff --git a/src/pt_snap_cli/core/dataset_files.py b/src/pt_snap_cli/core/dataset_files.py index e32a6d6..23f94b6 100644 --- a/src/pt_snap_cli/core/dataset_files.py +++ b/src/pt_snap_cli/core/dataset_files.py @@ -6,6 +6,7 @@ from pathlib import Path from .errors import DatabaseSchemaError +from .validation_budget import ValidationBudget, check_budget def require_readonly_member(root: Path, member: Path, *, immutable: bool = False) -> None: @@ -27,9 +28,12 @@ def require_readonly_member(root: Path, member: Path, *, immutable: bool = False raise DatabaseSchemaError(f"Dataset member uses persistent WAL mode: {member}") -def hash_file(path: Path) -> str: +def hash_file(path: Path, *, budget: ValidationBudget | None = None) -> str: + check_budget(budget) digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): + check_budget(budget) digest.update(chunk) + check_budget(budget) return digest.hexdigest() diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index 94126ec..b1d9da3 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -7,13 +7,18 @@ from __future__ import annotations +import copy import hashlib import json +import os +from collections import OrderedDict from dataclasses import dataclass from pathlib import Path +from threading import RLock from pt_snap_cli.core.dataset_contract import ( DatasetContractError, + DatasetManifest, DatasetValidation, DeviceRecord, QueryScope, @@ -28,10 +33,16 @@ from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, NativeManifest, + NativeSlice, parse_native_manifest, + require_native_members, validate_native_dataset, validate_native_scope, ) +from pt_snap_cli.core.validation_budget import ValidationBudget, check_budget, checked_rows + +_FileStamp = tuple[int, int, int, int, int, int, int] +_Generation = tuple[tuple[str, _FileStamp], ...] @dataclass(frozen=True) @@ -40,6 +51,25 @@ class ResolvedDataset: validation: DatasetValidation fingerprint: str ordered_frames_version: int | None = None + generation: _Generation | None = None + + def require_unchanged(self, budget: ValidationBudget | None = None) -> None: + """Guard a reused report/query resolution without repeating full validation.""" + if _root(self.root) != self.root: + raise DatabaseSchemaError("Dataset path changed during operation; retry.") + try: + current = _generation(self.root, self.validation.manifest, budget) + if self.generation is None or current != self.generation: + raise DatabaseSchemaError("Dataset members changed during operation; retry.") + if not _cacheable(current): + if ( + _content_fingerprint(self.root, self.validation.manifest, budget) + != self.fingerprint + or _generation(self.root, self.validation.manifest, budget) != current + ): + raise DatabaseSchemaError("Dataset content changed during operation; retry.") + except (DatasetContractError, OSError) as exc: + raise DatabaseSchemaError(f"Dataset changed during operation: {exc}") from exc @property def device_ids(self) -> list[int]: @@ -116,62 +146,207 @@ def to_dict(self) -> dict[str, object]: } +# This is deliberately process-local, bounded, and contains no open connections. +# Cross-call reuse is opt-in: unknown/coarse filesystems cannot supply a +# trustworthy change token merely because Python exposes st_ctime_ns. The caller +# must attest immutable publication and suitable change-time semantics explicitly. +_CACHE_LIMIT = 16 +_VALIDATED: OrderedDict[Path, tuple[_Generation, ResolvedDataset]] = OrderedDict() +_CACHE_LOCK = RLock() + + +def _stamp(path: Path) -> _FileStamp: + stat = path.stat() + return ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + stat.st_mode, + stat.st_nlink, + ) + + +def _root(path: Path | str) -> Path | None: + candidate = Path(path).expanduser().absolute() + if not candidate.is_dir() and candidate.name != "manifest.json": + return None + root = candidate if candidate.is_dir() else candidate.parent + if candidate.is_symlink() or root.resolve() != root: + raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") + if (root / "manifest.json").is_symlink(): + raise DatabaseSchemaError("Dataset manifest must not be a symlink.") + if not (root / "manifest.json").is_file(): + raise DatabaseSchemaError("Dataset manifest.json must be a regular file.") + return root + + +def _generation( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> _Generation: + """Always recheck aliases, transport, and membership before trusting a hit.""" + check_budget(budget) + if isinstance(manifest, NativeManifest): + require_native_members(root, manifest, budget=budget) + stamps = [(".", _stamp(root)), ("manifest.json", _stamp(root / "manifest.json"))] + for device in checked_rows(manifest.devices, budget): + directory = f"device_{device.device_id}" + if (root / directory).is_symlink(): + raise DatabaseSchemaError("Dataset device directory must not be a symlink.") + stamps.append((directory, _stamp(root / directory))) + for item in checked_rows(device.slices, budget): + member = root / item.file + _require_readonly_member( + root, member, immutable=not isinstance(manifest, NativeManifest) + ) + stamps.append((item.file, _stamp(member))) + check_budget(budget) + return tuple(stamps) + + +def _cacheable(generation: _Generation) -> bool: + # This is a caller contract, not an automatic inference from OS or filesystem + # metadata. Windows ctime historically means creation time, so stay strong. + return ( + os.environ.get("PT_SNAP_DATASET_CACHE") == "immutable" + and os.name == "posix" + and all(stamp[4] > 0 for _, stamp in generation) + ) + + +def _content_fingerprint( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> str: + """Strong verification for pinned operations without repeating row validation.""" + digest = hashlib.sha256() + digest.update(_hash_file(root / "manifest.json", budget=budget).encode("ascii")) + for device in checked_rows(manifest.devices, budget): + for item in checked_rows(device.slices, budget): + digest.update(item.file.encode("utf-8")) + digest.update(_hash_file(root / item.file, budget=budget).encode("ascii")) + return digest.hexdigest() + + class DatasetResolver: - """Stateless resolution: revalidate every call, retain no connections/manifest cache. + """Resolve and validate finalized datasets, reusing unchanged generations. - Content hashes (not cacheHash or size/mtime alone) form the Context generation. - Work is proportional to artifact size; no concurrent-publication lock is claimed. + By default every call validates and hashes content. Explicitly setting + PT_SNAP_DATASET_CACHE=immutable enables bounded process-local reuse under + the caller's immutable-publication/change-time contract. Even then paths, + members, sidecars and kernel change times are checked on every call. + Explicit validators always perform full validation. """ - def inspect(self, path: Path | str) -> ResolvedDataset | None: - candidate = Path(path).expanduser().absolute() - if not candidate.is_dir() and candidate.name != "manifest.json": + def inspect( + self, path: Path | str, *, budget: ValidationBudget | None = None + ) -> ResolvedDataset | None: + check_budget(budget) + root = _root(path) + if root is None: return None - root = candidate if candidate.is_dir() else candidate.parent - if candidate.is_symlink() or root.resolve() != root: - raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") try: + with _CACHE_LOCK: + cached = _VALIDATED.get(root) + if cached is not None: + previous, resolved = cached + manifest_unchanged = ( + _stamp(root / "manifest.json") == dict(previous)["manifest.json"] + ) + generation = ( + _generation(root, resolved.validation.manifest, budget) + if manifest_unchanged + else None + ) + if generation is not None and _cacheable(generation) and generation == previous: + with _CACHE_LOCK: + if root in _VALIDATED: + _VALIDATED.move_to_end(root) + result = copy.deepcopy(resolved) + check_budget(budget) + return result + with _CACHE_LOCK: + _VALIDATED.pop(root, None) manifest_path = root / "manifest.json" - if manifest_path.is_symlink(): - raise DatabaseSchemaError("Dataset manifest must not be a symlink.") - before = _hash_file(manifest_path) + manifest_stamp = _stamp(manifest_path) + before = _hash_file(manifest_path, budget=budget) with manifest_path.open(encoding="utf-8") as source: raw: object = json.load(source, object_pairs_hook=_unique_object) + check_budget(budget) planned = ( - parse_native_manifest(raw) + parse_native_manifest(raw, budget=budget) if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT - else parse_manifest(raw) + else parse_manifest(raw, budget=budget) ) - # mode=ro can create WAL/SHM files even after the last writer has - # checkpointed and removed them. Check every member before the first - # SQLite open, without following aliases or repairing the artifact. - for device in planned.devices: - for item in device.slices: - _require_readonly_member( - root, root / item.file, immutable=not isinstance(planned, NativeManifest) - ) + # Preflight every member before the first SQLite open. Capturing + # both ends also refuses publication during validation or hashing. + generation = _generation(root, planned, budget) + if manifest_stamp != _stamp(manifest_path): + raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") validation = ( - validate_native_dataset(root, planned) + validate_native_dataset(root, planned, budget=budget) if isinstance(planned, NativeManifest) - else validate_dataset(root) + else validate_dataset(root, budget=budget) ) digest = hashlib.sha256() digest.update(before.encode("ascii")) - for device in validation.manifest.devices: - for item in device.slices: - member = root / item.file - _require_readonly_member( - root, member, immutable=not isinstance(planned, NativeManifest) - ) + for device in checked_rows(validation.manifest.devices, budget): + for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) - digest.update(_hash_file(member).encode("ascii")) - if before != _hash_file(manifest_path): + # Native validation already verifies this digest before and + # after SQLite reads; do not hash every shard a third time. + content_hash = ( + item.sha256 + if isinstance(item, NativeSlice) + else _hash_file(root / item.file, budget=budget) + ) + digest.update(content_hash.encode("ascii")) + if before != _hash_file(manifest_path, budget=budget): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + if generation != _generation(root, planned, budget): + raise DatabaseSchemaError("Dataset members changed during inspection; retry.") extensions = raw.get("extensions") if isinstance(raw, dict) else None frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None version = frames.get("version") if isinstance(frames, dict) else None - return ResolvedDataset( - root, validation, digest.hexdigest(), version if type(version) is int else None + result = ResolvedDataset( + root, + validation, + digest.hexdigest(), + version if type(version) is int else None, + generation, ) + if _cacheable(generation): + with _CACHE_LOCK: + _VALIDATED[root] = (generation, copy.deepcopy(result)) + _VALIDATED.move_to_end(root) + while len(_VALIDATED) > _CACHE_LIMIT: + _VALIDATED.popitem(last=False) + return result except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: + with _CACHE_LOCK: + _VALIDATED.pop(root, None) raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc + + def completion_device_ids(self, path: Path | str) -> list[int] | None: + """Bounded manifest-only hints, never evidence of dataset validity. + + Completion must not open or hash shard contents. Large or invalid + manifests produce no hints; a real query still performs strong admission. + """ + root = _root(path) + if root is None: + return None + try: + with (root / "manifest.json").open("rb") as source: + data = source.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + return [] + raw: object = json.loads(data, object_pairs_hook=_unique_object) + manifest = ( + parse_native_manifest(raw) + if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT + else parse_manifest(raw) + ) + return sorted(device.device_id for device in manifest.devices) + except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError): + return [] diff --git a/src/pt_snap_cli/core/native_dataset_contract.py b/src/pt_snap_cli/core/native_dataset_contract.py index 729269f..ffbd8bc 100644 --- a/src/pt_snap_cli/core/native_dataset_contract.py +++ b/src/pt_snap_cli/core/native_dataset_contract.py @@ -40,6 +40,13 @@ from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata from .sharded_replay_service import ShardedReplayResult +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) NATIVE_FORMAT = "pt-snap-native-v2" NATIVE_MANIFEST_VERSION = 1 @@ -84,8 +91,11 @@ def _digest(value: object, loc: str) -> str: return result -def parse_native_manifest(value: object) -> NativeManifest: +def parse_native_manifest( + value: object, *, budget: ValidationBudget | None = None +) -> NativeManifest: + check_budget(budget) data = _object(value, "manifest") _need(data.get("format") == NATIVE_FORMAT, "format", "format", "unsupported native format") version = _integer(data.get("schemaVersion"), "schemaVersion", minimum=1) @@ -155,11 +165,11 @@ def parse_native_manifest(value: object) -> NativeManifest: ) omitted = tuple( _integer(v, "omittedDevices", maximum=2**31 - 1) - for v in _array(data.get("omittedDevices"), "omittedDevices") + for v in checked_rows(_array(data.get("omittedDevices"), "omittedDevices"), budget) ) _need(len(set(omitted)) == len(omitted), "omittedDevices", "duplicate", "duplicate device") devices: list[DeviceRecord] = [] - for key, raw in _object(data.get("devices"), "devices").items(): + for key, raw in checked_rows(_object(data.get("devices"), "devices").items(), budget): loc = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -181,7 +191,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) ready = tuple( _integer(v, loc + ".readySlices") - for v in _array(device.get("readySlices"), loc + ".readySlices") + for v in checked_rows(_array(device.get("readySlices"), loc + ".readySlices"), budget) ) _need( ready == tuple(range(len(records))), @@ -191,7 +201,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) slices: list[SliceRecord] = [] position, last_id = 0, -1 - for index, raw_slice in enumerate(records): + for index, raw_slice in enumerate(checked_rows(records, budget)): item = _object(raw_slice, loc) _need( type(item.get("index")) is int @@ -257,7 +267,9 @@ def read_native_manifest(root: Path) -> NativeManifest: raise DatasetContractError("manifest.json", "read", str(exc)) from exc -def require_native_members(root: Path, manifest: NativeManifest) -> None: +def require_native_members( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> None: """Ownership recognition before any replacement: never adopt unknown files/dirs.""" _need( root.resolve() == root and not root.is_symlink(), @@ -267,7 +279,7 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: ) expected = {"manifest.json", *(f"device_{d.device_id}" for d in manifest.devices)} _need( - {p.name for p in root.iterdir()} == expected, + {p.name for p in checked_rows(root.iterdir(), budget)} == expected, "dataset", "members", "unexpected/missing dataset member", @@ -281,18 +293,22 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: "unsafe device directory", ) _need( - {p.name for p in directory.iterdir()} == {Path(s.file).name for s in device.slices}, + {p.name for p in checked_rows(directory.iterdir(), budget)} + == {Path(s.file).name for s in device.slices}, str(directory), "members", "unexpected/missing shard", ) - for item in device.slices: + for item in checked_rows(device.slices, budget): _safe_file(root, item.file) -def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetValidation: +def validate_native_dataset( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> DatasetValidation: - require_native_members(root, manifest) + check_budget(budget) + require_native_members(root, manifest, budget=budget) expected_identity = cache_identity( manifest.metadata.source_sha256, manifest.metadata.requested_device, @@ -306,21 +322,32 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali ) # Check EVERY member before the FIRST SQLite open. No WAL/sidecars/repair. for device in manifest.devices: - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file _require_readonly_member(root, path) - _need(_hash_file(path) == item.sha256, item.file, "hash", "member content changed") + _need( + _hash_file(path, budget=budget) == item.sha256, + item.file, + "hash", + "member content changed", + ) boundaries = 0 for device in manifest.devices: ids: set[int] = set() lifetimes: dict[int, tuple[object, ...]] = {} - references: list[tuple[int, int]] = [] - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file try: - with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") _need( conn.execute("PRAGMA quick_check").fetchall() == [("ok",)], @@ -388,19 +415,28 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "dictionary", "invalid native dictionary", ) - real = [ - r[0] - for r in conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0 ORDER BY id') - ] + count, low, high = conn.execute( + f'SELECT COUNT(*), MIN(id), MAX(id) FROM "{trace}" WHERE id>=0' + ).fetchone() _need( - len(real) == item.end_position - item.start_position + 1 - and (real[0], real[-1]) == (item.start_event_id, item.end_event_id) - and not ids.intersection(real), + (count, low, high) + == ( + item.end_position - item.start_position + 1, + item.start_event_id, + item.end_event_id, + ), item.file, "event_identity", "real IDs/count differ from positions/ranges", ) - ids.update(real) + # Manifest intervals are disjoint; avoid a per-shard list + # and intersection copy in addition to the device ID set. + ids.update( + r[0] + for r in checked_rows( + conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0'), budget + ) + ) numeric = " OR ".join( f"typeof({name})!='integer'" for name, _ in TRACE_COLUMNS[:-1] ) @@ -425,17 +461,25 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali boundaries += conn.execute( f'SELECT COUNT(*) FROM "{trace}" WHERE id<0' ).fetchone()[0] - stacks = { - r[0]: r[1:] for r in conn.execute("SELECT * FROM pt_snap_block_reference") - } - blocks = conn.execute(f'SELECT * FROM "{block}"').fetchall() + mismatch = conn.execute( + f'SELECT b.id FROM "{block}" b WHERE NOT EXISTS ' + "(SELECT 1 FROM pt_snap_block_reference r WHERE r.blockId=b.id) " + "UNION ALL SELECT r.blockId FROM pt_snap_block_reference r " + f'WHERE NOT EXISTS (SELECT 1 FROM "{block}" b WHERE b.id=r.blockId) ' + "LIMIT 1" + ).fetchone() _need( - set(stacks) == {r[0] for r in blocks}, + mismatch is None, item.file, "extension", "reference/block membership differs", ) - for row in blocks: + rows = conn.execute( + f"SELECT b.*, r.stream, r.allocCallstack, r.freeCallstack " + f'FROM "{block}" b JOIN pt_snap_block_reference r ON r.blockId=b.id' + ) + for joined in checked_rows(rows, budget): + row, ref = joined[:7], joined[7:] bid, address, size, requested, state, alloc, free = row _need( all(type(v) is int for v in row) @@ -448,7 +492,6 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "block_identity", "unfinalized/invalid block", ) - ref = stacks[bid] _need( type(ref[0]) is int and all(v is None or isinstance(v, str) for v in ref[1:]), @@ -464,13 +507,17 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "conflicting finalized identity/reference", ) lifetimes[bid] = signature - references.append((alloc, free)) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc try: - with closing( - sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) - ) as metadata_conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as metadata_conn, + validation_progress(metadata_conn, budget), + ): metadata_conn.row_factory = sqlite3.Row _need( metadata_conn.execute( @@ -501,14 +548,18 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali raise DatasetContractError(item.file, "metadata", str(exc)) from exc _require_readonly_member(root, path) _need( - _hash_file(path) == item.sha256, + _hash_file(path, budget=budget) == item.sha256, item.file, "hash", "member changed during validation", ) _need( len(ids) == device.event_count - and all(v == -1 or v in ids for pair in references for v in pair), + and all( + v == -1 or v in ids + for lifetime in checked_rows(lifetimes.values(), budget) + for v in lifetime[3:5] + ), "devices", "event_identity", "count/lifetime references not in original device scope", diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 138e342..c02d6ed 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -233,7 +233,7 @@ def execute_query( if resolved.db_path is None: raise FocusNotConfiguredError("No database path specified and no database configured.") - dataset = DatasetResolver().inspect(resolved.db_path) + dataset = DatasetResolver().inspect(resolved.db_path, budget=budget) budget.remaining() scope: dict[str, object] | None = None query_params = params or {} diff --git a/src/pt_snap_cli/core/validation_budget.py b/src/pt_snap_cli/core/validation_budget.py new file mode 100644 index 0000000..ec60cf0 --- /dev/null +++ b/src/pt_snap_cli/core/validation_budget.py @@ -0,0 +1,79 @@ +"""Cancellation helpers shared by dataset validation and query execution. + +The small protocol avoids a dependency on the query/source resolver. Validators +can also be used at publication time without a query deadline. +""" + +from __future__ import annotations + +import math +import sqlite3 +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from typing import Protocol, TypeVar + +from .errors import QueryTimeoutError + + +class ValidationBudget(Protocol): + def remaining(self) -> float | None: ... + + +def check_budget(budget: ValidationBudget | None) -> None: + if budget is not None: + budget.remaining() + + +def sqlite_timeout(budget: ValidationBudget | None) -> float: + """SQLite's busy wait must not outlive the remaining shared deadline.""" + remaining = budget.remaining() if budget is not None else None + # SQLite truncates to whole milliseconds. Round up so an exhausted busy + # wait is classified as timeout, not a spurious schema/locked error just + # before the deadline (the additional wait is at most one millisecond). + return 5.0 if remaining is None else min(5.0, math.ceil(remaining * 1000) / 1000) + + +_T = TypeVar("_T") + + +def checked_rows(rows: Iterable[_T], budget: ValidationBudget | None) -> Iterator[_T]: + """Bound Python-side work between deadline checks, including cursor iteration.""" + check_budget(budget) + for index, row in enumerate(rows): + if index % 256 == 0: + check_budget(budget) + yield row + check_budget(budget) + + +@contextmanager +def validation_progress( + conn: sqlite3.Connection, budget: ValidationBudget | None +) -> Iterator[None]: + """Interrupt SQLite and preserve QUERY_TIMEOUT rather than a schema error.""" + check_budget(budget) + timed_out: QueryTimeoutError | None = None + + def progress() -> int: + nonlocal timed_out + try: + check_budget(budget) + except QueryTimeoutError as exc: + timed_out = exc + return 1 + return 0 + + if budget is not None: + conn.set_progress_handler(progress, 1000) + try: + try: + yield + except sqlite3.DatabaseError: + if timed_out is not None: + raise timed_out from None + check_budget(budget) + raise + check_budget(budget) + finally: + if budget is not None: + conn.set_progress_handler(None, 0) diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py new file mode 100644 index 0000000..af0287a --- /dev/null +++ b/tests/core/test_dataset_validation_cache.py @@ -0,0 +1,409 @@ +"""Work-count and cancellation regressions for dataset admission (issue #219).""" + +import json +import os +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.completion import complete_device_ids +from pt_snap_cli.core import dataset_contract, dataset_resolver, native_dataset_contract +from pt_snap_cli.core.dataset_contract import validate_dataset +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.validation_budget import validation_progress +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(autouse=True) +def clear_generations(monkeypatch): + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +@pytest.fixture(params=[False, True], ids=["compatibility", "native"]) +def artifact(tmp_path, request): + return native_dataset(tmp_path) if request.param else make_dataset(tmp_path / "dataset") + + +def forbidden(*args, **kwargs): + pytest.fail("warm resolution must not hash or validate shard contents") + + +def test_new_resolvers_reuse_verified_generation_without_hash_or_sql(artifact, monkeypatch): + first = DatasetResolver().inspect(artifact) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(native_dataset_contract, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + second = DatasetResolver().inspect(artifact / "manifest.json") + assert second == first and second is not first + second.require_unchanged() + + +@pytest.mark.parametrize("change", ["touch", "replace", "manifest"]) +def test_metadata_change_always_revalidates(artifact, monkeypatch, change): + first = DatasetResolver().inspect(artifact) + path = artifact / "manifest.json" if change == "manifest" else next(artifact.glob("*/*.db")) + if change == "replace": + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.replace(path) + else: + stat = path.stat() + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000)) + calls = [] + real = dataset_resolver._hash_file + + def tracked(*args, **kwargs): + calls.append(args[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + second = DatasetResolver().inspect(artifact) + assert first.fingerprint == second.fingerprint + assert artifact / "manifest.json" in calls + with pytest.raises(DatabaseSchemaError, match="changed during operation"): + first.require_unchanged() + + +def test_restored_mtime_size_does_not_hide_content_tampering(artifact): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + after = member.stat() + assert (before.st_ino, before.st_size, before.st_mtime_ns) == ( + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + assert before.st_ctime_ns != after.st_ctime_ns + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +@pytest.mark.parametrize("change", ["wal", "journal", "shm", "symlink", "manifest-symlink"]) +def test_cached_generation_still_rejects_sidecars_and_aliases(artifact, tmp_path, change): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + if change.endswith("symlink"): + path = artifact / "manifest.json" if change == "manifest-symlink" else member + saved = tmp_path / "saved" + path.rename(saved) + path.symlink_to(saved) + else: + Path(str(member) + "-" + change).touch() + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +def test_native_hashes_twice_on_admission_not_three_times(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + calls = [] + real = hash_file + + def tracked(path, **kwargs): + calls.append(path) + return real(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + monkeypatch.setattr(native_dataset_contract, "_hash_file", tracked) + DatasetResolver().inspect(root) + for member in root.glob("*/*.db"): + assert calls.count(member) == 2 + assert calls.count(root / "manifest.json") == 2 + + +def test_cached_manifest_cannot_be_mutated_by_a_previous_caller(tmp_path): + root = native_dataset(tmp_path) + first = DatasetResolver().inspect(root) + first.validation.manifest.identity["eventsPerSlice"] = "corrupted by caller" + second = DatasetResolver().inspect(root) + assert second.validation.manifest.identity["eventsPerSlice"] == 2 + + +def test_cache_is_bounded_and_untrusted_change_tokens_disable_reuse(tmp_path, monkeypatch): + monkeypatch.setattr(dataset_resolver, "_CACHE_LIMIT", 2) + for i in range(3): + DatasetResolver().inspect(make_dataset(tmp_path / str(i), devices=(0,), slices=1)) + assert len(dataset_resolver._VALIDATED) == 2 + root = make_dataset(tmp_path / "uncached", devices=(0,), slices=1) + monkeypatch.setattr(dataset_resolver, "_cacheable", lambda generation: False) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_completion_only_reads_bounded_manifest(artifact, monkeypatch): + monkeypatch.setenv("PT_SNAP_DB_PATH", str(artifact)) + expected = sorted(json.loads((artifact / "manifest.json").read_text())["devices"], key=int) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + assert complete_device_ids() == expected + # Completion candidates do not claim content integrity. + next(artifact.glob("*/*.db")).write_bytes(b"invalid member") + assert complete_device_ids() == expected + (artifact / "manifest.json").write_bytes(b" " * (1024 * 1024 + 1)) + assert complete_device_ids() == [] + + +class CancellableBudget: + def __init__(self): + self.cancelled = False + self.calls = 0 + + def remaining(self): + self.calls += 1 + if self.cancelled: + raise QueryTimeoutError("Query timed out after 0.01 seconds") + return 1.0 + + +def test_hash_checks_between_chunks_and_closes_file(tmp_path, monkeypatch): + path = tmp_path / "large" + path.write_bytes(b"x" * (4 * 1024 * 1024)) + budget = CancellableBudget() + original = budget.remaining + opened = [] + real_open = Path.open + + def tracked_open(self, *args, **kwargs): + result = real_open(self, *args, **kwargs) + opened.append(result) + return result + + def remaining(): + budget.cancelled = budget.calls >= 2 + return original() + + monkeypatch.setattr(Path, "open", tracked_open) + monkeypatch.setattr(budget, "remaining", remaining) + with pytest.raises(QueryTimeoutError): + hash_file(path, budget=budget) + assert budget.calls == 3 + assert len(opened) == 1 and opened[0].closed + assert hash_file(path) # Failure did not retain resources. + + +def test_sql_progress_translates_timeout_and_clears_handler(): + budget = CancellableBudget() + with closing(sqlite3.connect(":memory:")) as conn: + with pytest.raises(QueryTimeoutError): + with validation_progress(conn, budget): + budget.cancelled = True + conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<10000) " + "SELECT sum(x) FROM n" + ).fetchone() + # Still cancelled: this would interrupt if the handler leaked. + assert conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) " + "SELECT sum(x) FROM n" + ).fetchone() == (500500,) + + +def large_compatibility(root, rows=1000): + root = make_dataset(root, devices=(0,), slices=1) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,100,8,0,8,8,16,'stack')", + ((i,) for i in range(rows)), + ) + conn.executemany( + "INSERT INTO block_0 VALUES (?,100,8,8,1,?,-1)", ((i, i) for i in range(rows)) + ) + path = root / "manifest.json" + data = json.loads(path.read_text()) + data["eventsPerSlice"] = rows + data["devices"]["0"]["eventCount"] = rows + data["devices"]["0"]["slices"][0]["endEventId"] = rows - 1 + path.write_text(json.dumps(data)) + return root + + +def test_python_validation_loop_cancels_before_finishing_and_closes_connection( + tmp_path, monkeypatch +): + root = large_compatibility(tmp_path / "dataset") + budget = CancellableBudget() + checked = [] + connections = [] + real_integer = dataset_contract._integer + real_connect = sqlite3.connect + + def integer(value, location, *args, **kwargs): + if ".block_0." in location: + checked.append(location) + budget.cancelled = True + return real_integer(value, location, *args, **kwargs) + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + connections.append(conn) + return conn + + monkeypatch.setattr(dataset_contract, "_integer", integer) + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + validate_dataset(root, budget=budget) + assert 0 < len(checked) <= 256 * 7 + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(dataset_contract, "_integer", real_integer) + assert validate_dataset(root).real_event_count == 1000 + + +def test_native_quick_check_can_be_interrupted_and_resolver_recovers(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + budget = CancellableBudget() + connections = [] + real_connect = sqlite3.connect + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + conn.set_trace_callback( + lambda sql: setattr(budget, "cancelled", True) if sql == "PRAGMA quick_check" else None + ) + connections.append(conn) + return conn + + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=budget) + assert root not in dataset_resolver._VALIDATED + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(sqlite3, "connect", real_connect) + assert DatasetResolver().inspect(root) is not None + + +def test_cold_query_passes_shared_deadline_and_cli_keeps_timeout_code(tmp_path, monkeypatch): + from typer.testing import CliRunner + + from pt_snap_cli.cli import app + from pt_snap_cli.core.query_service import QueryService + + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + original = dataset_resolver._hash_file + received = [] + + def expire(path, *, budget=None): + assert budget is not None + received.append(budget) + budget.started -= 20 + return original(path, budget=budget) + + monkeypatch.setattr(dataset_resolver, "_hash_file", expire) + service = QueryService() + try: + with pytest.raises(QueryTimeoutError): + service.execute_query("event", {"id": 0}, db_path=root, timeout_s=10) + finally: + service.close() + result = CliRunner().invoke( + app, ["query", str(root), "--template-use", "event", "--timeout", "10", "--json"] + ) + assert result.exit_code == 1 + assert json.loads(result.stderr)["error"]["code"] == "QUERY_TIMEOUT" + assert len(received) == 2 + assert root not in dataset_resolver._VALIDATED + monkeypatch.setattr(dataset_resolver, "_hash_file", original) + service = QueryService() + try: + assert service.execute_query("event", {"id": 0}, db_path=root).rows[0]["id"] == 0 + finally: + service.close() + + +def test_native_lock_wait_respects_deadline(tmp_path): + import time + + from pt_snap_cli.core.dataset_sources import QueryBudget + + root = native_dataset(tmp_path) + member = sorted(root.glob("*/*.db"))[0] + with closing(sqlite3.connect(member)) as writer: + writer.execute("BEGIN EXCLUSIVE") + start = time.monotonic() + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=QueryBudget(0.02, start)) + assert time.monotonic() - start < 1.0 + writer.rollback() + assert DatasetResolver().inspect(root) is not None + + +@pytest.mark.parametrize("mode", [None, "off", "unknown"]) +def test_default_and_unknown_cache_mode_repeat_strong_validation(tmp_path, monkeypatch, mode): + if mode is None: + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + else: + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + calls = [] + original = dataset_resolver._hash_file + + def tracked(path, **kwargs): + calls.append(path) + return original(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + first = DatasetResolver().inspect(root) + second = DatasetResolver().inspect(root) + assert first.fingerprint == second.fingerprint + assert calls.count(root / "device_0/slice_00000.db") == 2 + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_hashes_even_when_filesystem_tokens_look_unchanged( + tmp_path, monkeypatch +): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + resolved.require_unchanged() + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='changed'") + monkeypatch.setattr(dataset_resolver, "_generation", lambda *args: resolved.generation) + with pytest.raises(DatabaseSchemaError, match="content changed"): + resolved.require_unchanged() + + +def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + DatasetResolver().inspect(root) + assert root in dataset_resolver._VALIDATED + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + with pytest.raises(QueryTimeoutError): + budget = CancellableBudget() + budget.cancelled = True + DatasetResolver().inspect(root, budget=budget) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_normalizes_hash_io_failure(tmp_path, monkeypatch): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + + def fail(*args, **kwargs): + raise PermissionError("member became unreadable") + + monkeypatch.setattr(dataset_resolver, "_hash_file", fail) + with pytest.raises(DatabaseSchemaError, match="member became unreadable"): + resolved.require_unchanged() diff --git a/tests/test_dataset_focus.py b/tests/test_dataset_focus.py index 8b525f4..2c42d67 100644 --- a/tests/test_dataset_focus.py +++ b/tests/test_dataset_focus.py @@ -482,8 +482,8 @@ def test_manifest_change_during_validation_fails(tmp_path, monkeypatch): root = make_dataset(tmp_path / "dataset") original = module.validate_dataset - def validate(path): - result = original(path) + def validate(path, **kwargs): + result = original(path, **kwargs) manifest = root / "manifest.json" data = json.loads(manifest.read_text()) data["cacheHash"] = "replaced" From 3aa912a5f970f1c03cc9a9d68d5c1c35a177d947 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:04:55 +0800 Subject: [PATCH 2/2] docs: align report validation claims with current behavior --- docs/en/sharded-snapshotdb.md | 13 ++++++++----- docs/zh/sharded-snapshotdb.md | 9 ++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53f7ae4..a0e59b5 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -245,8 +245,10 @@ addresses all matching files even for a cross-slice range, but does not execute By default **every independent inspection** fully validates the finalized contract and hashes the manifest/members. File metadata alone is not content-integrity proof: unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. -Pinned report-operation guards also rehash content by default, without repeating -row validation. +Each query, including each query step within a multi-step report, calls `inspect` +again. Reports do not currently pin one `ResolvedDataset` across their steps or +invoke `ResolvedDataset.require_unchanged()`; there is no additional report-level +content-hash guard. `PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most 16 validated generations, without open connections. Set it only when you control @@ -258,13 +260,14 @@ the tool. Unset it to retain strong validation; unknown values also use strong m Windows and platforms without a usable POSIX change token remain in strong mode. With that explicit opt-in, new resolver/service instances can reuse validated -results. Every hit and pinned-operation guard still rechecks canonical paths, +results. Every cache hit during inspection still rechecks canonical paths, sidecars, native membership and each file's device, inode, size, mtime, **ctime**, mode and link count. These conservative checks catch ordinary replacements and in-place edits with restored mtime, but do not replace the immutable-file contract. For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: -work is O(shards), independent of event/block counts. Default inspections and cache -misses remain proportional to content. Explicit `validate_dataset` / +work is O(shards), independent of event/block counts. This metadata-only reuse also +applies to report query steps; a cache hit does not rehash content. Default inspections +and cache misses remain proportional to content. Explicit `validate_dataset` / `validate_native_dataset` calls always validate fully. Native admission verifies hashes before and after validation and reuses the verified hashes for its fingerprint, avoiding a third member-hash pass. diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index c2ee244..ad3c1d1 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -200,7 +200,9 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 -已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 +每次查询(包括多步报告中的每个查询步骤)都会再次调用 `inspect`。报告目前不会跨步骤 +固定同一个 `ResolvedDataset`,也不会调用 `ResolvedDataset.require_unchanged()`; +没有额外的报告级内容哈希守卫。 `PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, 不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 @@ -208,11 +210,12 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, 未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 -显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 +显式启用后,新建 resolver/service 可复用已校验结果。每次 inspect 缓存命中仍验证 规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode 和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), -不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` +不随 event/block 数增长。报告中的查询步骤也适用这种仅检查元数据的复用;命中时不会 +重新哈希内容。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` / `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 复用已验证摘要,不再做第三遍成员哈希。