From 56eb261b93fbdcf3e8dfd6418d8235afd9a8c30b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:03:40 +0800 Subject: [PATCH 1/2] perf: reuse content-verified dataset validation in process --- docs/en/querying.md | 8 +- docs/en/sharded-snapshotdb.md | 66 +++-- docs/zh/querying.md | 7 +- docs/zh/sharded-snapshotdb.md | 46 ++-- src/pt_snap_cli/core/dataset_resolver.py | 105 +++++-- tests/core/test_dataset_content_cache.py | 286 ++++++++++++++++++++ tests/core/test_dataset_validation_cache.py | 11 +- 7 files changed, 453 insertions(+), 76 deletions(-) create mode 100644 tests/core/test_dataset_content_cache.py diff --git a/docs/en/querying.md b/docs/en/querying.md index 1730893..3d68d68 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -153,10 +153,12 @@ fetched/output work is limited to **100000 rows / 64 MiB serialized values**, in repeated source reads. Exceeding either fails the entire operation, never an alleged complete partial result. Python materialization/sort are bounded by this work ceiling, **not** by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual -cells have overhead. Strong manifest/member validation and hashing remain the default -and share that deadline: hash chunks, Python row batches and SQLite progress handlers +cells have overhead. Strong manifest/member admission remains the default; each +inspection hashes all content, reusing prior in-process row validation only when +content and generation match. Warm work remains O(bytes + shards), and new CLI +processes are cold. Admission shares that deadline: hash chunks, Python row batches and SQLite progress handlers check cancellation. Individual filesystem calls are not OS-level hard-preempted. -`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into process-local validation reuse +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only validation reuse only when the caller guarantees immutable finalized files and reliable filesystem change times; see [dataset validation and cache contracts](sharded-snapshotdb.md). No temporary merge DB, repair or artifact write. diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53f7ae4..0edd799 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -242,32 +242,46 @@ full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. -By default **every independent inspection** fully validates the finalized contract -and hashes the manifest/members. File metadata alone is not content-integrity proof: -unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. -Pinned report-operation guards also rehash content by default, without repeating -row validation. - -`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most -16 validated generations, without open connections. Set it only when you control -finalized, immutable publication and trust your filesystem's kernel change-time -semantics. **Do not enable it for files that may be modified in place**, live writers, -unknown/coarse/network filesystems, or filesystem snapshots that may be rolled back. -This is a caller responsibility contract, not a filesystem capability detected by -the tool. Unset it to retain strong validation; unknown values also use strong mode. -Windows and platforms without a usable POSIX change token remain in strong mode. - -With that explicit opt-in, new resolver/service instances can reuse validated -results. Every hit and pinned-operation guard still rechecks canonical paths, -sidecars, native membership and each file's device, inode, size, mtime, **ctime**, -mode and link count. These conservative checks catch ordinary replacements and -in-place edits with restored mtime, but do not replace the immutable-file contract. -For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: -work is O(shards), independent of event/block counts. Default inspections and cache -misses remain proportional to content. Explicit `validate_dataset` / -`validate_native_dataset` calls always validate fully. Native admission verifies -hashes before and after validation and reuses the verified hashes for its fingerprint, -avoiding a third member-hash pass. +By default, the first inspection fully validates the finalized contract and hashes +its content before/after validation. A process-wide LRU holds at most 16 validated +generations, with no open connections or row-level validation collections. Each +subsequent inspection still hashes the manifest and **every member** and checks +canonical paths, sidecars, native membership and file metadata before/after hashing. +Only an unchanged generation with the same content fingerprint reuses its validated +result, avoiding repeated SQLite/schema/row validation. Changed metadata or content +falls back to complete validation; failures and cancellation evict the entry. + +Default warm work is **O(bytes + shards)**, not O(1) or independent of dataset size. +It saves repeated row-validation CPU and temporary memory, not full-file reads. +Every new CLI process is cold; this is not a persistent or cross-process cache. +Compatibility-v1 cold admission adds a pre-validation member hash to its existing +post-validation hash; native admission retains two passes, reusing verified hashes +for the fingerprint rather than adding a third pass. Pinned report-operation guards +also rehash content by default, without repeating row validation. Explicit +`validate_dataset` / `validate_native_dataset` calls always validate fully. + +File metadata alone is not content-integrity proof: unknown or coarse filesystems +may expose unchanged `ctime_ns` after a write. Unsetting `PT_SNAP_DATASET_CACHE` +selects the default content-verified cache. Setting it to `off` or an unknown value +fully revalidates on every inspection without caching; strong hashing remains active. + +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only reuse in the +same bounded process cache. Set it only when you control finalized, immutable +publication and trust your filesystem's kernel change-time semantics. **Do not +enable it for files that may be modified in place**, live writers, unknown/coarse/ +network filesystems, or filesystem snapshots that may be rolled back. This is a +caller responsibility contract, not a filesystem capability detected by the tool. +Windows and platforms without a usable POSIX change token fully revalidate in this +mode. An immutable-mode compatibility admission cannot seed default reuse until +it has passed the stronger content-bracketed admission. + +With that explicit opt-in, every hit and pinned-operation guard still rechecks +canonical paths, sidecars, native membership and each file's device, inode, size, +mtime, **ctime**, mode and link count. These conservative checks catch ordinary +replacements and in-place edits with restored mtime, but do not replace the +immutable-file contract. An opted-in hit performs no row scans or whole-file hashes: +work is O(shards), independent of event/block counts. None of these checks provides +a concurrent-writer lock or an atomic filesystem snapshot. The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, Python validation loops check every 256 rows, and SQLite validation (including diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 104cf01..52616c8 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -131,10 +131,11 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 精确总数共用递减 deadline;报告的峰值选择与归因明确共用**一个**预算。累计取回/输出工作上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ -SQLite、一个批次及单个 cell 有额外开销。默认仍对 manifest/成员执行完整校验和 hash, -并共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 +SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入:每次对 manifest/全部成员 +取哈希,仅当内容及 generation 一致时复用进程内逐行校验结果。热路径仍为 O(字节数 + +分片数),新 CLI 进程仍冷启动。准入共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 单个文件系统调用不提供 OS 级硬抢占。仅当调用方保证最终产物不可变且文件系统变更时间 -可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用进程内校验复用;参见 +可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用纯元数据校验复用;参见 [数据集校验与缓存合同](sharded-snapshotdb.md)。不生成合并临时 DB、不修复或写产物。 兼容导出器默认**每设备每片 500000 个真实事件**。该存储容量与 **100000 行 / 64 MiB diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index c2ee244..c606653 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -198,23 +198,35 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 -默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 -本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 -已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 - -`PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, -不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 -**可能被原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照 -不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, -未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 - -显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 -规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode -和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 -合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), -不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` -/ `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 -复用已验证摘要,不再做第三遍成员哈希。 +默认首次 inspect 完整校验最终化合同,并在校验前后对内容取哈希。进程级 LRU 最多保留 +16 个已校验 generation,不保留连接或逐行校验集合。后续 inspect 仍对 manifest 和 +**全部成员**取内容哈希,并在哈希前后检查规范路径、sidecar、原生成员集合和文件元数据。 +只有 generation 未变且完整 fingerprint 相同才复用结果,跳过重复 SQLite/schema/逐行 +校验。元数据或内容变化回退完整校验;失败或取消会清除对应缓存条目。 + +默认热路径仍是 **O(内容字节数 + 分片数)**,不是 O(1),也不独立于数据规模。节省的是 +重复逐行校验 CPU 和临时内存,不是整文件读取。每个新 CLI 进程仍为冷启动;没有持久化 +或跨进程缓存。兼容 v1 冷准入在已有校验后哈希外增加一遍校验前成员哈希;原生准入仍为 +两遍,fingerprint 复用已验证摘要,不增加第三遍。已固定 generation 的报告操作默认仍 +重新哈希内容,不重复逐行校验。显式 `validate_dataset` / `validate_native_dataset` +始终完整校验。 + +文件元数据本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 +`ctime_ns`。不设置 `PT_SNAP_DATASET_CACHE` 使用默认内容验证缓存;设为 `off` 或未知 +取值时每次完整重验且不缓存,仍执行强内容哈希。 + +`PT_SNAP_DATASET_CACHE=immutable` 显式选择同一有界进程缓存的纯元数据复用路径。 +仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。**可能被 +原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照不得 +启用**。这是调用方责任合同,不代表工具已检测文件系统能力。Windows 或没有可用 +POSIX change token 的平台在此模式下每次完整重验。immutable 模式准入的兼容数据必须 +先通过更强的前后内容哈希准入,才能用于默认缓存命中。 + +显式启用后,每次命中及报告 generation 检查仍验证规范路径、sidecar、原生成员集合, +以及文件的设备号、inode、size、mtime、**ctime**、mode 和硬链接数。这些保守检查可检测 +常规替换及恢复 mtime 的原地修改,但不能代替不可变文件合同。显式启用后的命中不扫描 +行、不做整文件哈希,工作量为 O(分片数),不随 event/block 数增长。这些检查均不提供 +并发 writer 锁或原子文件系统快照。 查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite 校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index b1d9da3..1f7ea7c 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -29,7 +29,12 @@ from pt_snap_cli.core.dataset_files import hash_file as _hash_file from pt_snap_cli.core.dataset_files import require_readonly_member as _require_readonly_member from pt_snap_cli.core.dataset_support import DATASET_SUPPORT -from pt_snap_cli.core.errors import DatabaseSchemaError, InvalidDeviceError, InvalidParameterError +from pt_snap_cli.core.errors import ( + DatabaseSchemaError, + InvalidDeviceError, + InvalidParameterError, + QueryTimeoutError, +) from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, NativeManifest, @@ -147,11 +152,20 @@ def to_dict(self) -> dict[str, object]: # This is deliberately process-local, bounded, and contains no open connections. -# Cross-call reuse is opt-in: unknown/coarse filesystems cannot supply a -# trustworthy change token merely because Python exposes st_ctime_ns. The caller -# must attest immutable publication and suitable change-time semantics explicitly. +# Default reuse verifies all content bytes; metadata alone never proves integrity. +# Skipping byte scans requires the explicit immutable-publication contract. _CACHE_LIMIT = 16 -_VALIDATED: OrderedDict[Path, tuple[_Generation, ResolvedDataset]] = OrderedDict() + + +@dataclass(frozen=True) +class _ValidatedEntry: + generation: _Generation + resolved: ResolvedDataset + # Only admissions bracketed by content hashes can seed default reuse. + content_verified: bool + + +_VALIDATED: OrderedDict[Path, _ValidatedEntry] = OrderedDict() _CACHE_LOCK = RLock() @@ -216,11 +230,17 @@ def _cacheable(generation: _Generation) -> bool: def _content_fingerprint( - root: Path, manifest: DatasetManifest, budget: ValidationBudget | None + root: Path, + manifest: DatasetManifest, + budget: ValidationBudget | None, + *, + manifest_hash: str | None = None, ) -> str: """Strong verification for pinned operations without repeating row validation.""" digest = hashlib.sha256() - digest.update(_hash_file(root / "manifest.json", budget=budget).encode("ascii")) + digest.update( + (manifest_hash or _hash_file(root / "manifest.json", budget=budget)).encode("ascii") + ) for device in checked_rows(manifest.devices, budget): for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) @@ -231,7 +251,9 @@ def _content_fingerprint( class DatasetResolver: """Resolve and validate finalized datasets, reusing unchanged generations. - By default every call validates and hashes content. Explicitly setting + By default every call hashes content; a matching, fully validated in-process + result skips repeated row validation. Warm work remains O(bytes + shards), + and new CLI processes are cold. Explicitly setting PT_SNAP_DATASET_CACHE=immutable enables bounded process-local reuse under the caller's immutable-publication/change-time contract. Even then paths, members, sidecars and kernel change times are checked on every call. @@ -241,15 +263,26 @@ class DatasetResolver: def inspect( self, path: Path | str, *, budget: ValidationBudget | None = None ) -> ResolvedDataset | None: - check_budget(budget) - root = _root(path) - if root is None: - return None + candidate = Path(path).expanduser().absolute() + with _CACHE_LOCK: + cache_key = ( + candidate.parent + if candidate.name == "manifest.json" and candidate not in _VALIDATED + else candidate + ) try: + check_budget(budget) + root = _root(path) + if root is None: + with _CACHE_LOCK: + _ = _VALIDATED.pop(cache_key, None) + return None + cache_key = root + strong_cache = os.environ.get("PT_SNAP_DATASET_CACHE") is None with _CACHE_LOCK: cached = _VALIDATED.get(root) if cached is not None: - previous, resolved = cached + previous, resolved = cached.generation, cached.resolved manifest_unchanged = ( _stamp(root / "manifest.json") == dict(previous)["manifest.json"] ) @@ -258,7 +291,18 @@ def inspect( if manifest_unchanged else None ) - if generation is not None and _cacheable(generation) and generation == previous: + reuse = False + if generation is not None and generation == previous: + if _cacheable(generation): + reuse = True + elif strong_cache and cached.content_verified: + fingerprint = _content_fingerprint( + root, resolved.validation.manifest, budget + ) + if _generation(root, resolved.validation.manifest, budget) != generation: + raise DatabaseSchemaError("Dataset changed during inspection; retry.") + reuse = fingerprint == resolved.fingerprint + if reuse: with _CACHE_LOCK: if root in _VALIDATED: _VALIDATED.move_to_end(root) @@ -266,7 +310,7 @@ def inspect( check_budget(budget) return result with _CACHE_LOCK: - _VALIDATED.pop(root, None) + _ = _VALIDATED.pop(root, None) manifest_path = root / "manifest.json" manifest_stamp = _stamp(manifest_path) before = _hash_file(manifest_path, budget=budget) @@ -283,6 +327,14 @@ def inspect( generation = _generation(root, planned, budget) if manifest_stamp != _stamp(manifest_path): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + # Compatibility manifests contain no shard digests. Bracket row + # validation with byte hashes before allowing its result to be reused. + # Native validation already checks both ends against declared hashes. + compatibility_before = ( + _content_fingerprint(root, planned, budget, manifest_hash=before) + if strong_cache and not isinstance(planned, NativeManifest) + else None + ) validation = ( validate_native_dataset(root, planned, budget=budget) if isinstance(planned, NativeManifest) @@ -301,6 +353,9 @@ def inspect( else _hash_file(root / item.file, budget=budget) ) digest.update(content_hash.encode("ascii")) + fingerprint = digest.hexdigest() + if compatibility_before is not None and compatibility_before != fingerprint: + raise DatabaseSchemaError("Dataset content changed during inspection; retry.") if before != _hash_file(manifest_path, budget=budget): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") if generation != _generation(root, planned, budget): @@ -311,21 +366,31 @@ def inspect( result = ResolvedDataset( root, validation, - digest.hexdigest(), + fingerprint, version if type(version) is int else None, generation, ) - if _cacheable(generation): + if strong_cache or _cacheable(generation): + entry = _ValidatedEntry( + generation, + copy.deepcopy(result), + isinstance(planned, NativeManifest) or compatibility_before is not None, + ) + check_budget(budget) with _CACHE_LOCK: - _VALIDATED[root] = (generation, copy.deepcopy(result)) + _VALIDATED[root] = entry _VALIDATED.move_to_end(root) while len(_VALIDATED) > _CACHE_LIMIT: - _VALIDATED.popitem(last=False) + _ = _VALIDATED.popitem(last=False) return result except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: with _CACHE_LOCK: - _VALIDATED.pop(root, None) + _ = _VALIDATED.pop(cache_key, None) raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc + except (DatabaseSchemaError, QueryTimeoutError): + with _CACHE_LOCK: + _ = _VALIDATED.pop(cache_key, None) + raise def completion_device_ids(self, path: Path | str) -> list[int] | None: """Bounded manifest-only hints, never evidence of dataset validity. diff --git a/tests/core/test_dataset_content_cache.py b/tests/core/test_dataset_content_cache.py new file mode 100644 index 0000000..ff5e7ad --- /dev/null +++ b/tests/core/test_dataset_content_cache.py @@ -0,0 +1,286 @@ +"""Default reuse must prove content equality, even with untrustworthy stat tokens.""" + +import os +import sqlite3 +from collections import Counter +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_resolver as resolver +from pt_snap_cli.core import native_dataset_contract as native +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from tests.core.test_dataset_validation_cache import CancellableBudget +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(autouse=True) +def default_cache(monkeypatch): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + resolver._VALIDATED.clear() + yield + resolver._VALIDATED.clear() + + +@pytest.fixture(params=[False, True], ids=["compatibility", "native"]) +def artifact(tmp_path, request): + return native_dataset(tmp_path) if request.param else make_dataset(tmp_path / "dataset") + + +def test_default_cold_and_warm_work_counts(artifact, monkeypatch): + hashes = Counter() + + def tracked(path, **kwargs): + hashes[path] += 1 + return hash_file(path, **kwargs) + + monkeypatch.setattr(resolver, "_hash_file", tracked) + monkeypatch.setattr(native, "_hash_file", tracked) + first = resolver.DatasetResolver().inspect(artifact) + for member in artifact.glob("*/*.db"): + assert hashes[member] == 2 + assert hashes[artifact / "manifest.json"] == 2 + hashes.clear() + + def forbidden(*args, **kwargs): + pytest.fail("content-verified warm hit must not repeat SQLite/row validation") + + monkeypatch.setattr(resolver, "validate_dataset", forbidden) + monkeypatch.setattr(resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + second = resolver.DatasetResolver().inspect(artifact / "manifest.json") + assert first == second and first is not second + for member in artifact.glob("*/*.db"): + assert hashes[member] == 1 + assert hashes[artifact / "manifest.json"] == 1 + # Report guards remain independently strong, even following a cache hit. + second.require_unchanged() + for member in artifact.glob("*/*.db"): + assert hashes[member] == 2 + + +def test_default_detects_tampering_with_all_change_tokens_hidden(artifact, monkeypatch): + resolved = resolver.DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + assert (member.stat().st_size, member.stat().st_mtime_ns) == ( + before.st_size, + before.st_mtime_ns, + ) + monkeypatch.setattr(resolver, "_generation", lambda *args: resolved.generation) + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +@pytest.mark.parametrize("change", ["touch", "replace", "manifest"]) +def test_default_metadata_changes_fully_revalidate(artifact, monkeypatch, change): + first = resolver.DatasetResolver().inspect(artifact) + member = artifact / "manifest.json" if change == "manifest" else next(artifact.glob("*/*.db")) + if change == "replace": + replacement = member.with_suffix(".replacement") + replacement.write_bytes(member.read_bytes()) + replacement.replace(member) + else: + before = member.stat() + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns + 1000000)) + calls = [] + for name in ("validate_dataset", "validate_native_dataset"): + original = getattr(resolver, name) + + def tracked(*args, _original=original, **kwargs): + calls.append(1) + return _original(*args, **kwargs) + + monkeypatch.setattr(resolver, name, tracked) + second = resolver.DatasetResolver().inspect(artifact) + assert first.fingerprint == second.fingerprint + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "change", ["wal", "journal", "shm", "member-alias", "root-alias", "missing"] +) +def test_default_preflight_failure_evicts_cached_entry(artifact, tmp_path, change): + resolver.DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + if change == "missing": + member.unlink() + elif change in ("member-alias", "root-alias"): + source = member if change == "member-alias" else artifact + saved = tmp_path / "renamed" + source.rename(saved) + source.symlink_to(saved, target_is_directory=change == "root-alias") + else: + Path(str(member) + "-" + change).touch() + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_native_unlisted_member_invalidates(tmp_path): + root = native_dataset(tmp_path) + resolver.DatasetResolver().inspect(root) + (root / "device_0" / "slice_99999.db").write_bytes(b"extra") + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(root) + assert root not in resolver._VALIDATED + + +def test_warm_hash_cancellation_evicts_and_next_call_recovers(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + budget = CancellableBudget() + original = resolver._hash_file + + def cancelled(path, **kwargs): + if path.suffix == ".db": + budget.cancelled = True + return original(path, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(resolver, "_hash_file", cancelled) + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(artifact, budget=budget) + assert artifact not in resolver._VALIDATED + assert resolver.DatasetResolver().inspect(artifact) is not None + + +def test_expired_budget_evicts_without_any_hash(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + budget = CancellableBudget() + budget.cancelled = True + + def forbidden(*args, **kwargs): + pytest.fail("expired deadline must precede file hashing") + + monkeypatch.setattr(resolver, "_hash_file", forbidden) + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(artifact, budget=budget) + assert artifact not in resolver._VALIDATED + + +def test_warm_concurrent_change_refuses_and_evicts(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + original = resolver._hash_file + + def changed(path, **kwargs): + value = original(path, **kwargs) + if path.suffix == ".db": + before = path.stat() + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 1000000)) + return value + + monkeypatch.setattr(resolver, "_hash_file", changed) + with pytest.raises(DatabaseSchemaError, match="changed during inspection"): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_compatibility_cold_hashes_bracket_validation(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + original = resolver.validate_dataset + + def changed(*args, **kwargs): + result = original(*args, **kwargs) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='valid changed text'") + return result + + monkeypatch.setattr(resolver, "validate_dataset", changed) + with pytest.raises(DatabaseSchemaError, match="content changed during inspection"): + resolver.DatasetResolver().inspect(root) + assert root not in resolver._VALIDATED + + +def test_default_cache_is_bounded_deep_copied_and_owns_no_connections(tmp_path, monkeypatch): + monkeypatch.setattr(resolver, "_CACHE_LIMIT", 2) + roots = [make_dataset(tmp_path / str(i), devices=(0,), slices=1) for i in range(3)] + for root in roots: + resolver.DatasetResolver().inspect(root) + assert list(resolver._VALIDATED) == roots[1:] + first = resolver.DatasetResolver().inspect(roots[1]) + object.__setattr__(first.validation, "real_event_count", -1) + assert resolver.DatasetResolver().inspect(roots[1]).validation.real_event_count == 2 + assert list(resolver._VALIDATED) == [roots[2], roots[1]] + # Cached entries only contain result data; file removal needs no cache close. + for root in roots: + for path in root.glob("*/*.db"): + path.unlink() + assert all(not tuple(root.glob("*/*.db")) for root in roots) + + +def test_immutable_v1_entry_cannot_seed_default_reuse(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + resolver.DatasetResolver().inspect(root) + assert not resolver._VALIDATED[root].content_verified + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + calls = [] + original = resolver.validate_dataset + + def tracked(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(resolver, "validate_dataset", tracked) + resolver.DatasetResolver().inspect(root) + resolver.DatasetResolver().inspect(root) + assert calls == [1] + assert resolver._VALIDATED[root].content_verified + + +def test_default_reuse_does_not_require_trusted_posix_tokens(artifact, monkeypatch): + monkeypatch.setattr(resolver, "_cacheable", lambda generation: False) + resolver.DatasetResolver().inspect(artifact) + assert resolver.DatasetResolver().inspect(artifact) is not None + assert artifact in resolver._VALIDATED + + +def test_valid_compatibility_content_change_revalidates_with_hidden_tokens(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + first = resolver.DatasetResolver().inspect(root) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='valid new content'") + monkeypatch.setattr(resolver, "_generation", lambda *args: first.generation) + calls = [] + original = resolver.validate_dataset + + def tracked(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(resolver, "validate_dataset", tracked) + second = resolver.DatasetResolver().inspect(root) + assert second.fingerprint != first.fingerprint + assert calls == [1] + assert resolver.DatasetResolver().inspect(root) == second + assert calls == [1] + + +def test_hash_io_failure_evicts_entry(artifact, monkeypatch): + resolver.DatasetResolver().inspect(artifact) + + def failed(*args, **kwargs): + raise PermissionError("unreadable member") + + monkeypatch.setattr(resolver, "_hash_file", failed) + with pytest.raises(DatabaseSchemaError, match="unreadable member"): + resolver.DatasetResolver().inspect(artifact) + assert artifact not in resolver._VALIDATED + + +def test_dataset_directory_named_manifest_cancellation_evicts_exact_entry(tmp_path): + root = make_dataset(tmp_path / "manifest.json", devices=(0,), slices=1) + resolver.DatasetResolver().inspect(root) + budget = CancellableBudget() + budget.cancelled = True + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(root, budget=budget) + assert root not in resolver._VALIDATED diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py index af0287a..094ecc2 100644 --- a/tests/core/test_dataset_validation_cache.py +++ b/tests/core/test_dataset_validation_cache.py @@ -347,12 +347,9 @@ def test_native_lock_wait_respects_deadline(tmp_path): assert DatasetResolver().inspect(root) is not None -@pytest.mark.parametrize("mode", [None, "off", "unknown"]) -def test_default_and_unknown_cache_mode_repeat_strong_validation(tmp_path, monkeypatch, mode): - if mode is None: - monkeypatch.delenv("PT_SNAP_DATASET_CACHE") - else: - monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) +@pytest.mark.parametrize("mode", ["off", "unknown"]) +def test_disabled_and_unknown_cache_mode_repeat_full_validation(tmp_path, monkeypatch, mode): + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) calls = [] original = dataset_resolver._hash_file @@ -387,7 +384,7 @@ def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch) root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) DatasetResolver().inspect(root) assert root in dataset_resolver._VALIDATED - monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "off") with pytest.raises(QueryTimeoutError): budget = CancellableBudget() budget.cancelled = True From a5298ea3f7131eec4d66a018b12cf87aaf72072d Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:18:23 +0800 Subject: [PATCH 2/2] feat: bound many-shard lifecycle reduction with private scratch storage --- README.md | 3 + README_zh.md | 2 + docs/en/querying.md | 31 +- docs/zh/querying.md | 23 +- src/pt_snap_cli/core/dataset_global.py | 2 +- .../core/dataset_lifecycle_connection.py | 88 ++++++ .../core/dataset_lifecycle_scratch.py | 191 ++++++++++++ src/pt_snap_cli/core/dataset_lifecycle_sql.py | 102 ++---- src/pt_snap_cli/core/dataset_sources.py | 3 + tests/core/test_dataset_lifecycle_scratch.py | 293 ++++++++++++++++++ 10 files changed, 641 insertions(+), 97 deletions(-) create mode 100644 src/pt_snap_cli/core/dataset_lifecycle_connection.py create mode 100644 src/pt_snap_cli/core/dataset_lifecycle_scratch.py create mode 100644 tests/core/test_dataset_lifecycle_scratch.py diff --git a/README.md b/README.md index 751670c..afacb19 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,9 @@ canonical stack statistics, deduplicated lifecycles, terminal leak candidates an `report peak-memory` share bounded core semantics; custom SQL/overrides remain unsupported on datasets. See the [global support matrix](docs/en/querying.md#dataset-global-built-in-support-p3) and [dataset attribution and coverage](docs/en/querying.md#dataset-point-event-attribution). +Lifecycle queries exceeding SQLite's attachment limit use a private, page-limited +temporary derived database; source shards stay read-only. See the global support +matrix for storage limits, cleanup and memory caveats. Explicit `pt-snap import snapshot.pkl --format msinsight --json` produces `snapshot.pkl.msinsight/` for the fixed msinsight revision's **same original diff --git a/README_zh.md b/README_zh.md index 66abca3..1681c1e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -62,6 +62,8 @@ pt-snap split snapshot.pkl --slices 4 --output snapshot-slices 终片泄漏候选及 `report peak-memory` 共用有界 core 语义;数据集仍不支持自定义 SQL/override。 详见[全局支持矩阵](docs/zh/querying.md#数据集全局内建支持p3)及 [数据集定点归因与覆盖](docs/zh/querying.md#数据集定点事件归因)。 +超过 SQLite 附加上限的生命周期查询使用私有、限制主库页数空间的临时派生数据库,源分片 +保持只读;存储限额、清理及内存边界见全局支持矩阵。 显式 `pt-snap import snapshot.pkl --format msinsight --json` 生成 `snapshot.pkl.msinsight/`,对齐固定 msinsight 版本的**同一原始 pickle + 邻接缓存**入口。 diff --git a/docs/en/querying.md b/docs/en/querying.md index 3d68d68..57d9383 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -161,7 +161,8 @@ check cancellation. Individual filesystem calls are not OS-level hard-preempted. `PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only validation reuse only when the caller guarantees immutable finalized files and reliable filesystem change times; see [dataset validation and cache contracts](sharded-snapshotdb.md). -No temporary merge DB, repair or artifact write. +Source artifacts are never repaired or written. A large lifecycle query may create a +private temporary derived database as described below. The compatible exporter defaults to **500000 real events per device shard**. That storage capacity is separate from the **100000-row / 64-MiB query work @@ -180,16 +181,32 @@ formatted text is not their identity. Block/leak/lifetime reductions use read-on attached shards for proof, conflicts and latest observations before pagination; the attachment limit (normally ten) counts selected observation shards plus any other shards owning their allocation/free references. Slice queries discover those -proof owners before filtering or paging; exceeding the limit retains the budgeted -source-reading path. SQLite internal scans and temporary space are not bounded by -the fetched-row/byte budget. Summary and page (or lifetime buckets) each evaluate -the full lifecycle CTE, sharing the same deadline. No intermediate merged database -is created. +proof owners before filtering or paging. Above the limit, a query-private temporary +SQLite database retains only distinct event references and one narrow state row per +lifecycle. Sources are attached read-only one at a time: reference ownership/actions +are proved first, then all observations are reduced and conflicts checked before +filters, totals or pagination. Only returned candidates hydrate complete source events. +The small attached-set path is unchanged. + +The temporary directory is private (0700), its derived database is created with 0600 +permissions, and normal completion, errors and cancellation close connections and +remove the directory. A killed process or failed filesystem cleanup can leave private +residue; this is not secure erasure. Its main database page ceiling defaults to +512 MiB, configurable through the internal `QueryBudget.max_scratch_db_bytes` field from +4096 bytes to 4 GiB; zero disables this path and retains the original budgeted source +reader. This is not a new CLI option. Exceeding the page ceiling or running out of +storage rejects the entire operation. The ceiling does **not** bound rollback journals, +SQLite sorting files, total temporary space or RSS. Main/source page caches request +2 MiB each; these are cache targets, not memory guarantees. Internal SQL scans and +temporary work are not counted as Python-fetched rows. All stages, summary/page or +lifetime buckets and source hydration share the same diminishing deadline. A million-event dataset can therefore answer aggregates and small pages within the default work budget. Use `-n 5` or an explicit finite `limit` for large row-returning queries. Unlimited output, large offsets, many distinct stack groups, ordered-frame -reads or the attachment-limit fallback can still exceed the budget and fail honestly. +reads or a disabled temporary reducer can still exceed the fetched budget and fail +honestly. Many-lifecycle reductions can independently exceed the derived-page limit +or deadline. The limit counts rows fetched into Python and serialized output, not internal SQL scans, SQLite sorting space or total process memory. diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 52616c8..5f390b8 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -136,7 +136,8 @@ SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入 分片数),新 CLI 进程仍冷启动。准入共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 单个文件系统调用不提供 OS 级硬抢占。仅当调用方保证最终产物不可变且文件系统变更时间 可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用纯元数据校验复用;参见 -[数据集校验与缓存合同](sharded-snapshotdb.md)。不生成合并临时 DB、不修复或写产物。 +[数据集校验与缓存合同](sharded-snapshotdb.md)。不修复或写入源产物;较大的生命周期查询 +可能生成下述私有临时派生数据库。 兼容导出器默认**每设备每片 500000 个真实事件**。该存储容量与 **100000 行 / 64 MiB 查询工作预算**独立:即使只有一个默认大小的分片,无足够筛选或分页的结果物化也可能 @@ -149,13 +150,23 @@ SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入 与总计数计算。规范化原始有序帧的身份不能用格式化文本替代,因此仍走受预算约束的来源 读取路径。block/leak/lifetime 通过只读附加分片先证明身份、检测冲突、选最新观察,再分页; 附加上限(通常十片)按选定观察分片与其分配/释放引用所属的其他分片合计。 -slice 查询在筛选、分页前发现所需的证明来源分片;超限时回退到原有受预算约束的来源 -读取路径。SQLite 内部扫描与临时空间不受取回行数/字节预算约束;汇总和分页(或存活时长 -分桶)各自执行完整的生命周期 CTE,共用同一 deadline。不创建中间合并数据库。 +slice 查询在筛选、分页前发现所需的证明来源分片。超限时使用查询专属的临时 SQLite +派生库,仅保留去重后的事件引用及每个生命周期的一条窄状态记录。源分片逐个只读挂载: +先证明引用的所属片及 action,再归约全部观察、检查冲突,之后才筛选、计数和分页。 +仅返回候选补全完整来源事件。未超过附加上限的原路径不变。 + +临时目录权限为 0700,派生文件以 0600 创建;正常完成、错误和取消均关闭连接并删除目录。 +强制杀进程或文件系统清理失败可能留下私有残余;这不是安全擦除。派生主库页数空间默认 +上限为 512 MiB,可通过内部 `QueryBudget.max_scratch_db_bytes` 字段设为 4096 字节至 +4 GiB;0 禁用该路径、沿用原有受预算约束的来源读取。这不是新增 CLI 选项。页数超限或 +磁盘满会拒绝整个查询。该上限**不包括**回滚 journal、SQLite 排序文件、全部临时空间或 +RSS。主库和源库各请求 2 MiB page cache,仅为缓存目标,不保证内存上限。SQL 内部扫描 +和临时计算不算 Python 取回行;全部阶段、汇总/分页或时长分桶及来源补全共用递减 deadline。 因此,百万事件数据集的聚合与小页面可在默认工作预算内完成。大量逐行结果请指定 `-n 5` -或有限 `limit`。无限制输出、大 offset、大量不同栈分组、有序帧读取及超过附加上限的回退 -路径仍可能超预算,并明确失败。预算统计进入 Python 的实际取回行和序列化输出,不限制 +或有限 `limit`。无限制输出、大 offset、大量不同栈分组、有序帧读取及禁用临时归约库后的 +来源读取仍可能超取回预算,并明确失败。大量生命周期归约也可能单独超出派生主库页数上限 +或 deadline。预算统计进入 Python 的实际取回行和序列化输出,不限制 SQL 内部扫描、SQLite 排序空间或整个进程内存。 ## 数据集定点事件归因 diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 43ec456..cf03b6e 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -1,4 +1,4 @@ -"""Exact built-in global reductions, without a temporary merge database. +"""Exact built-in global reductions, with bounded fetched work. Shard SQL reduces counters, groups, and candidate windows before Python merging. Fetched work is cumulatively bounded by QueryBudget, including source hydration diff --git a/src/pt_snap_cli/core/dataset_lifecycle_connection.py b/src/pt_snap_cli/core/dataset_lifecycle_connection.py new file mode 100644 index 0000000..71beed0 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_lifecycle_connection.py @@ -0,0 +1,88 @@ +"""Shared deadline-aware SQLite connection ownership for lifecycle reductions.""" + +from __future__ import annotations + +import math +import sqlite3 +import time + +from pt_snap_cli.core.dataset_contract import SliceRecord +from pt_snap_cli.core.dataset_files import require_readonly_member +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.errors import QueryExecutionError + + +class _AttachedSources: + def __init__( + self, + sources: DatasetSourceResolver, + items: list[SliceRecord] | None = None, + *, + database: str = ":memory:", + ) -> None: + self.sources = sources + self.items = sources.device.slices if items is None else items + remaining = sources.budget.remaining() + self.connection = sqlite3.connect( + database, uri=True, timeout=min(5.0, remaining) if remaining is not None else 5.0 + ) + self.connection.row_factory = sqlite3.Row + + def _configure_deadline(self): + budget = self.sources.budget + remaining = budget.remaining() + deadline = budget.started + budget.timeout_s if budget.timeout_s is not None else None + self.connection.set_progress_handler( + (lambda: int(time.monotonic() >= deadline)) if deadline is not None else None, 1000 + ) + # SQLite's default five-second lock wait must not outlive the operation. + wait_ms = math.ceil(1000 * min(5.0, remaining)) if remaining is not None else 5000 + self.connection.execute(f"PRAGMA busy_timeout={wait_ms}") + + def __enter__(self): + try: + self._configure_deadline() + self.connection.execute("PRAGMA trusted_schema=OFF") + for item in self.items: + self._configure_deadline() + path = self.sources.dataset.root / item.file + immutable = self.sources.dataset.callstack_layout == "v1" + require_readonly_member(self.sources.dataset.root, path, immutable=immutable) + uri = path.as_uri() + "?mode=ro" + ("&immutable=1" if immutable else "") + self.connection.execute(f'ATTACH DATABASE ? AS "s{item.index}"', [uri]) + self.connection.execute("PRAGMA query_only=ON") + self.sources.budget.remaining() + return self + except BaseException as exc: + self.connection.set_progress_handler(None, 0) + self.connection.close() + if isinstance(exc, sqlite3.Error): + self.sources.budget.remaining() + raise QueryExecutionError(str(exc)) from exc + raise + + def __exit__(self, exc_type, exc_value, traceback): + self.connection.set_progress_handler(None, 0) + self.connection.close() + return False + + def read(self, sql: str, values: list[object] | None = None): + budget = self.sources.budget + cursor = self.connection.cursor() + self.sources.query_count += 1 + try: + self._configure_deadline() + cursor.execute(sql, values or []) + result = [] + while batch := cursor.fetchmany(256): + rows = [dict(row) for row in batch] + budget.consume(rows) + result.extend(rows) + budget.remaining() + return result + except (sqlite3.Error, OverflowError) as exc: + budget.remaining() # Preserve the domain timeout on interruption. + raise QueryExecutionError(str(exc)) from exc + finally: + cursor.close() + self.connection.set_progress_handler(None, 0) diff --git a/src/pt_snap_cli/core/dataset_lifecycle_scratch.py b/src/pt_snap_cli/core/dataset_lifecycle_scratch.py new file mode 100644 index 0000000..e354af3 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_lifecycle_scratch.py @@ -0,0 +1,191 @@ +"""Private disk-backed lifecycle reduction when the read-only ATTACH set is too large. + +Only references and one narrow row per identity are retained. The derived main +DB has a page ceiling; this does NOT limit all SQLite journals/sort files or RSS. +The fetched/output budget and operation deadline remain independent safeguards. +Source databases are never writable, even while the derived database is built. +""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from contextlib import contextmanager +from pathlib import Path + +from pt_snap_cli.core.dataset_files import require_readonly_member +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources +from pt_snap_cli.core.errors import QueryExecutionError + +_PAGE_SIZE = 4096 +_MAX_SCRATCH_BYTES = 4 * 1024 * 1024 * 1024 +_CACHE_KIB = 2048 +_COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") + + +class _ScratchSources(_AttachedSources): + def execute(self, sql, values=()): + self._configure_deadline() + self.sources.query_count += 1 + try: + self.connection.execute(sql, values).close() + self.sources.budget.remaining() + except (sqlite3.Error, OverflowError) as exc: + self.sources.budget.remaining() + if getattr(exc, "sqlite_errorcode", None) == getattr( + sqlite3, "SQLITE_FULL", 13 + ) or "database or disk is full" in str(exc): + raise QueryExecutionError( + "Dataset lifecycle scratch space exhausted (derived database page limit " + "or disk full); no partial global result." + ) from exc + raise QueryExecutionError(str(exc)) from exc + + @contextmanager + def source(self, item): + path = self.sources.dataset.root / item.file + immutable = self.sources.dataset.callstack_layout == "v1" + require_readonly_member(self.sources.dataset.root, path, immutable=immutable) + uri = path.as_uri() + "?mode=ro" + ("&immutable=1" if immutable else "") + self.execute("ATTACH DATABASE ? AS src", [uri]) + try: + # Bound each source's independent page cache too. URI mode=ro is the + # write barrier; query_only is connection-wide and would block main. + self.execute(f"PRAGMA src.cache_size=-{_CACHE_KIB}") + self.execute("PRAGMA src.mmap_size=0") + yield + finally: + # Detach must also work after deadline expiry or interruption. The + # outer owner closes the connection if detach itself fails. + self.connection.set_progress_handler(None, 0) + self.connection.execute("DETACH DATABASE src").close() + + def build(self, items, maximum): + self.connection.isolation_level = None + self.connection.set_authorizer( + lambda action, first, second, database, trigger: ( + sqlite3.SQLITE_DENY + if action in (sqlite3.SQLITE_INSERT, sqlite3.SQLITE_UPDATE, sqlite3.SQLITE_DELETE) + and database != "main" + else sqlite3.SQLITE_OK + ) + ) + self.execute("PRAGMA trusted_schema=OFF") + self.execute(f"PRAGMA main.page_size={_PAGE_SIZE}") + self.execute(f"PRAGMA main.max_page_count={maximum // _PAGE_SIZE}") + self.execute(f"PRAGMA main.cache_size=-{_CACHE_KIB}") + self.execute("PRAGMA main.mmap_size=0") + self.execute("PRAGMA temp_store=FILE") + self.execute("PRAGMA journal_mode=DELETE") + self.execute( + "CREATE TABLE refs(id INTEGER PRIMARY KEY, allocation_proved INTEGER NOT NULL " + "DEFAULT 0, completion_proved INTEGER NOT NULL DEFAULT 0)" + ) + device = self.sources.device.device_id + for item in items: + with self.source(item): + for column in ("allocEventId", "freeEventId"): + self.execute( + f'INSERT OR IGNORE INTO refs(id) SELECT "{column}" ' + f'FROM src."block_{device}" WHERE "{column}">=0' + ) + # Range ownership is authoritative. Never accept a same-ID event from a + # non-owner, a boundary/sentinel event, or an event with the wrong action. + for item in self.sources.device.slices: + needed = self.read( + "SELECT EXISTS(SELECT 1 FROM refs WHERE id BETWEEN ? AND ?) AS needed", + [item.start_event_id, item.end_event_id], + )[0]["needed"] + if needed: + with self.source(item): + self.execute( + "UPDATE refs SET allocation_proved=EXISTS(SELECT 1 FROM " + f'src."trace_entry_{device}" e WHERE e.id=refs.id AND e.id>=0 ' + "AND e.action=4), completion_proved=EXISTS(SELECT 1 FROM " + f'src."trace_entry_{device}" e WHERE e.id=refs.id AND e.id>=0 ' + "AND e.action=6) WHERE id BETWEEN ? AND ?", + [item.start_event_id, item.end_event_id], + ) + self.execute( + "CREATE TABLE lifecycle_state(token TEXT PRIMARY KEY COLLATE BINARY, " + + ", ".join(f'"{column}" INTEGER' for column in _COLUMNS) + + ", observation_slice_index INTEGER NOT NULL, allocation_proved INTEGER NOT NULL, " + "known_free INTEGER, invariant_conflict INTEGER NOT NULL DEFAULT 0, " + "free_conflict INTEGER NOT NULL DEFAULT 0) WITHOUT ROWID" + ) + latest = ( + "(excluded.observation_slice_index>lifecycle_state.observation_slice_index OR " + "(excluded.observation_slice_index=lifecycle_state.observation_slice_index " + "AND excluded.id>lifecycle_state.id))" + ) + updates = ", ".join( + f'"{column}"=CASE WHEN {latest} THEN excluded."{column}" ' + f'ELSE lifecycle_state."{column}" END' + for column in (*_COLUMNS, "observation_slice_index") + ) + conflict = " OR ".join( + f'lifecycle_state."{column}" IS NOT excluded."{column}"' + for column in ("address", "size", "requestedSize", "allocEventId") + ) + for item in items: + with self.source(item): + self.execute( + "INSERT INTO lifecycle_state(token, " + + ", ".join(f'"{column}"' for column in _COLUMNS) + + ", observation_slice_index, allocation_proved, known_free) " + "SELECT CASE WHEN a.allocation_proved THEN 'alloc:' || b.allocEventId " + "WHEN b.allocEventId=-1 AND b.id<0 THEN 'preexisting:' || b.id " + f"ELSE 'unproved:slice:{item.index}:block:' || b.id END, " + + ", ".join(f'b."{column}"' for column in _COLUMNS) + + f", {item.index}, COALESCE(a.allocation_proved,0), " + "CASE WHEN f.completion_proved THEN b.freeEventId END " + f'FROM src."block_{device}" b LEFT JOIN refs a ON a.id=b.allocEventId ' + "LEFT JOIN refs f ON f.id=b.freeEventId WHERE 1 " + "ON CONFLICT(token) DO UPDATE SET " + f"invariant_conflict=lifecycle_state.invariant_conflict OR ({conflict}), " + "free_conflict=lifecycle_state.free_conflict OR " + "(lifecycle_state.known_free IS NOT NULL AND excluded.known_free IS NOT NULL " + "AND lifecycle_state.known_free IS NOT excluded.known_free), " + "known_free=COALESCE(lifecycle_state.known_free,excluded.known_free), " + + updates + ) + self.execute("PRAGMA query_only=ON") + + +def _lifecycle_cte(terminal): + return f"""WITH latest AS ( + SELECT id,address,size,requestedSize,state,allocEventId, + COALESCE(known_free,freeEventId) AS freeEventId, + observation_slice_index,allocation_proved,token, + known_free IS NOT NULL AS completion_proved,invariant_conflict,free_conflict + FROM lifecycle_state + ), lifecycles AS ( + SELECT *, (observation_slice_index={terminal} AND state IN (0,1) + AND (freeEventId IS NULL OR freeEventId<0)) AS terminal_survivor + FROM latest + ) """ + + +@contextmanager +def scratch_lifecycles(sources, items): + maximum = sources.budget.max_scratch_db_bytes + if type(maximum) is not int or not _PAGE_SIZE <= maximum <= _MAX_SCRATCH_BYTES: + raise QueryExecutionError( + "Dataset lifecycle scratch limit must be 0 (disabled) or between 4096 bytes " + "and 4 GiB; this bounds derived database pages, not total temporary space or RSS." + ) + try: + with tempfile.TemporaryDirectory(prefix="pt-snap-lifecycle-") as directory: + path = Path(directory) / "derived.db" + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + os.close(descriptor) + database = _ScratchSources(sources, [], database=str(path)) + try: + database.build(items, maximum) + yield database, _lifecycle_cte(sources.device.slices[-1].index) + finally: + database.__exit__(None, None, None) + except (OSError, sqlite3.Error) as exc: + sources.budget.remaining() + raise QueryExecutionError(f"Dataset lifecycle scratch storage failed: {exc}") from exc diff --git a/src/pt_snap_cli/core/dataset_lifecycle_sql.py b/src/pt_snap_cli/core/dataset_lifecycle_sql.py index 678b69e..48df7e7 100644 --- a/src/pt_snap_cli/core/dataset_lifecycle_sql.py +++ b/src/pt_snap_cli/core/dataset_lifecycle_sql.py @@ -4,96 +4,20 @@ latest-observation selection and page ranking; only aggregate/page rows cross the Python boundary. The fetched-row/byte budget does not bound SQLite's internal scans or temporary space. Summary and page/buckets each evaluate the full CTE; -all statements share the operation deadline. Larger required shard sets keep -the budgeted general implementation. +all statements share the operation deadline. Larger required shard sets use a +private, size-limited derived SQLite database; source members remain read-only. """ from __future__ import annotations -import math import sqlite3 -import time from contextlib import closing from typing import cast -from pt_snap_cli.core.dataset_contract import SliceRecord -from pt_snap_cli.core.dataset_files import require_readonly_member -from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources from pt_snap_cli.core.errors import QueryExecutionError -class _AttachedSources: - def __init__( - self, sources: DatasetSourceResolver, items: list[SliceRecord] | None = None - ) -> None: - self.sources = sources - self.items = sources.device.slices if items is None else items - remaining = sources.budget.remaining() - self.connection = sqlite3.connect( - ":memory:", uri=True, timeout=min(5.0, remaining) if remaining is not None else 5.0 - ) - self.connection.row_factory = sqlite3.Row - - def _configure_deadline(self): - budget = self.sources.budget - remaining = budget.remaining() - deadline = budget.started + budget.timeout_s if budget.timeout_s is not None else None - self.connection.set_progress_handler( - (lambda: int(time.monotonic() >= deadline)) if deadline is not None else None, 1000 - ) - # SQLite's default five-second lock wait must not outlive the operation. - wait_ms = math.ceil(1000 * min(5.0, remaining)) if remaining is not None else 5000 - self.connection.execute(f"PRAGMA busy_timeout={wait_ms}") - - def __enter__(self): - try: - self._configure_deadline() - self.connection.execute("PRAGMA trusted_schema=OFF") - for item in self.items: - self._configure_deadline() - path = self.sources.dataset.root / item.file - immutable = self.sources.dataset.callstack_layout == "v1" - require_readonly_member(self.sources.dataset.root, path, immutable=immutable) - uri = path.as_uri() + "?mode=ro" + ("&immutable=1" if immutable else "") - self.connection.execute(f'ATTACH DATABASE ? AS "s{item.index}"', [uri]) - self.connection.execute("PRAGMA query_only=ON") - self.sources.budget.remaining() - return self - except BaseException as exc: - self.connection.set_progress_handler(None, 0) - self.connection.close() - if isinstance(exc, sqlite3.Error): - self.sources.budget.remaining() - raise QueryExecutionError(str(exc)) from exc - raise - - def __exit__(self, exc_type, exc_value, traceback): - self.connection.set_progress_handler(None, 0) - self.connection.close() - return False - - def read(self, sql: str, values: list[object] | None = None): - budget = self.sources.budget - cursor = self.connection.cursor() - self.sources.query_count += 1 - try: - self._configure_deadline() - cursor.execute(sql, values or []) - result = [] - while batch := cursor.fetchmany(256): - rows = [dict(row) for row in batch] - budget.consume(rows) - result.extend(rows) - budget.remaining() - return result - except (sqlite3.Error, OverflowError) as exc: - budget.remaining() # Preserve the domain timeout on interruption. - raise QueryExecutionError(str(exc)) from exc - finally: - cursor.close() - self.connection.set_progress_handler(None, 0) - - def _attachment_limit() -> int: # getlimit is unavailable on supported Python 3.10. Ten is SQLite's default. with closing(sqlite3.connect(":memory:")) as connection: @@ -213,11 +137,23 @@ def _filter(template, params): def lifecycle_query(sources, items, template, params, scope, candidate_limit): required = _required_slices(sources, items, _attachment_limit()) - if required is None: - return None predicate, values = _filter(template, params) - cte = _lifecycle_cte(sources, items, required) - with _AttachedSources(sources, required) as database: + if required is None: + from pt_snap_cli.core.dataset_lifecycle_scratch import scratch_lifecycles + + if sources.budget.max_scratch_db_bytes == 0 or _attachment_limit() < 1: + return None + execution = scratch_lifecycles(sources, items) + else: + from contextlib import contextmanager + + @contextmanager + def attached_lifecycles(): + with _AttachedSources(sources, required) as database: + yield database, _lifecycle_cte(sources, items, required) + + execution = attached_lifecycles() + with execution as (database, cte): summary = database.read( cte + f"""SELECT COUNT(*) AS lifecycles, COALESCE(SUM(allocation_proved),0) AS allocation_events_resolved, diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 4907016..4dc49ec 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -68,6 +68,9 @@ class QueryBudget: max_work_bytes: int = 64 * 1024 * 1024 work_rows: int = 0 work_bytes: int = 0 + # Per-operation derived database pages, not total SQLite temp space or RSS. + # Zero retains the bounded in-Python fallback. Local/internal API only. + max_scratch_db_bytes: int = 512 * 1024 * 1024 def consume(self, rows: list[dict[str, object]]) -> None: """Bound cumulative fetched rows/serialized values, not merely output. diff --git a/tests/core/test_dataset_lifecycle_scratch.py b/tests/core/test_dataset_lifecycle_scratch.py new file mode 100644 index 0000000..c8902b1 --- /dev/null +++ b/tests/core/test_dataset_lifecycle_scratch.py @@ -0,0 +1,293 @@ +"""Many-owner lifecycle queries retain exact semantics and bounded private storage.""" + +import contextlib +import hashlib +import os +import sqlite3 +import time +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_lifecycle_scratch as scratch +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from tests.core.test_dataset_sql_pushdown import run_resolved +from tests.test_dataset_focus import make_dataset + + +def many_owners(tmp_path): + root = make_dataset(tmp_path / "many", devices=(0,), slices=12) + with contextlib.closing(sqlite3.connect(root / "device_0/slice_00011.db")) as conn, conn: + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, 100 + i * 8, i) for i in range(24)), + ) + return root + + +def limited_connections(monkeypatch, limit): + if not hasattr(sqlite3.Connection, "setlimit"): + pytest.skip("Real per-connection SQLite limits require Python 3.11+") + original = sqlite3.connect + + def connect(*args, **kwargs): + connection = original(*args, **kwargs) + connection.setlimit(sqlite3.SQLITE_LIMIT_ATTACHED, limit) + return connection + + monkeypatch.setattr(sqlite3, "connect", connect) + + +def capture_storage(monkeypatch, tmp_path): + parent = tmp_path / "scratch" + parent.mkdir() + original = scratch.tempfile.TemporaryDirectory + created = [] + + def directory(*args, **kwargs): + result = original(*args, dir=parent, **kwargs) + created.append(Path(result.name)) + return result + + monkeypatch.setattr(scratch.tempfile, "TemporaryDirectory", directory) + return parent, created + + +@pytest.mark.parametrize("limit", [1, 2]) +@pytest.mark.parametrize("template", ["block", "leak_detection", "freed_block_lifetime"]) +@pytest.mark.parametrize("slice_index", [None, 11]) +def test_real_attach_limit_and_many_proof_owners( + tmp_path, monkeypatch, limit, template, slice_index +): + if template == "leak_detection" and slice_index is not None: + pytest.skip("Slice leak query is intentionally unsupported") + root = many_owners(tmp_path) + resolved = DatasetResolver().inspect(root) + expected = run_resolved( + resolved, + template, + slice_index=slice_index, + budget=QueryBudget(None, time.monotonic(), max_scratch_db_bytes=0), + ) + hashes = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob("*.db")} + limited_connections(monkeypatch, limit) + parent, created = capture_storage(monkeypatch, tmp_path) + original = scratch._ScratchSources.build + observed = [] + + def build(database, items, maximum): + original(database, items, maximum) + path = Path(database.connection.execute("PRAGMA database_list").fetchone()[2]) + observed.append(database.connection) + assert os.stat(path.parent).st_mode & 0o777 == 0o700 + assert os.stat(path).st_mode & 0o777 == 0o600 + assert path.stat().st_size <= maximum + assert database.connection.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) == limit + + monkeypatch.setattr(scratch._ScratchSources, "build", build) + budget = QueryBudget(None, time.monotonic(), max_work_rows=128) + actual = run_resolved(resolved, template, slice_index=slice_index, budget=budget) + assert actual.rows == expected.rows + assert (actual.total, actual.has_more) == (expected.total, expected.has_more) + assert {k: v for k, v in actual.scope["source_coverage"].items() if k != "source_queries"} == { + k: v for k, v in expected.scope["source_coverage"].items() if k != "source_queries" + } + assert created and not list(parent.iterdir()) + assert all(not path.exists() for path in created) + for connection in observed: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + assert {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in hashes} == hashes + assert not any(root.rglob("*-journal")) + assert budget.work_rows <= 128 + + +@pytest.mark.parametrize("maximum", [-1, 1, 4095, 4 * 1024**3 + 1, True, 1.5]) +def test_invalid_scratch_budget_is_domain_error(tmp_path, maximum): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + with pytest.raises(QueryExecutionError, match="scratch limit"): + run_resolved( + resolved, + "block", + budget=QueryBudget(None, time.monotonic(), max_scratch_db_bytes=maximum), + ) + + +def test_actual_page_limit_failure_cleans_private_storage(tmp_path, monkeypatch): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + parent, created = capture_storage(monkeypatch, tmp_path) + with pytest.raises(QueryExecutionError, match="scratch space exhausted"): + run_resolved( + resolved, "block", budget=QueryBudget(None, time.monotonic(), max_scratch_db_bytes=4096) + ) + assert created and not list(parent.iterdir()) + + +@pytest.mark.parametrize( + "phase", ["INSERT OR IGNORE", "UPDATE refs", "INSERT INTO lifecycle_state", "PRAGMA query_only"] +) +@pytest.mark.parametrize( + "failure", [QueryTimeoutError("expired"), KeyboardInterrupt(), OSError("full")] +) +def test_phase_failures_leave_no_storage_or_connection(tmp_path, monkeypatch, phase, failure): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + parent, created = capture_storage(monkeypatch, tmp_path) + original = scratch._ScratchSources.execute + connections = [] + + def execute(database, sql, values=()): + if sql.startswith(phase): + connections.append(database.connection) + raise failure + return original(database, sql, values) + + monkeypatch.setattr(scratch._ScratchSources, "execute", execute) + expected = QueryExecutionError if isinstance(failure, OSError) else type(failure) + with pytest.raises(expected): + run_resolved(resolved, "block") + assert created and not list(parent.iterdir()) + for connection in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + + +def test_disabled_scratch_preserves_fetched_work_budget(tmp_path, monkeypatch): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + monkeypatch.setattr(scratch, "scratch_lifecycles", lambda *args: pytest.fail("disabled")) + with pytest.raises(QueryExecutionError, match="work budget"): + run_resolved( + resolved, + "block", + budget=QueryBudget(None, time.monotonic(), max_work_rows=3, max_scratch_db_bytes=0), + ) + + +def test_shared_deadline_expires_during_build_and_cleans_up(tmp_path, monkeypatch): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + parent, created = capture_storage(monkeypatch, tmp_path) + original = scratch._ScratchSources.execute + + def execute(database, sql, values=()): + if sql.startswith("UPDATE refs"): + database.sources.budget.started -= 10 + return original(database, sql, values) + + monkeypatch.setattr(scratch._ScratchSources, "execute", execute) + with pytest.raises(QueryTimeoutError): + run_resolved(resolved, "block", budget=QueryBudget(5, time.monotonic())) + assert created and not list(parent.iterdir()) + + +def test_scratch_source_writes_rejected_while_main_is_writable(tmp_path, monkeypatch): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + original = scratch._ScratchSources.source + visited = [] + + @contextlib.contextmanager + def source(database, item): + with original(database, item): + with pytest.raises(sqlite3.DatabaseError, match="authorized|readonly"): + database.connection.execute("DELETE FROM src.block_0") + visited.append(item.index) + yield + + monkeypatch.setattr(scratch._ScratchSources, "source", source) + run_resolved(resolved, "block") + assert len(set(visited)) == 12 + + +@pytest.mark.parametrize( + "variant", ["earlier_free", "null_invariant", "wrong_action", "same_slice_tie"] +) +@pytest.mark.parametrize("template", ["block", "leak_detection", "freed_block_lifetime"]) +def test_scratch_semantics_match_attached_path(tmp_path, monkeypatch, variant, template): + from tests.core.test_dataset_attribution import case + from tests.core.test_dataset_sql_pushdown import last_observation + + root = case(tmp_path, devices=(0,)) + # Mutate after admission to exercise reducer defenses independently of validation. + resolved = DatasetResolver().inspect(root) + if variant == "earlier_free": + with contextlib.closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + conn.execute("UPDATE block_0 SET state=0,freeEventId=-1 WHERE id=0") + elif variant == "null_invariant": + for path in root.glob("device_0/*.db"): + with contextlib.closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE block_0 SET requestedSize=NULL WHERE id=0") + elif variant == "wrong_action": + for path in root.glob("device_0/*.db"): + with contextlib.closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET action=5 WHERE id=0") + else: + with contextlib.closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + conn.execute( + "INSERT INTO block_0 SELECT 900,address,size,requestedSize,0,allocEventId,freeEventId " + "FROM block_0 WHERE id=0" + ) + params = {"offset": 1, "limit": 3} if template == "block" else {} + expected = run_resolved(resolved, template, params) + limited_connections(monkeypatch, 1) + actual = run_resolved(resolved, template, params) + assert actual.rows == expected.rows + assert (actual.total, actual.has_more) == (expected.total, expected.has_more) + + +@pytest.mark.parametrize("variant", ["invariant", "null", "free"]) +@pytest.mark.parametrize("params", [{"id": 7}, {"limit": 0}, {"offset": 100}]) +def test_scratch_conflicts_outside_page_still_fail(tmp_path, monkeypatch, variant, params): + from tests.core.test_dataset_attribution import case + from tests.core.test_dataset_sql_pushdown import last_observation + + root = case(tmp_path, devices=(0,)) + # Mutate after admission to exercise reducer defenses independently of validation. + resolved = DatasetResolver().inspect(root) + with contextlib.closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + if variant == "invariant": + conn.execute("UPDATE block_0 SET size=size+1 WHERE id=0") + elif variant == "null": + conn.execute("UPDATE block_0 SET requestedSize=NULL WHERE id=0") + else: + conn.execute("UPDATE block_0 SET freeEventId=4 WHERE id=0") + if variant == "free": + for path in root.glob("device_0/*.db"): + with contextlib.closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET action=6 WHERE id=4") + with pytest.raises(QueryExecutionError, match="Conflicting"): + run_resolved(resolved, "block", params) + limited_connections(monkeypatch, 1) + with pytest.raises(QueryExecutionError, match="Conflicting"): + run_resolved(resolved, "block", params) + + +@pytest.mark.parametrize("stage", ["open", "detach"]) +def test_setup_or_detach_sqlite_failure_cleans_storage(tmp_path, monkeypatch, stage): + resolved = DatasetResolver().inspect(many_owners(tmp_path)) + parent, created = capture_storage(monkeypatch, tmp_path) + original = sqlite3.connect + connections = [] + + class DetachFailure(sqlite3.Connection): + def execute(self, sql, parameters=()): + if sql == "DETACH DATABASE src": + raise sqlite3.OperationalError("injected detach failure") + return super().execute(sql, parameters) + + def connect(path, *args, **kwargs): + if str(path).endswith("derived.db"): + if stage == "open": + raise sqlite3.OperationalError("injected open failure") + kwargs["factory"] = DetachFailure + connection = original(path, *args, **kwargs) + connections.append(connection) + return connection + return original(path, *args, **kwargs) + + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryExecutionError, match=f"injected {stage} failure"): + run_resolved(resolved, "block") + assert created and not list(parent.iterdir()) + for connection in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1")