diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 0edd799..4bcb719 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -283,6 +283,24 @@ immutable-file contract. An opted-in hit performs no row scans or whole-file has work is O(shards), independent of event/block counts. None of these checks provides a concurrent-writer lock or an atomic filesystem snapshot. +The two-mode acceptance contract is measured by work, not warm wall-clock timing. +For fixed manifest/member membership, let **B** be total member bytes and **M** +manifest bytes. The hash-read budget (excluding small header preflights and SQLite +validation reads) is: + +| Admission | Default strong | Explicit immutable | +| --- | --- | --- | +| Compatibility-v1 cold | 2B + 2M; one full validator | B + 2M; one full validator | +| Native-v2 cold | 2B + 2M; one full validator | 2B + 2M; one full validator | +| Unchanged in-process warm | B + M; no full validator | No hash reads or full validator | + +Both modes revalidate after manifest/member replacement and evict failed or +cancelled admissions; a new process starts cold. If a filesystem hides **all** +change tokens after content tampering, default strong mode detects the changed +bytes, whereas immutable mode may reuse stale validation. Such input violates the +opt-in contract. These are process-local acceptance guarantees, not a commitment +to a persistent cache or dataset-size-independent strong validation. + The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, Python validation loops check every 256 rows, and SQLite validation (including `quick_check`) uses an interruptible progress handler, cleared on every exit. diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index c606653..760825f 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -228,6 +228,21 @@ POSIX change token 的平台在此模式下每次完整重验。immutable 模式 行、不做整文件哈希,工作量为 O(分片数),不随 event/block 数增长。这些检查均不提供 并发 writer 锁或原子文件系统快照。 +双模式验收以实际工作量为准,不以 warm wall-clock 耗时作为唯一证据。固定 manifest/ +成员集合,令 **B** 为所有成员总字节数、**M** 为 manifest 字节数;哈希读取预算如下 +(不含小规模 header 预检及 SQLite 校验读取): + +| 准入路径 | 默认强验证 | 显式 immutable | +| --- | --- | --- | +| 兼容 v1 cold | 2B + 2M;一次完整 validator | B + 2M;一次完整 validator | +| 原生 v2 cold | 2B + 2M;一次完整 validator | 2B + 2M;一次完整 validator | +| 同进程未变更 warm | B + M;无完整 validator | 无哈希读取、无完整 validator | + +两模式在 manifest/成员替换后重新完整校验;失败或取消的准入会驱逐缓存,新进程仍从 +cold 开始。如果内容篡改后文件系统隐藏了**所有** change token,默认强验证仍检测内容 +变化,而 immutable 可能复用旧校验结果;此输入违反显式 opt-in 合同。这些仅为进程内 +验收保证,不承诺持久缓存或与数据量无关的默认强验证。 + 查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite 校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 取消保留 `QUERY_TIMEOUT`,不会缓存不完整校验。设备 tab 补全只读取有界 manifest diff --git a/tests/core/test_dataset_integrity_modes.py b/tests/core/test_dataset_integrity_modes.py new file mode 100644 index 0000000..d9d1e75 --- /dev/null +++ b/tests/core/test_dataset_integrity_modes.py @@ -0,0 +1,262 @@ +"""Issue #219 acceptance: strong default versus explicit immutable publication.""" + +import json +import os +import sqlite3 +import subprocess +import sys +from collections import Counter +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_resolver as resolver +from pt_snap_cli.core import native_dataset_contract as native +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.query_service import QueryService +from tests.core.test_dataset_validation_cache import CancellableBudget +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(params=["strong", "immutable"]) +def mode(request, monkeypatch): + if request.param == "immutable": + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + else: + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + resolver._VALIDATED.clear() + yield request.param + resolver._VALIDATED.clear() + + +@pytest.fixture(params=["v1", "v2"]) +def artifact(request, tmp_path, mode): + root = make_dataset(tmp_path / "dataset") if request.param == "v1" else native_dataset(tmp_path) + resolver._VALIDATED.clear() # Import-time validation is not reader admission. + return root, request.param + + +@pytest.fixture +def work(monkeypatch): + """Count bytes actually returned by hash reads, excluding header preflights.""" + counts = Counter() + active = [] + real_open = Path.open + + class Reader: + def __init__(self, source, path): + self.source, self.path = source, path + + def __enter__(self): + self.source.__enter__() + return self + + def __exit__(self, *args): + return self.source.__exit__(*args) + + def read(self, *args): + data = self.source.read(*args) + counts[("bytes", self.path)] += len(data) + return data + + def opened(path, *args, **kwargs): + source = real_open(path, *args, **kwargs) + return Reader(source, path) if active else source + + def hashed(path, **kwargs): + counts[("hashes", path)] += 1 + active.append(path) + try: + return hash_file(path, **kwargs) + finally: + active.pop() + + monkeypatch.setattr(Path, "open", opened) + monkeypatch.setattr(resolver, "_hash_file", hashed) + monkeypatch.setattr(native, "_hash_file", hashed) + for name in ("validate_dataset", "validate_native_dataset"): + original = getattr(resolver, name) + + def validated(*args, _original=original, **kwargs): + counts["validators"] += 1 + return _original(*args, **kwargs) + + monkeypatch.setattr(resolver, name, validated) + return counts + + +def test_cold_warm_actual_hash_bytes_and_validator_counts(artifact, mode, work): + root, layout = artifact + members = sorted(root.glob("*/*.db")) + manifest = root / "manifest.json" + cold_member_passes = 1 if (layout, mode) == ("v1", "immutable") else 2 + evidence = { + "layout": layout, + "mode": mode, + "member_bytes": sum(p.stat().st_size for p in members), + } + for state in ("cold", "warm"): + work.clear() + assert resolver.DatasetResolver().inspect(root) is not None + passes = cold_member_passes if state == "cold" else int(mode == "strong") + for path in members: + assert work[("hashes", path)] == passes + assert work[("bytes", path)] == passes * path.stat().st_size + manifest_passes = 2 if state == "cold" else int(mode == "strong") + assert work[("bytes", manifest)] == manifest_passes * manifest.stat().st_size + assert work["validators"] == int(state == "cold") + evidence[state] = { + "hash_read_bytes": sum( + v for k, v in work.items() if isinstance(k, tuple) and k[0] == "bytes" + ), + "validators": work["validators"], + } + print(json.dumps(evidence, sort_keys=True)) + + +@pytest.mark.parametrize("target", ["member", "manifest"]) +def test_atomic_replacement_revalidates_even_identical_bytes(artifact, mode, work, target): + root, _ = artifact + resolver.DatasetResolver().inspect(root) + path = root / "manifest.json" if target == "manifest" else next(root.glob("*/*.db")) + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.replace(path) + work.clear() + resolver.DatasetResolver().inspect(root) + assert work["validators"] == 1 + assert work[("bytes", path)] >= path.stat().st_size + + +def test_hidden_change_tokens_document_immutable_trust_boundary(artifact, mode, monkeypatch): + root, _ = artifact + first = resolver.DatasetResolver().inspect(root) + member = next(root.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + assert (member.stat().st_size, member.stat().st_mtime_ns) == ( + before.st_size, + before.st_mtime_ns, + ) + # Simulate a filesystem concealing ctime too. This violates immutable mode's + # explicit caller contract; that mode must not be described as strong integrity. + monkeypatch.setattr(resolver, "_generation", lambda *args: first.generation) + if mode == "strong": + with pytest.raises(DatabaseSchemaError): + resolver.DatasetResolver().inspect(root) + assert root not in resolver._VALIDATED + else: + assert resolver.DatasetResolver().inspect(root) == first + first.require_unchanged() + + +@pytest.mark.parametrize("warm", [False, True], ids=["cold", "warm"]) +def test_expired_deadline_evicts_and_full_retry_closes_connections( + artifact, mode, work, monkeypatch, warm +): + root, _ = artifact + if warm: + resolver.DatasetResolver().inspect(root) + work.clear() + budget = CancellableBudget() + budget.cancelled = True + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(root, budget=budget) + assert not work and root not in resolver._VALIDATED + connections = [] + original = sqlite3.connect + + def connect(*args, **kwargs): + conn = original(*args, **kwargs) + connections.append(conn) + return conn + + monkeypatch.setattr(sqlite3, "connect", connect) + resolver.DatasetResolver().inspect(root) + assert work["validators"] == 1 and connections + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + + +def test_owned_query_cache_close_and_service_reuse(artifact, mode, monkeypatch): + root, layout = artifact + connections = [] + original = sqlite3.connect + + def connect(*args, **kwargs): + conn = original(*args, **kwargs) + connections.append(conn) + return conn + + monkeypatch.setattr(sqlite3, "connect", connect) + service = QueryService() + try: + for _ in range(2): + result = service.execute_query( + "event", {"id": 0 if layout == "v1" else 7}, db_path=root + ) + assert len(result.rows) == 1 + service.close() + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + finally: + service.close() + + +def test_new_process_is_cold_even_after_parent_admission(artifact, mode): + root, _ = artifact + resolver.DatasetResolver().inspect(root) + code = """ +import sys +from pt_snap_cli.core import dataset_resolver as r +assert not r._VALIDATED +calls = [] +for name in ('validate_dataset', 'validate_native_dataset'): + original = getattr(r, name) + def tracked(*args, _original=original, **kwargs): + calls.append(1) + return _original(*args, **kwargs) + setattr(r, name, tracked) +assert r.DatasetResolver().inspect(sys.argv[1]) is not None +assert calls == [1] +""" + result = subprocess.run( + [sys.executable, "-c", code, str(root)], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + + +def test_cancellation_inside_validation_closes_and_cannot_seed_retry( + artifact, mode, work, monkeypatch +): + root, _ = artifact + budget = CancellableBudget() + connections = [] + original = sqlite3.connect + + def connect(*args, **kwargs): + conn = original(*args, **kwargs) + conn.set_trace_callback(lambda sql: setattr(budget, "cancelled", True)) + connections.append(conn) + return conn + + with monkeypatch.context() as patch: + patch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + resolver.DatasetResolver().inspect(root, budget=budget) + assert work["validators"] == 1 and connections + assert root not in resolver._VALIDATED + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + work.clear() + resolver.DatasetResolver().inspect(root) + assert work["validators"] == 1 diff --git a/tests/core/test_dataset_lifecycle_scratch.py b/tests/core/test_dataset_lifecycle_scratch.py index c8902b1..1683a3a 100644 --- a/tests/core/test_dataset_lifecycle_scratch.py +++ b/tests/core/test_dataset_lifecycle_scratch.py @@ -291,3 +291,69 @@ def connect(path, *args, **kwargs): for connection in connections: with pytest.raises(sqlite3.ProgrammingError, match="closed"): connection.execute("SELECT 1") + + +@pytest.mark.parametrize("failure", ["rows", "bytes", "timeout", "cancel", "io"]) +def test_result_phase_failure_preserves_sources_cleans_storage_and_allows_retry( + tmp_path, monkeypatch, failure +): + """Exercise failure after construction, while the scratch owner is yielded.""" + root = many_owners(tmp_path) + resolved = DatasetResolver().inspect(root) + expected = run_resolved(resolved, "block") + hashes = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob("*.db")} + parent, created = capture_storage(monkeypatch, tmp_path) + original_build = scratch._ScratchSources.build + original_read = scratch._ScratchSources.read + connections = [] + ready = set() + + def build(database, items, maximum): + original_build(database, items, maximum) + connections.append(database.connection) + ready.add(id(database)) + budget = database.sources.budget + if failure == "rows": + budget.max_work_rows = budget.work_rows + elif failure == "bytes": + budget.max_work_bytes = budget.work_bytes + elif failure == "timeout": + budget.timeout_s = 1 + budget.started = time.monotonic() - 2 + + def read(database, sql, values=None): + if id(database) in ready: + if failure == "cancel": + raise KeyboardInterrupt() + if failure == "io": + raise sqlite3.OperationalError("injected result read I/O failure") + return original_read(database, sql, values) + + with monkeypatch.context() as fault: + fault.setattr(scratch._ScratchSources, "build", build) + fault.setattr(scratch._ScratchSources, "read", read) + error, message = { + "rows": (QueryExecutionError, "work budget exceeded"), + "bytes": (QueryExecutionError, "work budget exceeded"), + "timeout": (QueryTimeoutError, "timed out"), + "cancel": (KeyboardInterrupt, "^$"), + "io": (QueryExecutionError, "result read I/O failure"), + }[failure] + with pytest.raises(error, match=message): + run_resolved(resolved, "block") + + assert len(connections) == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connections[0].execute("SELECT 1") + assert created and not list(parent.iterdir()) + assert all(not path.exists() for path in created) + assert {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in hashes} == hashes + assert not any(root.rglob("*-journal")) + assert not any(root.rglob("*-wal")) + assert not any(root.rglob("*-shm")) + + retried = run_resolved(resolved, "block") + assert retried.rows == expected.rows + assert (retried.total, retried.has_more) == (expected.total, expected.has_more) + assert len(created) == 2 and not list(parent.iterdir()) + assert {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in hashes} == hashes