Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/en/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,24 @@ immutable-file contract. An opted-in hit performs no row scans or whole-file has
work is O(shards), independent of event/block counts. None of these checks provides
a concurrent-writer lock or an atomic filesystem snapshot.

The two-mode acceptance contract is measured by work, not warm wall-clock timing.
For fixed manifest/member membership, let **B** be total member bytes and **M**
manifest bytes. The hash-read budget (excluding small header preflights and SQLite
validation reads) is:

| Admission | Default strong | Explicit immutable |
| --- | --- | --- |
| Compatibility-v1 cold | 2B + 2M; one full validator | B + 2M; one full validator |
| Native-v2 cold | 2B + 2M; one full validator | 2B + 2M; one full validator |
| Unchanged in-process warm | B + M; no full validator | No hash reads or full validator |

Both modes revalidate after manifest/member replacement and evict failed or
cancelled admissions; a new process starts cold. If a filesystem hides **all**
change tokens after content tampering, default strong mode detects the changed
bytes, whereas immutable mode may reuse stale validation. Such input violates the
opt-in contract. These are process-local acceptance guarantees, not a commitment
to a persistent cache or dataset-size-independent strong validation.

The query's shared deadline reaches admission: hashing checks every 1 MiB chunk,
Python validation loops check every 256 rows, and SQLite validation (including
`quick_check`) uses an interruptible progress handler, cleared on every exit.
Expand Down
15 changes: 15 additions & 0 deletions docs/zh/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,21 @@ POSIX change token 的平台在此模式下每次完整重验。immutable 模式
行、不做整文件哈希,工作量为 O(分片数),不随 event/block 数增长。这些检查均不提供
并发 writer 锁或原子文件系统快照。

双模式验收以实际工作量为准,不以 warm wall-clock 耗时作为唯一证据。固定 manifest/
成员集合,令 **B** 为所有成员总字节数、**M** 为 manifest 字节数;哈希读取预算如下
(不含小规模 header 预检及 SQLite 校验读取):

| 准入路径 | 默认强验证 | 显式 immutable |
| --- | --- | --- |
| 兼容 v1 cold | 2B + 2M;一次完整 validator | B + 2M;一次完整 validator |
| 原生 v2 cold | 2B + 2M;一次完整 validator | 2B + 2M;一次完整 validator |
| 同进程未变更 warm | B + M;无完整 validator | 无哈希读取、无完整 validator |

两模式在 manifest/成员替换后重新完整校验;失败或取消的准入会驱逐缓存,新进程仍从
cold 开始。如果内容篡改后文件系统隐藏了**所有** change token,默认强验证仍检测内容
变化,而 immutable 可能复用旧校验结果;此输入违反显式 opt-in 合同。这些仅为进程内
验收保证,不承诺持久缓存或与数据量无关的默认强验证。

查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite
校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。
取消保留 `QUERY_TIMEOUT`,不会缓存不完整校验。设备 tab 补全只读取有界 manifest
Expand Down
262 changes: 262 additions & 0 deletions tests/core/test_dataset_integrity_modes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,262 @@
"""Issue #219 acceptance: strong default versus explicit immutable publication."""

import json
import os
import sqlite3
import subprocess
import sys
from collections import Counter
from contextlib import closing
from pathlib import Path

import pytest

from pt_snap_cli.core import dataset_resolver as resolver
from pt_snap_cli.core import native_dataset_contract as native
from pt_snap_cli.core.dataset_files import hash_file
from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError
from pt_snap_cli.core.query_service import QueryService
from tests.core.test_dataset_validation_cache import CancellableBudget
from tests.test_dataset_focus import make_dataset
from tests.test_native_dataset import dataset as native_dataset


@pytest.fixture(params=["strong", "immutable"])
def mode(request, monkeypatch):
if request.param == "immutable":
monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable")
else:
monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False)
resolver._VALIDATED.clear()
yield request.param
resolver._VALIDATED.clear()


@pytest.fixture(params=["v1", "v2"])
def artifact(request, tmp_path, mode):
root = make_dataset(tmp_path / "dataset") if request.param == "v1" else native_dataset(tmp_path)
resolver._VALIDATED.clear() # Import-time validation is not reader admission.
return root, request.param


@pytest.fixture
def work(monkeypatch):
"""Count bytes actually returned by hash reads, excluding header preflights."""
counts = Counter()
active = []
real_open = Path.open

class Reader:
def __init__(self, source, path):
self.source, self.path = source, path

def __enter__(self):
self.source.__enter__()
return self

def __exit__(self, *args):
return self.source.__exit__(*args)

def read(self, *args):
data = self.source.read(*args)
counts[("bytes", self.path)] += len(data)
return data

def opened(path, *args, **kwargs):
source = real_open(path, *args, **kwargs)
return Reader(source, path) if active else source

def hashed(path, **kwargs):
counts[("hashes", path)] += 1
active.append(path)
try:
return hash_file(path, **kwargs)
finally:
active.pop()

monkeypatch.setattr(Path, "open", opened)
monkeypatch.setattr(resolver, "_hash_file", hashed)
monkeypatch.setattr(native, "_hash_file", hashed)
for name in ("validate_dataset", "validate_native_dataset"):
original = getattr(resolver, name)

def validated(*args, _original=original, **kwargs):
counts["validators"] += 1
return _original(*args, **kwargs)

monkeypatch.setattr(resolver, name, validated)
return counts


def test_cold_warm_actual_hash_bytes_and_validator_counts(artifact, mode, work):
root, layout = artifact
members = sorted(root.glob("*/*.db"))
manifest = root / "manifest.json"
cold_member_passes = 1 if (layout, mode) == ("v1", "immutable") else 2
evidence = {
"layout": layout,
"mode": mode,
"member_bytes": sum(p.stat().st_size for p in members),
}
for state in ("cold", "warm"):
work.clear()
assert resolver.DatasetResolver().inspect(root) is not None
passes = cold_member_passes if state == "cold" else int(mode == "strong")
for path in members:
assert work[("hashes", path)] == passes
assert work[("bytes", path)] == passes * path.stat().st_size
manifest_passes = 2 if state == "cold" else int(mode == "strong")
assert work[("bytes", manifest)] == manifest_passes * manifest.stat().st_size
assert work["validators"] == int(state == "cold")
evidence[state] = {
"hash_read_bytes": sum(
v for k, v in work.items() if isinstance(k, tuple) and k[0] == "bytes"
),
"validators": work["validators"],
}
print(json.dumps(evidence, sort_keys=True))


@pytest.mark.parametrize("target", ["member", "manifest"])
def test_atomic_replacement_revalidates_even_identical_bytes(artifact, mode, work, target):
root, _ = artifact
resolver.DatasetResolver().inspect(root)
path = root / "manifest.json" if target == "manifest" else next(root.glob("*/*.db"))
replacement = path.with_suffix(".replacement")
replacement.write_bytes(path.read_bytes())
replacement.replace(path)
work.clear()
resolver.DatasetResolver().inspect(root)
assert work["validators"] == 1
assert work[("bytes", path)] >= path.stat().st_size


def test_hidden_change_tokens_document_immutable_trust_boundary(artifact, mode, monkeypatch):
root, _ = artifact
first = resolver.DatasetResolver().inspect(root)
member = next(root.glob("*/*.db"))
before = member.stat()
device = member.parent.name.removeprefix("device_")
with closing(sqlite3.connect(member)) as conn, conn:
conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0")
os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns))
assert (member.stat().st_size, member.stat().st_mtime_ns) == (
before.st_size,
before.st_mtime_ns,
)
# Simulate a filesystem concealing ctime too. This violates immutable mode's
# explicit caller contract; that mode must not be described as strong integrity.
monkeypatch.setattr(resolver, "_generation", lambda *args: first.generation)
if mode == "strong":
with pytest.raises(DatabaseSchemaError):
resolver.DatasetResolver().inspect(root)
assert root not in resolver._VALIDATED
else:
assert resolver.DatasetResolver().inspect(root) == first
first.require_unchanged()


@pytest.mark.parametrize("warm", [False, True], ids=["cold", "warm"])
def test_expired_deadline_evicts_and_full_retry_closes_connections(
artifact, mode, work, monkeypatch, warm
):
root, _ = artifact
if warm:
resolver.DatasetResolver().inspect(root)
work.clear()
budget = CancellableBudget()
budget.cancelled = True
with pytest.raises(QueryTimeoutError):
resolver.DatasetResolver().inspect(root, budget=budget)
assert not work and root not in resolver._VALIDATED
connections = []
original = sqlite3.connect

def connect(*args, **kwargs):
conn = original(*args, **kwargs)
connections.append(conn)
return conn

monkeypatch.setattr(sqlite3, "connect", connect)
resolver.DatasetResolver().inspect(root)
assert work["validators"] == 1 and connections
for conn in connections:
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
conn.execute("SELECT 1")


def test_owned_query_cache_close_and_service_reuse(artifact, mode, monkeypatch):
root, layout = artifact
connections = []
original = sqlite3.connect

def connect(*args, **kwargs):
conn = original(*args, **kwargs)
connections.append(conn)
return conn

monkeypatch.setattr(sqlite3, "connect", connect)
service = QueryService()
try:
for _ in range(2):
result = service.execute_query(
"event", {"id": 0 if layout == "v1" else 7}, db_path=root
)
assert len(result.rows) == 1
service.close()
for conn in connections:
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
conn.execute("SELECT 1")
finally:
service.close()


def test_new_process_is_cold_even_after_parent_admission(artifact, mode):
root, _ = artifact
resolver.DatasetResolver().inspect(root)
code = """
import sys
from pt_snap_cli.core import dataset_resolver as r
assert not r._VALIDATED
calls = []
for name in ('validate_dataset', 'validate_native_dataset'):
original = getattr(r, name)
def tracked(*args, _original=original, **kwargs):
calls.append(1)
return _original(*args, **kwargs)
setattr(r, name, tracked)
assert r.DatasetResolver().inspect(sys.argv[1]) is not None
assert calls == [1]
"""
result = subprocess.run(
[sys.executable, "-c", code, str(root)], capture_output=True, text=True, check=False
)
assert result.returncode == 0, result.stderr


def test_cancellation_inside_validation_closes_and_cannot_seed_retry(
artifact, mode, work, monkeypatch
):
root, _ = artifact
budget = CancellableBudget()
connections = []
original = sqlite3.connect

def connect(*args, **kwargs):
conn = original(*args, **kwargs)
conn.set_trace_callback(lambda sql: setattr(budget, "cancelled", True))
connections.append(conn)
return conn

with monkeypatch.context() as patch:
patch.setattr(sqlite3, "connect", connect)
with pytest.raises(QueryTimeoutError):
resolver.DatasetResolver().inspect(root, budget=budget)
assert work["validators"] == 1 and connections
assert root not in resolver._VALIDATED
for conn in connections:
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
conn.execute("SELECT 1")
work.clear()
resolver.DatasetResolver().inspect(root)
assert work["validators"] == 1
66 changes: 66 additions & 0 deletions tests/core/test_dataset_lifecycle_scratch.py
Original file line number Diff line number Diff line change
Expand Up @@ -291,3 +291,69 @@ def connect(path, *args, **kwargs):
for connection in connections:
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
connection.execute("SELECT 1")


@pytest.mark.parametrize("failure", ["rows", "bytes", "timeout", "cancel", "io"])
def test_result_phase_failure_preserves_sources_cleans_storage_and_allows_retry(
tmp_path, monkeypatch, failure
):
"""Exercise failure after construction, while the scratch owner is yielded."""
root = many_owners(tmp_path)
resolved = DatasetResolver().inspect(root)
expected = run_resolved(resolved, "block")
hashes = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob("*.db")}
parent, created = capture_storage(monkeypatch, tmp_path)
original_build = scratch._ScratchSources.build
original_read = scratch._ScratchSources.read
connections = []
ready = set()

def build(database, items, maximum):
original_build(database, items, maximum)
connections.append(database.connection)
ready.add(id(database))
budget = database.sources.budget
if failure == "rows":
budget.max_work_rows = budget.work_rows
elif failure == "bytes":
budget.max_work_bytes = budget.work_bytes
elif failure == "timeout":
budget.timeout_s = 1
budget.started = time.monotonic() - 2

def read(database, sql, values=None):
if id(database) in ready:
if failure == "cancel":
raise KeyboardInterrupt()
if failure == "io":
raise sqlite3.OperationalError("injected result read I/O failure")
return original_read(database, sql, values)

with monkeypatch.context() as fault:
fault.setattr(scratch._ScratchSources, "build", build)
fault.setattr(scratch._ScratchSources, "read", read)
error, message = {
"rows": (QueryExecutionError, "work budget exceeded"),
"bytes": (QueryExecutionError, "work budget exceeded"),
"timeout": (QueryTimeoutError, "timed out"),
"cancel": (KeyboardInterrupt, "^$"),
"io": (QueryExecutionError, "result read I/O failure"),
}[failure]
with pytest.raises(error, match=message):
run_resolved(resolved, "block")

assert len(connections) == 1
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
connections[0].execute("SELECT 1")
assert created and not list(parent.iterdir())
assert all(not path.exists() for path in created)
assert {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in hashes} == hashes
assert not any(root.rglob("*-journal"))
assert not any(root.rglob("*-wal"))
assert not any(root.rglob("*-shm"))

retried = run_resolved(resolved, "block")
assert retried.rows == expected.rows
assert (retried.total, retried.has_more) == (expected.total, expected.has_more)
assert len(created) == 2 and not list(parent.iterdir())
assert {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in hashes} == hashes
Loading