From c5ed2a8dff80318e3bd4ae267a48eb9a6909ae0b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:12:00 +0800 Subject: [PATCH 1/3] fix: narrow dataset trace projections by template (#220) --- src/pt_snap_cli/core/dataset_global.py | 22 ++++++++--- tests/core/test_dataset_projections.py | 54 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/core/test_dataset_projections.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index cbc73f6..0990b3d 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -68,11 +68,20 @@ def _scope( } -def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord]): +def _traces( + sources: DatasetSourceResolver, + low: int, + high: int, + items: list[SliceRecord], + columns: tuple[str, ...], +): + # Each built-in supplies only the fields it actually consumes. In particular, + # inline stack text must not spend the byte budget of a counter-only query. + projection = ", ".join(f'"{column}"' for column in columns) for item in items: rows = sources.read( sources.dataset.root / item.file, - f'SELECT * FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', [low, high], ) yield item, rows @@ -81,7 +90,7 @@ def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[Sli def _peaks(sources, low, high, items, template, scope): chosen: dict[str, dict[str, object]] = {} count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items): + for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): for row in rows: sources.budget.remaining() event = cast(int, row["id"]) @@ -149,7 +158,10 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items): + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + for _item, batch in _traces(sources, low, high, items, columns): for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -287,7 +299,7 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items): + for _item, batch in _traces(sources, low, high, items, ("id",)): refs = sources.events(cast(int, r["id"]) for r in batch) for source in refs.values(): sources.budget.remaining() diff --git a/tests/core/test_dataset_projections.py b/tests/core/test_dataset_projections.py new file mode 100644 index 0000000..6811d89 --- /dev/null +++ b/tests/core/test_dataset_projections.py @@ -0,0 +1,54 @@ +"""Per-template scan projections must not charge unrelated inline stack text.""" + +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +@pytest.mark.parametrize("template", ["memory_peak", "allocator_gap", "allocation"]) +def test_counter_query_budget_is_independent_of_inline_stack_length(tmp_path, template): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + observations = [] + for stack in ("x" * 16, "x" * 8192, "x" * 32768, "栈🙂" * 8192): + for path in root.glob("device_0/*.db"): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=?", (stack,)) + service = QueryService() + try: + budget = QueryBudget(None, time.monotonic(), max_work_bytes=8192) + result = service.execute_query(template, db_path=root, _budget=budget) + observations.append((result.rows, budget.work_rows, budget.work_bytes)) + finally: + service.close() + assert all(item == observations[0] for item in observations) + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_event_projection_preserves_callstack_sort_and_paging(tmp_path, direction): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + stacks = ["z" * 8192, None, "栈🙂" * 8192, "a" * 16384, "", "a" * 16384] + for index, stack in enumerate(stacks): + path = root / f"device_0/slice_{index // 2:05d}.db" + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", (stack, index)) + service = QueryService() + try: + params = {"order_by": "callstack", "order_dir": direction, "offset": 1, "limit": 3} + result = service.execute_query("event", params, db_path=root, exact_total=True) + expected = sorted( + range(len(stacks)), + key=lambda event: (stacks[event] is not None, stacks[event], event), + reverse=direction == "DESC", + )[1:4] + assert [row["id"] for row in result.rows] == expected + assert [row["callstack"] for row in result.rows] == [stacks[event] for event in expected] + assert result.total == 6 and result.has_more and result.truncated + assert all(len(row) == 9 for row in result.rows) + finally: + service.close() From 32639e130413e10e7016a024be828c2e0ab9d296 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:27:08 +0800 Subject: [PATCH 2/3] fix: make dataset validation cancellable and cache reuse explicit (#219) --- docs/en/sharded-snapshotdb.md | 44 +- docs/zh/sharded-snapshotdb.md | 30 +- src/pt_snap_cli/completion.py | 6 +- src/pt_snap_cli/core/dataset_contract.py | 49 ++- src/pt_snap_cli/core/dataset_files.py | 6 +- src/pt_snap_cli/core/dataset_resolver.py | 243 +++++++++-- .../core/native_dataset_contract.py | 125 ++++-- src/pt_snap_cli/core/query_service.py | 2 +- src/pt_snap_cli/core/validation_budget.py | 79 ++++ tests/core/test_dataset_validation_cache.py | 409 ++++++++++++++++++ tests/test_dataset_focus.py | 4 +- 11 files changed, 896 insertions(+), 101 deletions(-) create mode 100644 src/pt_snap_cli/core/validation_budget.py create mode 100644 tests/core/test_dataset_validation_cache.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 3948d19..53f7ae4 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -242,12 +242,44 @@ full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. -It revalidates the finalized P0 contract on each call and hashes the manifest and -members; no unbounded manifest/connection cache is retained. Hashing uses bounded -chunks and is proportional to artifact size, not a performance claim. Context LRU -size defaults to four; dataset generation changes invalidate contexts on next -lookup even when size/mtime are unchanged. Analyzer-owned caches close on exit; -injected caches remain caller-owned. Validation opens/closes slices sequentially. +By default **every independent inspection** fully validates the finalized contract +and hashes the manifest/members. File metadata alone is not content-integrity proof: +unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. +Pinned report-operation guards also rehash content by default, without repeating +row validation. + +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most +16 validated generations, without open connections. Set it only when you control +finalized, immutable publication and trust your filesystem's kernel change-time +semantics. **Do not enable it for files that may be modified in place**, live writers, +unknown/coarse/network filesystems, or filesystem snapshots that may be rolled back. +This is a caller responsibility contract, not a filesystem capability detected by +the tool. Unset it to retain strong validation; unknown values also use strong mode. +Windows and platforms without a usable POSIX change token remain in strong mode. + +With that explicit opt-in, new resolver/service instances can reuse validated +results. Every hit and pinned-operation guard still rechecks canonical paths, +sidecars, native membership and each file's device, inode, size, mtime, **ctime**, +mode and link count. These conservative checks catch ordinary replacements and +in-place edits with restored mtime, but do not replace the immutable-file contract. +For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: +work is O(shards), independent of event/block counts. Default inspections and cache +misses remain proportional to content. Explicit `validate_dataset` / +`validate_native_dataset` calls always validate fully. Native admission verifies +hashes before and after validation and reuses the verified hashes for its fingerprint, +avoiding a third member-hash pass. + +The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, +Python validation loops check every 256 rows, and SQLite validation (including +`quick_check`) uses an interruptible progress handler, cleared on every exit. +Cancellation raises `QUERY_TIMEOUT`; no partial validation is cached. Device tab +completion reads only a size-bounded manifest (up to 1 MiB), never member SQLite +contents or hashes. Its candidates are hints, not proof the dataset is valid. + +Context LRU size defaults to four; dataset content-generation changes invalidate +contexts on next lookup even when size/mtime are unchanged. Analyzer-owned caches +close on exit; injected caches remain caller-owned. Validation opens/closes slices +sequentially. Building, missing/inconsistent slices, unknown base versions and unsafe paths are rejected before focus writes or query execution. Before **any** SQLite open, diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 137d3fd..c2ee244 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -198,10 +198,32 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 -每次调用重新校验最终化 P0 合同并对 manifest/所有成员取内容哈希,不保存无界 manifest/ -连接缓存。哈希按有界块读取,工作量与产物大小成正比,不宣称性能收益。Context LRU 默认 -最多四个;数据集 generation 变化在下次 lookup 使旧 Context 失效,即使 size/mtime 相同。 -analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有;校验逐片打开并关闭连接。 +默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 +本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 +已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 + +`PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, +不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 +**可能被原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照 +不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, +未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 + +显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 +规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode +和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 +合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), +不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` +/ `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 +复用已验证摘要,不再做第三遍成员哈希。 + +查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite +校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 +取消保留 `QUERY_TIMEOUT`,不会缓存不完整校验。设备 tab 补全只读取有界 manifest +(最多 1 MiB),不读取成员 SQLite 内容或整文件哈希;候选仅为提示,不证明数据集有效。 + +Context LRU 默认最多四个;数据集内容 generation 变化在下次 lookup 使旧 Context 失效, +即使 size/mtime 相同。analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有; +校验逐片打开并关闭连接。 building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写入或查询前拒绝。 **任何** SQLite 打开之前,裸 P0 validator 与 resolver 均先检查**全部**成员/父目录别名及 diff --git a/src/pt_snap_cli/completion.py b/src/pt_snap_cli/completion.py index 797dfff..749104c 100644 --- a/src/pt_snap_cli/completion.py +++ b/src/pt_snap_cli/completion.py @@ -77,9 +77,9 @@ def complete_device_ids() -> list[str]: try: from pt_snap_cli.core.dataset_resolver import DatasetResolver - dataset = DatasetResolver().inspect(db_path) - if dataset is not None: - return [str(device) for device in dataset.device_ids] + devices = DatasetResolver().completion_device_ids(db_path) + if devices is not None: + return [str(device) for device in devices] uri = f"file:{db_path}?mode=ro" with closing(sqlite3.connect(uri, uri=True)) as conn: cursor = conn.cursor() diff --git a/src/pt_snap_cli/core/dataset_contract.py b/src/pt_snap_cli/core/dataset_contract.py index 9d8be4c..a8b03fd 100644 --- a/src/pt_snap_cli/core/dataset_contract.py +++ b/src/pt_snap_cli/core/dataset_contract.py @@ -18,6 +18,13 @@ from .dataset_files import require_readonly_member from .errors import DatabaseSchemaError +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) MANIFEST_SCHEMA_VERSION = 1 MSINSIGHT_REVISION = "101f65b877a267ffd5f66ea3834706057ba243e5" @@ -118,12 +125,15 @@ def require_complete(self) -> None: ) -def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetManifest: +def parse_manifest( + value: object, *, require_complete: bool = True, budget: ValidationBudget | None = None +) -> DatasetManifest: """Validate JSON-shaped data; inspection may explicitly accept building. Unknown extension declarations are ignored, never promoted to capabilities. This does not check files; use validate_dataset for finalized artifacts. """ + check_budget(budget) data = _object(value, "manifest") version = _integer(data.get("schemaVersion"), "schemaVersion", maximum=2**31 - 1) _need( @@ -137,7 +147,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa devices_data = _object(data.get("devices"), "devices") _need(bool(devices_data), "devices", "empty", "at least one event-bearing device required") devices: list[DeviceRecord] = [] - for key, value in devices_data.items(): + for key, value in checked_rows(devices_data.items(), budget): location = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -153,7 +163,9 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) ready = tuple( _integer(item, location + ".readySlices", maximum=size - 1) - for item in _array(device.get("readySlices"), location + ".readySlices") + for item in checked_rows( + _array(device.get("readySlices"), location + ".readySlices"), budget + ) ) _need( len(set(ready)) == len(ready), @@ -170,7 +182,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) slices: list[SliceRecord] = [] expected_start = 0 - for index, raw in enumerate(records): + for index, raw in enumerate(checked_rows(records, budget)): loc = f"{location}.slices[{index}]" item = _object(raw, loc) number = _integer(item.get("index"), loc + ".index", maximum=2**31 - 1) @@ -357,6 +369,7 @@ def _validate_slice( device: DeviceRecord, item: SliceRecord, identities: dict[tuple[int, int], tuple[object, ...]], + budget: ValidationBudget | None = None, ) -> int: loc = item.file trace, block = f"trace_entry_{device.device_id}", f"block_{device.device_id}" @@ -415,7 +428,7 @@ def _validate_slice( ).fetchone() _need(bad is None, loc, "event_values", "invalid scalar type or negative byte metric") boundary_count = conn.execute(f'SELECT COUNT(*) FROM "{trace}" WHERE id<0').fetchone()[0] - for row in conn.execute(f'SELECT * FROM "{block}"'): + for row in checked_rows(conn.execute(f'SELECT * FROM "{block}"'), budget): block_id, address, size, requested, state, alloc, free = row for name, value in zip( ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId"), @@ -455,12 +468,15 @@ def _validate_slice( return cast(int, boundary_count) -def validate_dataset(directory: Path | str) -> DatasetValidation: +def validate_dataset( + directory: Path | str, *, budget: ValidationBudget | None = None +) -> DatasetValidation: """Read a complete compatibility-v1 dataset without migration or pickle. Checks are validation, not a filesystem sandbox or concurrent publication lock. Missing pt_snap_metadata/extension tables are valid external base artifacts. """ + check_budget(budget) root = Path(directory).absolute() _need( root.resolve() == root and not root.is_symlink(), @@ -472,11 +488,11 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: path = _safe_file(root, "manifest.json") with path.open(encoding="utf-8") as source: data: object = json.load(source, object_pairs_hook=_unique_object) - manifest = parse_manifest(data) + manifest = parse_manifest(data, budget=budget) # Preflight ALL members before the first SQLite open, also for callers # using this bare P0 validator rather than DatasetResolver. for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): member = _safe_file(root, item.file) try: require_readonly_member(root, member, immutable=True) @@ -485,14 +501,21 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: identities: dict[tuple[int, int], tuple[object, ...]] = {} boundaries = 0 for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): db_path = _safe_file(root, item.file) try: - with closing( - sqlite3.connect(db_path.as_uri() + "?mode=ro&immutable=1", uri=True) - ) as conn: + with ( + closing( + sqlite3.connect( + db_path.as_uri() + "?mode=ro&immutable=1", + uri=True, + timeout=sqlite_timeout(budget), + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") - boundaries += _validate_slice(conn, device, item, identities) + boundaries += _validate_slice(conn, device, item, identities, budget) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc return DatasetValidation( diff --git a/src/pt_snap_cli/core/dataset_files.py b/src/pt_snap_cli/core/dataset_files.py index e32a6d6..23f94b6 100644 --- a/src/pt_snap_cli/core/dataset_files.py +++ b/src/pt_snap_cli/core/dataset_files.py @@ -6,6 +6,7 @@ from pathlib import Path from .errors import DatabaseSchemaError +from .validation_budget import ValidationBudget, check_budget def require_readonly_member(root: Path, member: Path, *, immutable: bool = False) -> None: @@ -27,9 +28,12 @@ def require_readonly_member(root: Path, member: Path, *, immutable: bool = False raise DatabaseSchemaError(f"Dataset member uses persistent WAL mode: {member}") -def hash_file(path: Path) -> str: +def hash_file(path: Path, *, budget: ValidationBudget | None = None) -> str: + check_budget(budget) digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): + check_budget(budget) digest.update(chunk) + check_budget(budget) return digest.hexdigest() diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index 94126ec..b1d9da3 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -7,13 +7,18 @@ from __future__ import annotations +import copy import hashlib import json +import os +from collections import OrderedDict from dataclasses import dataclass from pathlib import Path +from threading import RLock from pt_snap_cli.core.dataset_contract import ( DatasetContractError, + DatasetManifest, DatasetValidation, DeviceRecord, QueryScope, @@ -28,10 +33,16 @@ from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, NativeManifest, + NativeSlice, parse_native_manifest, + require_native_members, validate_native_dataset, validate_native_scope, ) +from pt_snap_cli.core.validation_budget import ValidationBudget, check_budget, checked_rows + +_FileStamp = tuple[int, int, int, int, int, int, int] +_Generation = tuple[tuple[str, _FileStamp], ...] @dataclass(frozen=True) @@ -40,6 +51,25 @@ class ResolvedDataset: validation: DatasetValidation fingerprint: str ordered_frames_version: int | None = None + generation: _Generation | None = None + + def require_unchanged(self, budget: ValidationBudget | None = None) -> None: + """Guard a reused report/query resolution without repeating full validation.""" + if _root(self.root) != self.root: + raise DatabaseSchemaError("Dataset path changed during operation; retry.") + try: + current = _generation(self.root, self.validation.manifest, budget) + if self.generation is None or current != self.generation: + raise DatabaseSchemaError("Dataset members changed during operation; retry.") + if not _cacheable(current): + if ( + _content_fingerprint(self.root, self.validation.manifest, budget) + != self.fingerprint + or _generation(self.root, self.validation.manifest, budget) != current + ): + raise DatabaseSchemaError("Dataset content changed during operation; retry.") + except (DatasetContractError, OSError) as exc: + raise DatabaseSchemaError(f"Dataset changed during operation: {exc}") from exc @property def device_ids(self) -> list[int]: @@ -116,62 +146,207 @@ def to_dict(self) -> dict[str, object]: } +# This is deliberately process-local, bounded, and contains no open connections. +# Cross-call reuse is opt-in: unknown/coarse filesystems cannot supply a +# trustworthy change token merely because Python exposes st_ctime_ns. The caller +# must attest immutable publication and suitable change-time semantics explicitly. +_CACHE_LIMIT = 16 +_VALIDATED: OrderedDict[Path, tuple[_Generation, ResolvedDataset]] = OrderedDict() +_CACHE_LOCK = RLock() + + +def _stamp(path: Path) -> _FileStamp: + stat = path.stat() + return ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + stat.st_mode, + stat.st_nlink, + ) + + +def _root(path: Path | str) -> Path | None: + candidate = Path(path).expanduser().absolute() + if not candidate.is_dir() and candidate.name != "manifest.json": + return None + root = candidate if candidate.is_dir() else candidate.parent + if candidate.is_symlink() or root.resolve() != root: + raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") + if (root / "manifest.json").is_symlink(): + raise DatabaseSchemaError("Dataset manifest must not be a symlink.") + if not (root / "manifest.json").is_file(): + raise DatabaseSchemaError("Dataset manifest.json must be a regular file.") + return root + + +def _generation( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> _Generation: + """Always recheck aliases, transport, and membership before trusting a hit.""" + check_budget(budget) + if isinstance(manifest, NativeManifest): + require_native_members(root, manifest, budget=budget) + stamps = [(".", _stamp(root)), ("manifest.json", _stamp(root / "manifest.json"))] + for device in checked_rows(manifest.devices, budget): + directory = f"device_{device.device_id}" + if (root / directory).is_symlink(): + raise DatabaseSchemaError("Dataset device directory must not be a symlink.") + stamps.append((directory, _stamp(root / directory))) + for item in checked_rows(device.slices, budget): + member = root / item.file + _require_readonly_member( + root, member, immutable=not isinstance(manifest, NativeManifest) + ) + stamps.append((item.file, _stamp(member))) + check_budget(budget) + return tuple(stamps) + + +def _cacheable(generation: _Generation) -> bool: + # This is a caller contract, not an automatic inference from OS or filesystem + # metadata. Windows ctime historically means creation time, so stay strong. + return ( + os.environ.get("PT_SNAP_DATASET_CACHE") == "immutable" + and os.name == "posix" + and all(stamp[4] > 0 for _, stamp in generation) + ) + + +def _content_fingerprint( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> str: + """Strong verification for pinned operations without repeating row validation.""" + digest = hashlib.sha256() + digest.update(_hash_file(root / "manifest.json", budget=budget).encode("ascii")) + for device in checked_rows(manifest.devices, budget): + for item in checked_rows(device.slices, budget): + digest.update(item.file.encode("utf-8")) + digest.update(_hash_file(root / item.file, budget=budget).encode("ascii")) + return digest.hexdigest() + + class DatasetResolver: - """Stateless resolution: revalidate every call, retain no connections/manifest cache. + """Resolve and validate finalized datasets, reusing unchanged generations. - Content hashes (not cacheHash or size/mtime alone) form the Context generation. - Work is proportional to artifact size; no concurrent-publication lock is claimed. + By default every call validates and hashes content. Explicitly setting + PT_SNAP_DATASET_CACHE=immutable enables bounded process-local reuse under + the caller's immutable-publication/change-time contract. Even then paths, + members, sidecars and kernel change times are checked on every call. + Explicit validators always perform full validation. """ - def inspect(self, path: Path | str) -> ResolvedDataset | None: - candidate = Path(path).expanduser().absolute() - if not candidate.is_dir() and candidate.name != "manifest.json": + def inspect( + self, path: Path | str, *, budget: ValidationBudget | None = None + ) -> ResolvedDataset | None: + check_budget(budget) + root = _root(path) + if root is None: return None - root = candidate if candidate.is_dir() else candidate.parent - if candidate.is_symlink() or root.resolve() != root: - raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") try: + with _CACHE_LOCK: + cached = _VALIDATED.get(root) + if cached is not None: + previous, resolved = cached + manifest_unchanged = ( + _stamp(root / "manifest.json") == dict(previous)["manifest.json"] + ) + generation = ( + _generation(root, resolved.validation.manifest, budget) + if manifest_unchanged + else None + ) + if generation is not None and _cacheable(generation) and generation == previous: + with _CACHE_LOCK: + if root in _VALIDATED: + _VALIDATED.move_to_end(root) + result = copy.deepcopy(resolved) + check_budget(budget) + return result + with _CACHE_LOCK: + _VALIDATED.pop(root, None) manifest_path = root / "manifest.json" - if manifest_path.is_symlink(): - raise DatabaseSchemaError("Dataset manifest must not be a symlink.") - before = _hash_file(manifest_path) + manifest_stamp = _stamp(manifest_path) + before = _hash_file(manifest_path, budget=budget) with manifest_path.open(encoding="utf-8") as source: raw: object = json.load(source, object_pairs_hook=_unique_object) + check_budget(budget) planned = ( - parse_native_manifest(raw) + parse_native_manifest(raw, budget=budget) if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT - else parse_manifest(raw) + else parse_manifest(raw, budget=budget) ) - # mode=ro can create WAL/SHM files even after the last writer has - # checkpointed and removed them. Check every member before the first - # SQLite open, without following aliases or repairing the artifact. - for device in planned.devices: - for item in device.slices: - _require_readonly_member( - root, root / item.file, immutable=not isinstance(planned, NativeManifest) - ) + # Preflight every member before the first SQLite open. Capturing + # both ends also refuses publication during validation or hashing. + generation = _generation(root, planned, budget) + if manifest_stamp != _stamp(manifest_path): + raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") validation = ( - validate_native_dataset(root, planned) + validate_native_dataset(root, planned, budget=budget) if isinstance(planned, NativeManifest) - else validate_dataset(root) + else validate_dataset(root, budget=budget) ) digest = hashlib.sha256() digest.update(before.encode("ascii")) - for device in validation.manifest.devices: - for item in device.slices: - member = root / item.file - _require_readonly_member( - root, member, immutable=not isinstance(planned, NativeManifest) - ) + for device in checked_rows(validation.manifest.devices, budget): + for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) - digest.update(_hash_file(member).encode("ascii")) - if before != _hash_file(manifest_path): + # Native validation already verifies this digest before and + # after SQLite reads; do not hash every shard a third time. + content_hash = ( + item.sha256 + if isinstance(item, NativeSlice) + else _hash_file(root / item.file, budget=budget) + ) + digest.update(content_hash.encode("ascii")) + if before != _hash_file(manifest_path, budget=budget): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + if generation != _generation(root, planned, budget): + raise DatabaseSchemaError("Dataset members changed during inspection; retry.") extensions = raw.get("extensions") if isinstance(raw, dict) else None frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None version = frames.get("version") if isinstance(frames, dict) else None - return ResolvedDataset( - root, validation, digest.hexdigest(), version if type(version) is int else None + result = ResolvedDataset( + root, + validation, + digest.hexdigest(), + version if type(version) is int else None, + generation, ) + if _cacheable(generation): + with _CACHE_LOCK: + _VALIDATED[root] = (generation, copy.deepcopy(result)) + _VALIDATED.move_to_end(root) + while len(_VALIDATED) > _CACHE_LIMIT: + _VALIDATED.popitem(last=False) + return result except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: + with _CACHE_LOCK: + _VALIDATED.pop(root, None) raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc + + def completion_device_ids(self, path: Path | str) -> list[int] | None: + """Bounded manifest-only hints, never evidence of dataset validity. + + Completion must not open or hash shard contents. Large or invalid + manifests produce no hints; a real query still performs strong admission. + """ + root = _root(path) + if root is None: + return None + try: + with (root / "manifest.json").open("rb") as source: + data = source.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + return [] + raw: object = json.loads(data, object_pairs_hook=_unique_object) + manifest = ( + parse_native_manifest(raw) + if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT + else parse_manifest(raw) + ) + return sorted(device.device_id for device in manifest.devices) + except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError): + return [] diff --git a/src/pt_snap_cli/core/native_dataset_contract.py b/src/pt_snap_cli/core/native_dataset_contract.py index 729269f..ffbd8bc 100644 --- a/src/pt_snap_cli/core/native_dataset_contract.py +++ b/src/pt_snap_cli/core/native_dataset_contract.py @@ -40,6 +40,13 @@ from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata from .sharded_replay_service import ShardedReplayResult +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) NATIVE_FORMAT = "pt-snap-native-v2" NATIVE_MANIFEST_VERSION = 1 @@ -84,8 +91,11 @@ def _digest(value: object, loc: str) -> str: return result -def parse_native_manifest(value: object) -> NativeManifest: +def parse_native_manifest( + value: object, *, budget: ValidationBudget | None = None +) -> NativeManifest: + check_budget(budget) data = _object(value, "manifest") _need(data.get("format") == NATIVE_FORMAT, "format", "format", "unsupported native format") version = _integer(data.get("schemaVersion"), "schemaVersion", minimum=1) @@ -155,11 +165,11 @@ def parse_native_manifest(value: object) -> NativeManifest: ) omitted = tuple( _integer(v, "omittedDevices", maximum=2**31 - 1) - for v in _array(data.get("omittedDevices"), "omittedDevices") + for v in checked_rows(_array(data.get("omittedDevices"), "omittedDevices"), budget) ) _need(len(set(omitted)) == len(omitted), "omittedDevices", "duplicate", "duplicate device") devices: list[DeviceRecord] = [] - for key, raw in _object(data.get("devices"), "devices").items(): + for key, raw in checked_rows(_object(data.get("devices"), "devices").items(), budget): loc = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -181,7 +191,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) ready = tuple( _integer(v, loc + ".readySlices") - for v in _array(device.get("readySlices"), loc + ".readySlices") + for v in checked_rows(_array(device.get("readySlices"), loc + ".readySlices"), budget) ) _need( ready == tuple(range(len(records))), @@ -191,7 +201,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) slices: list[SliceRecord] = [] position, last_id = 0, -1 - for index, raw_slice in enumerate(records): + for index, raw_slice in enumerate(checked_rows(records, budget)): item = _object(raw_slice, loc) _need( type(item.get("index")) is int @@ -257,7 +267,9 @@ def read_native_manifest(root: Path) -> NativeManifest: raise DatasetContractError("manifest.json", "read", str(exc)) from exc -def require_native_members(root: Path, manifest: NativeManifest) -> None: +def require_native_members( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> None: """Ownership recognition before any replacement: never adopt unknown files/dirs.""" _need( root.resolve() == root and not root.is_symlink(), @@ -267,7 +279,7 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: ) expected = {"manifest.json", *(f"device_{d.device_id}" for d in manifest.devices)} _need( - {p.name for p in root.iterdir()} == expected, + {p.name for p in checked_rows(root.iterdir(), budget)} == expected, "dataset", "members", "unexpected/missing dataset member", @@ -281,18 +293,22 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: "unsafe device directory", ) _need( - {p.name for p in directory.iterdir()} == {Path(s.file).name for s in device.slices}, + {p.name for p in checked_rows(directory.iterdir(), budget)} + == {Path(s.file).name for s in device.slices}, str(directory), "members", "unexpected/missing shard", ) - for item in device.slices: + for item in checked_rows(device.slices, budget): _safe_file(root, item.file) -def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetValidation: +def validate_native_dataset( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> DatasetValidation: - require_native_members(root, manifest) + check_budget(budget) + require_native_members(root, manifest, budget=budget) expected_identity = cache_identity( manifest.metadata.source_sha256, manifest.metadata.requested_device, @@ -306,21 +322,32 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali ) # Check EVERY member before the FIRST SQLite open. No WAL/sidecars/repair. for device in manifest.devices: - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file _require_readonly_member(root, path) - _need(_hash_file(path) == item.sha256, item.file, "hash", "member content changed") + _need( + _hash_file(path, budget=budget) == item.sha256, + item.file, + "hash", + "member content changed", + ) boundaries = 0 for device in manifest.devices: ids: set[int] = set() lifetimes: dict[int, tuple[object, ...]] = {} - references: list[tuple[int, int]] = [] - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file try: - with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") _need( conn.execute("PRAGMA quick_check").fetchall() == [("ok",)], @@ -388,19 +415,28 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "dictionary", "invalid native dictionary", ) - real = [ - r[0] - for r in conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0 ORDER BY id') - ] + count, low, high = conn.execute( + f'SELECT COUNT(*), MIN(id), MAX(id) FROM "{trace}" WHERE id>=0' + ).fetchone() _need( - len(real) == item.end_position - item.start_position + 1 - and (real[0], real[-1]) == (item.start_event_id, item.end_event_id) - and not ids.intersection(real), + (count, low, high) + == ( + item.end_position - item.start_position + 1, + item.start_event_id, + item.end_event_id, + ), item.file, "event_identity", "real IDs/count differ from positions/ranges", ) - ids.update(real) + # Manifest intervals are disjoint; avoid a per-shard list + # and intersection copy in addition to the device ID set. + ids.update( + r[0] + for r in checked_rows( + conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0'), budget + ) + ) numeric = " OR ".join( f"typeof({name})!='integer'" for name, _ in TRACE_COLUMNS[:-1] ) @@ -425,17 +461,25 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali boundaries += conn.execute( f'SELECT COUNT(*) FROM "{trace}" WHERE id<0' ).fetchone()[0] - stacks = { - r[0]: r[1:] for r in conn.execute("SELECT * FROM pt_snap_block_reference") - } - blocks = conn.execute(f'SELECT * FROM "{block}"').fetchall() + mismatch = conn.execute( + f'SELECT b.id FROM "{block}" b WHERE NOT EXISTS ' + "(SELECT 1 FROM pt_snap_block_reference r WHERE r.blockId=b.id) " + "UNION ALL SELECT r.blockId FROM pt_snap_block_reference r " + f'WHERE NOT EXISTS (SELECT 1 FROM "{block}" b WHERE b.id=r.blockId) ' + "LIMIT 1" + ).fetchone() _need( - set(stacks) == {r[0] for r in blocks}, + mismatch is None, item.file, "extension", "reference/block membership differs", ) - for row in blocks: + rows = conn.execute( + f"SELECT b.*, r.stream, r.allocCallstack, r.freeCallstack " + f'FROM "{block}" b JOIN pt_snap_block_reference r ON r.blockId=b.id' + ) + for joined in checked_rows(rows, budget): + row, ref = joined[:7], joined[7:] bid, address, size, requested, state, alloc, free = row _need( all(type(v) is int for v in row) @@ -448,7 +492,6 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "block_identity", "unfinalized/invalid block", ) - ref = stacks[bid] _need( type(ref[0]) is int and all(v is None or isinstance(v, str) for v in ref[1:]), @@ -464,13 +507,17 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "conflicting finalized identity/reference", ) lifetimes[bid] = signature - references.append((alloc, free)) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc try: - with closing( - sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) - ) as metadata_conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as metadata_conn, + validation_progress(metadata_conn, budget), + ): metadata_conn.row_factory = sqlite3.Row _need( metadata_conn.execute( @@ -501,14 +548,18 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali raise DatasetContractError(item.file, "metadata", str(exc)) from exc _require_readonly_member(root, path) _need( - _hash_file(path) == item.sha256, + _hash_file(path, budget=budget) == item.sha256, item.file, "hash", "member changed during validation", ) _need( len(ids) == device.event_count - and all(v == -1 or v in ids for pair in references for v in pair), + and all( + v == -1 or v in ids + for lifetime in checked_rows(lifetimes.values(), budget) + for v in lifetime[3:5] + ), "devices", "event_identity", "count/lifetime references not in original device scope", diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 138e342..c02d6ed 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -233,7 +233,7 @@ def execute_query( if resolved.db_path is None: raise FocusNotConfiguredError("No database path specified and no database configured.") - dataset = DatasetResolver().inspect(resolved.db_path) + dataset = DatasetResolver().inspect(resolved.db_path, budget=budget) budget.remaining() scope: dict[str, object] | None = None query_params = params or {} diff --git a/src/pt_snap_cli/core/validation_budget.py b/src/pt_snap_cli/core/validation_budget.py new file mode 100644 index 0000000..ec60cf0 --- /dev/null +++ b/src/pt_snap_cli/core/validation_budget.py @@ -0,0 +1,79 @@ +"""Cancellation helpers shared by dataset validation and query execution. + +The small protocol avoids a dependency on the query/source resolver. Validators +can also be used at publication time without a query deadline. +""" + +from __future__ import annotations + +import math +import sqlite3 +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from typing import Protocol, TypeVar + +from .errors import QueryTimeoutError + + +class ValidationBudget(Protocol): + def remaining(self) -> float | None: ... + + +def check_budget(budget: ValidationBudget | None) -> None: + if budget is not None: + budget.remaining() + + +def sqlite_timeout(budget: ValidationBudget | None) -> float: + """SQLite's busy wait must not outlive the remaining shared deadline.""" + remaining = budget.remaining() if budget is not None else None + # SQLite truncates to whole milliseconds. Round up so an exhausted busy + # wait is classified as timeout, not a spurious schema/locked error just + # before the deadline (the additional wait is at most one millisecond). + return 5.0 if remaining is None else min(5.0, math.ceil(remaining * 1000) / 1000) + + +_T = TypeVar("_T") + + +def checked_rows(rows: Iterable[_T], budget: ValidationBudget | None) -> Iterator[_T]: + """Bound Python-side work between deadline checks, including cursor iteration.""" + check_budget(budget) + for index, row in enumerate(rows): + if index % 256 == 0: + check_budget(budget) + yield row + check_budget(budget) + + +@contextmanager +def validation_progress( + conn: sqlite3.Connection, budget: ValidationBudget | None +) -> Iterator[None]: + """Interrupt SQLite and preserve QUERY_TIMEOUT rather than a schema error.""" + check_budget(budget) + timed_out: QueryTimeoutError | None = None + + def progress() -> int: + nonlocal timed_out + try: + check_budget(budget) + except QueryTimeoutError as exc: + timed_out = exc + return 1 + return 0 + + if budget is not None: + conn.set_progress_handler(progress, 1000) + try: + try: + yield + except sqlite3.DatabaseError: + if timed_out is not None: + raise timed_out from None + check_budget(budget) + raise + check_budget(budget) + finally: + if budget is not None: + conn.set_progress_handler(None, 0) diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py new file mode 100644 index 0000000..af0287a --- /dev/null +++ b/tests/core/test_dataset_validation_cache.py @@ -0,0 +1,409 @@ +"""Work-count and cancellation regressions for dataset admission (issue #219).""" + +import json +import os +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.completion import complete_device_ids +from pt_snap_cli.core import dataset_contract, dataset_resolver, native_dataset_contract +from pt_snap_cli.core.dataset_contract import validate_dataset +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.validation_budget import validation_progress +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(autouse=True) +def clear_generations(monkeypatch): + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +@pytest.fixture(params=[False, True], ids=["compatibility", "native"]) +def artifact(tmp_path, request): + return native_dataset(tmp_path) if request.param else make_dataset(tmp_path / "dataset") + + +def forbidden(*args, **kwargs): + pytest.fail("warm resolution must not hash or validate shard contents") + + +def test_new_resolvers_reuse_verified_generation_without_hash_or_sql(artifact, monkeypatch): + first = DatasetResolver().inspect(artifact) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(native_dataset_contract, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + second = DatasetResolver().inspect(artifact / "manifest.json") + assert second == first and second is not first + second.require_unchanged() + + +@pytest.mark.parametrize("change", ["touch", "replace", "manifest"]) +def test_metadata_change_always_revalidates(artifact, monkeypatch, change): + first = DatasetResolver().inspect(artifact) + path = artifact / "manifest.json" if change == "manifest" else next(artifact.glob("*/*.db")) + if change == "replace": + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.replace(path) + else: + stat = path.stat() + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000)) + calls = [] + real = dataset_resolver._hash_file + + def tracked(*args, **kwargs): + calls.append(args[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + second = DatasetResolver().inspect(artifact) + assert first.fingerprint == second.fingerprint + assert artifact / "manifest.json" in calls + with pytest.raises(DatabaseSchemaError, match="changed during operation"): + first.require_unchanged() + + +def test_restored_mtime_size_does_not_hide_content_tampering(artifact): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + after = member.stat() + assert (before.st_ino, before.st_size, before.st_mtime_ns) == ( + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + assert before.st_ctime_ns != after.st_ctime_ns + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +@pytest.mark.parametrize("change", ["wal", "journal", "shm", "symlink", "manifest-symlink"]) +def test_cached_generation_still_rejects_sidecars_and_aliases(artifact, tmp_path, change): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + if change.endswith("symlink"): + path = artifact / "manifest.json" if change == "manifest-symlink" else member + saved = tmp_path / "saved" + path.rename(saved) + path.symlink_to(saved) + else: + Path(str(member) + "-" + change).touch() + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +def test_native_hashes_twice_on_admission_not_three_times(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + calls = [] + real = hash_file + + def tracked(path, **kwargs): + calls.append(path) + return real(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + monkeypatch.setattr(native_dataset_contract, "_hash_file", tracked) + DatasetResolver().inspect(root) + for member in root.glob("*/*.db"): + assert calls.count(member) == 2 + assert calls.count(root / "manifest.json") == 2 + + +def test_cached_manifest_cannot_be_mutated_by_a_previous_caller(tmp_path): + root = native_dataset(tmp_path) + first = DatasetResolver().inspect(root) + first.validation.manifest.identity["eventsPerSlice"] = "corrupted by caller" + second = DatasetResolver().inspect(root) + assert second.validation.manifest.identity["eventsPerSlice"] == 2 + + +def test_cache_is_bounded_and_untrusted_change_tokens_disable_reuse(tmp_path, monkeypatch): + monkeypatch.setattr(dataset_resolver, "_CACHE_LIMIT", 2) + for i in range(3): + DatasetResolver().inspect(make_dataset(tmp_path / str(i), devices=(0,), slices=1)) + assert len(dataset_resolver._VALIDATED) == 2 + root = make_dataset(tmp_path / "uncached", devices=(0,), slices=1) + monkeypatch.setattr(dataset_resolver, "_cacheable", lambda generation: False) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_completion_only_reads_bounded_manifest(artifact, monkeypatch): + monkeypatch.setenv("PT_SNAP_DB_PATH", str(artifact)) + expected = sorted(json.loads((artifact / "manifest.json").read_text())["devices"], key=int) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + assert complete_device_ids() == expected + # Completion candidates do not claim content integrity. + next(artifact.glob("*/*.db")).write_bytes(b"invalid member") + assert complete_device_ids() == expected + (artifact / "manifest.json").write_bytes(b" " * (1024 * 1024 + 1)) + assert complete_device_ids() == [] + + +class CancellableBudget: + def __init__(self): + self.cancelled = False + self.calls = 0 + + def remaining(self): + self.calls += 1 + if self.cancelled: + raise QueryTimeoutError("Query timed out after 0.01 seconds") + return 1.0 + + +def test_hash_checks_between_chunks_and_closes_file(tmp_path, monkeypatch): + path = tmp_path / "large" + path.write_bytes(b"x" * (4 * 1024 * 1024)) + budget = CancellableBudget() + original = budget.remaining + opened = [] + real_open = Path.open + + def tracked_open(self, *args, **kwargs): + result = real_open(self, *args, **kwargs) + opened.append(result) + return result + + def remaining(): + budget.cancelled = budget.calls >= 2 + return original() + + monkeypatch.setattr(Path, "open", tracked_open) + monkeypatch.setattr(budget, "remaining", remaining) + with pytest.raises(QueryTimeoutError): + hash_file(path, budget=budget) + assert budget.calls == 3 + assert len(opened) == 1 and opened[0].closed + assert hash_file(path) # Failure did not retain resources. + + +def test_sql_progress_translates_timeout_and_clears_handler(): + budget = CancellableBudget() + with closing(sqlite3.connect(":memory:")) as conn: + with pytest.raises(QueryTimeoutError): + with validation_progress(conn, budget): + budget.cancelled = True + conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<10000) " + "SELECT sum(x) FROM n" + ).fetchone() + # Still cancelled: this would interrupt if the handler leaked. + assert conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) " + "SELECT sum(x) FROM n" + ).fetchone() == (500500,) + + +def large_compatibility(root, rows=1000): + root = make_dataset(root, devices=(0,), slices=1) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,100,8,0,8,8,16,'stack')", + ((i,) for i in range(rows)), + ) + conn.executemany( + "INSERT INTO block_0 VALUES (?,100,8,8,1,?,-1)", ((i, i) for i in range(rows)) + ) + path = root / "manifest.json" + data = json.loads(path.read_text()) + data["eventsPerSlice"] = rows + data["devices"]["0"]["eventCount"] = rows + data["devices"]["0"]["slices"][0]["endEventId"] = rows - 1 + path.write_text(json.dumps(data)) + return root + + +def test_python_validation_loop_cancels_before_finishing_and_closes_connection( + tmp_path, monkeypatch +): + root = large_compatibility(tmp_path / "dataset") + budget = CancellableBudget() + checked = [] + connections = [] + real_integer = dataset_contract._integer + real_connect = sqlite3.connect + + def integer(value, location, *args, **kwargs): + if ".block_0." in location: + checked.append(location) + budget.cancelled = True + return real_integer(value, location, *args, **kwargs) + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + connections.append(conn) + return conn + + monkeypatch.setattr(dataset_contract, "_integer", integer) + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + validate_dataset(root, budget=budget) + assert 0 < len(checked) <= 256 * 7 + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(dataset_contract, "_integer", real_integer) + assert validate_dataset(root).real_event_count == 1000 + + +def test_native_quick_check_can_be_interrupted_and_resolver_recovers(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + budget = CancellableBudget() + connections = [] + real_connect = sqlite3.connect + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + conn.set_trace_callback( + lambda sql: setattr(budget, "cancelled", True) if sql == "PRAGMA quick_check" else None + ) + connections.append(conn) + return conn + + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=budget) + assert root not in dataset_resolver._VALIDATED + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(sqlite3, "connect", real_connect) + assert DatasetResolver().inspect(root) is not None + + +def test_cold_query_passes_shared_deadline_and_cli_keeps_timeout_code(tmp_path, monkeypatch): + from typer.testing import CliRunner + + from pt_snap_cli.cli import app + from pt_snap_cli.core.query_service import QueryService + + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + original = dataset_resolver._hash_file + received = [] + + def expire(path, *, budget=None): + assert budget is not None + received.append(budget) + budget.started -= 20 + return original(path, budget=budget) + + monkeypatch.setattr(dataset_resolver, "_hash_file", expire) + service = QueryService() + try: + with pytest.raises(QueryTimeoutError): + service.execute_query("event", {"id": 0}, db_path=root, timeout_s=10) + finally: + service.close() + result = CliRunner().invoke( + app, ["query", str(root), "--template-use", "event", "--timeout", "10", "--json"] + ) + assert result.exit_code == 1 + assert json.loads(result.stderr)["error"]["code"] == "QUERY_TIMEOUT" + assert len(received) == 2 + assert root not in dataset_resolver._VALIDATED + monkeypatch.setattr(dataset_resolver, "_hash_file", original) + service = QueryService() + try: + assert service.execute_query("event", {"id": 0}, db_path=root).rows[0]["id"] == 0 + finally: + service.close() + + +def test_native_lock_wait_respects_deadline(tmp_path): + import time + + from pt_snap_cli.core.dataset_sources import QueryBudget + + root = native_dataset(tmp_path) + member = sorted(root.glob("*/*.db"))[0] + with closing(sqlite3.connect(member)) as writer: + writer.execute("BEGIN EXCLUSIVE") + start = time.monotonic() + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=QueryBudget(0.02, start)) + assert time.monotonic() - start < 1.0 + writer.rollback() + assert DatasetResolver().inspect(root) is not None + + +@pytest.mark.parametrize("mode", [None, "off", "unknown"]) +def test_default_and_unknown_cache_mode_repeat_strong_validation(tmp_path, monkeypatch, mode): + if mode is None: + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + else: + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + calls = [] + original = dataset_resolver._hash_file + + def tracked(path, **kwargs): + calls.append(path) + return original(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + first = DatasetResolver().inspect(root) + second = DatasetResolver().inspect(root) + assert first.fingerprint == second.fingerprint + assert calls.count(root / "device_0/slice_00000.db") == 2 + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_hashes_even_when_filesystem_tokens_look_unchanged( + tmp_path, monkeypatch +): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + resolved.require_unchanged() + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='changed'") + monkeypatch.setattr(dataset_resolver, "_generation", lambda *args: resolved.generation) + with pytest.raises(DatabaseSchemaError, match="content changed"): + resolved.require_unchanged() + + +def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + DatasetResolver().inspect(root) + assert root in dataset_resolver._VALIDATED + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + with pytest.raises(QueryTimeoutError): + budget = CancellableBudget() + budget.cancelled = True + DatasetResolver().inspect(root, budget=budget) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_normalizes_hash_io_failure(tmp_path, monkeypatch): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + + def fail(*args, **kwargs): + raise PermissionError("member became unreadable") + + monkeypatch.setattr(dataset_resolver, "_hash_file", fail) + with pytest.raises(DatabaseSchemaError, match="member became unreadable"): + resolved.require_unchanged() diff --git a/tests/test_dataset_focus.py b/tests/test_dataset_focus.py index 8b525f4..2c42d67 100644 --- a/tests/test_dataset_focus.py +++ b/tests/test_dataset_focus.py @@ -482,8 +482,8 @@ def test_manifest_change_during_validation_fails(tmp_path, monkeypatch): root = make_dataset(tmp_path / "dataset") original = module.validate_dataset - def validate(path): - result = original(path) + def validate(path, **kwargs): + result = original(path, **kwargs) manifest = root / "manifest.json" data = json.loads(manifest.read_text()) data["cacheHash"] = "replaced" From ef641382eb7ea5441408d3055fa437fbfa7183fc Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:22:06 +0800 Subject: [PATCH 3/3] fix: preserve NULL peak report semantics and update skill workflow --- skills/pt-snap-memory-peak-breakdown/SKILL.md | 24 +++++---- .../pt-snap-memory-peak-breakdown/SKILL.md | 24 +++++---- src/pt_snap_cli/core/report_service.py | 10 ++-- tests/core/test_report_service.py | 52 +++++++++++++++++++ .../test_memory_peak_breakdown_contract.py | 14 +++++ tests/test_peak_report_completeness.py | 33 ++++++++++++ 6 files changed, 136 insertions(+), 21 deletions(-) diff --git a/skills/pt-snap-memory-peak-breakdown/SKILL.md b/skills/pt-snap-memory-peak-breakdown/SKILL.md index 1d7a275..791ec41 100644 --- a/skills/pt-snap-memory-peak-breakdown/SKILL.md +++ b/skills/pt-snap-memory-peak-breakdown/SKILL.md @@ -244,19 +244,25 @@ pt-snap report peak-memory "" --device --metric reserved --include- Keep the JSON in the response. The report composes these product templates: -- `memory_peak` finds the active, allocated, and reserved peak values and event - IDs. -- `allocator_gap` returns each metric's counters and gaps at that same metric's - peak event. +- `allocator_gap` runs once to find the active, allocated, and reserved peak + values and event IDs, plus each metric's same-event counters and gaps. The + report projects its six-field `peak` section from this result, preserving + `memory_peak` semantics without running a separate `memory_peak` query. - `active_memory_callstack_at_event` groups blocks active at the report's selected metric event by allocation callstack. -From the three results, record each metric's own peak value and event ID. Peak -ties resolve to the earliest event ID. Record same-event counters and gaps from +Both steps share one resolved database/device, budget, and dataset generation. +If the selected metric has no peak value or event ID, the report returns empty +attribution without running the attribution query; never invent an event for an +empty range or an all-NULL counter. Other metrics can still have valid peaks. + +From the three metric reports, record each metric's own peak value and event ID. +Peak ties resolve to the earliest event ID. Record same-event counters and gaps from `allocator_gap`; never subtract independently occurring peak values as if they -occurred together. The repeated `peak` and `allocator_gap` sections should agree -across all three reports; the metric-specific `event_id` and `callstack_groups` -are the selected-event breakdown. +occurred together. The repeated `peak` sections should agree across all three +reports. For metrics with a valid peak, the `allocator_gap` sections also agree; +an empty-attribution report has `allocator_gap: null`. The metric-specific +`event_id` and `callstack_groups` are the selected-event breakdown. Read report `has_more`, `truncated`, `total_is_exact`, and `effective_params` before claiming a complete composition. A full dynamic `top_n` window means diff --git a/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md b/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md index 1d7a275..791ec41 100644 --- a/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md +++ b/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md @@ -244,19 +244,25 @@ pt-snap report peak-memory "" --device --metric reserved --include- Keep the JSON in the response. The report composes these product templates: -- `memory_peak` finds the active, allocated, and reserved peak values and event - IDs. -- `allocator_gap` returns each metric's counters and gaps at that same metric's - peak event. +- `allocator_gap` runs once to find the active, allocated, and reserved peak + values and event IDs, plus each metric's same-event counters and gaps. The + report projects its six-field `peak` section from this result, preserving + `memory_peak` semantics without running a separate `memory_peak` query. - `active_memory_callstack_at_event` groups blocks active at the report's selected metric event by allocation callstack. -From the three results, record each metric's own peak value and event ID. Peak -ties resolve to the earliest event ID. Record same-event counters and gaps from +Both steps share one resolved database/device, budget, and dataset generation. +If the selected metric has no peak value or event ID, the report returns empty +attribution without running the attribution query; never invent an event for an +empty range or an all-NULL counter. Other metrics can still have valid peaks. + +From the three metric reports, record each metric's own peak value and event ID. +Peak ties resolve to the earliest event ID. Record same-event counters and gaps from `allocator_gap`; never subtract independently occurring peak values as if they -occurred together. The repeated `peak` and `allocator_gap` sections should agree -across all three reports; the metric-specific `event_id` and `callstack_groups` -are the selected-event breakdown. +occurred together. The repeated `peak` sections should agree across all three +reports. For metrics with a valid peak, the `allocator_gap` sections also agree; +an empty-attribution report has `allocator_gap: null`. The metric-specific +`event_id` and `callstack_groups` are the selected-event breakdown. Read report `has_more`, `truncated`, `total_is_exact`, and `effective_params` before claiming a complete composition. A full dynamic `top_n` window means diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index d6b6e51..6a823a5 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -110,10 +110,14 @@ def peak_memory_report( start_dir=start_dir, ) gap = gap_result.rows[0] if gap_result.rows else None - # allocator_gap selects the same independent, earliest-tie peak events. - # Keep the report's peak object at the historical six-field contract. + # allocator_gap selects the same independent, earliest-tie peak events + # for non-NULL counters. Preserve memory_peak's six-field contract, + # including NULL event IDs when a standalone counter is entirely NULL. peak = {field: gap.get(field) for field in _PEAK_FIELDS} if gap else {} - # Peak event IDs remain INTEGER (or NULL for an empty trace). + for peak_metric, event_field in _EVENT_ID_BY_METRIC.items(): + if peak and peak[f"peak_{peak_metric}"] is None: + peak[event_field] = None + # No attribution event exists for an empty trace or a NULL peak. event_id = cast(int | None, peak.get(_EVENT_ID_BY_METRIC[metric])) attribution_params: dict[str, object] = { "event_id": event_id, diff --git a/tests/core/test_report_service.py b/tests/core/test_report_service.py index 88e2a83..e692970 100644 --- a/tests/core/test_report_service.py +++ b/tests/core/test_report_service.py @@ -1,10 +1,12 @@ """Tests for peak memory report service.""" import sqlite3 +from contextlib import closing from pathlib import Path import pytest +from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates @@ -121,6 +123,56 @@ def test_peak_memory_report_can_exclude_static(report_db: Path) -> None: } +@pytest.mark.parametrize("metric", [None, "active", "allocated", "reserved"]) +@pytest.mark.parametrize("bounded", [False, True]) +def test_null_peak_matches_memory_peak_without_attribution( + report_db: Path, monkeypatch, metric, bounded: bool +) -> None: + selected_metric = metric or "active" + bounds = {"start_id": 2, "end_id": 3} if bounded else {} + with closing(sqlite3.connect(report_db)) as conn, conn: + where = " WHERE id BETWEEN 2 AND 3" if bounded else "" + conn.execute(f"UPDATE trace_entry_0 SET {selected_metric} = NULL{where}") + expected = QueryService().execute_query("memory_peak", params=bounds, db_path=report_db) + service = ReportService() + calls = [] + execute = service._query_service.execute_query + + def tracked_execute(template, *args, **kwargs): + calls.append(template) + return execute(template, *args, **kwargs) + + def unexpected_attribution(*args, **kwargs): + pytest.fail("A NULL peak must not invent an attribution event") + + monkeypatch.setattr(service._query_service, "execute_query", tracked_execute) + monkeypatch.setattr(service, "event_attribution", unexpected_attribution) + report = service.peak_memory_report( + report_db, **bounds, **({"metric": metric} if metric is not None else {}) + ) + + assert report.peak == expected.rows[0] + assert report.event_id is None and report.allocator_gap is None + assert report.callstack_groups == [] and report.total_is_exact + assert report.effective_params["event_id"] is None + assert not report.has_more and not report.truncated + assert calls == ["allocator_gap"] + + +@pytest.mark.parametrize("active", [None, 0]) +def test_report_preserves_null_unselected_peak_and_zero_peak(report_db: Path, active) -> None: + with closing(sqlite3.connect(report_db)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET active = ?", (active,)) + expected = QueryService().execute_query("memory_peak", db_path=report_db) + metric = "reserved" if active is None else "active" + report = ReportService().peak_memory_report(report_db, metric=metric) + + assert report.peak == expected.rows[0] + assert report.event_id == (3 if active is None else 1) + assert report.allocator_gap is not None + assert report.callstack_groups + + def test_peak_memory_report_rejects_invalid_metric(report_db: Path) -> None: with pytest.raises(ValueError, match="Invalid metric"): ReportService().peak_memory_report(report_db, metric="invalid") # type: ignore[arg-type] diff --git a/tests/skills/test_memory_peak_breakdown_contract.py b/tests/skills/test_memory_peak_breakdown_contract.py index c6f6505..034c64c 100644 --- a/tests/skills/test_memory_peak_breakdown_contract.py +++ b/tests/skills/test_memory_peak_breakdown_contract.py @@ -141,6 +141,20 @@ def test_full_trace_workflow_runs_all_metrics_with_static_limit_and_json() -> No assert "active_blocks_at_event` at the chosen metric's peak event" in skill +def test_report_workflow_documents_reused_peak_query_and_null_semantics() -> None: + normalized = _normalized_skill() + + assert "`allocator_gap` runs once" in normalized + assert "without running a separate `memory_peak` query" in normalized + assert ( + "Both steps share one resolved database/device, budget, and dataset generation" + in normalized + ) + assert "empty attribution without running the attribution query" in normalized + assert "empty range or an all-NULL counter" in normalized + assert "an empty-attribution report has `allocator_gap: null`" in normalized + + def test_range_fallback_and_peak_event_semantics_are_explicit() -> None: skill = _skill() normalized = _normalized_skill() diff --git a/tests/test_peak_report_completeness.py b/tests/test_peak_report_completeness.py index ef91130..6495e4c 100644 --- a/tests/test_peak_report_completeness.py +++ b/tests/test_peak_report_completeness.py @@ -129,6 +129,39 @@ def test_report_completeness_and_same_event_coverage( assert ("Increase --limit" in result.stdout) == partial +@pytest.mark.parametrize("bounded", [False, True]) +@pytest.mark.parametrize("json_output", [True, False]) +def test_default_report_has_no_attribution_for_null_active_peak( + attribution_db: Path, bounded: bool, json_output: bool +) -> None: + bounds = {"start_id": 2, "end_id": 3} if bounded else {} + with closing(sqlite3.connect(attribution_db)) as conn, conn: + where = " WHERE id BETWEEN 2 AND 3" if bounded else "" + conn.execute(f"UPDATE trace_entry_0 SET active = NULL{where}") + args = ["report", "peak-memory", str(attribution_db)] + if bounded: + args += ["--start-id", "2", "--end-id", "3"] + result = CliRunner().invoke(app, args + (["--json"] if json_output else [])) + assert result.exit_code == 0, result.output + if json_output: + payload = json.loads(result.stdout) + with SnapshotAnalyzer(attribution_db) as analyzer: + peak = analyzer.execute_query("memory_peak", params=bounds) + assert payload["peak"] == peak["rows"][0] + assert payload["metric"] == "active" + assert payload["event_id"] is None and payload["allocator_gap"] is None + assert payload["callstack_groups"] == [] + assert payload["included_bytes"] == 0 + assert payload["active_bytes_at_event"] is None + assert payload["coverage_percent"] is None + assert payload["effective_params"]["event_id"] is None + assert payload["total_is_exact"] is True + assert payload["has_more"] is False and payload["truncated"] is False + else: + assert "Attribution: unavailable (no peak event)" in result.stdout + assert "No active memory callstack groups found" in result.stdout + + @pytest.mark.parametrize("state", ["empty", "null", "zero"]) @pytest.mark.parametrize("json_output", [True, False]) def test_report_unknown_coverage(attribution_db: Path, state: str, json_output: bool) -> None: