From c5ed2a8dff80318e3bd4ae267a48eb9a6909ae0b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:12:00 +0800 Subject: [PATCH 1/8] fix: narrow dataset trace projections by template (#220) --- src/pt_snap_cli/core/dataset_global.py | 22 ++++++++--- tests/core/test_dataset_projections.py | 54 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/core/test_dataset_projections.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index cbc73f6..0990b3d 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -68,11 +68,20 @@ def _scope( } -def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord]): +def _traces( + sources: DatasetSourceResolver, + low: int, + high: int, + items: list[SliceRecord], + columns: tuple[str, ...], +): + # Each built-in supplies only the fields it actually consumes. In particular, + # inline stack text must not spend the byte budget of a counter-only query. + projection = ", ".join(f'"{column}"' for column in columns) for item in items: rows = sources.read( sources.dataset.root / item.file, - f'SELECT * FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', [low, high], ) yield item, rows @@ -81,7 +90,7 @@ def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[Sli def _peaks(sources, low, high, items, template, scope): chosen: dict[str, dict[str, object]] = {} count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items): + for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): for row in rows: sources.budget.remaining() event = cast(int, row["id"]) @@ -149,7 +158,10 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items): + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + for _item, batch in _traces(sources, low, high, items, columns): for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -287,7 +299,7 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items): + for _item, batch in _traces(sources, low, high, items, ("id",)): refs = sources.events(cast(int, r["id"]) for r in batch) for source in refs.values(): sources.budget.remaining() diff --git a/tests/core/test_dataset_projections.py b/tests/core/test_dataset_projections.py new file mode 100644 index 0000000..6811d89 --- /dev/null +++ b/tests/core/test_dataset_projections.py @@ -0,0 +1,54 @@ +"""Per-template scan projections must not charge unrelated inline stack text.""" + +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +@pytest.mark.parametrize("template", ["memory_peak", "allocator_gap", "allocation"]) +def test_counter_query_budget_is_independent_of_inline_stack_length(tmp_path, template): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + observations = [] + for stack in ("x" * 16, "x" * 8192, "x" * 32768, "栈🙂" * 8192): + for path in root.glob("device_0/*.db"): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=?", (stack,)) + service = QueryService() + try: + budget = QueryBudget(None, time.monotonic(), max_work_bytes=8192) + result = service.execute_query(template, db_path=root, _budget=budget) + observations.append((result.rows, budget.work_rows, budget.work_bytes)) + finally: + service.close() + assert all(item == observations[0] for item in observations) + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_event_projection_preserves_callstack_sort_and_paging(tmp_path, direction): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + stacks = ["z" * 8192, None, "栈🙂" * 8192, "a" * 16384, "", "a" * 16384] + for index, stack in enumerate(stacks): + path = root / f"device_0/slice_{index // 2:05d}.db" + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", (stack, index)) + service = QueryService() + try: + params = {"order_by": "callstack", "order_dir": direction, "offset": 1, "limit": 3} + result = service.execute_query("event", params, db_path=root, exact_total=True) + expected = sorted( + range(len(stacks)), + key=lambda event: (stacks[event] is not None, stacks[event], event), + reverse=direction == "DESC", + )[1:4] + assert [row["id"] for row in result.rows] == expected + assert [row["callstack"] for row in result.rows] == [stacks[event] for event in expected] + assert result.total == 6 and result.has_more and result.truncated + assert all(len(row) == 9 for row in result.rows) + finally: + service.close() From b1a2d068e18689cd9aea62d71aca36bf7292c9b9 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:14:26 +0800 Subject: [PATCH 2/8] fix: reuse budgeted event sources within dataset queries (#222) --- src/pt_snap_cli/core/dataset_global.py | 32 ++++-- src/pt_snap_cli/core/dataset_sources.py | 87 ++++++++++----- tests/core/test_dataset_source_reuse.py | 134 ++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 tests/core/test_dataset_source_reuse.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 0990b3d..a7ab3ec 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -73,15 +73,26 @@ def _traces( low: int, high: int, items: list[SliceRecord], - columns: tuple[str, ...], + columns: tuple[str, ...] = ("id", *METRICS), + *, + with_sources: bool = False, ): - # Each built-in supplies only the fields it actually consumes. In particular, - # inline stack text must not spend the byte budget of a counter-only query. - projection = ", ".join(f'"{column}"' for column in columns) + projection = ", ".join(f't."{column}"' for column in columns) for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if with_sources: + projection = ( + "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" + ) + if sources.dataset.callstack_layout == "v2": + sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + sql = f"SELECT {projection}, t.callstack FROM {trace} t" + else: + sql = f"SELECT {projection} FROM {trace} t" rows = sources.read( sources.dataset.root / item.file, - f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", [low, high], ) yield item, rows @@ -158,10 +169,9 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - columns = ("id", *METRICS) - if template == "event": - columns += ("action", "address", "size", "stream") - for _item, batch in _traces(sources, low, high, items, columns): + for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): + if template == "event": + sources.events_from_rows(_item, batch) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -299,8 +309,8 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, ("id",)): - refs = sources.events(cast(int, r["id"]) for r in batch) + for _item, batch in _traces(sources, low, high, items, with_sources=True): + refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 7a8e3f0..3532752 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -1,7 +1,7 @@ """Read-only, batched device/event source lookup over a validated dataset. No address joins, foreign-event insertion, text-to-frame reconstruction, or retained -source cache. The caller owns the Context LRU and the entire operation's deadline. +cross-operation source cache. The caller owns the Context LRU and the entire operation's deadline. """ from __future__ import annotations @@ -140,6 +140,9 @@ def __init__( self.budget = budget self.query_count = 0 self._starts = [s.start_event_id for s in self.device.slices] + # Lifetime is one budgeted operation. Each retained source was charged + # when fetched; reusing the same object performs no further database work. + self._events: dict[int, EventSource] = {} @property def namespace(self) -> str: @@ -203,12 +206,15 @@ def read( def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: grouped: dict[int, list[int]] = defaultdict(list) + result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() + if event in self._events: + result[event] = self._events[event] + continue item = self.owner(event) if item is not None: grouped[item.index].append(event) - result: dict[int, EventSource] = {} for index, ids in grouped.items(): item = self.device.slices[index] path = self.dataset.root / item.file @@ -224,32 +230,57 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - for row in rows: - event_id = cast(int, row["id"]) - text = row.get("callstack") - captured = isinstance(text, str) and text != "" - local_id = cast(int | None, row.pop("callstackId", None)) - # V2 IDs select text WITHIN the owning DB; they are provenance, - # not dataset grouping keys. Global interning equivalence uses - # the full captured text, never a shortened display prefix. - identity = ( - "text:sha256:" - + hashlib.sha256( - (text if isinstance(text, str) else "").encode("utf-8") - ).hexdigest() - ) - result[event_id] = EventSource( - row, - index, - ( - f"{self.namespace}:{identity}" - if captured - else f"{self.namespace}:category:missing" - ), - "captured" if captured else "missing", - local_stack_id=local_id, - ) - self._attach_frames(path, ids, result) + result.update(self.events_from_rows(item, rows)) + self.budget.remaining() + return result + + def events_from_rows( + self, item: SliceRecord, rows: list[dict[str, object]] + ) -> dict[int, EventSource]: + """Resolve already charged full event rows without fetching them again. + + Native rows must include joined callstack text and the local callstackId. + Frame reads still incur the same work budget as events() source lookup. + """ + result: dict[int, EventSource] = {} + for original in rows: + self.budget.remaining() + event_id = cast(int, original["id"]) + if event_id in self._events: + result[event_id] = self._events[event_id] + continue + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") + row = dict(original) + event_id = cast(int, row["id"]) + text = row.get("callstack") + captured = isinstance(text, str) and text != "" + local_id = cast(int | None, row.pop("callstackId", None)) + # V2 IDs select text WITHIN the owning DB; they are provenance, + # not dataset grouping keys. Global interning equivalence uses + # the full captured text, never a shortened display prefix. + identity = ( + "text:sha256:" + + hashlib.sha256( + (text if isinstance(text, str) else "").encode("utf-8") + ).hexdigest() + ) + result[event_id] = EventSource( + row, + item.index, + ( + f"{self.namespace}:{identity}" + if captured + else f"{self.namespace}:category:missing" + ), + "captured" if captured else "missing", + local_stack_id=local_id, + ) + fresh = {event: source for event, source in result.items() if event not in self._events} + if fresh: + self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) + self._events.update(fresh) + result.update(fresh) self.budget.remaining() return result diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py new file mode 100644 index 0000000..87c8265 --- /dev/null +++ b/tests/core/test_dataset_source_reuse.py @@ -0,0 +1,134 @@ +"""Operation-local reuse preserves real read accounting and source identity.""" + +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +def dataset(tmp_path, count, stack="shared", blocks=False): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["eventsPerSlice"] = count + manifest["devices"]["0"]["eventCount"] = count + manifest["devices"]["0"]["slices"][0]["endEventId"] = count - 1 + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,8,8,16,?)", + ((i, i * 8, stack) for i in range(count)), + ) + if blocks: + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(count)), + ) + return root + + +@pytest.mark.parametrize("template", ["callstack_analysis", "event"]) +def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): + root = dataset(tmp_path, 60_000) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + result = service.execute_query( + template, + params=( + {"min_count": 1} + if template == "callstack_analysis" + else { + "order_by": "callstack", + "limit": 1, + } + ), + db_path=root, + _budget=budget, + ) + finally: + service.close() + assert result.returned == 1 + assert budget.work_rows == 60_001 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 60_000 + + +def test_target_and_distinct_events_reuse_only_identical_sources(tmp_path): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + budget = QueryBudget(None, time.monotonic()) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, budget) + sources.target(3) + first = sources.events([0, 1, 2, 3]) + reads, work = sources.query_count, budget.work_rows + repeated = sources.events([3, 2, 1, 0, 3]) + assert sources.query_count == reads + assert budget.work_rows == work == 4 + assert repeated[3] is first[3] + assert first[0].event["id"] != first[1].event["id"] + assert first[0].stack_id == first[1].stack_id + + +@pytest.mark.parametrize("count, succeeds", [(49_999, True), (50_000, False)]) +def test_distinct_block_and_allocation_work_still_counts(tmp_path, count, succeeds): + root = dataset(tmp_path, count, blocks=True) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows == 100_001 + finally: + service.close() + + +@pytest.mark.parametrize("count, succeeds", [(3_000, True), (5_000, False)]) +def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds): + root = dataset(tmp_path, count, stack="x" * 16_384, blocks=True) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows < budget.max_work_rows + assert budget.work_bytes > budget.max_work_bytes From eda49166421dcab5c73fe7d107504122d701928c Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:17:49 +0800 Subject: [PATCH 3/8] fix: defer event page hydration and reuse frame schema checks --- src/pt_snap_cli/core/dataset_global.py | 19 ++++--- src/pt_snap_cli/core/dataset_sources.py | 47 +++++++++++------ tests/core/test_dataset_source_reuse.py | 68 +++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index a7ab3ec..857cfa8 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -169,9 +169,13 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): - if template == "event": - sources.events_from_rows(_item, batch) + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + with_sources = template == "event" and params["order_by"] == "callstack" + for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): + if with_sources: + sources.events_from_rows(_item, batch, with_frames=False) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -188,13 +192,8 @@ def _events(sources, low, high, items, params, template, scope): "last_event_id": last, "real_event_count": count, } - if template == "event": - # Only returned events need formatted text; resolve separately after the - # global window below. `order_by=callstack` requires text BEFORE sorting. - if params["order_by"] == "callstack": - refs = sources.events(cast(int, r["id"]) for r in rows) - for row in rows: - row["callstack"] = refs[cast(int, row["id"])].event.get("callstack") + # callstack sorting already has joined text; resolving sources here would + # eagerly fetch ordered frames that only the returned page needs. _sort(rows, params["order_by"], params["order_dir"] == "DESC") return rows diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3532752..3a22a5d 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -143,6 +143,8 @@ def __init__( # Lifetime is one budgeted operation. Each retained source was charged # when fetched; reusing the same object performs no further database work. self._events: dict[int, EventSource] = {} + self._frames_resolved: set[int] = set() + self._frame_schemas: dict[Path, bool] = {} @property def namespace(self) -> str: @@ -230,12 +232,17 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - result.update(self.events_from_rows(item, rows)) + result.update(self.events_from_rows(item, rows, with_frames=False)) + frame_groups: dict[int, list[int]] = defaultdict(list) + for event, source in result.items(): + frame_groups[source.slice_index].append(event) + for index, ids in frame_groups.items(): + self._ensure_frames(self.device.slices[index], ids) self.budget.remaining() - return result + return {event: self._events[event] for event in result} def events_from_rows( - self, item: SliceRecord, rows: list[dict[str, object]] + self, item: SliceRecord, rows: list[dict[str, object]], *, with_frames: bool = True ) -> dict[int, EventSource]: """Resolve already charged full event rows without fetching them again. @@ -277,12 +284,17 @@ def events_from_rows( local_stack_id=local_id, ) fresh = {event: source for event, source in result.items() if event not in self._events} - if fresh: - self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) self._events.update(fresh) - result.update(fresh) + if with_frames: + self._ensure_frames(item, list(result)) self.budget.remaining() - return result + return {event: self._events[event] for event in result} + + def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: + pending = [event for event in ids if event not in self._frames_resolved] + if pending: + self._attach_frames(self.dataset.root / item.file, pending, self._events) + self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: # This is a NEW, explicit read contract, not a guessed original-producer @@ -297,14 +309,19 @@ def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSou ("frameJson", "TEXT", 0), ], } - for table, columns in expected.items(): - kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) - schema = self.read(path, f'PRAGMA table_info("{table}")') - if ( - kind != [{"type": "table"}] - or [(r["name"], r["type"], r["pk"]) for r in schema] != columns - ): - return # Unknown/absent schemas never imply precision. + if path not in self._frame_schemas: + self._frame_schemas[path] = True + for table, columns in expected.items(): + kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = self.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + self._frame_schemas[path] = False + break # Unknown/absent schemas never imply precision. + if not self._frame_schemas[path]: + return for start in range(0, len(ids), SOURCE_BATCH_SIZE): batch = ids[start : start + SOURCE_BATCH_SIZE] placeholders = ",".join("?" for _ in batch) diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 87c8265..3923079 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -132,3 +132,71 @@ def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds ) assert budget.work_rows < budget.max_work_rows assert budget.work_bytes > budget.max_work_bytes + + +def test_id_sorted_page_does_not_materialize_unneeded_stack_text(tmp_path): + root = dataset(tmp_path, 10_000, stack="x" * 8192) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + params={"order_by": "id", "limit": 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_bytes < 2_000_000 + + +def framed_dataset(tmp_path, count): + root = dataset(tmp_path, count) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT," + "PRIMARY KEY(eventId,frameIndex));" + ) + conn.executemany( + "INSERT INTO pt_snap_frame_coverage VALUES (?,1)", ((i,) for i in range(count)) + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", ((i, '{"name":"f"}') for i in range(count)) + ) + return root + + +def test_frame_schema_is_checked_once_across_source_batches(tmp_path, monkeypatch): + root = framed_dataset(tmp_path, 600) + resolved = DatasetResolver().inspect(root) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + first = sources.events(range(512)) + second = sources.events(range(512, 600)) + assert all( + source.frames == [{"name": "f"}] for source in [*first.values(), *second.values()] + ) + assert sum("PRAGMA table_info" in sql for sql in queries) == 2 + assert sum("sqlite_master" in sql for sql in queries) == 2 + + +def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): + root = framed_dataset(tmp_path, 600) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget + ) + assert result.rows[0]["frames"] == [{"name": "f"}] + # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. + assert budget.work_rows == 610 From 32639e130413e10e7016a024be828c2e0ab9d296 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:27:08 +0800 Subject: [PATCH 4/8] fix: make dataset validation cancellable and cache reuse explicit (#219) --- docs/en/sharded-snapshotdb.md | 44 +- docs/zh/sharded-snapshotdb.md | 30 +- src/pt_snap_cli/completion.py | 6 +- src/pt_snap_cli/core/dataset_contract.py | 49 ++- src/pt_snap_cli/core/dataset_files.py | 6 +- src/pt_snap_cli/core/dataset_resolver.py | 243 +++++++++-- .../core/native_dataset_contract.py | 125 ++++-- src/pt_snap_cli/core/query_service.py | 2 +- src/pt_snap_cli/core/validation_budget.py | 79 ++++ tests/core/test_dataset_validation_cache.py | 409 ++++++++++++++++++ tests/test_dataset_focus.py | 4 +- 11 files changed, 896 insertions(+), 101 deletions(-) create mode 100644 src/pt_snap_cli/core/validation_budget.py create mode 100644 tests/core/test_dataset_validation_cache.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 3948d19..53f7ae4 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -242,12 +242,44 @@ full-text stack-statistics versions are documented with the same contracts. `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. -It revalidates the finalized P0 contract on each call and hashes the manifest and -members; no unbounded manifest/connection cache is retained. Hashing uses bounded -chunks and is proportional to artifact size, not a performance claim. Context LRU -size defaults to four; dataset generation changes invalidate contexts on next -lookup even when size/mtime are unchanged. Analyzer-owned caches close on exit; -injected caches remain caller-owned. Validation opens/closes slices sequentially. +By default **every independent inspection** fully validates the finalized contract +and hashes the manifest/members. File metadata alone is not content-integrity proof: +unknown or coarse filesystems may expose unchanged `ctime_ns` after a write. +Pinned report-operation guards also rehash content by default, without repeating +row validation. + +`PT_SNAP_DATASET_CACHE=immutable` explicitly opts into a process-wide LRU of at most +16 validated generations, without open connections. Set it only when you control +finalized, immutable publication and trust your filesystem's kernel change-time +semantics. **Do not enable it for files that may be modified in place**, live writers, +unknown/coarse/network filesystems, or filesystem snapshots that may be rolled back. +This is a caller responsibility contract, not a filesystem capability detected by +the tool. Unset it to retain strong validation; unknown values also use strong mode. +Windows and platforms without a usable POSIX change token remain in strong mode. + +With that explicit opt-in, new resolver/service instances can reuse validated +results. Every hit and pinned-operation guard still rechecks canonical paths, +sidecars, native membership and each file's device, inode, size, mtime, **ctime**, +mode and link count. These conservative checks catch ordinary replacements and +in-place edits with restored mtime, but do not replace the immutable-file contract. +For a fixed shard count, an opted-in hit performs no row scans or whole-file hashes: +work is O(shards), independent of event/block counts. Default inspections and cache +misses remain proportional to content. Explicit `validate_dataset` / +`validate_native_dataset` calls always validate fully. Native admission verifies +hashes before and after validation and reuses the verified hashes for its fingerprint, +avoiding a third member-hash pass. + +The query's shared deadline reaches admission: hashing checks every 1 MiB chunk, +Python validation loops check every 256 rows, and SQLite validation (including +`quick_check`) uses an interruptible progress handler, cleared on every exit. +Cancellation raises `QUERY_TIMEOUT`; no partial validation is cached. Device tab +completion reads only a size-bounded manifest (up to 1 MiB), never member SQLite +contents or hashes. Its candidates are hints, not proof the dataset is valid. + +Context LRU size defaults to four; dataset content-generation changes invalidate +contexts on next lookup even when size/mtime are unchanged. Analyzer-owned caches +close on exit; injected caches remain caller-owned. Validation opens/closes slices +sequentially. Building, missing/inconsistent slices, unknown base versions and unsafe paths are rejected before focus writes or query execution. Before **any** SQLite open, diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 137d3fd..c2ee244 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -198,10 +198,32 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 -每次调用重新校验最终化 P0 合同并对 manifest/所有成员取内容哈希,不保存无界 manifest/ -连接缓存。哈希按有界块读取,工作量与产物大小成正比,不宣称性能收益。Context LRU 默认 -最多四个;数据集 generation 变化在下次 lookup 使旧 Context 失效,即使 size/mtime 相同。 -analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有;校验逐片打开并关闭连接。 +默认**每次独立 inspect 都完整校验**最终化合同,并对 manifest/成员取内容哈希。文件元数据 +本身不是内容完整性证明:未知或低精度文件系统可能在写入后仍返回相同 `ctime_ns`。 +已固定 generation 的报告操作默认也重新哈希内容,但不重复逐行校验。 + +`PT_SNAP_DATASET_CACHE=immutable` 显式启用最多 16 个已校验 generation 的进程级 LRU, +不保留连接。仅在你控制最终化、不可变发布,且信任文件系统内核 change-time 语义时启用。 +**可能被原地修改的文件、live writer、未知/低精度/网络文件系统或可能回滚的文件系统快照 +不得启用**。这是调用方责任合同,不代表工具已检测文件系统能力。不设置该变量即保留强校验, +未知取值也使用强校验;Windows 或没有可用 POSIX change token 的平台始终使用强校验。 + +显式启用后,新建 resolver/service 可复用已校验结果。每次命中及报告 generation 检查仍验证 +规范路径、sidecar、原生成员集合,以及文件的设备号、inode、size、mtime、**ctime**、mode +和硬链接数。这些保守检查可检测常规替换及恢复 mtime 的原地修改,但不能代替不可变文件 +合同。固定分片数时,显式启用后的命中不扫描行、不做整文件哈希,工作量为 O(分片数), +不随 event/block 数增长。默认 inspect 及未命中仍与内容规模成正比。显式 `validate_dataset` +/ `validate_native_dataset` 始终完整校验。原生准入保留校验前后两遍哈希,fingerprint 直接 +复用已验证摘要,不再做第三遍成员哈希。 + +查询共用 deadline 覆盖准入:哈希每个 1 MiB 块、Python 校验每 256 行检查一次;SQLite +校验(含 `quick_check`)使用可中断 progress handler,并在所有退出路径清除。 +取消保留 `QUERY_TIMEOUT`,不会缓存不完整校验。设备 tab 补全只读取有界 manifest +(最多 1 MiB),不读取成员 SQLite 内容或整文件哈希;候选仅为提示,不证明数据集有效。 + +Context LRU 默认最多四个;数据集内容 generation 变化在下次 lookup 使旧 Context 失效, +即使 size/mtime 相同。analyzer 自有缓存在退出时关闭,注入缓存仍由调用者所有; +校验逐片打开并关闭连接。 building、缺片/矛盾片、未知基础版本、不安全路径在 focus 写入或查询前拒绝。 **任何** SQLite 打开之前,裸 P0 validator 与 resolver 均先检查**全部**成员/父目录别名及 diff --git a/src/pt_snap_cli/completion.py b/src/pt_snap_cli/completion.py index 797dfff..749104c 100644 --- a/src/pt_snap_cli/completion.py +++ b/src/pt_snap_cli/completion.py @@ -77,9 +77,9 @@ def complete_device_ids() -> list[str]: try: from pt_snap_cli.core.dataset_resolver import DatasetResolver - dataset = DatasetResolver().inspect(db_path) - if dataset is not None: - return [str(device) for device in dataset.device_ids] + devices = DatasetResolver().completion_device_ids(db_path) + if devices is not None: + return [str(device) for device in devices] uri = f"file:{db_path}?mode=ro" with closing(sqlite3.connect(uri, uri=True)) as conn: cursor = conn.cursor() diff --git a/src/pt_snap_cli/core/dataset_contract.py b/src/pt_snap_cli/core/dataset_contract.py index 9d8be4c..a8b03fd 100644 --- a/src/pt_snap_cli/core/dataset_contract.py +++ b/src/pt_snap_cli/core/dataset_contract.py @@ -18,6 +18,13 @@ from .dataset_files import require_readonly_member from .errors import DatabaseSchemaError +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) MANIFEST_SCHEMA_VERSION = 1 MSINSIGHT_REVISION = "101f65b877a267ffd5f66ea3834706057ba243e5" @@ -118,12 +125,15 @@ def require_complete(self) -> None: ) -def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetManifest: +def parse_manifest( + value: object, *, require_complete: bool = True, budget: ValidationBudget | None = None +) -> DatasetManifest: """Validate JSON-shaped data; inspection may explicitly accept building. Unknown extension declarations are ignored, never promoted to capabilities. This does not check files; use validate_dataset for finalized artifacts. """ + check_budget(budget) data = _object(value, "manifest") version = _integer(data.get("schemaVersion"), "schemaVersion", maximum=2**31 - 1) _need( @@ -137,7 +147,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa devices_data = _object(data.get("devices"), "devices") _need(bool(devices_data), "devices", "empty", "at least one event-bearing device required") devices: list[DeviceRecord] = [] - for key, value in devices_data.items(): + for key, value in checked_rows(devices_data.items(), budget): location = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -153,7 +163,9 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) ready = tuple( _integer(item, location + ".readySlices", maximum=size - 1) - for item in _array(device.get("readySlices"), location + ".readySlices") + for item in checked_rows( + _array(device.get("readySlices"), location + ".readySlices"), budget + ) ) _need( len(set(ready)) == len(ready), @@ -170,7 +182,7 @@ def parse_manifest(value: object, *, require_complete: bool = True) -> DatasetMa ) slices: list[SliceRecord] = [] expected_start = 0 - for index, raw in enumerate(records): + for index, raw in enumerate(checked_rows(records, budget)): loc = f"{location}.slices[{index}]" item = _object(raw, loc) number = _integer(item.get("index"), loc + ".index", maximum=2**31 - 1) @@ -357,6 +369,7 @@ def _validate_slice( device: DeviceRecord, item: SliceRecord, identities: dict[tuple[int, int], tuple[object, ...]], + budget: ValidationBudget | None = None, ) -> int: loc = item.file trace, block = f"trace_entry_{device.device_id}", f"block_{device.device_id}" @@ -415,7 +428,7 @@ def _validate_slice( ).fetchone() _need(bad is None, loc, "event_values", "invalid scalar type or negative byte metric") boundary_count = conn.execute(f'SELECT COUNT(*) FROM "{trace}" WHERE id<0').fetchone()[0] - for row in conn.execute(f'SELECT * FROM "{block}"'): + for row in checked_rows(conn.execute(f'SELECT * FROM "{block}"'), budget): block_id, address, size, requested, state, alloc, free = row for name, value in zip( ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId"), @@ -455,12 +468,15 @@ def _validate_slice( return cast(int, boundary_count) -def validate_dataset(directory: Path | str) -> DatasetValidation: +def validate_dataset( + directory: Path | str, *, budget: ValidationBudget | None = None +) -> DatasetValidation: """Read a complete compatibility-v1 dataset without migration or pickle. Checks are validation, not a filesystem sandbox or concurrent publication lock. Missing pt_snap_metadata/extension tables are valid external base artifacts. """ + check_budget(budget) root = Path(directory).absolute() _need( root.resolve() == root and not root.is_symlink(), @@ -472,11 +488,11 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: path = _safe_file(root, "manifest.json") with path.open(encoding="utf-8") as source: data: object = json.load(source, object_pairs_hook=_unique_object) - manifest = parse_manifest(data) + manifest = parse_manifest(data, budget=budget) # Preflight ALL members before the first SQLite open, also for callers # using this bare P0 validator rather than DatasetResolver. for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): member = _safe_file(root, item.file) try: require_readonly_member(root, member, immutable=True) @@ -485,14 +501,21 @@ def validate_dataset(directory: Path | str) -> DatasetValidation: identities: dict[tuple[int, int], tuple[object, ...]] = {} boundaries = 0 for device in manifest.devices: - for item in device.slices: + for item in checked_rows(device.slices, budget): db_path = _safe_file(root, item.file) try: - with closing( - sqlite3.connect(db_path.as_uri() + "?mode=ro&immutable=1", uri=True) - ) as conn: + with ( + closing( + sqlite3.connect( + db_path.as_uri() + "?mode=ro&immutable=1", + uri=True, + timeout=sqlite_timeout(budget), + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") - boundaries += _validate_slice(conn, device, item, identities) + boundaries += _validate_slice(conn, device, item, identities, budget) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc return DatasetValidation( diff --git a/src/pt_snap_cli/core/dataset_files.py b/src/pt_snap_cli/core/dataset_files.py index e32a6d6..23f94b6 100644 --- a/src/pt_snap_cli/core/dataset_files.py +++ b/src/pt_snap_cli/core/dataset_files.py @@ -6,6 +6,7 @@ from pathlib import Path from .errors import DatabaseSchemaError +from .validation_budget import ValidationBudget, check_budget def require_readonly_member(root: Path, member: Path, *, immutable: bool = False) -> None: @@ -27,9 +28,12 @@ def require_readonly_member(root: Path, member: Path, *, immutable: bool = False raise DatabaseSchemaError(f"Dataset member uses persistent WAL mode: {member}") -def hash_file(path: Path) -> str: +def hash_file(path: Path, *, budget: ValidationBudget | None = None) -> str: + check_budget(budget) digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): + check_budget(budget) digest.update(chunk) + check_budget(budget) return digest.hexdigest() diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index 94126ec..b1d9da3 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -7,13 +7,18 @@ from __future__ import annotations +import copy import hashlib import json +import os +from collections import OrderedDict from dataclasses import dataclass from pathlib import Path +from threading import RLock from pt_snap_cli.core.dataset_contract import ( DatasetContractError, + DatasetManifest, DatasetValidation, DeviceRecord, QueryScope, @@ -28,10 +33,16 @@ from pt_snap_cli.core.native_dataset_contract import ( NATIVE_FORMAT, NativeManifest, + NativeSlice, parse_native_manifest, + require_native_members, validate_native_dataset, validate_native_scope, ) +from pt_snap_cli.core.validation_budget import ValidationBudget, check_budget, checked_rows + +_FileStamp = tuple[int, int, int, int, int, int, int] +_Generation = tuple[tuple[str, _FileStamp], ...] @dataclass(frozen=True) @@ -40,6 +51,25 @@ class ResolvedDataset: validation: DatasetValidation fingerprint: str ordered_frames_version: int | None = None + generation: _Generation | None = None + + def require_unchanged(self, budget: ValidationBudget | None = None) -> None: + """Guard a reused report/query resolution without repeating full validation.""" + if _root(self.root) != self.root: + raise DatabaseSchemaError("Dataset path changed during operation; retry.") + try: + current = _generation(self.root, self.validation.manifest, budget) + if self.generation is None or current != self.generation: + raise DatabaseSchemaError("Dataset members changed during operation; retry.") + if not _cacheable(current): + if ( + _content_fingerprint(self.root, self.validation.manifest, budget) + != self.fingerprint + or _generation(self.root, self.validation.manifest, budget) != current + ): + raise DatabaseSchemaError("Dataset content changed during operation; retry.") + except (DatasetContractError, OSError) as exc: + raise DatabaseSchemaError(f"Dataset changed during operation: {exc}") from exc @property def device_ids(self) -> list[int]: @@ -116,62 +146,207 @@ def to_dict(self) -> dict[str, object]: } +# This is deliberately process-local, bounded, and contains no open connections. +# Cross-call reuse is opt-in: unknown/coarse filesystems cannot supply a +# trustworthy change token merely because Python exposes st_ctime_ns. The caller +# must attest immutable publication and suitable change-time semantics explicitly. +_CACHE_LIMIT = 16 +_VALIDATED: OrderedDict[Path, tuple[_Generation, ResolvedDataset]] = OrderedDict() +_CACHE_LOCK = RLock() + + +def _stamp(path: Path) -> _FileStamp: + stat = path.stat() + return ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + stat.st_mode, + stat.st_nlink, + ) + + +def _root(path: Path | str) -> Path | None: + candidate = Path(path).expanduser().absolute() + if not candidate.is_dir() and candidate.name != "manifest.json": + return None + root = candidate if candidate.is_dir() else candidate.parent + if candidate.is_symlink() or root.resolve() != root: + raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") + if (root / "manifest.json").is_symlink(): + raise DatabaseSchemaError("Dataset manifest must not be a symlink.") + if not (root / "manifest.json").is_file(): + raise DatabaseSchemaError("Dataset manifest.json must be a regular file.") + return root + + +def _generation( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> _Generation: + """Always recheck aliases, transport, and membership before trusting a hit.""" + check_budget(budget) + if isinstance(manifest, NativeManifest): + require_native_members(root, manifest, budget=budget) + stamps = [(".", _stamp(root)), ("manifest.json", _stamp(root / "manifest.json"))] + for device in checked_rows(manifest.devices, budget): + directory = f"device_{device.device_id}" + if (root / directory).is_symlink(): + raise DatabaseSchemaError("Dataset device directory must not be a symlink.") + stamps.append((directory, _stamp(root / directory))) + for item in checked_rows(device.slices, budget): + member = root / item.file + _require_readonly_member( + root, member, immutable=not isinstance(manifest, NativeManifest) + ) + stamps.append((item.file, _stamp(member))) + check_budget(budget) + return tuple(stamps) + + +def _cacheable(generation: _Generation) -> bool: + # This is a caller contract, not an automatic inference from OS or filesystem + # metadata. Windows ctime historically means creation time, so stay strong. + return ( + os.environ.get("PT_SNAP_DATASET_CACHE") == "immutable" + and os.name == "posix" + and all(stamp[4] > 0 for _, stamp in generation) + ) + + +def _content_fingerprint( + root: Path, manifest: DatasetManifest, budget: ValidationBudget | None +) -> str: + """Strong verification for pinned operations without repeating row validation.""" + digest = hashlib.sha256() + digest.update(_hash_file(root / "manifest.json", budget=budget).encode("ascii")) + for device in checked_rows(manifest.devices, budget): + for item in checked_rows(device.slices, budget): + digest.update(item.file.encode("utf-8")) + digest.update(_hash_file(root / item.file, budget=budget).encode("ascii")) + return digest.hexdigest() + + class DatasetResolver: - """Stateless resolution: revalidate every call, retain no connections/manifest cache. + """Resolve and validate finalized datasets, reusing unchanged generations. - Content hashes (not cacheHash or size/mtime alone) form the Context generation. - Work is proportional to artifact size; no concurrent-publication lock is claimed. + By default every call validates and hashes content. Explicitly setting + PT_SNAP_DATASET_CACHE=immutable enables bounded process-local reuse under + the caller's immutable-publication/change-time contract. Even then paths, + members, sidecars and kernel change times are checked on every call. + Explicit validators always perform full validation. """ - def inspect(self, path: Path | str) -> ResolvedDataset | None: - candidate = Path(path).expanduser().absolute() - if not candidate.is_dir() and candidate.name != "manifest.json": + def inspect( + self, path: Path | str, *, budget: ValidationBudget | None = None + ) -> ResolvedDataset | None: + check_budget(budget) + root = _root(path) + if root is None: return None - root = candidate if candidate.is_dir() else candidate.parent - if candidate.is_symlink() or root.resolve() != root: - raise DatabaseSchemaError("Dataset must use canonical non-symlink paths.") try: + with _CACHE_LOCK: + cached = _VALIDATED.get(root) + if cached is not None: + previous, resolved = cached + manifest_unchanged = ( + _stamp(root / "manifest.json") == dict(previous)["manifest.json"] + ) + generation = ( + _generation(root, resolved.validation.manifest, budget) + if manifest_unchanged + else None + ) + if generation is not None and _cacheable(generation) and generation == previous: + with _CACHE_LOCK: + if root in _VALIDATED: + _VALIDATED.move_to_end(root) + result = copy.deepcopy(resolved) + check_budget(budget) + return result + with _CACHE_LOCK: + _VALIDATED.pop(root, None) manifest_path = root / "manifest.json" - if manifest_path.is_symlink(): - raise DatabaseSchemaError("Dataset manifest must not be a symlink.") - before = _hash_file(manifest_path) + manifest_stamp = _stamp(manifest_path) + before = _hash_file(manifest_path, budget=budget) with manifest_path.open(encoding="utf-8") as source: raw: object = json.load(source, object_pairs_hook=_unique_object) + check_budget(budget) planned = ( - parse_native_manifest(raw) + parse_native_manifest(raw, budget=budget) if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT - else parse_manifest(raw) + else parse_manifest(raw, budget=budget) ) - # mode=ro can create WAL/SHM files even after the last writer has - # checkpointed and removed them. Check every member before the first - # SQLite open, without following aliases or repairing the artifact. - for device in planned.devices: - for item in device.slices: - _require_readonly_member( - root, root / item.file, immutable=not isinstance(planned, NativeManifest) - ) + # Preflight every member before the first SQLite open. Capturing + # both ends also refuses publication during validation or hashing. + generation = _generation(root, planned, budget) + if manifest_stamp != _stamp(manifest_path): + raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") validation = ( - validate_native_dataset(root, planned) + validate_native_dataset(root, planned, budget=budget) if isinstance(planned, NativeManifest) - else validate_dataset(root) + else validate_dataset(root, budget=budget) ) digest = hashlib.sha256() digest.update(before.encode("ascii")) - for device in validation.manifest.devices: - for item in device.slices: - member = root / item.file - _require_readonly_member( - root, member, immutable=not isinstance(planned, NativeManifest) - ) + for device in checked_rows(validation.manifest.devices, budget): + for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) - digest.update(_hash_file(member).encode("ascii")) - if before != _hash_file(manifest_path): + # Native validation already verifies this digest before and + # after SQLite reads; do not hash every shard a third time. + content_hash = ( + item.sha256 + if isinstance(item, NativeSlice) + else _hash_file(root / item.file, budget=budget) + ) + digest.update(content_hash.encode("ascii")) + if before != _hash_file(manifest_path, budget=budget): raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + if generation != _generation(root, planned, budget): + raise DatabaseSchemaError("Dataset members changed during inspection; retry.") extensions = raw.get("extensions") if isinstance(raw, dict) else None frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None version = frames.get("version") if isinstance(frames, dict) else None - return ResolvedDataset( - root, validation, digest.hexdigest(), version if type(version) is int else None + result = ResolvedDataset( + root, + validation, + digest.hexdigest(), + version if type(version) is int else None, + generation, ) + if _cacheable(generation): + with _CACHE_LOCK: + _VALIDATED[root] = (generation, copy.deepcopy(result)) + _VALIDATED.move_to_end(root) + while len(_VALIDATED) > _CACHE_LIMIT: + _VALIDATED.popitem(last=False) + return result except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError) as exc: + with _CACHE_LOCK: + _VALIDATED.pop(root, None) raise DatabaseSchemaError(f"Invalid dataset: {exc}") from exc + + def completion_device_ids(self, path: Path | str) -> list[int] | None: + """Bounded manifest-only hints, never evidence of dataset validity. + + Completion must not open or hash shard contents. Large or invalid + manifests produce no hints; a real query still performs strong admission. + """ + root = _root(path) + if root is None: + return None + try: + with (root / "manifest.json").open("rb") as source: + data = source.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + return [] + raw: object = json.loads(data, object_pairs_hook=_unique_object) + manifest = ( + parse_native_manifest(raw) + if isinstance(raw, dict) and raw.get("format") == NATIVE_FORMAT + else parse_manifest(raw) + ) + return sorted(device.device_id for device in manifest.devices) + except (DatasetContractError, OSError, UnicodeError, json.JSONDecodeError): + return [] diff --git a/src/pt_snap_cli/core/native_dataset_contract.py b/src/pt_snap_cli/core/native_dataset_contract.py index 729269f..ffbd8bc 100644 --- a/src/pt_snap_cli/core/native_dataset_contract.py +++ b/src/pt_snap_cli/core/native_dataset_contract.py @@ -40,6 +40,13 @@ from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata from .sharded_replay_service import ShardedReplayResult +from .validation_budget import ( + ValidationBudget, + check_budget, + checked_rows, + sqlite_timeout, + validation_progress, +) NATIVE_FORMAT = "pt-snap-native-v2" NATIVE_MANIFEST_VERSION = 1 @@ -84,8 +91,11 @@ def _digest(value: object, loc: str) -> str: return result -def parse_native_manifest(value: object) -> NativeManifest: +def parse_native_manifest( + value: object, *, budget: ValidationBudget | None = None +) -> NativeManifest: + check_budget(budget) data = _object(value, "manifest") _need(data.get("format") == NATIVE_FORMAT, "format", "format", "unsupported native format") version = _integer(data.get("schemaVersion"), "schemaVersion", minimum=1) @@ -155,11 +165,11 @@ def parse_native_manifest(value: object) -> NativeManifest: ) omitted = tuple( _integer(v, "omittedDevices", maximum=2**31 - 1) - for v in _array(data.get("omittedDevices"), "omittedDevices") + for v in checked_rows(_array(data.get("omittedDevices"), "omittedDevices"), budget) ) _need(len(set(omitted)) == len(omitted), "omittedDevices", "duplicate", "duplicate device") devices: list[DeviceRecord] = [] - for key, raw in _object(data.get("devices"), "devices").items(): + for key, raw in checked_rows(_object(data.get("devices"), "devices").items(), budget): loc = f"devices.{key}" _need( isinstance(key, str) and re.fullmatch(r"0|[1-9][0-9]*", key) is not None, @@ -181,7 +191,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) ready = tuple( _integer(v, loc + ".readySlices") - for v in _array(device.get("readySlices"), loc + ".readySlices") + for v in checked_rows(_array(device.get("readySlices"), loc + ".readySlices"), budget) ) _need( ready == tuple(range(len(records))), @@ -191,7 +201,7 @@ def parse_native_manifest(value: object) -> NativeManifest: ) slices: list[SliceRecord] = [] position, last_id = 0, -1 - for index, raw_slice in enumerate(records): + for index, raw_slice in enumerate(checked_rows(records, budget)): item = _object(raw_slice, loc) _need( type(item.get("index")) is int @@ -257,7 +267,9 @@ def read_native_manifest(root: Path) -> NativeManifest: raise DatasetContractError("manifest.json", "read", str(exc)) from exc -def require_native_members(root: Path, manifest: NativeManifest) -> None: +def require_native_members( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> None: """Ownership recognition before any replacement: never adopt unknown files/dirs.""" _need( root.resolve() == root and not root.is_symlink(), @@ -267,7 +279,7 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: ) expected = {"manifest.json", *(f"device_{d.device_id}" for d in manifest.devices)} _need( - {p.name for p in root.iterdir()} == expected, + {p.name for p in checked_rows(root.iterdir(), budget)} == expected, "dataset", "members", "unexpected/missing dataset member", @@ -281,18 +293,22 @@ def require_native_members(root: Path, manifest: NativeManifest) -> None: "unsafe device directory", ) _need( - {p.name for p in directory.iterdir()} == {Path(s.file).name for s in device.slices}, + {p.name for p in checked_rows(directory.iterdir(), budget)} + == {Path(s.file).name for s in device.slices}, str(directory), "members", "unexpected/missing shard", ) - for item in device.slices: + for item in checked_rows(device.slices, budget): _safe_file(root, item.file) -def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetValidation: +def validate_native_dataset( + root: Path, manifest: NativeManifest, *, budget: ValidationBudget | None = None +) -> DatasetValidation: - require_native_members(root, manifest) + check_budget(budget) + require_native_members(root, manifest, budget=budget) expected_identity = cache_identity( manifest.metadata.source_sha256, manifest.metadata.requested_device, @@ -306,21 +322,32 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali ) # Check EVERY member before the FIRST SQLite open. No WAL/sidecars/repair. for device in manifest.devices: - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file _require_readonly_member(root, path) - _need(_hash_file(path) == item.sha256, item.file, "hash", "member content changed") + _need( + _hash_file(path, budget=budget) == item.sha256, + item.file, + "hash", + "member content changed", + ) boundaries = 0 for device in manifest.devices: ids: set[int] = set() lifetimes: dict[int, tuple[object, ...]] = {} - references: list[tuple[int, int]] = [] - for raw in device.slices: + for raw in checked_rows(device.slices, budget): item = cast(NativeSlice, raw) path = root / item.file try: - with closing(sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)) as conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as conn, + validation_progress(conn, budget), + ): conn.execute("PRAGMA query_only=ON") _need( conn.execute("PRAGMA quick_check").fetchall() == [("ok",)], @@ -388,19 +415,28 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "dictionary", "invalid native dictionary", ) - real = [ - r[0] - for r in conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0 ORDER BY id') - ] + count, low, high = conn.execute( + f'SELECT COUNT(*), MIN(id), MAX(id) FROM "{trace}" WHERE id>=0' + ).fetchone() _need( - len(real) == item.end_position - item.start_position + 1 - and (real[0], real[-1]) == (item.start_event_id, item.end_event_id) - and not ids.intersection(real), + (count, low, high) + == ( + item.end_position - item.start_position + 1, + item.start_event_id, + item.end_event_id, + ), item.file, "event_identity", "real IDs/count differ from positions/ranges", ) - ids.update(real) + # Manifest intervals are disjoint; avoid a per-shard list + # and intersection copy in addition to the device ID set. + ids.update( + r[0] + for r in checked_rows( + conn.execute(f'SELECT id FROM "{trace}" WHERE id>=0'), budget + ) + ) numeric = " OR ".join( f"typeof({name})!='integer'" for name, _ in TRACE_COLUMNS[:-1] ) @@ -425,17 +461,25 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali boundaries += conn.execute( f'SELECT COUNT(*) FROM "{trace}" WHERE id<0' ).fetchone()[0] - stacks = { - r[0]: r[1:] for r in conn.execute("SELECT * FROM pt_snap_block_reference") - } - blocks = conn.execute(f'SELECT * FROM "{block}"').fetchall() + mismatch = conn.execute( + f'SELECT b.id FROM "{block}" b WHERE NOT EXISTS ' + "(SELECT 1 FROM pt_snap_block_reference r WHERE r.blockId=b.id) " + "UNION ALL SELECT r.blockId FROM pt_snap_block_reference r " + f'WHERE NOT EXISTS (SELECT 1 FROM "{block}" b WHERE b.id=r.blockId) ' + "LIMIT 1" + ).fetchone() _need( - set(stacks) == {r[0] for r in blocks}, + mismatch is None, item.file, "extension", "reference/block membership differs", ) - for row in blocks: + rows = conn.execute( + f"SELECT b.*, r.stream, r.allocCallstack, r.freeCallstack " + f'FROM "{block}" b JOIN pt_snap_block_reference r ON r.blockId=b.id' + ) + for joined in checked_rows(rows, budget): + row, ref = joined[:7], joined[7:] bid, address, size, requested, state, alloc, free = row _need( all(type(v) is int for v in row) @@ -448,7 +492,6 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "block_identity", "unfinalized/invalid block", ) - ref = stacks[bid] _need( type(ref[0]) is int and all(v is None or isinstance(v, str) for v in ref[1:]), @@ -464,13 +507,17 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali "conflicting finalized identity/reference", ) lifetimes[bid] = signature - references.append((alloc, free)) except sqlite3.DatabaseError as exc: raise DatasetContractError(item.file, "sqlite", str(exc)) from exc try: - with closing( - sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) - ) as metadata_conn: + with ( + closing( + sqlite3.connect( + path.as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as metadata_conn, + validation_progress(metadata_conn, budget), + ): metadata_conn.row_factory = sqlite3.Row _need( metadata_conn.execute( @@ -501,14 +548,18 @@ def validate_native_dataset(root: Path, manifest: NativeManifest) -> DatasetVali raise DatasetContractError(item.file, "metadata", str(exc)) from exc _require_readonly_member(root, path) _need( - _hash_file(path) == item.sha256, + _hash_file(path, budget=budget) == item.sha256, item.file, "hash", "member changed during validation", ) _need( len(ids) == device.event_count - and all(v == -1 or v in ids for pair in references for v in pair), + and all( + v == -1 or v in ids + for lifetime in checked_rows(lifetimes.values(), budget) + for v in lifetime[3:5] + ), "devices", "event_identity", "count/lifetime references not in original device scope", diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 138e342..c02d6ed 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -233,7 +233,7 @@ def execute_query( if resolved.db_path is None: raise FocusNotConfiguredError("No database path specified and no database configured.") - dataset = DatasetResolver().inspect(resolved.db_path) + dataset = DatasetResolver().inspect(resolved.db_path, budget=budget) budget.remaining() scope: dict[str, object] | None = None query_params = params or {} diff --git a/src/pt_snap_cli/core/validation_budget.py b/src/pt_snap_cli/core/validation_budget.py new file mode 100644 index 0000000..ec60cf0 --- /dev/null +++ b/src/pt_snap_cli/core/validation_budget.py @@ -0,0 +1,79 @@ +"""Cancellation helpers shared by dataset validation and query execution. + +The small protocol avoids a dependency on the query/source resolver. Validators +can also be used at publication time without a query deadline. +""" + +from __future__ import annotations + +import math +import sqlite3 +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from typing import Protocol, TypeVar + +from .errors import QueryTimeoutError + + +class ValidationBudget(Protocol): + def remaining(self) -> float | None: ... + + +def check_budget(budget: ValidationBudget | None) -> None: + if budget is not None: + budget.remaining() + + +def sqlite_timeout(budget: ValidationBudget | None) -> float: + """SQLite's busy wait must not outlive the remaining shared deadline.""" + remaining = budget.remaining() if budget is not None else None + # SQLite truncates to whole milliseconds. Round up so an exhausted busy + # wait is classified as timeout, not a spurious schema/locked error just + # before the deadline (the additional wait is at most one millisecond). + return 5.0 if remaining is None else min(5.0, math.ceil(remaining * 1000) / 1000) + + +_T = TypeVar("_T") + + +def checked_rows(rows: Iterable[_T], budget: ValidationBudget | None) -> Iterator[_T]: + """Bound Python-side work between deadline checks, including cursor iteration.""" + check_budget(budget) + for index, row in enumerate(rows): + if index % 256 == 0: + check_budget(budget) + yield row + check_budget(budget) + + +@contextmanager +def validation_progress( + conn: sqlite3.Connection, budget: ValidationBudget | None +) -> Iterator[None]: + """Interrupt SQLite and preserve QUERY_TIMEOUT rather than a schema error.""" + check_budget(budget) + timed_out: QueryTimeoutError | None = None + + def progress() -> int: + nonlocal timed_out + try: + check_budget(budget) + except QueryTimeoutError as exc: + timed_out = exc + return 1 + return 0 + + if budget is not None: + conn.set_progress_handler(progress, 1000) + try: + try: + yield + except sqlite3.DatabaseError: + if timed_out is not None: + raise timed_out from None + check_budget(budget) + raise + check_budget(budget) + finally: + if budget is not None: + conn.set_progress_handler(None, 0) diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py new file mode 100644 index 0000000..af0287a --- /dev/null +++ b/tests/core/test_dataset_validation_cache.py @@ -0,0 +1,409 @@ +"""Work-count and cancellation regressions for dataset admission (issue #219).""" + +import json +import os +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.completion import complete_device_ids +from pt_snap_cli.core import dataset_contract, dataset_resolver, native_dataset_contract +from pt_snap_cli.core.dataset_contract import validate_dataset +from pt_snap_cli.core.dataset_files import hash_file +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.validation_budget import validation_progress +from tests.test_dataset_focus import make_dataset +from tests.test_native_dataset import dataset as native_dataset + + +@pytest.fixture(autouse=True) +def clear_generations(monkeypatch): + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +@pytest.fixture(params=[False, True], ids=["compatibility", "native"]) +def artifact(tmp_path, request): + return native_dataset(tmp_path) if request.param else make_dataset(tmp_path / "dataset") + + +def forbidden(*args, **kwargs): + pytest.fail("warm resolution must not hash or validate shard contents") + + +def test_new_resolvers_reuse_verified_generation_without_hash_or_sql(artifact, monkeypatch): + first = DatasetResolver().inspect(artifact) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(native_dataset_contract, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + second = DatasetResolver().inspect(artifact / "manifest.json") + assert second == first and second is not first + second.require_unchanged() + + +@pytest.mark.parametrize("change", ["touch", "replace", "manifest"]) +def test_metadata_change_always_revalidates(artifact, monkeypatch, change): + first = DatasetResolver().inspect(artifact) + path = artifact / "manifest.json" if change == "manifest" else next(artifact.glob("*/*.db")) + if change == "replace": + replacement = path.with_suffix(".replacement") + replacement.write_bytes(path.read_bytes()) + replacement.replace(path) + else: + stat = path.stat() + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000)) + calls = [] + real = dataset_resolver._hash_file + + def tracked(*args, **kwargs): + calls.append(args[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + second = DatasetResolver().inspect(artifact) + assert first.fingerprint == second.fingerprint + assert artifact / "manifest.json" in calls + with pytest.raises(DatabaseSchemaError, match="changed during operation"): + first.require_unchanged() + + +def test_restored_mtime_size_does_not_hide_content_tampering(artifact): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + before = member.stat() + device = member.parent.name.removeprefix("device_") + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute(f"UPDATE trace_entry_{device} SET action=100 WHERE id>=0") + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + after = member.stat() + assert (before.st_ino, before.st_size, before.st_mtime_ns) == ( + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + assert before.st_ctime_ns != after.st_ctime_ns + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +@pytest.mark.parametrize("change", ["wal", "journal", "shm", "symlink", "manifest-symlink"]) +def test_cached_generation_still_rejects_sidecars_and_aliases(artifact, tmp_path, change): + DatasetResolver().inspect(artifact) + member = next(artifact.glob("*/*.db")) + if change.endswith("symlink"): + path = artifact / "manifest.json" if change == "manifest-symlink" else member + saved = tmp_path / "saved" + path.rename(saved) + path.symlink_to(saved) + else: + Path(str(member) + "-" + change).touch() + with pytest.raises(DatabaseSchemaError): + DatasetResolver().inspect(artifact) + + +def test_native_hashes_twice_on_admission_not_three_times(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + calls = [] + real = hash_file + + def tracked(path, **kwargs): + calls.append(path) + return real(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + monkeypatch.setattr(native_dataset_contract, "_hash_file", tracked) + DatasetResolver().inspect(root) + for member in root.glob("*/*.db"): + assert calls.count(member) == 2 + assert calls.count(root / "manifest.json") == 2 + + +def test_cached_manifest_cannot_be_mutated_by_a_previous_caller(tmp_path): + root = native_dataset(tmp_path) + first = DatasetResolver().inspect(root) + first.validation.manifest.identity["eventsPerSlice"] = "corrupted by caller" + second = DatasetResolver().inspect(root) + assert second.validation.manifest.identity["eventsPerSlice"] == 2 + + +def test_cache_is_bounded_and_untrusted_change_tokens_disable_reuse(tmp_path, monkeypatch): + monkeypatch.setattr(dataset_resolver, "_CACHE_LIMIT", 2) + for i in range(3): + DatasetResolver().inspect(make_dataset(tmp_path / str(i), devices=(0,), slices=1)) + assert len(dataset_resolver._VALIDATED) == 2 + root = make_dataset(tmp_path / "uncached", devices=(0,), slices=1) + monkeypatch.setattr(dataset_resolver, "_cacheable", lambda generation: False) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_completion_only_reads_bounded_manifest(artifact, monkeypatch): + monkeypatch.setenv("PT_SNAP_DB_PATH", str(artifact)) + expected = sorted(json.loads((artifact / "manifest.json").read_text())["devices"], key=int) + monkeypatch.setattr(dataset_resolver, "_hash_file", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_dataset", forbidden) + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", forbidden) + monkeypatch.setattr(sqlite3, "connect", forbidden) + assert complete_device_ids() == expected + # Completion candidates do not claim content integrity. + next(artifact.glob("*/*.db")).write_bytes(b"invalid member") + assert complete_device_ids() == expected + (artifact / "manifest.json").write_bytes(b" " * (1024 * 1024 + 1)) + assert complete_device_ids() == [] + + +class CancellableBudget: + def __init__(self): + self.cancelled = False + self.calls = 0 + + def remaining(self): + self.calls += 1 + if self.cancelled: + raise QueryTimeoutError("Query timed out after 0.01 seconds") + return 1.0 + + +def test_hash_checks_between_chunks_and_closes_file(tmp_path, monkeypatch): + path = tmp_path / "large" + path.write_bytes(b"x" * (4 * 1024 * 1024)) + budget = CancellableBudget() + original = budget.remaining + opened = [] + real_open = Path.open + + def tracked_open(self, *args, **kwargs): + result = real_open(self, *args, **kwargs) + opened.append(result) + return result + + def remaining(): + budget.cancelled = budget.calls >= 2 + return original() + + monkeypatch.setattr(Path, "open", tracked_open) + monkeypatch.setattr(budget, "remaining", remaining) + with pytest.raises(QueryTimeoutError): + hash_file(path, budget=budget) + assert budget.calls == 3 + assert len(opened) == 1 and opened[0].closed + assert hash_file(path) # Failure did not retain resources. + + +def test_sql_progress_translates_timeout_and_clears_handler(): + budget = CancellableBudget() + with closing(sqlite3.connect(":memory:")) as conn: + with pytest.raises(QueryTimeoutError): + with validation_progress(conn, budget): + budget.cancelled = True + conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<10000) " + "SELECT sum(x) FROM n" + ).fetchone() + # Still cancelled: this would interrupt if the handler leaked. + assert conn.execute( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) " + "SELECT sum(x) FROM n" + ).fetchone() == (500500,) + + +def large_compatibility(root, rows=1000): + root = make_dataset(root, devices=(0,), slices=1) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,100,8,0,8,8,16,'stack')", + ((i,) for i in range(rows)), + ) + conn.executemany( + "INSERT INTO block_0 VALUES (?,100,8,8,1,?,-1)", ((i, i) for i in range(rows)) + ) + path = root / "manifest.json" + data = json.loads(path.read_text()) + data["eventsPerSlice"] = rows + data["devices"]["0"]["eventCount"] = rows + data["devices"]["0"]["slices"][0]["endEventId"] = rows - 1 + path.write_text(json.dumps(data)) + return root + + +def test_python_validation_loop_cancels_before_finishing_and_closes_connection( + tmp_path, monkeypatch +): + root = large_compatibility(tmp_path / "dataset") + budget = CancellableBudget() + checked = [] + connections = [] + real_integer = dataset_contract._integer + real_connect = sqlite3.connect + + def integer(value, location, *args, **kwargs): + if ".block_0." in location: + checked.append(location) + budget.cancelled = True + return real_integer(value, location, *args, **kwargs) + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + connections.append(conn) + return conn + + monkeypatch.setattr(dataset_contract, "_integer", integer) + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + validate_dataset(root, budget=budget) + assert 0 < len(checked) <= 256 * 7 + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(dataset_contract, "_integer", real_integer) + assert validate_dataset(root).real_event_count == 1000 + + +def test_native_quick_check_can_be_interrupted_and_resolver_recovers(tmp_path, monkeypatch): + root = native_dataset(tmp_path) + budget = CancellableBudget() + connections = [] + real_connect = sqlite3.connect + + def connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + conn.set_trace_callback( + lambda sql: setattr(budget, "cancelled", True) if sql == "PRAGMA quick_check" else None + ) + connections.append(conn) + return conn + + monkeypatch.setattr(sqlite3, "connect", connect) + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=budget) + assert root not in dataset_resolver._VALIDATED + for conn in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + conn.execute("SELECT 1") + monkeypatch.setattr(sqlite3, "connect", real_connect) + assert DatasetResolver().inspect(root) is not None + + +def test_cold_query_passes_shared_deadline_and_cli_keeps_timeout_code(tmp_path, monkeypatch): + from typer.testing import CliRunner + + from pt_snap_cli.cli import app + from pt_snap_cli.core.query_service import QueryService + + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + original = dataset_resolver._hash_file + received = [] + + def expire(path, *, budget=None): + assert budget is not None + received.append(budget) + budget.started -= 20 + return original(path, budget=budget) + + monkeypatch.setattr(dataset_resolver, "_hash_file", expire) + service = QueryService() + try: + with pytest.raises(QueryTimeoutError): + service.execute_query("event", {"id": 0}, db_path=root, timeout_s=10) + finally: + service.close() + result = CliRunner().invoke( + app, ["query", str(root), "--template-use", "event", "--timeout", "10", "--json"] + ) + assert result.exit_code == 1 + assert json.loads(result.stderr)["error"]["code"] == "QUERY_TIMEOUT" + assert len(received) == 2 + assert root not in dataset_resolver._VALIDATED + monkeypatch.setattr(dataset_resolver, "_hash_file", original) + service = QueryService() + try: + assert service.execute_query("event", {"id": 0}, db_path=root).rows[0]["id"] == 0 + finally: + service.close() + + +def test_native_lock_wait_respects_deadline(tmp_path): + import time + + from pt_snap_cli.core.dataset_sources import QueryBudget + + root = native_dataset(tmp_path) + member = sorted(root.glob("*/*.db"))[0] + with closing(sqlite3.connect(member)) as writer: + writer.execute("BEGIN EXCLUSIVE") + start = time.monotonic() + with pytest.raises(QueryTimeoutError): + DatasetResolver().inspect(root, budget=QueryBudget(0.02, start)) + assert time.monotonic() - start < 1.0 + writer.rollback() + assert DatasetResolver().inspect(root) is not None + + +@pytest.mark.parametrize("mode", [None, "off", "unknown"]) +def test_default_and_unknown_cache_mode_repeat_strong_validation(tmp_path, monkeypatch, mode): + if mode is None: + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + else: + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", mode) + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + calls = [] + original = dataset_resolver._hash_file + + def tracked(path, **kwargs): + calls.append(path) + return original(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", tracked) + first = DatasetResolver().inspect(root) + second = DatasetResolver().inspect(root) + assert first.fingerprint == second.fingerprint + assert calls.count(root / "device_0/slice_00000.db") == 2 + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_hashes_even_when_filesystem_tokens_look_unchanged( + tmp_path, monkeypatch +): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + resolved.require_unchanged() + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack='changed'") + monkeypatch.setattr(dataset_resolver, "_generation", lambda *args: resolved.generation) + with pytest.raises(DatabaseSchemaError, match="content changed"): + resolved.require_unchanged() + + +def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + DatasetResolver().inspect(root) + assert root in dataset_resolver._VALIDATED + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + with pytest.raises(QueryTimeoutError): + budget = CancellableBudget() + budget.cancelled = True + DatasetResolver().inspect(root, budget=budget) + DatasetResolver().inspect(root) + assert root not in dataset_resolver._VALIDATED + + +def test_strong_operation_guard_normalizes_hash_io_failure(tmp_path, monkeypatch): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + + def fail(*args, **kwargs): + raise PermissionError("member became unreadable") + + monkeypatch.setattr(dataset_resolver, "_hash_file", fail) + with pytest.raises(DatabaseSchemaError, match="member became unreadable"): + resolved.require_unchanged() diff --git a/tests/test_dataset_focus.py b/tests/test_dataset_focus.py index 8b525f4..2c42d67 100644 --- a/tests/test_dataset_focus.py +++ b/tests/test_dataset_focus.py @@ -482,8 +482,8 @@ def test_manifest_change_during_validation_fails(tmp_path, monkeypatch): root = make_dataset(tmp_path / "dataset") original = module.validate_dataset - def validate(path): - result = original(path) + def validate(path, **kwargs): + result = original(path, **kwargs) manifest = root / "manifest.json" data = json.loads(manifest.read_text()) data["cacheHash"] = "replaced" From 98fac137d26f99872252de5a3877b04d25202f56 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:31:09 +0800 Subject: [PATCH 5/8] fix: push bounded dataset-global queries into SQLite (#218) --- docs/en/querying.md | 20 +- docs/zh/querying.md | 15 +- src/pt_snap_cli/core/dataset_global.py | 404 +++++++++++------- src/pt_snap_cli/core/dataset_lifecycle_sql.py | 273 ++++++++++++ tests/core/test_dataset_source_reuse.py | 6 +- tests/core/test_dataset_sql_pushdown.py | 291 +++++++++++++ 6 files changed, 857 insertions(+), 152 deletions(-) create mode 100644 src/pt_snap_cli/core/dataset_lifecycle_sql.py create mode 100644 tests/core/test_dataset_sql_pushdown.py diff --git a/docs/en/querying.md b/docs/en/querying.md index 60d92ed..60c6cc7 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -151,11 +151,29 @@ Context LRU. One diminishing deadline includes all shards, sources, grouping and totals; reports explicitly share **one** budget across their three queries. Cumulative fetched/output work is limited to **100000 rows / 64 MiB serialized values**, including repeated source reads. Exceeding either fails the entire operation, never an alleged -complete partial result. Materialization/sort are bounded by this work ceiling, **not** +complete partial result. Python materialization/sort are bounded by this work ceiling, **not** by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual cells have overhead. Manifest hashing/validation remains proportional to artifact size and phase-checked, not hard-preempted. No temporary merge DB, repair or artifact write. +Global peak queries fetch scalar range evidence and at most three counter rows per +shard. Event/allocation pages apply SQL filters and ordering per shard, merge at most +`offset + limit` candidates per shard, and count matches separately. Range evidence +is measured before non-range filters. Ordinary text-stack statistics merge full +per-shard groups before global thresholds; averages use summed sizes and counts. +Canonical ordered-frame statistics retain the bounded source-reading path because +formatted text is not their identity. Block/leak/lifetime reductions use read-only +attached shards for proof, conflicts and latest observations before pagination; +when the device exceeds SQLite's attachment limit (normally ten), they retain the +budgeted source-reading path. No intermediate merged database is created. + +A million-event dataset can therefore answer aggregates and small pages within the +default work budget. Use `-n 5` or an explicit finite `limit` for large row-returning +queries. Unlimited output, large offsets, many distinct stack groups, ordered-frame +reads or the attachment-limit fallback can still exceed the budget and fail honestly. +The limit counts rows fetched into Python and serialized output, not internal SQL +scans, SQLite sorting space or total process memory. + ## Dataset point-event attribution A complete compatibility-v1 or native-v2 directory/manifest supports `event` diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 69c243e..a504c35 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -130,10 +130,23 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 数据集读取先分离批次再查外片来源,使用借用的有界 Context LRU。所有片、来源、分组、 精确总数共用递减 deadline;报告三条查询明确共用**一个**预算。累计取回/输出工作上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 -结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ +结论。Python 物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ SQLite、一个批次及单个 cell 有额外开销。manifest hash/校验仍与产物大小成正比且仅在 阶段边界检查,不硬抢占。不生成合并临时 DB、不修复或写产物。 +全局峰值查询每片只取范围汇总及最多三条计数器记录。event/allocation 在各片 SQL 中筛选、 +排序,每片最多取 `offset + limit` 个候选后全局归并,并单独统计匹配总数;实际范围证据在 +非范围筛选前计算。普通文本栈统计先合并各片完整分组,再应用全局阈值,平均值由总大小 +与总计数计算。规范化原始有序帧的身份不能用格式化文本替代,因此仍走受预算约束的来源 +读取路径。block/leak/lifetime 通过只读附加分片先证明身份、检测冲突、选最新观察,再分页; +设备分片数超出 SQLite 的附加上限(通常十片)时,回退到原有受预算约束的来源读取路径。 +不创建中间合并数据库。 + +因此,百万事件数据集的聚合与小页面可在默认工作预算内完成。大量逐行结果请指定 `-n 5` +或有限 `limit`。无限制输出、大 offset、大量不同栈分组、有序帧读取及超过附加上限的回退 +路径仍可能超预算,并明确失败。预算统计进入 Python 的实际取回行和序列化输出,不限制 +SQL 内部扫描、SQLite 排序空间或整个进程内存。 + ## 数据集定点事件归因 完整兼容 v1 或原生 v2 目录/manifest 支持 `event` 寻址,以及接受真实 `event_id` 的 diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 857cfa8..43ec456 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -1,8 +1,9 @@ -"""Explicit built-in global algorithms; no local caps or temporary merge database. +"""Exact built-in global reductions, without a temporary merge database. -Fetched work is cumulatively bounded by QueryBudget, including sources and final -output. Sorting/materialization is O(bounded work), NOT O(max_rows). A budget -failure rejects the operation rather than returning an alleged global answer. +Shard SQL reduces counters, groups, and candidate windows before Python merging. +Fetched work is cumulatively bounded by QueryBudget, including source hydration +and final output. Unlimited/high-cardinality results can still exhaust the budget; +a failure rejects the operation rather than returning an alleged global answer. """ from __future__ import annotations @@ -11,6 +12,7 @@ from typing import cast from pt_snap_cli.core.dataset_contract import QueryScope, SliceRecord +from pt_snap_cli.core.dataset_lifecycle_sql import lifecycle_query from pt_snap_cli.core.dataset_sources import DatasetSourceResolver from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError from pt_snap_cli.core.models import QueryResult @@ -68,47 +70,50 @@ def _scope( } -def _traces( - sources: DatasetSourceResolver, - low: int, - high: int, - items: list[SliceRecord], - columns: tuple[str, ...] = ("id", *METRICS), - *, - with_sources: bool = False, -): - projection = ", ".join(f't."{column}"' for column in columns) +def _range_statistics(sources, low, high, items, scope, predicate="1", values=None): + """Range evidence is deliberately collected BEFORE non-range filters.""" + count = matching = 0 + first = last = None for item in items: - trace = f'"trace_entry_{sources.device.device_id}"' - if with_sources: - projection = ( - "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" - ) - if sources.dataset.callstack_layout == "v2": - sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" - else: - sql = f"SELECT {projection}, t.callstack FROM {trace} t" - else: - sql = f"SELECT {projection} FROM {trace} t" - rows = sources.read( + row = sources.read( sources.dataset.root / item.file, - sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", - [low, high], - ) - yield item, rows + f"SELECT MIN(id) AS first, MAX(id) AS last, COUNT(*) AS count, " + f"COALESCE(SUM(CASE WHEN {predicate} THEN 1 ELSE 0 END), 0) AS matching " + f'FROM "trace_entry_{sources.device.device_id}" ' + "WHERE id>=0 AND id>=? AND id<=?", + [*(values or []), low, high], + )[0] + count += row["count"] + matching += row["matching"] + if row["first"] is not None: + first = row["first"] if first is None else min(first, row["first"]) + last = row["last"] if last is None else max(last, row["last"]) + scope["actual_range"] = { + "first_event_id": first, + "last_event_id": last, + "real_event_count": count, + } + return matching def _peaks(sources, low, high, items, template, scope): + _range_statistics(sources, low, high, items, scope) chosen: dict[str, dict[str, object]] = {} - count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): - for row in rows: - sources.budget.remaining() - event = cast(int, row["id"]) - count += 1 - first = event if first is None else min(first, event) - last = event if last is None else max(last, event) - for metric in METRICS: + # These index-friendly queries fetch at most three narrow rows per shard. + # NULL counters never nominate an event, matching MAX + equality semantics. + for item in items: + for metric in METRICS: + rows = sources.read( + sources.dataset.root / item.file, + f"SELECT id, allocated, active, reserved " + f'FROM "trace_entry_{sources.device.device_id}" ' + f'WHERE id>=0 AND id>=? AND id<=? AND "{metric}" IS NOT NULL ' + f'ORDER BY "{metric}" DESC, id ASC LIMIT 1', + [low, high], + ) + if rows: + row = rows[0] + event = cast(int, row["id"]) if metric not in chosen or (cast(int, row[metric]), -event) > ( cast(int, chosen[metric][metric]), -cast(int, chosen[metric]["id"]), @@ -125,7 +130,9 @@ def _peaks(sources, low, high, items, template, scope): result[f"{other}_at_{metric}_peak"] = row.get(other) for other in ("active", "allocated"): result[f"reserved_{other}_gap_at_{metric}_peak"] = ( - cast(int, row["reserved"]) - cast(int, row[other]) if row else None + cast(int, row["reserved"]) - cast(int, row[other]) + if row and row["reserved"] is not None and row[other] is not None + else None ) if template == "allocator_gap": ids = [result[f"peak_{m}_event_id"] for m in METRICS] @@ -135,17 +142,31 @@ def _peaks(sources, low, high, items, template, scope): result[f"peak_{a}_event_id"] is not None and result[f"peak_{a}_event_id"] == result[f"peak_{b}_event_id"] ) - scope["actual_range"] = { - "first_event_id": first, - "last_event_id": last, - "real_event_count": count, - } scope["peak_owner_slices"] = { m: sources.owner(cast(int, r["id"])).index for m, r in chosen.items() } return [result] +def _predicates(params, fields, alias=""): + clauses, values = [], [] + for field in fields: + for prefix, operator in (("", "="), ("min_", ">="), ("max_", "<=")): + value = params.get(prefix + field) + if value is not None: + clauses.append(f'{alias}"{field}" {operator} ?') + values.append(value) + return " AND ".join(clauses) or "1", values + + +def _window(params, max_rows): + offset = max(0, cast(int, params.get("offset", 0))) + limit = cast(int, params.get("limit", -1)) + if max_rows is not None and max_rows > 0: + limit = min(limit, max_rows) if limit >= 0 else max_rows + return offset, limit + + def _matches(row: dict[str, object], params: dict[str, object], fields: tuple[str, ...]) -> bool: for field in fields: value = params.get(field) @@ -166,36 +187,45 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: rows.sort(key=lambda r: (r[key] is not None, r[key]), reverse=descending) -def _events(sources, low, high, items, params, template, scope): +def _events(sources, low, high, items, params, template, scope, candidate_limit): + fields = METRICS + if template == "event": + fields += ("id", "action", "address", "size", "stream") + predicate, values = _predicates(params, fields) + total = _range_statistics(sources, low, high, items, scope, predicate, values) rows = [] - first, last, count = None, None, 0 + order = params["order_by"] + direction = params["order_dir"] columns = ("id", *METRICS) if template == "event": - columns += ("action", "address", "size", "stream") - with_sources = template == "event" and params["order_by"] == "callstack" - for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): - if with_sources: - sources.events_from_rows(_item, batch, with_frames=False) - for row in batch: - sources.budget.remaining() - event = cast(int, row["id"]) - first = event if first is None else min(first, event) - last = event if last is None else max(last, event) - count += 1 - fields = ("allocated", "active", "reserved") - if template == "event": - fields += ("id", "action", "address", "size", "stream") - if _matches(row, params, fields): - rows.append(row) - scope["actual_range"] = { - "first_event_id": first, - "last_event_id": last, - "real_event_count": count, - } - # callstack sorting already has joined text; resolving sources here would - # eagerly fetch ordered frames that only the returned page needs. - _sort(rows, params["order_by"], params["order_dir"] == "DESC") - return rows + columns = ("id", "action", "address", "size", "stream", *METRICS) + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + projection = ", ".join(f't."{column}"' for column in columns) + join = "" + if order == "callstack": + if sources.dataset.callstack_layout == "v2": + projection += ", t.callstackId, cs.callstack" + join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + projection += ", t.callstack" + row_predicate, row_values = _predicates(params, fields, "t.") + # Dataset text identity/order is exact, independent of source collation. + sort_column = '"callstack" COLLATE BINARY' if order == "callstack" else f'"{order}"' + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {row_predicate} " + f"ORDER BY {sort_column} {direction}, t.id {direction} LIMIT ?", + [low, high, *row_values, candidate_limit], + ) + if order == "callstack": + # Full candidate text has already crossed the budget boundary. Reuse + # its event data; ordered frames are needed only for the final page. + sources.events_from_rows(item, batch, with_frames=False) + rows.extend(batch) + _sort(rows, order, direction == "DESC") + return rows, total def _lifecycles(sources, items, scope): @@ -305,11 +335,103 @@ def _lifecycles(sources, items, scope): return rows +def _text_stacks(sources, low, high, items, params, scope): + """Merge shard sufficient statistics, never local HAVING/ranking windows.""" + grouped = {} + events = 0 + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if sources.dataset.callstack_layout == "v2": + table = f"{trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + text, present = "cs.callstack", "t.callstackId IS NOT NULL" + else: + table, text, present = f"{trace} t", "t.callstack", "t.callstack IS NOT NULL" + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {text} AS callstack, COUNT(*) AS alloc_count, SUM(t.size) AS total_size, " + f"MAX(t.size) AS max_size, MIN(t.id) AS stack_event_id FROM {table} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {present} GROUP BY {text} COLLATE BINARY", + [low, high], + ) + for local in batch: + sources.budget.remaining() + value = local["callstack"] + identity = ( + f"{sources.namespace}:text:sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" + if isinstance(value, str) + else f"{sources.namespace}:category:missing" + ) + kind = "captured" if isinstance(value, str) and value != "" else "missing" + row = grouped.get(identity) + if row is None: + row = dict(local) + row.update( + { + "callstack": value if value is not None else "[missing callstack]", + "source_stack_id": identity, + "stack_kind": kind, + "text_kind": kind, + "frames_status": "text_only", + } + ) + grouped[identity] = row + else: + row["alloc_count"] += local["alloc_count"] + row["total_size"] += local["total_size"] + row["max_size"] = max(row["max_size"], local["max_size"]) + row["stack_event_id"] = min(row["stack_event_id"], local["stack_event_id"]) + events += local["alloc_count"] + return _finish_stacks(sources, grouped, params, scope, events, 0) + + +def _finish_stacks(sources, grouped, params, scope, events, ordered): + rows = [ + r + for r in grouped.values() + if r["alloc_count"] >= params["min_count"] and r["total_size"] >= params["min_size"] + ] + for row in rows: + row["avg_size"] = row["total_size"] / row["alloc_count"] + rows.sort( + key=lambda r: ( + -r["total_size"], + r["callstack"] or "", + r["stack_kind"] == "missing", + r["source_stack_id"], + ) + ) + scope["source_coverage"] = { + "range_complete": True, + "coverage_scope": "all_real_stack_bearing_actions_before_global_thresholds", + "stack_events": events, + "ordered_frame_events": ordered, + "source_queries": sources.query_count, + } + return rows + + def _stacks(sources, low, high, items, params, scope): + if sources.dataset.ordered_frames_version != 1: + return _text_stacks(sources, low, high, items, params, scope) grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, with_sources=True): - refs = sources.events_from_rows(_item, batch) + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + columns = ("id", "action", "address", "size", "stream", *METRICS) + projection = ", ".join(f't."{column}"' for column in columns) + join = "" + if sources.dataset.callstack_layout == "v2": + projection += ", t.callstackId, cs.callstack" + join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + projection += ", t.callstack" + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", + [low, high], + ) + refs = sources.events_from_rows(item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") @@ -347,29 +469,7 @@ def _stacks(sources, low, high, items, params, scope): row["text_kind"] = "captured" events += 1 ordered += source.frames is not None - rows = [ - r - for r in grouped.values() - if r["alloc_count"] >= params["min_count"] and r["total_size"] >= params["min_size"] - ] - for row in rows: - row["avg_size"] = row["total_size"] / row["alloc_count"] - rows.sort( - key=lambda r: ( - -r["total_size"], - r["callstack"] or "", - r["stack_kind"] == "missing", - r["source_stack_id"], - ) - ) - scope["source_coverage"] = { - "range_complete": True, - "coverage_scope": "all_real_stack_bearing_actions_before_global_thresholds", - "stack_events": events, - "ordered_frame_events": ordered, - "source_queries": sources.query_count, - } - return rows + return _finish_stacks(sources, grouped, params, scope, events, ordered) def global_query( @@ -386,10 +486,15 @@ def global_query( ) low, high, items = _range(sources, range_params, slice_index) scope = _scope(sources, low, high, items) + offset, limit = _window(params, max_rows) + candidate_limit = offset + limit if limit >= 0 else -1 + total_override = None if template in ("memory_peak", "allocator_gap"): rows = _peaks(sources, low, high, items, template, scope) elif template in ("event", "allocation"): - rows = _events(sources, low, high, items, params, template, scope) + rows, total_override = _events( + sources, low, high, items, params, template, scope, candidate_limit + ) elif template == "callstack_analysis": rows = _stacks(sources, low, high, items, params, scope) elif template == "preexisting_live": @@ -407,56 +512,61 @@ def global_query( raise QueryExecutionError( "Dataset leak_detection requires terminal dataset scope, not --slice; select an explicit member for local observations." ) - rows = _lifecycles(sources, items, scope) - if template == "block": - rows = [ - r - for r in rows - if _matches( - r, - params, - ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"), - ) - ] - _sort(rows, cast(str, params["order_by"]), params["order_dir"] == "DESC") - elif template == "leak_detection": - rows = [ - r - for r in rows - if r["terminal_survivor"] - and r["allocation_source"] is not None - and cast(int, r["size"]) >= cast(int, params["min_size"]) - ] - _sort(rows, "size", True) - elif template == "freed_block_lifetime": - buckets = {} - for row in rows: - if row["allocation_source"] is None or row["free_source"] is None: - continue - distance = cast(int, row["freeEventId"]) - cast(int, row["allocEventId"]) - if distance < 0: - continue - index = next( - (i for i, cap in enumerate((1000, 5000, 20000, 100000)) if distance < cap), 4 - ) - bucket = buckets.setdefault( - index, - { - "lifetime_events": ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[index], - "block_count": 0, - "size_bytes": 0, - }, - ) - bucket["block_count"] += 1 - bucket["size_bytes"] += row["size"] - rows = [buckets[i] for i in sorted(buckets)] + pushed = lifecycle_query(sources, items, template, params, scope, candidate_limit) + if pushed is not None: + rows, total_override = pushed else: - raise QueryExecutionError("No dataset-global merge implementation for this template.") - total = len(rows) - offset = max(0, cast(int, params.get("offset", 0))) - limit = cast(int, params.get("limit", -1)) - if max_rows is not None and max_rows > 0: - limit = min(limit, max_rows) if limit >= 0 else max_rows + rows = _lifecycles(sources, items, scope) + if template == "block": + rows = [ + r + for r in rows + if _matches( + r, + params, + ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"), + ) + ] + _sort(rows, cast(str, params["order_by"]), params["order_dir"] == "DESC") + elif template == "leak_detection": + rows = [ + r + for r in rows + if r["terminal_survivor"] + and r["allocation_source"] is not None + and cast(int, r["size"]) >= cast(int, params["min_size"]) + ] + _sort(rows, "size", True) + elif template == "freed_block_lifetime": + buckets = {} + for row in rows: + if row["allocation_source"] is None or row["free_source"] is None: + continue + distance = cast(int, row["freeEventId"]) - cast(int, row["allocEventId"]) + if distance < 0: + continue + index = next( + (i for i, cap in enumerate((1000, 5000, 20000, 100000)) if distance < cap), + 4, + ) + bucket = buckets.setdefault( + index, + { + "lifetime_events": ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[ + index + ], + "block_count": 0, + "size_bytes": 0, + }, + ) + bucket["block_count"] += 1 + bucket["size_bytes"] += row["size"] + rows = [buckets[i] for i in sorted(buckets)] + else: + raise QueryExecutionError( + "No dataset-global merge implementation for this template." + ) + total = len(rows) if total_override is None else total_override selected = rows[offset : offset + limit] if limit >= 0 else rows[offset:] if template == "event": refs = sources.events(cast(int, r["id"]) for r in selected) diff --git a/src/pt_snap_cli/core/dataset_lifecycle_sql.py b/src/pt_snap_cli/core/dataset_lifecycle_sql.py new file mode 100644 index 0000000..9caf2e1 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_lifecycle_sql.py @@ -0,0 +1,273 @@ +"""Read-only relational lifecycle reduction over a bounded set of attached shards. + +No rows are copied to a merge database. SQLite performs proof, conflict checks, +latest-observation selection and page ranking; only aggregate/page rows cross the +Python boundary. Larger shard sets keep the budgeted general implementation. +""" + +from __future__ import annotations + +import math +import sqlite3 +import time +from contextlib import closing +from typing import cast + +from pt_snap_cli.core.dataset_files import require_readonly_member +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.errors import QueryExecutionError + + +class _AttachedSources: + def __init__(self, sources: DatasetSourceResolver) -> None: + self.sources = sources + remaining = sources.budget.remaining() + self.connection = sqlite3.connect( + ":memory:", uri=True, timeout=min(5.0, remaining) if remaining is not None else 5.0 + ) + self.connection.row_factory = sqlite3.Row + + def _configure_deadline(self): + budget = self.sources.budget + remaining = budget.remaining() + deadline = budget.started + budget.timeout_s if budget.timeout_s is not None else None + self.connection.set_progress_handler( + (lambda: int(time.monotonic() >= deadline)) if deadline is not None else None, 1000 + ) + # SQLite's default five-second lock wait must not outlive the operation. + wait_ms = math.ceil(1000 * min(5.0, remaining)) if remaining is not None else 5000 + self.connection.execute(f"PRAGMA busy_timeout={wait_ms}") + + def __enter__(self): + try: + self._configure_deadline() + self.connection.execute("PRAGMA trusted_schema=OFF") + for item in self.sources.device.slices: + self._configure_deadline() + path = self.sources.dataset.root / item.file + immutable = self.sources.dataset.callstack_layout == "v1" + require_readonly_member(self.sources.dataset.root, path, immutable=immutable) + uri = path.as_uri() + "?mode=ro" + ("&immutable=1" if immutable else "") + self.connection.execute(f'ATTACH DATABASE ? AS "s{item.index}"', [uri]) + self.connection.execute("PRAGMA query_only=ON") + self.sources.budget.remaining() + return self + except BaseException as exc: + self.connection.set_progress_handler(None, 0) + self.connection.close() + if isinstance(exc, sqlite3.Error): + self.sources.budget.remaining() + raise QueryExecutionError(str(exc)) from exc + raise + + def __exit__(self, exc_type, exc_value, traceback): + self.connection.set_progress_handler(None, 0) + self.connection.close() + return False + + def read(self, sql: str, values: list[object] | None = None): + budget = self.sources.budget + cursor = self.connection.cursor() + self.sources.query_count += 1 + try: + self._configure_deadline() + cursor.execute(sql, values or []) + result = [] + while batch := cursor.fetchmany(256): + rows = [dict(row) for row in batch] + budget.consume(rows) + result.extend(rows) + budget.remaining() + return result + except sqlite3.Error as exc: + budget.remaining() # Preserve the domain timeout on interruption. + raise QueryExecutionError(str(exc)) from exc + finally: + cursor.close() + self.connection.set_progress_handler(None, 0) + + +def _attachment_limit() -> int: + # getlimit is unavailable on supported Python 3.10. Ten is SQLite's default. + with closing(sqlite3.connect(":memory:")) as connection: + getter = getattr(connection, "getlimit", None) + category = getattr(sqlite3, "SQLITE_LIMIT_ATTACHED", None) + return getter(category) if getter is not None and category is not None else 10 + + +def _proof(sources, field, action): + device = sources.device.device_id + cases = " ".join( + f'WHEN b."{field}" BETWEEN {item.start_event_id} AND {item.end_event_id} ' + f'THEN EXISTS(SELECT 1 FROM "s{item.index}"."trace_entry_{device}" e ' + f'WHERE e.id=b."{field}" AND e.id>=0 AND e.action={action})' + for item in sources.device.slices + ) + return f"CASE {cases} ELSE 0 END" + + +def _lifecycle_cte(sources, items): + allocated = _proof(sources, "allocEventId", 4) + completed = _proof(sources, "freeEventId", 6) + observations = " UNION ALL ".join( + f"SELECT b.*, {item.index} AS observation_slice_index, " + f"{allocated} AS allocation_proved, " + f"CASE WHEN {completed} THEN b.freeEventId END AS completed_free " + f'FROM "s{item.index}"."block_{sources.device.device_id}" b' + for item in items + ) + invariants = ("address", "size", "requestedSize", "allocEventId") + conflicts = " OR ".join( + f'(MIN("{column}") IS NOT MAX("{column}") OR ' + f'(COUNT("{column}")>0 AND COUNT("{column}")=?", [params["min_size"]] + if template == "freed_block_lifetime": + return "allocation_proved AND completion_proved AND freeEventId>=allocEventId", [] + clauses, values = [], [] + for field in ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"): + for prefix, operator in (("", "="), ("min_", ">="), ("max_", "<=")): + value = params.get(prefix + field) + if value is not None: + clauses.append(f'"{field}" {operator} ?') + values.append(value) + return " AND ".join(clauses) or "1", values + + +def lifecycle_query(sources, items, template, params, scope, candidate_limit): + if len(sources.device.slices) > _attachment_limit(): + return None + predicate, values = _filter(template, params) + cte = _lifecycle_cte(sources, items) + with _AttachedSources(sources) as database: + summary = database.read( + cte + f"""SELECT COUNT(*) AS lifecycles, + COALESCE(SUM(allocation_proved),0) AS allocation_events_resolved, + COALESCE(SUM(completion_proved),0) AS free_events_resolved, + COALESCE(SUM(token LIKE 'unproved:%'),0) AS unproved_identities, + COALESCE(SUM(allocEventId>=0 AND NOT allocation_proved),0) AS missing_allocations, + COALESCE(MAX(invariant_conflict),0) AS invariant_conflict, + COALESCE(MAX(free_conflict),0) AS free_conflict, + COALESCE(SUM(CASE WHEN {predicate} THEN 1 ELSE 0 END),0) AS matching + FROM lifecycles""", + values, + )[0] + if summary["invariant_conflict"]: + raise QueryExecutionError("Conflicting observations for a proved lifecycle identity.") + if summary["free_conflict"]: + raise QueryExecutionError("Conflicting free-completion events for one lifecycle.") + scope["source_coverage"] = { + "coverage_scope": "all_deduplicated_lifecycles_before_filters_and_page", + "range_complete": True, + **{ + key: summary[key] + for key in ( + "lifecycles", + "allocation_events_resolved", + "free_events_resolved", + "unproved_identities", + ) + }, + "allocation_source_complete": not summary["missing_allocations"], + "source_queries": sources.query_count, + } + if template == "freed_block_lifetime": + buckets = database.read( + cte + f"""SELECT CASE + WHEN freeEventId-allocEventId<1000 THEN 0 + WHEN freeEventId-allocEventId<5000 THEN 1 + WHEN freeEventId-allocEventId<20000 THEN 2 + WHEN freeEventId-allocEventId<100000 THEN 3 ELSE 4 END AS bucket, + COUNT(*) AS block_count, SUM(size) AS size_bytes + FROM lifecycles WHERE {predicate} GROUP BY bucket ORDER BY bucket""", + values, + ) + for row in buckets: + row["lifetime_events"] = ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[ + row.pop("bucket") + ] + scope["source_coverage"]["source_queries"] = sources.query_count + return buckets, len(buckets) + order = params["order_by"] if template == "block" else "size" + direction = params["order_dir"] if template == "block" else "DESC" + rows = database.read( + cte + f"SELECT * FROM lifecycles WHERE {predicate} " + f'ORDER BY "{order}" {direction}, id {direction}, token {direction} LIMIT ?', + [*values, candidate_limit], + ) + refs = sources.events( + cast(int, row[field]) + for row in rows + for field in ("allocEventId", "freeEventId") + if type(row[field]) is int and cast(int, row[field]) >= 0 + ) + for row in rows: + alloc, free = row["allocEventId"], row["freeEventId"] + allocation = refs.get(alloc) if row.pop("allocation_proved") else None + completion = refs.get(free) if row.pop("completion_proved") else None + token = row.pop("token") + row.pop("invariant_conflict") + row.pop("free_conflict") + row.update( + { + "lifecycle_id": f"{sources.namespace}:{token}", + "identity_status": ( + "proved_allocation_event" + if allocation + else ( + "stable_negative_token" + if token.startswith("preexisting:") + else "unproved_shard_local" + ) + ), + "allocation_source": allocation.to_dict() if allocation else None, + "allocation_source_status": ( + "resolved" + if allocation + else ("unknown_preexisting" if alloc == -1 else "missing") + ), + "free_source": completion.to_dict() if completion else None, + "free_source_status": ( + "resolved" + if completion + else ("live_or_unknown" if free is None or free < 0 else "missing") + ), + "state_scope": "latest_slice_observation", + "terminal_survivor": bool(row["terminal_survivor"]), + "category": "dynamic" if alloc >= 0 else "static" if free == -1 else "preexisting", + } + ) + scope["source_coverage"]["source_queries"] = sources.query_count + return rows, summary["matching"] diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 3923079..5d617f0 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -58,7 +58,7 @@ def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): finally: service.close() assert result.returned == 1 - assert budget.work_rows == 60_001 + assert budget.work_rows <= 60_001 if template == "callstack_analysis": assert result.rows[0]["alloc_count"] == 60_000 @@ -198,5 +198,5 @@ def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget ) assert result.rows[0]["frames"] == [{"name": "f"}] - # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. - assert budget.work_rows == 610 + # SQL pushdown may reduce trace work; only one returned frame is hydrated. + assert budget.work_rows <= 610 diff --git a/tests/core/test_dataset_sql_pushdown.py b/tests/core/test_dataset_sql_pushdown.py new file mode 100644 index 0000000..9512581 --- /dev/null +++ b/tests/core/test_dataset_sql_pushdown.py @@ -0,0 +1,291 @@ +"""Bounded global SQL reductions preserve full-scope evidence and lifecycle proof.""" + +import dataclasses +import hashlib +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core import dataset_lifecycle_sql +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_global import global_query +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.query.registry import get_query +from tests.core.test_dataset_attribution import case +from tests.core.test_dataset_import import source_options +from tests.test_dataset_focus import make_dataset + + +def large_dataset(tmp_path, per_shard=60_000): + root = make_dataset(tmp_path / "large", devices=(0,), slices=2) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["eventsPerSlice"] = per_shard + manifest["devices"]["0"]["eventCount"] = 2 * per_shard + for index, record in enumerate(manifest["devices"]["0"]["slices"]): + low, high = index * per_shard, (index + 1) * per_shard - 1 + record.update(startEventId=low, endEventId=high) + with closing(sqlite3.connect(root / record["file"])) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,?,?,?,'shared')", + ((i, i * 8, i * 8, i * 8, i * 16) for i in range(low, high + 1)), + ) + # Carry-in live blocks are repeated; future allocations never appear. + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(high + 1)), + ) + manifest_path.write_text(json.dumps(manifest)) + return root + + +@pytest.fixture(scope="module") +def large_root(tmp_path_factory): + return large_dataset(tmp_path_factory.mktemp("sql-pushdown")) + + +@pytest.mark.parametrize( + "template", + ["memory_peak", "event", "allocation", "callstack_analysis", "block", "leak_detection"], +) +def test_large_real_scope_fetches_only_aggregates_and_bounded_candidates(large_root, template): + budget = QueryBudget(None, time.monotonic(), max_work_rows=64) + with closing(QueryService()) as service: + result = service.execute_query( + template, db_path=large_root, max_rows=5, exact_total=True, _budget=budget + ) + expected_total = 1 if template in ("memory_peak", "callstack_analysis") else 120_000 + assert result.total == expected_total + assert result.returned == min(expected_total, 5) + assert result.has_more == (expected_total > 5) + assert budget.work_rows <= 64 + if template in ("block", "leak_detection"): + coverage = result.scope["source_coverage"] + assert coverage["lifecycles"] == 120_000 + assert coverage["allocation_events_resolved"] == 120_000 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 120_000 + assert result.rows[0]["total_size"] == 960_000 + + +def test_unbounded_output_still_exhausts_real_fetched_work_budget(large_root): + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query("allocation", db_path=large_root, _budget=budget) + assert budget.work_rows > budget.max_work_rows + + +def test_actual_range_precedes_filters_and_empty_window_is_exact(tmp_path): + root = make_dataset(tmp_path / "range", devices=(0,), slices=3) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + {"min_id": 1, "max_id": 4, "action": 99, "limit": 0}, + db_path=root, + exact_total=True, + ) + assert result.rows == [] and result.total == 0 and not result.has_more + assert result.scope["actual_range"] == { + "first_event_id": 1, + "last_event_id": 4, + "real_event_count": 4, + } + + +def run_resolved(resolved, template, params=None, budget=None): + config = get_query(template) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver( + resolved, 0, cache, budget or QueryBudget(None, time.monotonic()) + ) + return global_query( + source, + template, + config.validate_params(params or {}), + 5, + True, + None, + config.semantics_version, + ) + + +@pytest.mark.parametrize("template", ["block", "leak_detection", "freed_block_lifetime"]) +def test_attachment_ceiling_fallback_preserves_lifecycle_results(tmp_path, monkeypatch, template): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + pushed = run_resolved(resolved, template) + monkeypatch.setattr(dataset_lifecycle_sql, "_attachment_limit", lambda: 1) + fallback = run_resolved(resolved, template) + assert pushed.rows == fallback.rows + assert pushed.total == fallback.total and pushed.has_more == fallback.has_more + first, second = pushed.scope["source_coverage"], fallback.scope["source_coverage"] + assert {k: v for k, v in first.items() if k != "source_queries"} == { + k: v for k, v in second.items() if k != "source_queries" + } + + +def last_observation(root, block_id): + for path in sorted(root.glob("device_0/*.db"), reverse=True): + with closing(sqlite3.connect(path)) as conn: + if conn.execute("SELECT 1 FROM block_0 WHERE id=?", [block_id]).fetchone(): + return path + raise AssertionError(f"No observation for block {block_id}") + + +def test_lifecycle_conflicts_are_checked_before_filters_and_page(tmp_path): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + with closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + assert conn.execute("UPDATE block_0 SET size=size+1 WHERE id=0").rowcount == 1 + with pytest.raises(QueryExecutionError, match="Conflicting observations"): + run_resolved(resolved, "block", {"id": 7}) + + +def test_latest_state_and_prior_proved_completion_survive_sql_reduction(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + with closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + assert conn.execute("UPDATE block_0 SET state=0,freeEventId=-1 WHERE id=0").rowcount == 1 + params = {"id": 0} + pushed = run_resolved(resolved, "block", params) + monkeypatch.setattr(dataset_lifecycle_sql, "_attachment_limit", lambda: 1) + fallback = run_resolved(resolved, "block", params) + assert pushed.rows == fallback.rows + assert pushed.rows[0]["state"] == 0 + assert pushed.rows[0]["freeEventId"] == 10 + assert pushed.rows[0]["free_source"]["event"]["action"] == 6 + assert not pushed.rows[0]["terminal_survivor"] + + +def test_attached_members_are_read_only_and_connection_is_closed(tmp_path): + resolved = DatasetResolver().inspect(make_dataset(tmp_path / "readonly", devices=(0,))) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + with dataset_lifecycle_sql._AttachedSources(source) as attached: + with pytest.raises(sqlite3.OperationalError, match="readonly"): + attached.connection.execute("DELETE FROM s0.trace_entry_0") + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + + +def test_attached_query_timeout_clears_handler_and_closes_connection(tmp_path): + resolved = DatasetResolver().inspect(make_dataset(tmp_path / "timeout", devices=(0,))) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(0.05, time.monotonic())) + with dataset_lifecycle_sql._AttachedSources(source) as attached: + with pytest.raises(QueryTimeoutError): + attached.read( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n " + "WHERE x<100000000) SELECT SUM(x) FROM n" + ) + assert attached.connection.execute("SELECT 1").fetchone()[0] == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + + +def test_callstack_order_reuses_already_fetched_candidate_events(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "reuse", devices=(0,), slices=3) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(QueryService()) as service: + result = service.execute_query("event", {"order_by": "callstack", "limit": 1}, db_path=root) + assert result.returned == 1 + assert sum("ORDER BY" in sql and "trace_entry" in sql for sql in queries) == 3 + assert not any("WHERE t.id IN" in sql for sql in queries) + + +def test_attaching_locked_member_respects_remaining_deadline(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "locked", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + # Exercise the native non-immutable opening mode with an actual SQLite lock. + monkeypatch.setattr(type(resolved), "callstack_layout", property(lambda self: "v2")) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as locked: + locked.execute("BEGIN EXCLUSIVE") + with closing(ContextCache()) as cache: + start = time.monotonic() + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(0.05, start)) + attached = dataset_lifecycle_sql._AttachedSources(source) + with pytest.raises(QueryTimeoutError): + with attached: + attached.read("SELECT COUNT(*) FROM s0.trace_entry_0") + assert time.monotonic() - start < 0.5 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + locked.rollback() + + +@pytest.mark.parametrize("layout", ["v1", "v2"]) +@pytest.mark.parametrize("collation,texts", [("NOCASE", ("a", "A")), ("RTRIM", ("a ", "a"))]) +def test_global_text_identity_and_candidates_ignore_source_collation( + tmp_path, layout, collation, texts +): + if layout == "v1": + root = make_dataset(tmp_path / "collated", devices=(0,), slices=1) + else: + root = ( + ImportService() + .import_snapshot( + dataclasses.replace(source_options(tmp_path), events_per_slice=99, set_focus=False) + ) + .db_path + ) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + item = manifest["devices"]["0"]["slices"][0] + member = root / item["file"] + with closing(sqlite3.connect(member)) as conn, conn: + ids = [r[0] for r in conn.execute("SELECT id FROM trace_entry_0 WHERE id>=0 ORDER BY id")] + if layout == "v1": + original = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='trace_entry_0'" + ).fetchone()[0] + conn.execute("ALTER TABLE trace_entry_0 RENAME TO original_trace") + conn.execute(original.replace("callstack TEXT", f"callstack TEXT COLLATE {collation}")) + conn.execute("INSERT INTO trace_entry_0 SELECT * FROM original_trace") + conn.execute("DROP TABLE original_trace") + conn.execute("UPDATE trace_entry_0 SET callstack=?", [texts[0]]) + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", [texts[1], ids[1]]) + else: + conn.execute("DROP TABLE callstack") + conn.execute( + f"CREATE TABLE callstack (id INTEGER PRIMARY KEY,callstack TEXT COLLATE {collation})" + ) + conn.executemany("INSERT INTO callstack VALUES (?,?)", enumerate(texts)) + conn.execute("UPDATE trace_entry_0 SET callstackId=0") + conn.execute("UPDATE trace_entry_0 SET callstackId=1 WHERE id=?", [ids[1]]) + conn.execute("UPDATE trace_entry_0 SET size=1 WHERE id>=0") + if layout == "v2": + item["sha256"] = hashlib.sha256(member.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + expected_text = {event: texts[int(event == ids[1])] for event in ids} + with closing(QueryService()) as service: + groups = service.execute_query("callstack_analysis", {"min_count": 1}, db_path=root) + assert {r["callstack"]: r["alloc_count"] for r in groups.rows} == { + texts[0]: len(ids) - 1, + texts[1]: 1, + } + assert len({r["source_stack_id"] for r in groups.rows}) == 2 + for direction in ("ASC", "DESC"): + result = service.execute_query( + "event", {"order_by": "callstack", "order_dir": direction, "limit": 1}, db_path=root + ) + expected = sorted( + ids, key=lambda event: (expected_text[event], event), reverse=direction == "DESC" + )[0] + assert result.rows[0]["id"] == expected + assert result.rows[0]["callstack"] == expected_text[expected] From 973843032d61a999ec9d00b544f8bb89d1ff49e5 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:19:36 +0800 Subject: [PATCH 6/8] fix: preserve dataset source cache validation contracts --- src/pt_snap_cli/core/dataset_global.py | 12 ++++++--- src/pt_snap_cli/core/dataset_sources.py | 8 +++--- tests/core/test_dataset_global.py | 36 +++++++++++++++++++++++++ tests/core/test_dataset_source_reuse.py | 26 ++++++++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 857cfa8..eea697f 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -81,9 +81,6 @@ def _traces( for item in items: trace = f'"trace_entry_{sources.device.device_id}"' if with_sources: - projection = ( - "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" - ) if sources.dataset.callstack_layout == "v2": sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" else: @@ -308,7 +305,14 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, with_sources=True): + for _item, batch in _traces( + sources, + low, + high, + items, + ("id", *METRICS, "action", "address", "size", "stream"), + with_sources=True, + ): refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3a22a5d..4907016 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -211,7 +211,7 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() - if event in self._events: + if type(event) is int and event in self._events: result[event] = self._events[event] continue item = self.owner(event) @@ -248,16 +248,18 @@ def events_from_rows( Native rows must include joined callstack text and the local callstackId. Frame reads still incur the same work budget as events() source lookup. + Sources remain cached until this resolver operation ends. Serialized + value accounting excludes Python object overhead and is not an RSS cap. """ result: dict[int, EventSource] = {} for original in rows: self.budget.remaining() event_id = cast(int, original["id"]) + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") if event_id in self._events: result[event_id] = self._events[event_id] continue - if self.owner(event_id) != item: - raise QueryExecutionError("Source event does not belong to the selected shard.") row = dict(original) event_id = cast(int, row["id"]) text = row.get("callstack") diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 82a0dc5..86332b2 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -547,3 +547,39 @@ def test_cli_api_report_global_parity_and_range_options(tmp_path): result = json.loads(cli.stdout) assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15} assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10 + + +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_callstack_sorted_event_page_matches_standalone(tmp_path, native, direction): + if native: + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + importer = ImportService() + root = importer.import_snapshot(request).db_path + single = importer.import_snapshot( + dataclasses.replace(request, events_per_slice=None) + ).db_path + else: + root = case(tmp_path) + single = standalone(root, tmp_path) + # Standalone includes synthetic negative boundary IDs; compare real events. + params = { + "min_id": 7 if native else 0, + "order_by": "callstack", + "order_dir": direction, + "offset": 1, + "limit": 3, + } + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in ((0, 1) if native else (0,)): + actual = dataset.execute_query("event", params, device_id=device, exact_total=True) + expected = reference.execute_query("event", params, device_id=device, exact_total=True) + assert actual["rows"] + keys = list(expected["rows"][0]) + assert projected(actual["rows"], keys) == expected["rows"] + assert all("callstackId" not in row for row in actual["rows"]) + assert actual["total"] == expected["total"] + assert actual["has_more"] == expected["has_more"] + assert actual["truncated"] == expected["truncated"] + assert hashes(root) == before diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 3923079..07e6e32 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -200,3 +200,29 @@ def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): assert result.rows[0]["frames"] == [{"name": "f"}] # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. assert budget.work_rows == 610 + + +@pytest.mark.parametrize("invalid", [True, 1.0]) +def test_cached_event_lookup_preserves_exact_integer_validation(tmp_path, invalid): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + assert sources.events([invalid]) == {} + expected = sources.events([1]) + reads = sources.query_count + assert sources.events([invalid]) == {} + assert sources.events([1]) == expected + assert sources.query_count == reads + + +@pytest.mark.parametrize("warm_cache", [False, True]) +def test_preloaded_rows_reject_wrong_shard_even_when_cached(tmp_path, warm_cache): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=2) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + if warm_cache: + assert 0 in sources.events([0]) + with pytest.raises(QueryExecutionError, match="does not belong"): + sources.events_from_rows(sources.device.slices[1], [{"id": 0}]) From ef641382eb7ea5441408d3055fa437fbfa7183fc Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:22:06 +0800 Subject: [PATCH 7/8] fix: preserve NULL peak report semantics and update skill workflow --- skills/pt-snap-memory-peak-breakdown/SKILL.md | 24 +++++---- .../pt-snap-memory-peak-breakdown/SKILL.md | 24 +++++---- src/pt_snap_cli/core/report_service.py | 10 ++-- tests/core/test_report_service.py | 52 +++++++++++++++++++ .../test_memory_peak_breakdown_contract.py | 14 +++++ tests/test_peak_report_completeness.py | 33 ++++++++++++ 6 files changed, 136 insertions(+), 21 deletions(-) diff --git a/skills/pt-snap-memory-peak-breakdown/SKILL.md b/skills/pt-snap-memory-peak-breakdown/SKILL.md index 1d7a275..791ec41 100644 --- a/skills/pt-snap-memory-peak-breakdown/SKILL.md +++ b/skills/pt-snap-memory-peak-breakdown/SKILL.md @@ -244,19 +244,25 @@ pt-snap report peak-memory "" --device --metric reserved --include- Keep the JSON in the response. The report composes these product templates: -- `memory_peak` finds the active, allocated, and reserved peak values and event - IDs. -- `allocator_gap` returns each metric's counters and gaps at that same metric's - peak event. +- `allocator_gap` runs once to find the active, allocated, and reserved peak + values and event IDs, plus each metric's same-event counters and gaps. The + report projects its six-field `peak` section from this result, preserving + `memory_peak` semantics without running a separate `memory_peak` query. - `active_memory_callstack_at_event` groups blocks active at the report's selected metric event by allocation callstack. -From the three results, record each metric's own peak value and event ID. Peak -ties resolve to the earliest event ID. Record same-event counters and gaps from +Both steps share one resolved database/device, budget, and dataset generation. +If the selected metric has no peak value or event ID, the report returns empty +attribution without running the attribution query; never invent an event for an +empty range or an all-NULL counter. Other metrics can still have valid peaks. + +From the three metric reports, record each metric's own peak value and event ID. +Peak ties resolve to the earliest event ID. Record same-event counters and gaps from `allocator_gap`; never subtract independently occurring peak values as if they -occurred together. The repeated `peak` and `allocator_gap` sections should agree -across all three reports; the metric-specific `event_id` and `callstack_groups` -are the selected-event breakdown. +occurred together. The repeated `peak` sections should agree across all three +reports. For metrics with a valid peak, the `allocator_gap` sections also agree; +an empty-attribution report has `allocator_gap: null`. The metric-specific +`event_id` and `callstack_groups` are the selected-event breakdown. Read report `has_more`, `truncated`, `total_is_exact`, and `effective_params` before claiming a complete composition. A full dynamic `top_n` window means diff --git a/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md b/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md index 1d7a275..791ec41 100644 --- a/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md +++ b/src/pt_snap_cli/bundled_skills/pt-snap-memory-peak-breakdown/SKILL.md @@ -244,19 +244,25 @@ pt-snap report peak-memory "" --device --metric reserved --include- Keep the JSON in the response. The report composes these product templates: -- `memory_peak` finds the active, allocated, and reserved peak values and event - IDs. -- `allocator_gap` returns each metric's counters and gaps at that same metric's - peak event. +- `allocator_gap` runs once to find the active, allocated, and reserved peak + values and event IDs, plus each metric's same-event counters and gaps. The + report projects its six-field `peak` section from this result, preserving + `memory_peak` semantics without running a separate `memory_peak` query. - `active_memory_callstack_at_event` groups blocks active at the report's selected metric event by allocation callstack. -From the three results, record each metric's own peak value and event ID. Peak -ties resolve to the earliest event ID. Record same-event counters and gaps from +Both steps share one resolved database/device, budget, and dataset generation. +If the selected metric has no peak value or event ID, the report returns empty +attribution without running the attribution query; never invent an event for an +empty range or an all-NULL counter. Other metrics can still have valid peaks. + +From the three metric reports, record each metric's own peak value and event ID. +Peak ties resolve to the earliest event ID. Record same-event counters and gaps from `allocator_gap`; never subtract independently occurring peak values as if they -occurred together. The repeated `peak` and `allocator_gap` sections should agree -across all three reports; the metric-specific `event_id` and `callstack_groups` -are the selected-event breakdown. +occurred together. The repeated `peak` sections should agree across all three +reports. For metrics with a valid peak, the `allocator_gap` sections also agree; +an empty-attribution report has `allocator_gap: null`. The metric-specific +`event_id` and `callstack_groups` are the selected-event breakdown. Read report `has_more`, `truncated`, `total_is_exact`, and `effective_params` before claiming a complete composition. A full dynamic `top_n` window means diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index d6b6e51..6a823a5 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -110,10 +110,14 @@ def peak_memory_report( start_dir=start_dir, ) gap = gap_result.rows[0] if gap_result.rows else None - # allocator_gap selects the same independent, earliest-tie peak events. - # Keep the report's peak object at the historical six-field contract. + # allocator_gap selects the same independent, earliest-tie peak events + # for non-NULL counters. Preserve memory_peak's six-field contract, + # including NULL event IDs when a standalone counter is entirely NULL. peak = {field: gap.get(field) for field in _PEAK_FIELDS} if gap else {} - # Peak event IDs remain INTEGER (or NULL for an empty trace). + for peak_metric, event_field in _EVENT_ID_BY_METRIC.items(): + if peak and peak[f"peak_{peak_metric}"] is None: + peak[event_field] = None + # No attribution event exists for an empty trace or a NULL peak. event_id = cast(int | None, peak.get(_EVENT_ID_BY_METRIC[metric])) attribution_params: dict[str, object] = { "event_id": event_id, diff --git a/tests/core/test_report_service.py b/tests/core/test_report_service.py index 88e2a83..e692970 100644 --- a/tests/core/test_report_service.py +++ b/tests/core/test_report_service.py @@ -1,10 +1,12 @@ """Tests for peak memory report service.""" import sqlite3 +from contextlib import closing from pathlib import Path import pytest +from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates @@ -121,6 +123,56 @@ def test_peak_memory_report_can_exclude_static(report_db: Path) -> None: } +@pytest.mark.parametrize("metric", [None, "active", "allocated", "reserved"]) +@pytest.mark.parametrize("bounded", [False, True]) +def test_null_peak_matches_memory_peak_without_attribution( + report_db: Path, monkeypatch, metric, bounded: bool +) -> None: + selected_metric = metric or "active" + bounds = {"start_id": 2, "end_id": 3} if bounded else {} + with closing(sqlite3.connect(report_db)) as conn, conn: + where = " WHERE id BETWEEN 2 AND 3" if bounded else "" + conn.execute(f"UPDATE trace_entry_0 SET {selected_metric} = NULL{where}") + expected = QueryService().execute_query("memory_peak", params=bounds, db_path=report_db) + service = ReportService() + calls = [] + execute = service._query_service.execute_query + + def tracked_execute(template, *args, **kwargs): + calls.append(template) + return execute(template, *args, **kwargs) + + def unexpected_attribution(*args, **kwargs): + pytest.fail("A NULL peak must not invent an attribution event") + + monkeypatch.setattr(service._query_service, "execute_query", tracked_execute) + monkeypatch.setattr(service, "event_attribution", unexpected_attribution) + report = service.peak_memory_report( + report_db, **bounds, **({"metric": metric} if metric is not None else {}) + ) + + assert report.peak == expected.rows[0] + assert report.event_id is None and report.allocator_gap is None + assert report.callstack_groups == [] and report.total_is_exact + assert report.effective_params["event_id"] is None + assert not report.has_more and not report.truncated + assert calls == ["allocator_gap"] + + +@pytest.mark.parametrize("active", [None, 0]) +def test_report_preserves_null_unselected_peak_and_zero_peak(report_db: Path, active) -> None: + with closing(sqlite3.connect(report_db)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET active = ?", (active,)) + expected = QueryService().execute_query("memory_peak", db_path=report_db) + metric = "reserved" if active is None else "active" + report = ReportService().peak_memory_report(report_db, metric=metric) + + assert report.peak == expected.rows[0] + assert report.event_id == (3 if active is None else 1) + assert report.allocator_gap is not None + assert report.callstack_groups + + def test_peak_memory_report_rejects_invalid_metric(report_db: Path) -> None: with pytest.raises(ValueError, match="Invalid metric"): ReportService().peak_memory_report(report_db, metric="invalid") # type: ignore[arg-type] diff --git a/tests/skills/test_memory_peak_breakdown_contract.py b/tests/skills/test_memory_peak_breakdown_contract.py index c6f6505..034c64c 100644 --- a/tests/skills/test_memory_peak_breakdown_contract.py +++ b/tests/skills/test_memory_peak_breakdown_contract.py @@ -141,6 +141,20 @@ def test_full_trace_workflow_runs_all_metrics_with_static_limit_and_json() -> No assert "active_blocks_at_event` at the chosen metric's peak event" in skill +def test_report_workflow_documents_reused_peak_query_and_null_semantics() -> None: + normalized = _normalized_skill() + + assert "`allocator_gap` runs once" in normalized + assert "without running a separate `memory_peak` query" in normalized + assert ( + "Both steps share one resolved database/device, budget, and dataset generation" + in normalized + ) + assert "empty attribution without running the attribution query" in normalized + assert "empty range or an all-NULL counter" in normalized + assert "an empty-attribution report has `allocator_gap: null`" in normalized + + def test_range_fallback_and_peak_event_semantics_are_explicit() -> None: skill = _skill() normalized = _normalized_skill() diff --git a/tests/test_peak_report_completeness.py b/tests/test_peak_report_completeness.py index ef91130..6495e4c 100644 --- a/tests/test_peak_report_completeness.py +++ b/tests/test_peak_report_completeness.py @@ -129,6 +129,39 @@ def test_report_completeness_and_same_event_coverage( assert ("Increase --limit" in result.stdout) == partial +@pytest.mark.parametrize("bounded", [False, True]) +@pytest.mark.parametrize("json_output", [True, False]) +def test_default_report_has_no_attribution_for_null_active_peak( + attribution_db: Path, bounded: bool, json_output: bool +) -> None: + bounds = {"start_id": 2, "end_id": 3} if bounded else {} + with closing(sqlite3.connect(attribution_db)) as conn, conn: + where = " WHERE id BETWEEN 2 AND 3" if bounded else "" + conn.execute(f"UPDATE trace_entry_0 SET active = NULL{where}") + args = ["report", "peak-memory", str(attribution_db)] + if bounded: + args += ["--start-id", "2", "--end-id", "3"] + result = CliRunner().invoke(app, args + (["--json"] if json_output else [])) + assert result.exit_code == 0, result.output + if json_output: + payload = json.loads(result.stdout) + with SnapshotAnalyzer(attribution_db) as analyzer: + peak = analyzer.execute_query("memory_peak", params=bounds) + assert payload["peak"] == peak["rows"][0] + assert payload["metric"] == "active" + assert payload["event_id"] is None and payload["allocator_gap"] is None + assert payload["callstack_groups"] == [] + assert payload["included_bytes"] == 0 + assert payload["active_bytes_at_event"] is None + assert payload["coverage_percent"] is None + assert payload["effective_params"]["event_id"] is None + assert payload["total_is_exact"] is True + assert payload["has_more"] is False and payload["truncated"] is False + else: + assert "Attribution: unavailable (no peak event)" in result.stdout + assert "No active memory callstack groups found" in result.stdout + + @pytest.mark.parametrize("state", ["empty", "null", "zero"]) @pytest.mark.parametrize("json_output", [True, False]) def test_report_unknown_coverage(attribution_db: Path, state: str, json_output: bool) -> None: From 1c4198de0badfcfd4174201ad678e436f30b438e Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:28:51 +0800 Subject: [PATCH 8/8] fix: normalize dataset scalar rounding overflow --- src/pt_snap_cli/core/dataset_attribution.py | 25 +++++++++++---- tests/core/test_dataset_realistic.py | 35 ++++++++++++++++++++- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index ece501f..1d806f3 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -3,10 +3,12 @@ from __future__ import annotations import sqlite3 -from contextlib import closing +from collections.abc import Iterator +from contextlib import closing, contextmanager from typing import cast from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, EventSource +from pt_snap_cli.core.errors import QueryExecutionError from pt_snap_cli.core.models import QueryResult from pt_snap_cli.core.stack_summary import summarize_stacks @@ -144,10 +146,7 @@ def event_attribution( # Use the same SQLite arithmetic/ROUND implementation as standalone SQL. # A Decimal conversion of a binary float changes some near-half values. # This connection evaluates scalars only: no source reads or merge tables. - with ( - closing(sqlite3.connect(":memory:")) as connection, - closing(connection.cursor()) as cursor, - ): + with _rounding_cursor(sources) as cursor: for row in selected: sources.budget.remaining() row["percent_of_active_blocks"] = cursor.execute( @@ -255,7 +254,7 @@ def _groups( # Reuse one scalar-only engine for this pass and close it on timeout as well # as success. Derived values do not add source IO; final output is still # charged by event_attribution's existing consume(selected) boundary. - with closing(sqlite3.connect(":memory:")) as connection, closing(connection.cursor()) as cursor: + with _rounding_cursor(sources) as cursor: for row in rows: sources.budget.remaining() row["size_gib"], row["requested_gib"] = cursor.execute( @@ -277,3 +276,17 @@ def summarize_dataset_stacks(rows: list[dict[str, object]], budget: int) -> None for row, (identity, kind) in zip(rows, identities, strict=True): row["stack_id"] = identity row["stack_kind"] = kind + + +@contextmanager +def _rounding_cursor(sources: DatasetSourceResolver) -> Iterator[sqlite3.Cursor]: + """Keep scalar arithmetic failures on the same domain boundary as shard SQL.""" + try: + with ( + closing(sqlite3.connect(":memory:")) as connection, + closing(connection.cursor()) as cursor, + ): + yield cursor + except (sqlite3.Error, OverflowError) as exc: + sources.budget.remaining() # Preserve a deadline that expired during SQLite work. + raise QueryExecutionError(str(exc)) from exc diff --git a/tests/core/test_dataset_realistic.py b/tests/core/test_dataset_realistic.py index 30290a9..2a38823 100644 --- a/tests/core/test_dataset_realistic.py +++ b/tests/core/test_dataset_realistic.py @@ -11,12 +11,14 @@ from pathlib import Path import pytest +from typer.testing import CliRunner from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app from pt_snap_cli.core import dataset_attribution from pt_snap_cli.core.dataset_contract import validate_dataset from pt_snap_cli.core.dataset_sources import QueryBudget -from pt_snap_cli.core.errors import QueryTimeoutError +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.models import ImportOptions from pt_snap_cli.core.query_service import QueryService @@ -184,6 +186,37 @@ def test_half_near_half_and_large_values_match_actual_sqlite_rounding(tmp_path, assert hashes(tmp_path) == before +@pytest.mark.parametrize("same_stack", [True, False], ids=["gib", "percentages"]) +def test_rounding_int64_aggregate_overflow_is_a_domain_error(tmp_path, same_stack): + root, single = rounding_case(tmp_path, (1, 1)) + for path in [single, *root.glob("device_0/*.db")]: + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute("UPDATE block_0 SET size=?, requestedSize=?", (2**62, 2**62)) + if same_stack: + connection.execute("UPDATE trace_entry_0 SET callstack='same'") + validate_dataset(root) + before = hashes(tmp_path) + for path in (root, single): + with SnapshotAnalyzer(path) as analyzer, pytest.raises(QueryExecutionError): + analyzer.execute_query("active_memory_callstack_at_event", {"event_id": 1, "top_n": -1}) + result = CliRunner().invoke( + app, + [ + "query", + str(path), + "--template-use", + "active_memory_callstack_at_event", + "--params", + '{"event_id": 1, "top_n": -1}', + "--json", + ], + ) + assert result.exit_code == 1 and result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "QUERY_FAILED" + assert "Traceback" not in result.stderr + assert hashes(tmp_path) == before + + @pytest.mark.parametrize("rounding_pass", [1, 2], ids=["gib", "percentages"]) def test_rounding_timeout_closes_scalar_connection_and_cursor(tmp_path, monkeypatch, rounding_pass): root, _ = rounding_case(tmp_path, (1, 127))