From c5ed2a8dff80318e3bd4ae267a48eb9a6909ae0b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:12:00 +0800 Subject: [PATCH 1/5] fix: narrow dataset trace projections by template (#220) --- src/pt_snap_cli/core/dataset_global.py | 22 ++++++++--- tests/core/test_dataset_projections.py | 54 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/core/test_dataset_projections.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index cbc73f6..0990b3d 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -68,11 +68,20 @@ def _scope( } -def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord]): +def _traces( + sources: DatasetSourceResolver, + low: int, + high: int, + items: list[SliceRecord], + columns: tuple[str, ...], +): + # Each built-in supplies only the fields it actually consumes. In particular, + # inline stack text must not spend the byte budget of a counter-only query. + projection = ", ".join(f'"{column}"' for column in columns) for item in items: rows = sources.read( sources.dataset.root / item.file, - f'SELECT * FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', [low, high], ) yield item, rows @@ -81,7 +90,7 @@ def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[Sli def _peaks(sources, low, high, items, template, scope): chosen: dict[str, dict[str, object]] = {} count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items): + for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): for row in rows: sources.budget.remaining() event = cast(int, row["id"]) @@ -149,7 +158,10 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items): + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + for _item, batch in _traces(sources, low, high, items, columns): for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -287,7 +299,7 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items): + for _item, batch in _traces(sources, low, high, items, ("id",)): refs = sources.events(cast(int, r["id"]) for r in batch) for source in refs.values(): sources.budget.remaining() diff --git a/tests/core/test_dataset_projections.py b/tests/core/test_dataset_projections.py new file mode 100644 index 0000000..6811d89 --- /dev/null +++ b/tests/core/test_dataset_projections.py @@ -0,0 +1,54 @@ +"""Per-template scan projections must not charge unrelated inline stack text.""" + +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +@pytest.mark.parametrize("template", ["memory_peak", "allocator_gap", "allocation"]) +def test_counter_query_budget_is_independent_of_inline_stack_length(tmp_path, template): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + observations = [] + for stack in ("x" * 16, "x" * 8192, "x" * 32768, "栈🙂" * 8192): + for path in root.glob("device_0/*.db"): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=?", (stack,)) + service = QueryService() + try: + budget = QueryBudget(None, time.monotonic(), max_work_bytes=8192) + result = service.execute_query(template, db_path=root, _budget=budget) + observations.append((result.rows, budget.work_rows, budget.work_bytes)) + finally: + service.close() + assert all(item == observations[0] for item in observations) + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_event_projection_preserves_callstack_sort_and_paging(tmp_path, direction): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + stacks = ["z" * 8192, None, "栈🙂" * 8192, "a" * 16384, "", "a" * 16384] + for index, stack in enumerate(stacks): + path = root / f"device_0/slice_{index // 2:05d}.db" + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", (stack, index)) + service = QueryService() + try: + params = {"order_by": "callstack", "order_dir": direction, "offset": 1, "limit": 3} + result = service.execute_query("event", params, db_path=root, exact_total=True) + expected = sorted( + range(len(stacks)), + key=lambda event: (stacks[event] is not None, stacks[event], event), + reverse=direction == "DESC", + )[1:4] + assert [row["id"] for row in result.rows] == expected + assert [row["callstack"] for row in result.rows] == [stacks[event] for event in expected] + assert result.total == 6 and result.has_more and result.truncated + assert all(len(row) == 9 for row in result.rows) + finally: + service.close() From b1a2d068e18689cd9aea62d71aca36bf7292c9b9 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:14:26 +0800 Subject: [PATCH 2/5] fix: reuse budgeted event sources within dataset queries (#222) --- src/pt_snap_cli/core/dataset_global.py | 32 ++++-- src/pt_snap_cli/core/dataset_sources.py | 87 ++++++++++----- tests/core/test_dataset_source_reuse.py | 134 ++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 tests/core/test_dataset_source_reuse.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 0990b3d..a7ab3ec 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -73,15 +73,26 @@ def _traces( low: int, high: int, items: list[SliceRecord], - columns: tuple[str, ...], + columns: tuple[str, ...] = ("id", *METRICS), + *, + with_sources: bool = False, ): - # Each built-in supplies only the fields it actually consumes. In particular, - # inline stack text must not spend the byte budget of a counter-only query. - projection = ", ".join(f'"{column}"' for column in columns) + projection = ", ".join(f't."{column}"' for column in columns) for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if with_sources: + projection = ( + "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" + ) + if sources.dataset.callstack_layout == "v2": + sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + sql = f"SELECT {projection}, t.callstack FROM {trace} t" + else: + sql = f"SELECT {projection} FROM {trace} t" rows = sources.read( sources.dataset.root / item.file, - f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", [low, high], ) yield item, rows @@ -158,10 +169,9 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - columns = ("id", *METRICS) - if template == "event": - columns += ("action", "address", "size", "stream") - for _item, batch in _traces(sources, low, high, items, columns): + for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): + if template == "event": + sources.events_from_rows(_item, batch) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -299,8 +309,8 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, ("id",)): - refs = sources.events(cast(int, r["id"]) for r in batch) + for _item, batch in _traces(sources, low, high, items, with_sources=True): + refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 7a8e3f0..3532752 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -1,7 +1,7 @@ """Read-only, batched device/event source lookup over a validated dataset. No address joins, foreign-event insertion, text-to-frame reconstruction, or retained -source cache. The caller owns the Context LRU and the entire operation's deadline. +cross-operation source cache. The caller owns the Context LRU and the entire operation's deadline. """ from __future__ import annotations @@ -140,6 +140,9 @@ def __init__( self.budget = budget self.query_count = 0 self._starts = [s.start_event_id for s in self.device.slices] + # Lifetime is one budgeted operation. Each retained source was charged + # when fetched; reusing the same object performs no further database work. + self._events: dict[int, EventSource] = {} @property def namespace(self) -> str: @@ -203,12 +206,15 @@ def read( def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: grouped: dict[int, list[int]] = defaultdict(list) + result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() + if event in self._events: + result[event] = self._events[event] + continue item = self.owner(event) if item is not None: grouped[item.index].append(event) - result: dict[int, EventSource] = {} for index, ids in grouped.items(): item = self.device.slices[index] path = self.dataset.root / item.file @@ -224,32 +230,57 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - for row in rows: - event_id = cast(int, row["id"]) - text = row.get("callstack") - captured = isinstance(text, str) and text != "" - local_id = cast(int | None, row.pop("callstackId", None)) - # V2 IDs select text WITHIN the owning DB; they are provenance, - # not dataset grouping keys. Global interning equivalence uses - # the full captured text, never a shortened display prefix. - identity = ( - "text:sha256:" - + hashlib.sha256( - (text if isinstance(text, str) else "").encode("utf-8") - ).hexdigest() - ) - result[event_id] = EventSource( - row, - index, - ( - f"{self.namespace}:{identity}" - if captured - else f"{self.namespace}:category:missing" - ), - "captured" if captured else "missing", - local_stack_id=local_id, - ) - self._attach_frames(path, ids, result) + result.update(self.events_from_rows(item, rows)) + self.budget.remaining() + return result + + def events_from_rows( + self, item: SliceRecord, rows: list[dict[str, object]] + ) -> dict[int, EventSource]: + """Resolve already charged full event rows without fetching them again. + + Native rows must include joined callstack text and the local callstackId. + Frame reads still incur the same work budget as events() source lookup. + """ + result: dict[int, EventSource] = {} + for original in rows: + self.budget.remaining() + event_id = cast(int, original["id"]) + if event_id in self._events: + result[event_id] = self._events[event_id] + continue + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") + row = dict(original) + event_id = cast(int, row["id"]) + text = row.get("callstack") + captured = isinstance(text, str) and text != "" + local_id = cast(int | None, row.pop("callstackId", None)) + # V2 IDs select text WITHIN the owning DB; they are provenance, + # not dataset grouping keys. Global interning equivalence uses + # the full captured text, never a shortened display prefix. + identity = ( + "text:sha256:" + + hashlib.sha256( + (text if isinstance(text, str) else "").encode("utf-8") + ).hexdigest() + ) + result[event_id] = EventSource( + row, + item.index, + ( + f"{self.namespace}:{identity}" + if captured + else f"{self.namespace}:category:missing" + ), + "captured" if captured else "missing", + local_stack_id=local_id, + ) + fresh = {event: source for event, source in result.items() if event not in self._events} + if fresh: + self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) + self._events.update(fresh) + result.update(fresh) self.budget.remaining() return result diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py new file mode 100644 index 0000000..87c8265 --- /dev/null +++ b/tests/core/test_dataset_source_reuse.py @@ -0,0 +1,134 @@ +"""Operation-local reuse preserves real read accounting and source identity.""" + +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +def dataset(tmp_path, count, stack="shared", blocks=False): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["eventsPerSlice"] = count + manifest["devices"]["0"]["eventCount"] = count + manifest["devices"]["0"]["slices"][0]["endEventId"] = count - 1 + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,8,8,16,?)", + ((i, i * 8, stack) for i in range(count)), + ) + if blocks: + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(count)), + ) + return root + + +@pytest.mark.parametrize("template", ["callstack_analysis", "event"]) +def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): + root = dataset(tmp_path, 60_000) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + result = service.execute_query( + template, + params=( + {"min_count": 1} + if template == "callstack_analysis" + else { + "order_by": "callstack", + "limit": 1, + } + ), + db_path=root, + _budget=budget, + ) + finally: + service.close() + assert result.returned == 1 + assert budget.work_rows == 60_001 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 60_000 + + +def test_target_and_distinct_events_reuse_only_identical_sources(tmp_path): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + budget = QueryBudget(None, time.monotonic()) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, budget) + sources.target(3) + first = sources.events([0, 1, 2, 3]) + reads, work = sources.query_count, budget.work_rows + repeated = sources.events([3, 2, 1, 0, 3]) + assert sources.query_count == reads + assert budget.work_rows == work == 4 + assert repeated[3] is first[3] + assert first[0].event["id"] != first[1].event["id"] + assert first[0].stack_id == first[1].stack_id + + +@pytest.mark.parametrize("count, succeeds", [(49_999, True), (50_000, False)]) +def test_distinct_block_and_allocation_work_still_counts(tmp_path, count, succeeds): + root = dataset(tmp_path, count, blocks=True) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows == 100_001 + finally: + service.close() + + +@pytest.mark.parametrize("count, succeeds", [(3_000, True), (5_000, False)]) +def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds): + root = dataset(tmp_path, count, stack="x" * 16_384, blocks=True) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows < budget.max_work_rows + assert budget.work_bytes > budget.max_work_bytes From eda49166421dcab5c73fe7d107504122d701928c Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:17:49 +0800 Subject: [PATCH 3/5] fix: defer event page hydration and reuse frame schema checks --- src/pt_snap_cli/core/dataset_global.py | 19 ++++--- src/pt_snap_cli/core/dataset_sources.py | 47 +++++++++++------ tests/core/test_dataset_source_reuse.py | 68 +++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index a7ab3ec..857cfa8 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -169,9 +169,13 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): - if template == "event": - sources.events_from_rows(_item, batch) + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + with_sources = template == "event" and params["order_by"] == "callstack" + for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): + if with_sources: + sources.events_from_rows(_item, batch, with_frames=False) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -188,13 +192,8 @@ def _events(sources, low, high, items, params, template, scope): "last_event_id": last, "real_event_count": count, } - if template == "event": - # Only returned events need formatted text; resolve separately after the - # global window below. `order_by=callstack` requires text BEFORE sorting. - if params["order_by"] == "callstack": - refs = sources.events(cast(int, r["id"]) for r in rows) - for row in rows: - row["callstack"] = refs[cast(int, row["id"])].event.get("callstack") + # callstack sorting already has joined text; resolving sources here would + # eagerly fetch ordered frames that only the returned page needs. _sort(rows, params["order_by"], params["order_dir"] == "DESC") return rows diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3532752..3a22a5d 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -143,6 +143,8 @@ def __init__( # Lifetime is one budgeted operation. Each retained source was charged # when fetched; reusing the same object performs no further database work. self._events: dict[int, EventSource] = {} + self._frames_resolved: set[int] = set() + self._frame_schemas: dict[Path, bool] = {} @property def namespace(self) -> str: @@ -230,12 +232,17 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - result.update(self.events_from_rows(item, rows)) + result.update(self.events_from_rows(item, rows, with_frames=False)) + frame_groups: dict[int, list[int]] = defaultdict(list) + for event, source in result.items(): + frame_groups[source.slice_index].append(event) + for index, ids in frame_groups.items(): + self._ensure_frames(self.device.slices[index], ids) self.budget.remaining() - return result + return {event: self._events[event] for event in result} def events_from_rows( - self, item: SliceRecord, rows: list[dict[str, object]] + self, item: SliceRecord, rows: list[dict[str, object]], *, with_frames: bool = True ) -> dict[int, EventSource]: """Resolve already charged full event rows without fetching them again. @@ -277,12 +284,17 @@ def events_from_rows( local_stack_id=local_id, ) fresh = {event: source for event, source in result.items() if event not in self._events} - if fresh: - self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) self._events.update(fresh) - result.update(fresh) + if with_frames: + self._ensure_frames(item, list(result)) self.budget.remaining() - return result + return {event: self._events[event] for event in result} + + def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: + pending = [event for event in ids if event not in self._frames_resolved] + if pending: + self._attach_frames(self.dataset.root / item.file, pending, self._events) + self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: # This is a NEW, explicit read contract, not a guessed original-producer @@ -297,14 +309,19 @@ def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSou ("frameJson", "TEXT", 0), ], } - for table, columns in expected.items(): - kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) - schema = self.read(path, f'PRAGMA table_info("{table}")') - if ( - kind != [{"type": "table"}] - or [(r["name"], r["type"], r["pk"]) for r in schema] != columns - ): - return # Unknown/absent schemas never imply precision. + if path not in self._frame_schemas: + self._frame_schemas[path] = True + for table, columns in expected.items(): + kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = self.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + self._frame_schemas[path] = False + break # Unknown/absent schemas never imply precision. + if not self._frame_schemas[path]: + return for start in range(0, len(ids), SOURCE_BATCH_SIZE): batch = ids[start : start + SOURCE_BATCH_SIZE] placeholders = ",".join("?" for _ in batch) diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 87c8265..3923079 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -132,3 +132,71 @@ def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds ) assert budget.work_rows < budget.max_work_rows assert budget.work_bytes > budget.max_work_bytes + + +def test_id_sorted_page_does_not_materialize_unneeded_stack_text(tmp_path): + root = dataset(tmp_path, 10_000, stack="x" * 8192) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + params={"order_by": "id", "limit": 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_bytes < 2_000_000 + + +def framed_dataset(tmp_path, count): + root = dataset(tmp_path, count) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT," + "PRIMARY KEY(eventId,frameIndex));" + ) + conn.executemany( + "INSERT INTO pt_snap_frame_coverage VALUES (?,1)", ((i,) for i in range(count)) + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", ((i, '{"name":"f"}') for i in range(count)) + ) + return root + + +def test_frame_schema_is_checked_once_across_source_batches(tmp_path, monkeypatch): + root = framed_dataset(tmp_path, 600) + resolved = DatasetResolver().inspect(root) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + first = sources.events(range(512)) + second = sources.events(range(512, 600)) + assert all( + source.frames == [{"name": "f"}] for source in [*first.values(), *second.values()] + ) + assert sum("PRAGMA table_info" in sql for sql in queries) == 2 + assert sum("sqlite_master" in sql for sql in queries) == 2 + + +def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): + root = framed_dataset(tmp_path, 600) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget + ) + assert result.rows[0]["frames"] == [{"name": "f"}] + # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. + assert budget.work_rows == 610 From 98fac137d26f99872252de5a3877b04d25202f56 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:31:09 +0800 Subject: [PATCH 4/5] fix: push bounded dataset-global queries into SQLite (#218) --- docs/en/querying.md | 20 +- docs/zh/querying.md | 15 +- src/pt_snap_cli/core/dataset_global.py | 404 +++++++++++------- src/pt_snap_cli/core/dataset_lifecycle_sql.py | 273 ++++++++++++ tests/core/test_dataset_source_reuse.py | 6 +- tests/core/test_dataset_sql_pushdown.py | 291 +++++++++++++ 6 files changed, 857 insertions(+), 152 deletions(-) create mode 100644 src/pt_snap_cli/core/dataset_lifecycle_sql.py create mode 100644 tests/core/test_dataset_sql_pushdown.py diff --git a/docs/en/querying.md b/docs/en/querying.md index 60d92ed..60c6cc7 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -151,11 +151,29 @@ Context LRU. One diminishing deadline includes all shards, sources, grouping and totals; reports explicitly share **one** budget across their three queries. Cumulative fetched/output work is limited to **100000 rows / 64 MiB serialized values**, including repeated source reads. Exceeding either fails the entire operation, never an alleged -complete partial result. Materialization/sort are bounded by this work ceiling, **not** +complete partial result. Python materialization/sort are bounded by this work ceiling, **not** by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual cells have overhead. Manifest hashing/validation remains proportional to artifact size and phase-checked, not hard-preempted. No temporary merge DB, repair or artifact write. +Global peak queries fetch scalar range evidence and at most three counter rows per +shard. Event/allocation pages apply SQL filters and ordering per shard, merge at most +`offset + limit` candidates per shard, and count matches separately. Range evidence +is measured before non-range filters. Ordinary text-stack statistics merge full +per-shard groups before global thresholds; averages use summed sizes and counts. +Canonical ordered-frame statistics retain the bounded source-reading path because +formatted text is not their identity. Block/leak/lifetime reductions use read-only +attached shards for proof, conflicts and latest observations before pagination; +when the device exceeds SQLite's attachment limit (normally ten), they retain the +budgeted source-reading path. No intermediate merged database is created. + +A million-event dataset can therefore answer aggregates and small pages within the +default work budget. Use `-n 5` or an explicit finite `limit` for large row-returning +queries. Unlimited output, large offsets, many distinct stack groups, ordered-frame +reads or the attachment-limit fallback can still exceed the budget and fail honestly. +The limit counts rows fetched into Python and serialized output, not internal SQL +scans, SQLite sorting space or total process memory. + ## Dataset point-event attribution A complete compatibility-v1 or native-v2 directory/manifest supports `event` diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 69c243e..a504c35 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -130,10 +130,23 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 数据集读取先分离批次再查外片来源,使用借用的有界 Context LRU。所有片、来源、分组、 精确总数共用递减 deadline;报告三条查询明确共用**一个**预算。累计取回/输出工作上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 -结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ +结论。Python 物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ SQLite、一个批次及单个 cell 有额外开销。manifest hash/校验仍与产物大小成正比且仅在 阶段边界检查,不硬抢占。不生成合并临时 DB、不修复或写产物。 +全局峰值查询每片只取范围汇总及最多三条计数器记录。event/allocation 在各片 SQL 中筛选、 +排序,每片最多取 `offset + limit` 个候选后全局归并,并单独统计匹配总数;实际范围证据在 +非范围筛选前计算。普通文本栈统计先合并各片完整分组,再应用全局阈值,平均值由总大小 +与总计数计算。规范化原始有序帧的身份不能用格式化文本替代,因此仍走受预算约束的来源 +读取路径。block/leak/lifetime 通过只读附加分片先证明身份、检测冲突、选最新观察,再分页; +设备分片数超出 SQLite 的附加上限(通常十片)时,回退到原有受预算约束的来源读取路径。 +不创建中间合并数据库。 + +因此,百万事件数据集的聚合与小页面可在默认工作预算内完成。大量逐行结果请指定 `-n 5` +或有限 `limit`。无限制输出、大 offset、大量不同栈分组、有序帧读取及超过附加上限的回退 +路径仍可能超预算,并明确失败。预算统计进入 Python 的实际取回行和序列化输出,不限制 +SQL 内部扫描、SQLite 排序空间或整个进程内存。 + ## 数据集定点事件归因 完整兼容 v1 或原生 v2 目录/manifest 支持 `event` 寻址,以及接受真实 `event_id` 的 diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 857cfa8..43ec456 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -1,8 +1,9 @@ -"""Explicit built-in global algorithms; no local caps or temporary merge database. +"""Exact built-in global reductions, without a temporary merge database. -Fetched work is cumulatively bounded by QueryBudget, including sources and final -output. Sorting/materialization is O(bounded work), NOT O(max_rows). A budget -failure rejects the operation rather than returning an alleged global answer. +Shard SQL reduces counters, groups, and candidate windows before Python merging. +Fetched work is cumulatively bounded by QueryBudget, including source hydration +and final output. Unlimited/high-cardinality results can still exhaust the budget; +a failure rejects the operation rather than returning an alleged global answer. """ from __future__ import annotations @@ -11,6 +12,7 @@ from typing import cast from pt_snap_cli.core.dataset_contract import QueryScope, SliceRecord +from pt_snap_cli.core.dataset_lifecycle_sql import lifecycle_query from pt_snap_cli.core.dataset_sources import DatasetSourceResolver from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError from pt_snap_cli.core.models import QueryResult @@ -68,47 +70,50 @@ def _scope( } -def _traces( - sources: DatasetSourceResolver, - low: int, - high: int, - items: list[SliceRecord], - columns: tuple[str, ...] = ("id", *METRICS), - *, - with_sources: bool = False, -): - projection = ", ".join(f't."{column}"' for column in columns) +def _range_statistics(sources, low, high, items, scope, predicate="1", values=None): + """Range evidence is deliberately collected BEFORE non-range filters.""" + count = matching = 0 + first = last = None for item in items: - trace = f'"trace_entry_{sources.device.device_id}"' - if with_sources: - projection = ( - "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" - ) - if sources.dataset.callstack_layout == "v2": - sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" - else: - sql = f"SELECT {projection}, t.callstack FROM {trace} t" - else: - sql = f"SELECT {projection} FROM {trace} t" - rows = sources.read( + row = sources.read( sources.dataset.root / item.file, - sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", - [low, high], - ) - yield item, rows + f"SELECT MIN(id) AS first, MAX(id) AS last, COUNT(*) AS count, " + f"COALESCE(SUM(CASE WHEN {predicate} THEN 1 ELSE 0 END), 0) AS matching " + f'FROM "trace_entry_{sources.device.device_id}" ' + "WHERE id>=0 AND id>=? AND id<=?", + [*(values or []), low, high], + )[0] + count += row["count"] + matching += row["matching"] + if row["first"] is not None: + first = row["first"] if first is None else min(first, row["first"]) + last = row["last"] if last is None else max(last, row["last"]) + scope["actual_range"] = { + "first_event_id": first, + "last_event_id": last, + "real_event_count": count, + } + return matching def _peaks(sources, low, high, items, template, scope): + _range_statistics(sources, low, high, items, scope) chosen: dict[str, dict[str, object]] = {} - count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): - for row in rows: - sources.budget.remaining() - event = cast(int, row["id"]) - count += 1 - first = event if first is None else min(first, event) - last = event if last is None else max(last, event) - for metric in METRICS: + # These index-friendly queries fetch at most three narrow rows per shard. + # NULL counters never nominate an event, matching MAX + equality semantics. + for item in items: + for metric in METRICS: + rows = sources.read( + sources.dataset.root / item.file, + f"SELECT id, allocated, active, reserved " + f'FROM "trace_entry_{sources.device.device_id}" ' + f'WHERE id>=0 AND id>=? AND id<=? AND "{metric}" IS NOT NULL ' + f'ORDER BY "{metric}" DESC, id ASC LIMIT 1', + [low, high], + ) + if rows: + row = rows[0] + event = cast(int, row["id"]) if metric not in chosen or (cast(int, row[metric]), -event) > ( cast(int, chosen[metric][metric]), -cast(int, chosen[metric]["id"]), @@ -125,7 +130,9 @@ def _peaks(sources, low, high, items, template, scope): result[f"{other}_at_{metric}_peak"] = row.get(other) for other in ("active", "allocated"): result[f"reserved_{other}_gap_at_{metric}_peak"] = ( - cast(int, row["reserved"]) - cast(int, row[other]) if row else None + cast(int, row["reserved"]) - cast(int, row[other]) + if row and row["reserved"] is not None and row[other] is not None + else None ) if template == "allocator_gap": ids = [result[f"peak_{m}_event_id"] for m in METRICS] @@ -135,17 +142,31 @@ def _peaks(sources, low, high, items, template, scope): result[f"peak_{a}_event_id"] is not None and result[f"peak_{a}_event_id"] == result[f"peak_{b}_event_id"] ) - scope["actual_range"] = { - "first_event_id": first, - "last_event_id": last, - "real_event_count": count, - } scope["peak_owner_slices"] = { m: sources.owner(cast(int, r["id"])).index for m, r in chosen.items() } return [result] +def _predicates(params, fields, alias=""): + clauses, values = [], [] + for field in fields: + for prefix, operator in (("", "="), ("min_", ">="), ("max_", "<=")): + value = params.get(prefix + field) + if value is not None: + clauses.append(f'{alias}"{field}" {operator} ?') + values.append(value) + return " AND ".join(clauses) or "1", values + + +def _window(params, max_rows): + offset = max(0, cast(int, params.get("offset", 0))) + limit = cast(int, params.get("limit", -1)) + if max_rows is not None and max_rows > 0: + limit = min(limit, max_rows) if limit >= 0 else max_rows + return offset, limit + + def _matches(row: dict[str, object], params: dict[str, object], fields: tuple[str, ...]) -> bool: for field in fields: value = params.get(field) @@ -166,36 +187,45 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: rows.sort(key=lambda r: (r[key] is not None, r[key]), reverse=descending) -def _events(sources, low, high, items, params, template, scope): +def _events(sources, low, high, items, params, template, scope, candidate_limit): + fields = METRICS + if template == "event": + fields += ("id", "action", "address", "size", "stream") + predicate, values = _predicates(params, fields) + total = _range_statistics(sources, low, high, items, scope, predicate, values) rows = [] - first, last, count = None, None, 0 + order = params["order_by"] + direction = params["order_dir"] columns = ("id", *METRICS) if template == "event": - columns += ("action", "address", "size", "stream") - with_sources = template == "event" and params["order_by"] == "callstack" - for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): - if with_sources: - sources.events_from_rows(_item, batch, with_frames=False) - for row in batch: - sources.budget.remaining() - event = cast(int, row["id"]) - first = event if first is None else min(first, event) - last = event if last is None else max(last, event) - count += 1 - fields = ("allocated", "active", "reserved") - if template == "event": - fields += ("id", "action", "address", "size", "stream") - if _matches(row, params, fields): - rows.append(row) - scope["actual_range"] = { - "first_event_id": first, - "last_event_id": last, - "real_event_count": count, - } - # callstack sorting already has joined text; resolving sources here would - # eagerly fetch ordered frames that only the returned page needs. - _sort(rows, params["order_by"], params["order_dir"] == "DESC") - return rows + columns = ("id", "action", "address", "size", "stream", *METRICS) + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + projection = ", ".join(f't."{column}"' for column in columns) + join = "" + if order == "callstack": + if sources.dataset.callstack_layout == "v2": + projection += ", t.callstackId, cs.callstack" + join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + projection += ", t.callstack" + row_predicate, row_values = _predicates(params, fields, "t.") + # Dataset text identity/order is exact, independent of source collation. + sort_column = '"callstack" COLLATE BINARY' if order == "callstack" else f'"{order}"' + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {row_predicate} " + f"ORDER BY {sort_column} {direction}, t.id {direction} LIMIT ?", + [low, high, *row_values, candidate_limit], + ) + if order == "callstack": + # Full candidate text has already crossed the budget boundary. Reuse + # its event data; ordered frames are needed only for the final page. + sources.events_from_rows(item, batch, with_frames=False) + rows.extend(batch) + _sort(rows, order, direction == "DESC") + return rows, total def _lifecycles(sources, items, scope): @@ -305,11 +335,103 @@ def _lifecycles(sources, items, scope): return rows +def _text_stacks(sources, low, high, items, params, scope): + """Merge shard sufficient statistics, never local HAVING/ranking windows.""" + grouped = {} + events = 0 + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if sources.dataset.callstack_layout == "v2": + table = f"{trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + text, present = "cs.callstack", "t.callstackId IS NOT NULL" + else: + table, text, present = f"{trace} t", "t.callstack", "t.callstack IS NOT NULL" + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {text} AS callstack, COUNT(*) AS alloc_count, SUM(t.size) AS total_size, " + f"MAX(t.size) AS max_size, MIN(t.id) AS stack_event_id FROM {table} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {present} GROUP BY {text} COLLATE BINARY", + [low, high], + ) + for local in batch: + sources.budget.remaining() + value = local["callstack"] + identity = ( + f"{sources.namespace}:text:sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" + if isinstance(value, str) + else f"{sources.namespace}:category:missing" + ) + kind = "captured" if isinstance(value, str) and value != "" else "missing" + row = grouped.get(identity) + if row is None: + row = dict(local) + row.update( + { + "callstack": value if value is not None else "[missing callstack]", + "source_stack_id": identity, + "stack_kind": kind, + "text_kind": kind, + "frames_status": "text_only", + } + ) + grouped[identity] = row + else: + row["alloc_count"] += local["alloc_count"] + row["total_size"] += local["total_size"] + row["max_size"] = max(row["max_size"], local["max_size"]) + row["stack_event_id"] = min(row["stack_event_id"], local["stack_event_id"]) + events += local["alloc_count"] + return _finish_stacks(sources, grouped, params, scope, events, 0) + + +def _finish_stacks(sources, grouped, params, scope, events, ordered): + rows = [ + r + for r in grouped.values() + if r["alloc_count"] >= params["min_count"] and r["total_size"] >= params["min_size"] + ] + for row in rows: + row["avg_size"] = row["total_size"] / row["alloc_count"] + rows.sort( + key=lambda r: ( + -r["total_size"], + r["callstack"] or "", + r["stack_kind"] == "missing", + r["source_stack_id"], + ) + ) + scope["source_coverage"] = { + "range_complete": True, + "coverage_scope": "all_real_stack_bearing_actions_before_global_thresholds", + "stack_events": events, + "ordered_frame_events": ordered, + "source_queries": sources.query_count, + } + return rows + + def _stacks(sources, low, high, items, params, scope): + if sources.dataset.ordered_frames_version != 1: + return _text_stacks(sources, low, high, items, params, scope) grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, with_sources=True): - refs = sources.events_from_rows(_item, batch) + for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + columns = ("id", "action", "address", "size", "stream", *METRICS) + projection = ", ".join(f't."{column}"' for column in columns) + join = "" + if sources.dataset.callstack_layout == "v2": + projection += ", t.callstackId, cs.callstack" + join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + projection += ", t.callstack" + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", + [low, high], + ) + refs = sources.events_from_rows(item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") @@ -347,29 +469,7 @@ def _stacks(sources, low, high, items, params, scope): row["text_kind"] = "captured" events += 1 ordered += source.frames is not None - rows = [ - r - for r in grouped.values() - if r["alloc_count"] >= params["min_count"] and r["total_size"] >= params["min_size"] - ] - for row in rows: - row["avg_size"] = row["total_size"] / row["alloc_count"] - rows.sort( - key=lambda r: ( - -r["total_size"], - r["callstack"] or "", - r["stack_kind"] == "missing", - r["source_stack_id"], - ) - ) - scope["source_coverage"] = { - "range_complete": True, - "coverage_scope": "all_real_stack_bearing_actions_before_global_thresholds", - "stack_events": events, - "ordered_frame_events": ordered, - "source_queries": sources.query_count, - } - return rows + return _finish_stacks(sources, grouped, params, scope, events, ordered) def global_query( @@ -386,10 +486,15 @@ def global_query( ) low, high, items = _range(sources, range_params, slice_index) scope = _scope(sources, low, high, items) + offset, limit = _window(params, max_rows) + candidate_limit = offset + limit if limit >= 0 else -1 + total_override = None if template in ("memory_peak", "allocator_gap"): rows = _peaks(sources, low, high, items, template, scope) elif template in ("event", "allocation"): - rows = _events(sources, low, high, items, params, template, scope) + rows, total_override = _events( + sources, low, high, items, params, template, scope, candidate_limit + ) elif template == "callstack_analysis": rows = _stacks(sources, low, high, items, params, scope) elif template == "preexisting_live": @@ -407,56 +512,61 @@ def global_query( raise QueryExecutionError( "Dataset leak_detection requires terminal dataset scope, not --slice; select an explicit member for local observations." ) - rows = _lifecycles(sources, items, scope) - if template == "block": - rows = [ - r - for r in rows - if _matches( - r, - params, - ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"), - ) - ] - _sort(rows, cast(str, params["order_by"]), params["order_dir"] == "DESC") - elif template == "leak_detection": - rows = [ - r - for r in rows - if r["terminal_survivor"] - and r["allocation_source"] is not None - and cast(int, r["size"]) >= cast(int, params["min_size"]) - ] - _sort(rows, "size", True) - elif template == "freed_block_lifetime": - buckets = {} - for row in rows: - if row["allocation_source"] is None or row["free_source"] is None: - continue - distance = cast(int, row["freeEventId"]) - cast(int, row["allocEventId"]) - if distance < 0: - continue - index = next( - (i for i, cap in enumerate((1000, 5000, 20000, 100000)) if distance < cap), 4 - ) - bucket = buckets.setdefault( - index, - { - "lifetime_events": ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[index], - "block_count": 0, - "size_bytes": 0, - }, - ) - bucket["block_count"] += 1 - bucket["size_bytes"] += row["size"] - rows = [buckets[i] for i in sorted(buckets)] + pushed = lifecycle_query(sources, items, template, params, scope, candidate_limit) + if pushed is not None: + rows, total_override = pushed else: - raise QueryExecutionError("No dataset-global merge implementation for this template.") - total = len(rows) - offset = max(0, cast(int, params.get("offset", 0))) - limit = cast(int, params.get("limit", -1)) - if max_rows is not None and max_rows > 0: - limit = min(limit, max_rows) if limit >= 0 else max_rows + rows = _lifecycles(sources, items, scope) + if template == "block": + rows = [ + r + for r in rows + if _matches( + r, + params, + ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"), + ) + ] + _sort(rows, cast(str, params["order_by"]), params["order_dir"] == "DESC") + elif template == "leak_detection": + rows = [ + r + for r in rows + if r["terminal_survivor"] + and r["allocation_source"] is not None + and cast(int, r["size"]) >= cast(int, params["min_size"]) + ] + _sort(rows, "size", True) + elif template == "freed_block_lifetime": + buckets = {} + for row in rows: + if row["allocation_source"] is None or row["free_source"] is None: + continue + distance = cast(int, row["freeEventId"]) - cast(int, row["allocEventId"]) + if distance < 0: + continue + index = next( + (i for i, cap in enumerate((1000, 5000, 20000, 100000)) if distance < cap), + 4, + ) + bucket = buckets.setdefault( + index, + { + "lifetime_events": ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[ + index + ], + "block_count": 0, + "size_bytes": 0, + }, + ) + bucket["block_count"] += 1 + bucket["size_bytes"] += row["size"] + rows = [buckets[i] for i in sorted(buckets)] + else: + raise QueryExecutionError( + "No dataset-global merge implementation for this template." + ) + total = len(rows) if total_override is None else total_override selected = rows[offset : offset + limit] if limit >= 0 else rows[offset:] if template == "event": refs = sources.events(cast(int, r["id"]) for r in selected) diff --git a/src/pt_snap_cli/core/dataset_lifecycle_sql.py b/src/pt_snap_cli/core/dataset_lifecycle_sql.py new file mode 100644 index 0000000..9caf2e1 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_lifecycle_sql.py @@ -0,0 +1,273 @@ +"""Read-only relational lifecycle reduction over a bounded set of attached shards. + +No rows are copied to a merge database. SQLite performs proof, conflict checks, +latest-observation selection and page ranking; only aggregate/page rows cross the +Python boundary. Larger shard sets keep the budgeted general implementation. +""" + +from __future__ import annotations + +import math +import sqlite3 +import time +from contextlib import closing +from typing import cast + +from pt_snap_cli.core.dataset_files import require_readonly_member +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.errors import QueryExecutionError + + +class _AttachedSources: + def __init__(self, sources: DatasetSourceResolver) -> None: + self.sources = sources + remaining = sources.budget.remaining() + self.connection = sqlite3.connect( + ":memory:", uri=True, timeout=min(5.0, remaining) if remaining is not None else 5.0 + ) + self.connection.row_factory = sqlite3.Row + + def _configure_deadline(self): + budget = self.sources.budget + remaining = budget.remaining() + deadline = budget.started + budget.timeout_s if budget.timeout_s is not None else None + self.connection.set_progress_handler( + (lambda: int(time.monotonic() >= deadline)) if deadline is not None else None, 1000 + ) + # SQLite's default five-second lock wait must not outlive the operation. + wait_ms = math.ceil(1000 * min(5.0, remaining)) if remaining is not None else 5000 + self.connection.execute(f"PRAGMA busy_timeout={wait_ms}") + + def __enter__(self): + try: + self._configure_deadline() + self.connection.execute("PRAGMA trusted_schema=OFF") + for item in self.sources.device.slices: + self._configure_deadline() + path = self.sources.dataset.root / item.file + immutable = self.sources.dataset.callstack_layout == "v1" + require_readonly_member(self.sources.dataset.root, path, immutable=immutable) + uri = path.as_uri() + "?mode=ro" + ("&immutable=1" if immutable else "") + self.connection.execute(f'ATTACH DATABASE ? AS "s{item.index}"', [uri]) + self.connection.execute("PRAGMA query_only=ON") + self.sources.budget.remaining() + return self + except BaseException as exc: + self.connection.set_progress_handler(None, 0) + self.connection.close() + if isinstance(exc, sqlite3.Error): + self.sources.budget.remaining() + raise QueryExecutionError(str(exc)) from exc + raise + + def __exit__(self, exc_type, exc_value, traceback): + self.connection.set_progress_handler(None, 0) + self.connection.close() + return False + + def read(self, sql: str, values: list[object] | None = None): + budget = self.sources.budget + cursor = self.connection.cursor() + self.sources.query_count += 1 + try: + self._configure_deadline() + cursor.execute(sql, values or []) + result = [] + while batch := cursor.fetchmany(256): + rows = [dict(row) for row in batch] + budget.consume(rows) + result.extend(rows) + budget.remaining() + return result + except sqlite3.Error as exc: + budget.remaining() # Preserve the domain timeout on interruption. + raise QueryExecutionError(str(exc)) from exc + finally: + cursor.close() + self.connection.set_progress_handler(None, 0) + + +def _attachment_limit() -> int: + # getlimit is unavailable on supported Python 3.10. Ten is SQLite's default. + with closing(sqlite3.connect(":memory:")) as connection: + getter = getattr(connection, "getlimit", None) + category = getattr(sqlite3, "SQLITE_LIMIT_ATTACHED", None) + return getter(category) if getter is not None and category is not None else 10 + + +def _proof(sources, field, action): + device = sources.device.device_id + cases = " ".join( + f'WHEN b."{field}" BETWEEN {item.start_event_id} AND {item.end_event_id} ' + f'THEN EXISTS(SELECT 1 FROM "s{item.index}"."trace_entry_{device}" e ' + f'WHERE e.id=b."{field}" AND e.id>=0 AND e.action={action})' + for item in sources.device.slices + ) + return f"CASE {cases} ELSE 0 END" + + +def _lifecycle_cte(sources, items): + allocated = _proof(sources, "allocEventId", 4) + completed = _proof(sources, "freeEventId", 6) + observations = " UNION ALL ".join( + f"SELECT b.*, {item.index} AS observation_slice_index, " + f"{allocated} AS allocation_proved, " + f"CASE WHEN {completed} THEN b.freeEventId END AS completed_free " + f'FROM "s{item.index}"."block_{sources.device.device_id}" b' + for item in items + ) + invariants = ("address", "size", "requestedSize", "allocEventId") + conflicts = " OR ".join( + f'(MIN("{column}") IS NOT MAX("{column}") OR ' + f'(COUNT("{column}")>0 AND COUNT("{column}")=?", [params["min_size"]] + if template == "freed_block_lifetime": + return "allocation_proved AND completion_proved AND freeEventId>=allocEventId", [] + clauses, values = [], [] + for field in ("id", "address", "size", "requestedSize", "allocEventId", "freeEventId"): + for prefix, operator in (("", "="), ("min_", ">="), ("max_", "<=")): + value = params.get(prefix + field) + if value is not None: + clauses.append(f'"{field}" {operator} ?') + values.append(value) + return " AND ".join(clauses) or "1", values + + +def lifecycle_query(sources, items, template, params, scope, candidate_limit): + if len(sources.device.slices) > _attachment_limit(): + return None + predicate, values = _filter(template, params) + cte = _lifecycle_cte(sources, items) + with _AttachedSources(sources) as database: + summary = database.read( + cte + f"""SELECT COUNT(*) AS lifecycles, + COALESCE(SUM(allocation_proved),0) AS allocation_events_resolved, + COALESCE(SUM(completion_proved),0) AS free_events_resolved, + COALESCE(SUM(token LIKE 'unproved:%'),0) AS unproved_identities, + COALESCE(SUM(allocEventId>=0 AND NOT allocation_proved),0) AS missing_allocations, + COALESCE(MAX(invariant_conflict),0) AS invariant_conflict, + COALESCE(MAX(free_conflict),0) AS free_conflict, + COALESCE(SUM(CASE WHEN {predicate} THEN 1 ELSE 0 END),0) AS matching + FROM lifecycles""", + values, + )[0] + if summary["invariant_conflict"]: + raise QueryExecutionError("Conflicting observations for a proved lifecycle identity.") + if summary["free_conflict"]: + raise QueryExecutionError("Conflicting free-completion events for one lifecycle.") + scope["source_coverage"] = { + "coverage_scope": "all_deduplicated_lifecycles_before_filters_and_page", + "range_complete": True, + **{ + key: summary[key] + for key in ( + "lifecycles", + "allocation_events_resolved", + "free_events_resolved", + "unproved_identities", + ) + }, + "allocation_source_complete": not summary["missing_allocations"], + "source_queries": sources.query_count, + } + if template == "freed_block_lifetime": + buckets = database.read( + cte + f"""SELECT CASE + WHEN freeEventId-allocEventId<1000 THEN 0 + WHEN freeEventId-allocEventId<5000 THEN 1 + WHEN freeEventId-allocEventId<20000 THEN 2 + WHEN freeEventId-allocEventId<100000 THEN 3 ELSE 4 END AS bucket, + COUNT(*) AS block_count, SUM(size) AS size_bytes + FROM lifecycles WHERE {predicate} GROUP BY bucket ORDER BY bucket""", + values, + ) + for row in buckets: + row["lifetime_events"] = ("<1k", "1k-5k", "5k-20k", "20k-100k", ">=100k")[ + row.pop("bucket") + ] + scope["source_coverage"]["source_queries"] = sources.query_count + return buckets, len(buckets) + order = params["order_by"] if template == "block" else "size" + direction = params["order_dir"] if template == "block" else "DESC" + rows = database.read( + cte + f"SELECT * FROM lifecycles WHERE {predicate} " + f'ORDER BY "{order}" {direction}, id {direction}, token {direction} LIMIT ?', + [*values, candidate_limit], + ) + refs = sources.events( + cast(int, row[field]) + for row in rows + for field in ("allocEventId", "freeEventId") + if type(row[field]) is int and cast(int, row[field]) >= 0 + ) + for row in rows: + alloc, free = row["allocEventId"], row["freeEventId"] + allocation = refs.get(alloc) if row.pop("allocation_proved") else None + completion = refs.get(free) if row.pop("completion_proved") else None + token = row.pop("token") + row.pop("invariant_conflict") + row.pop("free_conflict") + row.update( + { + "lifecycle_id": f"{sources.namespace}:{token}", + "identity_status": ( + "proved_allocation_event" + if allocation + else ( + "stable_negative_token" + if token.startswith("preexisting:") + else "unproved_shard_local" + ) + ), + "allocation_source": allocation.to_dict() if allocation else None, + "allocation_source_status": ( + "resolved" + if allocation + else ("unknown_preexisting" if alloc == -1 else "missing") + ), + "free_source": completion.to_dict() if completion else None, + "free_source_status": ( + "resolved" + if completion + else ("live_or_unknown" if free is None or free < 0 else "missing") + ), + "state_scope": "latest_slice_observation", + "terminal_survivor": bool(row["terminal_survivor"]), + "category": "dynamic" if alloc >= 0 else "static" if free == -1 else "preexisting", + } + ) + scope["source_coverage"]["source_queries"] = sources.query_count + return rows, summary["matching"] diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 3923079..5d617f0 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -58,7 +58,7 @@ def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): finally: service.close() assert result.returned == 1 - assert budget.work_rows == 60_001 + assert budget.work_rows <= 60_001 if template == "callstack_analysis": assert result.rows[0]["alloc_count"] == 60_000 @@ -198,5 +198,5 @@ def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget ) assert result.rows[0]["frames"] == [{"name": "f"}] - # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. - assert budget.work_rows == 610 + # SQL pushdown may reduce trace work; only one returned frame is hydrated. + assert budget.work_rows <= 610 diff --git a/tests/core/test_dataset_sql_pushdown.py b/tests/core/test_dataset_sql_pushdown.py new file mode 100644 index 0000000..9512581 --- /dev/null +++ b/tests/core/test_dataset_sql_pushdown.py @@ -0,0 +1,291 @@ +"""Bounded global SQL reductions preserve full-scope evidence and lifecycle proof.""" + +import dataclasses +import hashlib +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core import dataset_lifecycle_sql +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_global import global_query +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.query.registry import get_query +from tests.core.test_dataset_attribution import case +from tests.core.test_dataset_import import source_options +from tests.test_dataset_focus import make_dataset + + +def large_dataset(tmp_path, per_shard=60_000): + root = make_dataset(tmp_path / "large", devices=(0,), slices=2) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["eventsPerSlice"] = per_shard + manifest["devices"]["0"]["eventCount"] = 2 * per_shard + for index, record in enumerate(manifest["devices"]["0"]["slices"]): + low, high = index * per_shard, (index + 1) * per_shard - 1 + record.update(startEventId=low, endEventId=high) + with closing(sqlite3.connect(root / record["file"])) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,?,?,?,'shared')", + ((i, i * 8, i * 8, i * 8, i * 16) for i in range(low, high + 1)), + ) + # Carry-in live blocks are repeated; future allocations never appear. + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(high + 1)), + ) + manifest_path.write_text(json.dumps(manifest)) + return root + + +@pytest.fixture(scope="module") +def large_root(tmp_path_factory): + return large_dataset(tmp_path_factory.mktemp("sql-pushdown")) + + +@pytest.mark.parametrize( + "template", + ["memory_peak", "event", "allocation", "callstack_analysis", "block", "leak_detection"], +) +def test_large_real_scope_fetches_only_aggregates_and_bounded_candidates(large_root, template): + budget = QueryBudget(None, time.monotonic(), max_work_rows=64) + with closing(QueryService()) as service: + result = service.execute_query( + template, db_path=large_root, max_rows=5, exact_total=True, _budget=budget + ) + expected_total = 1 if template in ("memory_peak", "callstack_analysis") else 120_000 + assert result.total == expected_total + assert result.returned == min(expected_total, 5) + assert result.has_more == (expected_total > 5) + assert budget.work_rows <= 64 + if template in ("block", "leak_detection"): + coverage = result.scope["source_coverage"] + assert coverage["lifecycles"] == 120_000 + assert coverage["allocation_events_resolved"] == 120_000 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 120_000 + assert result.rows[0]["total_size"] == 960_000 + + +def test_unbounded_output_still_exhausts_real_fetched_work_budget(large_root): + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query("allocation", db_path=large_root, _budget=budget) + assert budget.work_rows > budget.max_work_rows + + +def test_actual_range_precedes_filters_and_empty_window_is_exact(tmp_path): + root = make_dataset(tmp_path / "range", devices=(0,), slices=3) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + {"min_id": 1, "max_id": 4, "action": 99, "limit": 0}, + db_path=root, + exact_total=True, + ) + assert result.rows == [] and result.total == 0 and not result.has_more + assert result.scope["actual_range"] == { + "first_event_id": 1, + "last_event_id": 4, + "real_event_count": 4, + } + + +def run_resolved(resolved, template, params=None, budget=None): + config = get_query(template) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver( + resolved, 0, cache, budget or QueryBudget(None, time.monotonic()) + ) + return global_query( + source, + template, + config.validate_params(params or {}), + 5, + True, + None, + config.semantics_version, + ) + + +@pytest.mark.parametrize("template", ["block", "leak_detection", "freed_block_lifetime"]) +def test_attachment_ceiling_fallback_preserves_lifecycle_results(tmp_path, monkeypatch, template): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + pushed = run_resolved(resolved, template) + monkeypatch.setattr(dataset_lifecycle_sql, "_attachment_limit", lambda: 1) + fallback = run_resolved(resolved, template) + assert pushed.rows == fallback.rows + assert pushed.total == fallback.total and pushed.has_more == fallback.has_more + first, second = pushed.scope["source_coverage"], fallback.scope["source_coverage"] + assert {k: v for k, v in first.items() if k != "source_queries"} == { + k: v for k, v in second.items() if k != "source_queries" + } + + +def last_observation(root, block_id): + for path in sorted(root.glob("device_0/*.db"), reverse=True): + with closing(sqlite3.connect(path)) as conn: + if conn.execute("SELECT 1 FROM block_0 WHERE id=?", [block_id]).fetchone(): + return path + raise AssertionError(f"No observation for block {block_id}") + + +def test_lifecycle_conflicts_are_checked_before_filters_and_page(tmp_path): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + with closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + assert conn.execute("UPDATE block_0 SET size=size+1 WHERE id=0").rowcount == 1 + with pytest.raises(QueryExecutionError, match="Conflicting observations"): + run_resolved(resolved, "block", {"id": 7}) + + +def test_latest_state_and_prior_proved_completion_survive_sql_reduction(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + resolved = DatasetResolver().inspect(root) + with closing(sqlite3.connect(last_observation(root, 0))) as conn, conn: + assert conn.execute("UPDATE block_0 SET state=0,freeEventId=-1 WHERE id=0").rowcount == 1 + params = {"id": 0} + pushed = run_resolved(resolved, "block", params) + monkeypatch.setattr(dataset_lifecycle_sql, "_attachment_limit", lambda: 1) + fallback = run_resolved(resolved, "block", params) + assert pushed.rows == fallback.rows + assert pushed.rows[0]["state"] == 0 + assert pushed.rows[0]["freeEventId"] == 10 + assert pushed.rows[0]["free_source"]["event"]["action"] == 6 + assert not pushed.rows[0]["terminal_survivor"] + + +def test_attached_members_are_read_only_and_connection_is_closed(tmp_path): + resolved = DatasetResolver().inspect(make_dataset(tmp_path / "readonly", devices=(0,))) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + with dataset_lifecycle_sql._AttachedSources(source) as attached: + with pytest.raises(sqlite3.OperationalError, match="readonly"): + attached.connection.execute("DELETE FROM s0.trace_entry_0") + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + + +def test_attached_query_timeout_clears_handler_and_closes_connection(tmp_path): + resolved = DatasetResolver().inspect(make_dataset(tmp_path / "timeout", devices=(0,))) + with closing(ContextCache()) as cache: + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(0.05, time.monotonic())) + with dataset_lifecycle_sql._AttachedSources(source) as attached: + with pytest.raises(QueryTimeoutError): + attached.read( + "WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n " + "WHERE x<100000000) SELECT SUM(x) FROM n" + ) + assert attached.connection.execute("SELECT 1").fetchone()[0] == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + + +def test_callstack_order_reuses_already_fetched_candidate_events(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "reuse", devices=(0,), slices=3) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(QueryService()) as service: + result = service.execute_query("event", {"order_by": "callstack", "limit": 1}, db_path=root) + assert result.returned == 1 + assert sum("ORDER BY" in sql and "trace_entry" in sql for sql in queries) == 3 + assert not any("WHERE t.id IN" in sql for sql in queries) + + +def test_attaching_locked_member_respects_remaining_deadline(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "locked", devices=(0,), slices=1) + resolved = DatasetResolver().inspect(root) + # Exercise the native non-immutable opening mode with an actual SQLite lock. + monkeypatch.setattr(type(resolved), "callstack_layout", property(lambda self: "v2")) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as locked: + locked.execute("BEGIN EXCLUSIVE") + with closing(ContextCache()) as cache: + start = time.monotonic() + source = DatasetSourceResolver(resolved, 0, cache, QueryBudget(0.05, start)) + attached = dataset_lifecycle_sql._AttachedSources(source) + with pytest.raises(QueryTimeoutError): + with attached: + attached.read("SELECT COUNT(*) FROM s0.trace_entry_0") + assert time.monotonic() - start < 0.5 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + attached.connection.execute("SELECT 1") + locked.rollback() + + +@pytest.mark.parametrize("layout", ["v1", "v2"]) +@pytest.mark.parametrize("collation,texts", [("NOCASE", ("a", "A")), ("RTRIM", ("a ", "a"))]) +def test_global_text_identity_and_candidates_ignore_source_collation( + tmp_path, layout, collation, texts +): + if layout == "v1": + root = make_dataset(tmp_path / "collated", devices=(0,), slices=1) + else: + root = ( + ImportService() + .import_snapshot( + dataclasses.replace(source_options(tmp_path), events_per_slice=99, set_focus=False) + ) + .db_path + ) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + item = manifest["devices"]["0"]["slices"][0] + member = root / item["file"] + with closing(sqlite3.connect(member)) as conn, conn: + ids = [r[0] for r in conn.execute("SELECT id FROM trace_entry_0 WHERE id>=0 ORDER BY id")] + if layout == "v1": + original = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='trace_entry_0'" + ).fetchone()[0] + conn.execute("ALTER TABLE trace_entry_0 RENAME TO original_trace") + conn.execute(original.replace("callstack TEXT", f"callstack TEXT COLLATE {collation}")) + conn.execute("INSERT INTO trace_entry_0 SELECT * FROM original_trace") + conn.execute("DROP TABLE original_trace") + conn.execute("UPDATE trace_entry_0 SET callstack=?", [texts[0]]) + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", [texts[1], ids[1]]) + else: + conn.execute("DROP TABLE callstack") + conn.execute( + f"CREATE TABLE callstack (id INTEGER PRIMARY KEY,callstack TEXT COLLATE {collation})" + ) + conn.executemany("INSERT INTO callstack VALUES (?,?)", enumerate(texts)) + conn.execute("UPDATE trace_entry_0 SET callstackId=0") + conn.execute("UPDATE trace_entry_0 SET callstackId=1 WHERE id=?", [ids[1]]) + conn.execute("UPDATE trace_entry_0 SET size=1 WHERE id>=0") + if layout == "v2": + item["sha256"] = hashlib.sha256(member.read_bytes()).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + expected_text = {event: texts[int(event == ids[1])] for event in ids} + with closing(QueryService()) as service: + groups = service.execute_query("callstack_analysis", {"min_count": 1}, db_path=root) + assert {r["callstack"]: r["alloc_count"] for r in groups.rows} == { + texts[0]: len(ids) - 1, + texts[1]: 1, + } + assert len({r["source_stack_id"] for r in groups.rows}) == 2 + for direction in ("ASC", "DESC"): + result = service.execute_query( + "event", {"order_by": "callstack", "order_dir": direction, "limit": 1}, db_path=root + ) + expected = sorted( + ids, key=lambda event: (expected_text[event], event), reverse=direction == "DESC" + )[0] + assert result.rows[0]["id"] == expected + assert result.rows[0]["callstack"] == expected_text[expected] From 973843032d61a999ec9d00b544f8bb89d1ff49e5 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:19:36 +0800 Subject: [PATCH 5/5] fix: preserve dataset source cache validation contracts --- src/pt_snap_cli/core/dataset_global.py | 12 ++++++--- src/pt_snap_cli/core/dataset_sources.py | 8 +++--- tests/core/test_dataset_global.py | 36 +++++++++++++++++++++++++ tests/core/test_dataset_source_reuse.py | 26 ++++++++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 857cfa8..eea697f 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -81,9 +81,6 @@ def _traces( for item in items: trace = f'"trace_entry_{sources.device.device_id}"' if with_sources: - projection = ( - "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" - ) if sources.dataset.callstack_layout == "v2": sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" else: @@ -308,7 +305,14 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, with_sources=True): + for _item, batch in _traces( + sources, + low, + high, + items, + ("id", *METRICS, "action", "address", "size", "stream"), + with_sources=True, + ): refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3a22a5d..4907016 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -211,7 +211,7 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() - if event in self._events: + if type(event) is int and event in self._events: result[event] = self._events[event] continue item = self.owner(event) @@ -248,16 +248,18 @@ def events_from_rows( Native rows must include joined callstack text and the local callstackId. Frame reads still incur the same work budget as events() source lookup. + Sources remain cached until this resolver operation ends. Serialized + value accounting excludes Python object overhead and is not an RSS cap. """ result: dict[int, EventSource] = {} for original in rows: self.budget.remaining() event_id = cast(int, original["id"]) + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") if event_id in self._events: result[event_id] = self._events[event_id] continue - if self.owner(event_id) != item: - raise QueryExecutionError("Source event does not belong to the selected shard.") row = dict(original) event_id = cast(int, row["id"]) text = row.get("callstack") diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 82a0dc5..86332b2 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -547,3 +547,39 @@ def test_cli_api_report_global_parity_and_range_options(tmp_path): result = json.loads(cli.stdout) assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15} assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10 + + +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_callstack_sorted_event_page_matches_standalone(tmp_path, native, direction): + if native: + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + importer = ImportService() + root = importer.import_snapshot(request).db_path + single = importer.import_snapshot( + dataclasses.replace(request, events_per_slice=None) + ).db_path + else: + root = case(tmp_path) + single = standalone(root, tmp_path) + # Standalone includes synthetic negative boundary IDs; compare real events. + params = { + "min_id": 7 if native else 0, + "order_by": "callstack", + "order_dir": direction, + "offset": 1, + "limit": 3, + } + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in ((0, 1) if native else (0,)): + actual = dataset.execute_query("event", params, device_id=device, exact_total=True) + expected = reference.execute_query("event", params, device_id=device, exact_total=True) + assert actual["rows"] + keys = list(expected["rows"][0]) + assert projected(actual["rows"], keys) == expected["rows"] + assert all("callstackId" not in row for row in actual["rows"]) + assert actual["total"] == expected["total"] + assert actual["has_more"] == expected["has_more"] + assert actual["truncated"] == expected["truncated"] + assert hashes(root) == before diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 3923079..07e6e32 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -200,3 +200,29 @@ def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): assert result.rows[0]["frames"] == [{"name": "f"}] # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. assert budget.work_rows == 610 + + +@pytest.mark.parametrize("invalid", [True, 1.0]) +def test_cached_event_lookup_preserves_exact_integer_validation(tmp_path, invalid): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + assert sources.events([invalid]) == {} + expected = sources.events([1]) + reads = sources.query_count + assert sources.events([invalid]) == {} + assert sources.events([1]) == expected + assert sources.query_count == reads + + +@pytest.mark.parametrize("warm_cache", [False, True]) +def test_preloaded_rows_reject_wrong_shard_even_when_cached(tmp_path, warm_cache): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=2) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + if warm_cache: + assert 0 in sources.events([0]) + with pytest.raises(QueryExecutionError, match="does not belong"): + sources.events_from_rows(sources.device.slices[1], [{"id": 0}])