From c5ed2a8dff80318e3bd4ae267a48eb9a6909ae0b Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:12:00 +0800 Subject: [PATCH 1/4] fix: narrow dataset trace projections by template (#220) --- src/pt_snap_cli/core/dataset_global.py | 22 ++++++++--- tests/core/test_dataset_projections.py | 54 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/core/test_dataset_projections.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index cbc73f6..0990b3d 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -68,11 +68,20 @@ def _scope( } -def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[SliceRecord]): +def _traces( + sources: DatasetSourceResolver, + low: int, + high: int, + items: list[SliceRecord], + columns: tuple[str, ...], +): + # Each built-in supplies only the fields it actually consumes. In particular, + # inline stack text must not spend the byte budget of a counter-only query. + projection = ", ".join(f'"{column}"' for column in columns) for item in items: rows = sources.read( sources.dataset.root / item.file, - f'SELECT * FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', [low, high], ) yield item, rows @@ -81,7 +90,7 @@ def _traces(sources: DatasetSourceResolver, low: int, high: int, items: list[Sli def _peaks(sources, low, high, items, template, scope): chosen: dict[str, dict[str, object]] = {} count, first, last = 0, None, None - for _item, rows in _traces(sources, low, high, items): + for _item, rows in _traces(sources, low, high, items, ("id", *METRICS)): for row in rows: sources.budget.remaining() event = cast(int, row["id"]) @@ -149,7 +158,10 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items): + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + for _item, batch in _traces(sources, low, high, items, columns): for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -287,7 +299,7 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items): + for _item, batch in _traces(sources, low, high, items, ("id",)): refs = sources.events(cast(int, r["id"]) for r in batch) for source in refs.values(): sources.budget.remaining() diff --git a/tests/core/test_dataset_projections.py b/tests/core/test_dataset_projections.py new file mode 100644 index 0000000..6811d89 --- /dev/null +++ b/tests/core/test_dataset_projections.py @@ -0,0 +1,54 @@ +"""Per-template scan projections must not charge unrelated inline stack text.""" + +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +@pytest.mark.parametrize("template", ["memory_peak", "allocator_gap", "allocation"]) +def test_counter_query_budget_is_independent_of_inline_stack_length(tmp_path, template): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + observations = [] + for stack in ("x" * 16, "x" * 8192, "x" * 32768, "栈🙂" * 8192): + for path in root.glob("device_0/*.db"): + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=?", (stack,)) + service = QueryService() + try: + budget = QueryBudget(None, time.monotonic(), max_work_bytes=8192) + result = service.execute_query(template, db_path=root, _budget=budget) + observations.append((result.rows, budget.work_rows, budget.work_bytes)) + finally: + service.close() + assert all(item == observations[0] for item in observations) + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_event_projection_preserves_callstack_sort_and_paging(tmp_path, direction): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=3) + stacks = ["z" * 8192, None, "栈🙂" * 8192, "a" * 16384, "", "a" * 16384] + for index, stack in enumerate(stacks): + path = root / f"device_0/slice_{index // 2:05d}.db" + with closing(sqlite3.connect(path)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id=?", (stack, index)) + service = QueryService() + try: + params = {"order_by": "callstack", "order_dir": direction, "offset": 1, "limit": 3} + result = service.execute_query("event", params, db_path=root, exact_total=True) + expected = sorted( + range(len(stacks)), + key=lambda event: (stacks[event] is not None, stacks[event], event), + reverse=direction == "DESC", + )[1:4] + assert [row["id"] for row in result.rows] == expected + assert [row["callstack"] for row in result.rows] == [stacks[event] for event in expected] + assert result.total == 6 and result.has_more and result.truncated + assert all(len(row) == 9 for row in result.rows) + finally: + service.close() From b1a2d068e18689cd9aea62d71aca36bf7292c9b9 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:14:26 +0800 Subject: [PATCH 2/4] fix: reuse budgeted event sources within dataset queries (#222) --- src/pt_snap_cli/core/dataset_global.py | 32 ++++-- src/pt_snap_cli/core/dataset_sources.py | 87 ++++++++++----- tests/core/test_dataset_source_reuse.py | 134 ++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 tests/core/test_dataset_source_reuse.py diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 0990b3d..a7ab3ec 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -73,15 +73,26 @@ def _traces( low: int, high: int, items: list[SliceRecord], - columns: tuple[str, ...], + columns: tuple[str, ...] = ("id", *METRICS), + *, + with_sources: bool = False, ): - # Each built-in supplies only the fields it actually consumes. In particular, - # inline stack text must not spend the byte budget of a counter-only query. - projection = ", ".join(f'"{column}"' for column in columns) + projection = ", ".join(f't."{column}"' for column in columns) for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if with_sources: + projection = ( + "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" + ) + if sources.dataset.callstack_layout == "v2": + sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + sql = f"SELECT {projection}, t.callstack FROM {trace} t" + else: + sql = f"SELECT {projection} FROM {trace} t" rows = sources.read( sources.dataset.root / item.file, - f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", [low, high], ) yield item, rows @@ -158,10 +169,9 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - columns = ("id", *METRICS) - if template == "event": - columns += ("action", "address", "size", "stream") - for _item, batch in _traces(sources, low, high, items, columns): + for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): + if template == "event": + sources.events_from_rows(_item, batch) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -299,8 +309,8 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, ("id",)): - refs = sources.events(cast(int, r["id"]) for r in batch) + for _item, batch in _traces(sources, low, high, items, with_sources=True): + refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 7a8e3f0..3532752 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -1,7 +1,7 @@ """Read-only, batched device/event source lookup over a validated dataset. No address joins, foreign-event insertion, text-to-frame reconstruction, or retained -source cache. The caller owns the Context LRU and the entire operation's deadline. +cross-operation source cache. The caller owns the Context LRU and the entire operation's deadline. """ from __future__ import annotations @@ -140,6 +140,9 @@ def __init__( self.budget = budget self.query_count = 0 self._starts = [s.start_event_id for s in self.device.slices] + # Lifetime is one budgeted operation. Each retained source was charged + # when fetched; reusing the same object performs no further database work. + self._events: dict[int, EventSource] = {} @property def namespace(self) -> str: @@ -203,12 +206,15 @@ def read( def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: grouped: dict[int, list[int]] = defaultdict(list) + result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() + if event in self._events: + result[event] = self._events[event] + continue item = self.owner(event) if item is not None: grouped[item.index].append(event) - result: dict[int, EventSource] = {} for index, ids in grouped.items(): item = self.device.slices[index] path = self.dataset.root / item.file @@ -224,32 +230,57 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - for row in rows: - event_id = cast(int, row["id"]) - text = row.get("callstack") - captured = isinstance(text, str) and text != "" - local_id = cast(int | None, row.pop("callstackId", None)) - # V2 IDs select text WITHIN the owning DB; they are provenance, - # not dataset grouping keys. Global interning equivalence uses - # the full captured text, never a shortened display prefix. - identity = ( - "text:sha256:" - + hashlib.sha256( - (text if isinstance(text, str) else "").encode("utf-8") - ).hexdigest() - ) - result[event_id] = EventSource( - row, - index, - ( - f"{self.namespace}:{identity}" - if captured - else f"{self.namespace}:category:missing" - ), - "captured" if captured else "missing", - local_stack_id=local_id, - ) - self._attach_frames(path, ids, result) + result.update(self.events_from_rows(item, rows)) + self.budget.remaining() + return result + + def events_from_rows( + self, item: SliceRecord, rows: list[dict[str, object]] + ) -> dict[int, EventSource]: + """Resolve already charged full event rows without fetching them again. + + Native rows must include joined callstack text and the local callstackId. + Frame reads still incur the same work budget as events() source lookup. + """ + result: dict[int, EventSource] = {} + for original in rows: + self.budget.remaining() + event_id = cast(int, original["id"]) + if event_id in self._events: + result[event_id] = self._events[event_id] + continue + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") + row = dict(original) + event_id = cast(int, row["id"]) + text = row.get("callstack") + captured = isinstance(text, str) and text != "" + local_id = cast(int | None, row.pop("callstackId", None)) + # V2 IDs select text WITHIN the owning DB; they are provenance, + # not dataset grouping keys. Global interning equivalence uses + # the full captured text, never a shortened display prefix. + identity = ( + "text:sha256:" + + hashlib.sha256( + (text if isinstance(text, str) else "").encode("utf-8") + ).hexdigest() + ) + result[event_id] = EventSource( + row, + item.index, + ( + f"{self.namespace}:{identity}" + if captured + else f"{self.namespace}:category:missing" + ), + "captured" if captured else "missing", + local_stack_id=local_id, + ) + fresh = {event: source for event, source in result.items() if event not in self._events} + if fresh: + self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) + self._events.update(fresh) + result.update(fresh) self.budget.remaining() return result diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py new file mode 100644 index 0000000..87c8265 --- /dev/null +++ b/tests/core/test_dataset_source_reuse.py @@ -0,0 +1,134 @@ +"""Operation-local reuse preserves real read accounting and source identity.""" + +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +def dataset(tmp_path, count, stack="shared", blocks=False): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["eventsPerSlice"] = count + manifest["devices"]["0"]["eventCount"] = count + manifest["devices"]["0"]["slices"][0]["endEventId"] = count - 1 + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,8,8,16,?)", + ((i, i * 8, stack) for i in range(count)), + ) + if blocks: + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(count)), + ) + return root + + +@pytest.mark.parametrize("template", ["callstack_analysis", "event"]) +def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): + root = dataset(tmp_path, 60_000) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + result = service.execute_query( + template, + params=( + {"min_count": 1} + if template == "callstack_analysis" + else { + "order_by": "callstack", + "limit": 1, + } + ), + db_path=root, + _budget=budget, + ) + finally: + service.close() + assert result.returned == 1 + assert budget.work_rows == 60_001 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 60_000 + + +def test_target_and_distinct_events_reuse_only_identical_sources(tmp_path): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + budget = QueryBudget(None, time.monotonic()) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, budget) + sources.target(3) + first = sources.events([0, 1, 2, 3]) + reads, work = sources.query_count, budget.work_rows + repeated = sources.events([3, 2, 1, 0, 3]) + assert sources.query_count == reads + assert budget.work_rows == work == 4 + assert repeated[3] is first[3] + assert first[0].event["id"] != first[1].event["id"] + assert first[0].stack_id == first[1].stack_id + + +@pytest.mark.parametrize("count, succeeds", [(49_999, True), (50_000, False)]) +def test_distinct_block_and_allocation_work_still_counts(tmp_path, count, succeeds): + root = dataset(tmp_path, count, blocks=True) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows == 100_001 + finally: + service.close() + + +@pytest.mark.parametrize("count, succeeds", [(3_000, True), (5_000, False)]) +def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds): + root = dataset(tmp_path, count, stack="x" * 16_384, blocks=True) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows < budget.max_work_rows + assert budget.work_bytes > budget.max_work_bytes From eda49166421dcab5c73fe7d107504122d701928c Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:17:49 +0800 Subject: [PATCH 3/4] fix: defer event page hydration and reuse frame schema checks --- src/pt_snap_cli/core/dataset_global.py | 19 ++++--- src/pt_snap_cli/core/dataset_sources.py | 47 +++++++++++------ tests/core/test_dataset_source_reuse.py | 68 +++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index a7ab3ec..857cfa8 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -169,9 +169,13 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: def _events(sources, low, high, items, params, template, scope): rows = [] first, last, count = None, None, 0 - for _item, batch in _traces(sources, low, high, items, with_sources=template == "event"): - if template == "event": - sources.events_from_rows(_item, batch) + columns = ("id", *METRICS) + if template == "event": + columns += ("action", "address", "size", "stream") + with_sources = template == "event" and params["order_by"] == "callstack" + for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): + if with_sources: + sources.events_from_rows(_item, batch, with_frames=False) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -188,13 +192,8 @@ def _events(sources, low, high, items, params, template, scope): "last_event_id": last, "real_event_count": count, } - if template == "event": - # Only returned events need formatted text; resolve separately after the - # global window below. `order_by=callstack` requires text BEFORE sorting. - if params["order_by"] == "callstack": - refs = sources.events(cast(int, r["id"]) for r in rows) - for row in rows: - row["callstack"] = refs[cast(int, row["id"])].event.get("callstack") + # callstack sorting already has joined text; resolving sources here would + # eagerly fetch ordered frames that only the returned page needs. _sort(rows, params["order_by"], params["order_dir"] == "DESC") return rows diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3532752..3a22a5d 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -143,6 +143,8 @@ def __init__( # Lifetime is one budgeted operation. Each retained source was charged # when fetched; reusing the same object performs no further database work. self._events: dict[int, EventSource] = {} + self._frames_resolved: set[int] = set() + self._frame_schemas: dict[Path, bool] = {} @property def namespace(self) -> str: @@ -230,12 +232,17 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - result.update(self.events_from_rows(item, rows)) + result.update(self.events_from_rows(item, rows, with_frames=False)) + frame_groups: dict[int, list[int]] = defaultdict(list) + for event, source in result.items(): + frame_groups[source.slice_index].append(event) + for index, ids in frame_groups.items(): + self._ensure_frames(self.device.slices[index], ids) self.budget.remaining() - return result + return {event: self._events[event] for event in result} def events_from_rows( - self, item: SliceRecord, rows: list[dict[str, object]] + self, item: SliceRecord, rows: list[dict[str, object]], *, with_frames: bool = True ) -> dict[int, EventSource]: """Resolve already charged full event rows without fetching them again. @@ -277,12 +284,17 @@ def events_from_rows( local_stack_id=local_id, ) fresh = {event: source for event, source in result.items() if event not in self._events} - if fresh: - self._attach_frames(self.dataset.root / item.file, list(fresh), fresh) self._events.update(fresh) - result.update(fresh) + if with_frames: + self._ensure_frames(item, list(result)) self.budget.remaining() - return result + return {event: self._events[event] for event in result} + + def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: + pending = [event for event in ids if event not in self._frames_resolved] + if pending: + self._attach_frames(self.dataset.root / item.file, pending, self._events) + self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: # This is a NEW, explicit read contract, not a guessed original-producer @@ -297,14 +309,19 @@ def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSou ("frameJson", "TEXT", 0), ], } - for table, columns in expected.items(): - kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) - schema = self.read(path, f'PRAGMA table_info("{table}")') - if ( - kind != [{"type": "table"}] - or [(r["name"], r["type"], r["pk"]) for r in schema] != columns - ): - return # Unknown/absent schemas never imply precision. + if path not in self._frame_schemas: + self._frame_schemas[path] = True + for table, columns in expected.items(): + kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = self.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + self._frame_schemas[path] = False + break # Unknown/absent schemas never imply precision. + if not self._frame_schemas[path]: + return for start in range(0, len(ids), SOURCE_BATCH_SIZE): batch = ids[start : start + SOURCE_BATCH_SIZE] placeholders = ",".join("?" for _ in batch) diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 87c8265..3923079 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -132,3 +132,71 @@ def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds ) assert budget.work_rows < budget.max_work_rows assert budget.work_bytes > budget.max_work_bytes + + +def test_id_sorted_page_does_not_materialize_unneeded_stack_text(tmp_path): + root = dataset(tmp_path, 10_000, stack="x" * 8192) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + params={"order_by": "id", "limit": 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_bytes < 2_000_000 + + +def framed_dataset(tmp_path, count): + root = dataset(tmp_path, count) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT," + "PRIMARY KEY(eventId,frameIndex));" + ) + conn.executemany( + "INSERT INTO pt_snap_frame_coverage VALUES (?,1)", ((i,) for i in range(count)) + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", ((i, '{"name":"f"}') for i in range(count)) + ) + return root + + +def test_frame_schema_is_checked_once_across_source_batches(tmp_path, monkeypatch): + root = framed_dataset(tmp_path, 600) + resolved = DatasetResolver().inspect(root) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + first = sources.events(range(512)) + second = sources.events(range(512, 600)) + assert all( + source.frames == [{"name": "f"}] for source in [*first.values(), *second.values()] + ) + assert sum("PRAGMA table_info" in sql for sql in queries) == 2 + assert sum("sqlite_master" in sql for sql in queries) == 2 + + +def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): + root = framed_dataset(tmp_path, 600) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget + ) + assert result.rows[0]["frames"] == [{"name": "f"}] + # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. + assert budget.work_rows == 610 From 973843032d61a999ec9d00b544f8bb89d1ff49e5 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:19:36 +0800 Subject: [PATCH 4/4] fix: preserve dataset source cache validation contracts --- src/pt_snap_cli/core/dataset_global.py | 12 ++++++--- src/pt_snap_cli/core/dataset_sources.py | 8 +++--- tests/core/test_dataset_global.py | 36 +++++++++++++++++++++++++ tests/core/test_dataset_source_reuse.py | 26 ++++++++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 857cfa8..eea697f 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -81,9 +81,6 @@ def _traces( for item in items: trace = f'"trace_entry_{sources.device.device_id}"' if with_sources: - projection = ( - "t.id, t.action, t.address, t.size, t.stream, t.allocated, t.active, t.reserved" - ) if sources.dataset.callstack_layout == "v2": sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" else: @@ -308,7 +305,14 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, with_sources=True): + for _item, batch in _traces( + sources, + low, + high, + items, + ("id", *METRICS, "action", "address", "size", "stream"), + with_sources=True, + ): refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 3a22a5d..4907016 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -211,7 +211,7 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() - if event in self._events: + if type(event) is int and event in self._events: result[event] = self._events[event] continue item = self.owner(event) @@ -248,16 +248,18 @@ def events_from_rows( Native rows must include joined callstack text and the local callstackId. Frame reads still incur the same work budget as events() source lookup. + Sources remain cached until this resolver operation ends. Serialized + value accounting excludes Python object overhead and is not an RSS cap. """ result: dict[int, EventSource] = {} for original in rows: self.budget.remaining() event_id = cast(int, original["id"]) + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") if event_id in self._events: result[event_id] = self._events[event_id] continue - if self.owner(event_id) != item: - raise QueryExecutionError("Source event does not belong to the selected shard.") row = dict(original) event_id = cast(int, row["id"]) text = row.get("callstack") diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 82a0dc5..86332b2 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -547,3 +547,39 @@ def test_cli_api_report_global_parity_and_range_options(tmp_path): result = json.loads(cli.stdout) assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15} assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10 + + +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_callstack_sorted_event_page_matches_standalone(tmp_path, native, direction): + if native: + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + importer = ImportService() + root = importer.import_snapshot(request).db_path + single = importer.import_snapshot( + dataclasses.replace(request, events_per_slice=None) + ).db_path + else: + root = case(tmp_path) + single = standalone(root, tmp_path) + # Standalone includes synthetic negative boundary IDs; compare real events. + params = { + "min_id": 7 if native else 0, + "order_by": "callstack", + "order_dir": direction, + "offset": 1, + "limit": 3, + } + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in ((0, 1) if native else (0,)): + actual = dataset.execute_query("event", params, device_id=device, exact_total=True) + expected = reference.execute_query("event", params, device_id=device, exact_total=True) + assert actual["rows"] + keys = list(expected["rows"][0]) + assert projected(actual["rows"], keys) == expected["rows"] + assert all("callstackId" not in row for row in actual["rows"]) + assert actual["total"] == expected["total"] + assert actual["has_more"] == expected["has_more"] + assert actual["truncated"] == expected["truncated"] + assert hashes(root) == before diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index 3923079..07e6e32 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -200,3 +200,29 @@ def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): assert result.rows[0]["frames"] == [{"name": "f"}] # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. assert budget.work_rows == 610 + + +@pytest.mark.parametrize("invalid", [True, 1.0]) +def test_cached_event_lookup_preserves_exact_integer_validation(tmp_path, invalid): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + assert sources.events([invalid]) == {} + expected = sources.events([1]) + reads = sources.query_count + assert sources.events([invalid]) == {} + assert sources.events([1]) == expected + assert sources.query_count == reads + + +@pytest.mark.parametrize("warm_cache", [False, True]) +def test_preloaded_rows_reject_wrong_shard_even_when_cached(tmp_path, warm_cache): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=2) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + if warm_cache: + assert 0 in sources.events([0]) + with pytest.raises(QueryExecutionError, match="does not belong"): + sources.events_from_rows(sources.device.slices[1], [{"id": 0}])