diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 0990b3d..eea697f 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -73,15 +73,23 @@ def _traces( low: int, high: int, items: list[SliceRecord], - columns: tuple[str, ...], + columns: tuple[str, ...] = ("id", *METRICS), + *, + with_sources: bool = False, ): - # Each built-in supplies only the fields it actually consumes. In particular, - # inline stack text must not spend the byte budget of a counter-only query. - projection = ", ".join(f'"{column}"' for column in columns) + projection = ", ".join(f't."{column}"' for column in columns) for item in items: + trace = f'"trace_entry_{sources.device.device_id}"' + if with_sources: + if sources.dataset.callstack_layout == "v2": + sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id" + else: + sql = f"SELECT {projection}, t.callstack FROM {trace} t" + else: + sql = f"SELECT {projection} FROM {trace} t" rows = sources.read( sources.dataset.root / item.file, - f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id', + sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", [low, high], ) yield item, rows @@ -161,7 +169,10 @@ def _events(sources, low, high, items, params, template, scope): columns = ("id", *METRICS) if template == "event": columns += ("action", "address", "size", "stream") - for _item, batch in _traces(sources, low, high, items, columns): + with_sources = template == "event" and params["order_by"] == "callstack" + for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources): + if with_sources: + sources.events_from_rows(_item, batch, with_frames=False) for row in batch: sources.budget.remaining() event = cast(int, row["id"]) @@ -178,13 +189,8 @@ def _events(sources, low, high, items, params, template, scope): "last_event_id": last, "real_event_count": count, } - if template == "event": - # Only returned events need formatted text; resolve separately after the - # global window below. `order_by=callstack` requires text BEFORE sorting. - if params["order_by"] == "callstack": - refs = sources.events(cast(int, r["id"]) for r in rows) - for row in rows: - row["callstack"] = refs[cast(int, row["id"])].event.get("callstack") + # callstack sorting already has joined text; resolving sources here would + # eagerly fetch ordered frames that only the returned page needs. _sort(rows, params["order_by"], params["order_dir"] == "DESC") return rows @@ -299,8 +305,15 @@ def _lifecycles(sources, items, scope): def _stacks(sources, low, high, items, params, scope): grouped = {} events = ordered = 0 - for _item, batch in _traces(sources, low, high, items, ("id",)): - refs = sources.events(cast(int, r["id"]) for r in batch) + for _item, batch in _traces( + sources, + low, + high, + items, + ("id", *METRICS, "action", "address", "size", "stream"), + with_sources=True, + ): + refs = sources.events_from_rows(_item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 7a8e3f0..4907016 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -1,7 +1,7 @@ """Read-only, batched device/event source lookup over a validated dataset. No address joins, foreign-event insertion, text-to-frame reconstruction, or retained -source cache. The caller owns the Context LRU and the entire operation's deadline. +cross-operation source cache. The caller owns the Context LRU and the entire operation's deadline. """ from __future__ import annotations @@ -140,6 +140,11 @@ def __init__( self.budget = budget self.query_count = 0 self._starts = [s.start_event_id for s in self.device.slices] + # Lifetime is one budgeted operation. Each retained source was charged + # when fetched; reusing the same object performs no further database work. + self._events: dict[int, EventSource] = {} + self._frames_resolved: set[int] = set() + self._frame_schemas: dict[Path, bool] = {} @property def namespace(self) -> str: @@ -203,12 +208,15 @@ def read( def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: grouped: dict[int, list[int]] = defaultdict(list) + result: dict[int, EventSource] = {} for event in sorted(set(event_ids)): self.budget.remaining() + if type(event) is int and event in self._events: + result[event] = self._events[event] + continue item = self.owner(event) if item is not None: grouped[item.index].append(event) - result: dict[int, EventSource] = {} for index, ids in grouped.items(): item = self.device.slices[index] path = self.dataset.root / item.file @@ -224,34 +232,71 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]: rows = self.read( path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch) ) - for row in rows: - event_id = cast(int, row["id"]) - text = row.get("callstack") - captured = isinstance(text, str) and text != "" - local_id = cast(int | None, row.pop("callstackId", None)) - # V2 IDs select text WITHIN the owning DB; they are provenance, - # not dataset grouping keys. Global interning equivalence uses - # the full captured text, never a shortened display prefix. - identity = ( - "text:sha256:" - + hashlib.sha256( - (text if isinstance(text, str) else "").encode("utf-8") - ).hexdigest() - ) - result[event_id] = EventSource( - row, - index, - ( - f"{self.namespace}:{identity}" - if captured - else f"{self.namespace}:category:missing" - ), - "captured" if captured else "missing", - local_stack_id=local_id, - ) - self._attach_frames(path, ids, result) + result.update(self.events_from_rows(item, rows, with_frames=False)) + frame_groups: dict[int, list[int]] = defaultdict(list) + for event, source in result.items(): + frame_groups[source.slice_index].append(event) + for index, ids in frame_groups.items(): + self._ensure_frames(self.device.slices[index], ids) + self.budget.remaining() + return {event: self._events[event] for event in result} + + def events_from_rows( + self, item: SliceRecord, rows: list[dict[str, object]], *, with_frames: bool = True + ) -> dict[int, EventSource]: + """Resolve already charged full event rows without fetching them again. + + Native rows must include joined callstack text and the local callstackId. + Frame reads still incur the same work budget as events() source lookup. + Sources remain cached until this resolver operation ends. Serialized + value accounting excludes Python object overhead and is not an RSS cap. + """ + result: dict[int, EventSource] = {} + for original in rows: + self.budget.remaining() + event_id = cast(int, original["id"]) + if self.owner(event_id) != item: + raise QueryExecutionError("Source event does not belong to the selected shard.") + if event_id in self._events: + result[event_id] = self._events[event_id] + continue + row = dict(original) + event_id = cast(int, row["id"]) + text = row.get("callstack") + captured = isinstance(text, str) and text != "" + local_id = cast(int | None, row.pop("callstackId", None)) + # V2 IDs select text WITHIN the owning DB; they are provenance, + # not dataset grouping keys. Global interning equivalence uses + # the full captured text, never a shortened display prefix. + identity = ( + "text:sha256:" + + hashlib.sha256( + (text if isinstance(text, str) else "").encode("utf-8") + ).hexdigest() + ) + result[event_id] = EventSource( + row, + item.index, + ( + f"{self.namespace}:{identity}" + if captured + else f"{self.namespace}:category:missing" + ), + "captured" if captured else "missing", + local_stack_id=local_id, + ) + fresh = {event: source for event, source in result.items() if event not in self._events} + self._events.update(fresh) + if with_frames: + self._ensure_frames(item, list(result)) self.budget.remaining() - return result + return {event: self._events[event] for event in result} + + def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: + pending = [event for event in ids if event not in self._frames_resolved] + if pending: + self._attach_frames(self.dataset.root / item.file, pending, self._events) + self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: # This is a NEW, explicit read contract, not a guessed original-producer @@ -266,14 +311,19 @@ def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSou ("frameJson", "TEXT", 0), ], } - for table, columns in expected.items(): - kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) - schema = self.read(path, f'PRAGMA table_info("{table}")') - if ( - kind != [{"type": "table"}] - or [(r["name"], r["type"], r["pk"]) for r in schema] != columns - ): - return # Unknown/absent schemas never imply precision. + if path not in self._frame_schemas: + self._frame_schemas[path] = True + for table, columns in expected.items(): + kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = self.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + self._frame_schemas[path] = False + break # Unknown/absent schemas never imply precision. + if not self._frame_schemas[path]: + return for start in range(0, len(ids), SOURCE_BATCH_SIZE): batch = ids[start : start + SOURCE_BATCH_SIZE] placeholders = ",".join("?" for _ in batch) diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 82a0dc5..86332b2 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -547,3 +547,39 @@ def test_cli_api_report_global_parity_and_range_options(tmp_path): result = json.loads(cli.stdout) assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15} assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10 + + +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_callstack_sorted_event_page_matches_standalone(tmp_path, native, direction): + if native: + request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False) + importer = ImportService() + root = importer.import_snapshot(request).db_path + single = importer.import_snapshot( + dataclasses.replace(request, events_per_slice=None) + ).db_path + else: + root = case(tmp_path) + single = standalone(root, tmp_path) + # Standalone includes synthetic negative boundary IDs; compare real events. + params = { + "min_id": 7 if native else 0, + "order_by": "callstack", + "order_dir": direction, + "offset": 1, + "limit": 3, + } + before = hashes(root) + with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference: + for device in ((0, 1) if native else (0,)): + actual = dataset.execute_query("event", params, device_id=device, exact_total=True) + expected = reference.execute_query("event", params, device_id=device, exact_total=True) + assert actual["rows"] + keys = list(expected["rows"][0]) + assert projected(actual["rows"], keys) == expected["rows"] + assert all("callstackId" not in row for row in actual["rows"]) + assert actual["total"] == expected["total"] + assert actual["has_more"] == expected["has_more"] + assert actual["truncated"] == expected["truncated"] + assert hashes(root) == before diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py new file mode 100644 index 0000000..07e6e32 --- /dev/null +++ b/tests/core/test_dataset_source_reuse.py @@ -0,0 +1,228 @@ +"""Operation-local reuse preserves real read accounting and source identity.""" + +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_service import QueryService +from tests.test_dataset_focus import make_dataset + + +def dataset(tmp_path, count, stack="shared", blocks=False): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["eventsPerSlice"] = count + manifest["devices"]["0"]["eventCount"] = count + manifest["devices"]["0"]["slices"][0]["endEventId"] = count - 1 + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM trace_entry_0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,?,8,0,8,8,16,?)", + ((i, i * 8, stack) for i in range(count)), + ) + if blocks: + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,8,8,1,?,-1)", + ((i, i * 8, i) for i in range(count)), + ) + return root + + +@pytest.mark.parametrize("template", ["callstack_analysis", "event"]) +def test_sixty_thousand_events_are_not_read_twice(tmp_path, template): + root = dataset(tmp_path, 60_000) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + result = service.execute_query( + template, + params=( + {"min_count": 1} + if template == "callstack_analysis" + else { + "order_by": "callstack", + "limit": 1, + } + ), + db_path=root, + _budget=budget, + ) + finally: + service.close() + assert result.returned == 1 + assert budget.work_rows == 60_001 + if template == "callstack_analysis": + assert result.rows[0]["alloc_count"] == 60_000 + + +def test_target_and_distinct_events_reuse_only_identical_sources(tmp_path): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + budget = QueryBudget(None, time.monotonic()) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, budget) + sources.target(3) + first = sources.events([0, 1, 2, 3]) + reads, work = sources.query_count, budget.work_rows + repeated = sources.events([3, 2, 1, 0, 3]) + assert sources.query_count == reads + assert budget.work_rows == work == 4 + assert repeated[3] is first[3] + assert first[0].event["id"] != first[1].event["id"] + assert first[0].stack_id == first[1].stack_id + + +@pytest.mark.parametrize("count, succeeds", [(49_999, True), (50_000, False)]) +def test_distinct_block_and_allocation_work_still_counts(tmp_path, count, succeeds): + root = dataset(tmp_path, count, blocks=True) + budget = QueryBudget(None, time.monotonic()) + service = QueryService() + try: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows == 100_001 + finally: + service.close() + + +@pytest.mark.parametrize("count, succeeds", [(3_000, True), (5_000, False)]) +def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds): + root = dataset(tmp_path, count, stack="x" * 16_384, blocks=True) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + if succeeds: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_rows == 2 * count + 1 + else: + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert budget.work_rows < budget.max_work_rows + assert budget.work_bytes > budget.max_work_bytes + + +def test_id_sorted_page_does_not_materialize_unneeded_stack_text(tmp_path): + root = dataset(tmp_path, 10_000, stack="x" * 8192) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + params={"order_by": "id", "limit": 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert budget.work_bytes < 2_000_000 + + +def framed_dataset(tmp_path, count): + root = dataset(tmp_path, count) + manifest = json.loads((root / "manifest.json").read_text()) + manifest["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + (root / "manifest.json").write_text(json.dumps(manifest)) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT," + "PRIMARY KEY(eventId,frameIndex));" + ) + conn.executemany( + "INSERT INTO pt_snap_frame_coverage VALUES (?,1)", ((i,) for i in range(count)) + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,0,?)", ((i, '{"name":"f"}') for i in range(count)) + ) + return root + + +def test_frame_schema_is_checked_once_across_source_batches(tmp_path, monkeypatch): + root = framed_dataset(tmp_path, 600) + resolved = DatasetResolver().inspect(root) + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + queries.append(sql) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + first = sources.events(range(512)) + second = sources.events(range(512, 600)) + assert all( + source.frames == [{"name": "f"}] for source in [*first.values(), *second.values()] + ) + assert sum("PRAGMA table_info" in sql for sql in queries) == 2 + assert sum("sqlite_master" in sql for sql in queries) == 2 + + +def test_callstack_sort_loads_frames_only_for_returned_page(tmp_path): + root = framed_dataset(tmp_path, 600) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", params={"order_by": "callstack", "limit": 1}, db_path=root, _budget=budget + ) + assert result.rows[0]["frames"] == [{"name": "f"}] + # 600 trace rows + 7 schema rows + 1 coverage + 1 frame + 1 output. + assert budget.work_rows == 610 + + +@pytest.mark.parametrize("invalid", [True, 1.0]) +def test_cached_event_lookup_preserves_exact_integer_validation(tmp_path, invalid): + root = dataset(tmp_path, 4) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + assert sources.events([invalid]) == {} + expected = sources.events([1]) + reads = sources.query_count + assert sources.events([invalid]) == {} + assert sources.events([1]) == expected + assert sources.query_count == reads + + +@pytest.mark.parametrize("warm_cache", [False, True]) +def test_preloaded_rows_reject_wrong_shard_even_when_cached(tmp_path, warm_cache): + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=2) + resolved = DatasetResolver().inspect(root) + with closing(ContextCache()) as cache: + sources = DatasetSourceResolver(resolved, 0, cache, QueryBudget(None, time.monotonic())) + if warm_cache: + assert 0 in sources.events([0]) + with pytest.raises(QueryExecutionError, match="does not belong"): + sources.events_from_rows(sources.device.slices[1], [{"id": 0}])