Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 28 additions & 15 deletions src/pt_snap_cli/core/dataset_global.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,23 @@ def _traces(
low: int,
high: int,
items: list[SliceRecord],
columns: tuple[str, ...],
columns: tuple[str, ...] = ("id", *METRICS),
*,
with_sources: bool = False,
):
# Each built-in supplies only the fields it actually consumes. In particular,
# inline stack text must not spend the byte budget of a counter-only query.
projection = ", ".join(f'"{column}"' for column in columns)
projection = ", ".join(f't."{column}"' for column in columns)
for item in items:
trace = f'"trace_entry_{sources.device.device_id}"'
if with_sources:
if sources.dataset.callstack_layout == "v2":
sql = f"SELECT {projection}, t.callstackId, cs.callstack FROM {trace} t LEFT JOIN callstack cs ON t.callstackId=cs.id"
else:
sql = f"SELECT {projection}, t.callstack FROM {trace} t"
else:
sql = f"SELECT {projection} FROM {trace} t"
rows = sources.read(
sources.dataset.root / item.file,
f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" WHERE id>=0 AND id>=? AND id<=? ORDER BY id',
sql + " WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id",
[low, high],
)
yield item, rows
Expand Down Expand Up @@ -161,7 +169,10 @@ def _events(sources, low, high, items, params, template, scope):
columns = ("id", *METRICS)
if template == "event":
columns += ("action", "address", "size", "stream")
for _item, batch in _traces(sources, low, high, items, columns):
with_sources = template == "event" and params["order_by"] == "callstack"
for _item, batch in _traces(sources, low, high, items, columns, with_sources=with_sources):
if with_sources:
sources.events_from_rows(_item, batch, with_frames=False)
for row in batch:
sources.budget.remaining()
event = cast(int, row["id"])
Expand All @@ -178,13 +189,8 @@ def _events(sources, low, high, items, params, template, scope):
"last_event_id": last,
"real_event_count": count,
}
if template == "event":
# Only returned events need formatted text; resolve separately after the
# global window below. `order_by=callstack` requires text BEFORE sorting.
if params["order_by"] == "callstack":
refs = sources.events(cast(int, r["id"]) for r in rows)
for row in rows:
row["callstack"] = refs[cast(int, row["id"])].event.get("callstack")
# callstack sorting already has joined text; resolving sources here would
# eagerly fetch ordered frames that only the returned page needs.
_sort(rows, params["order_by"], params["order_dir"] == "DESC")
return rows

Expand Down Expand Up @@ -299,8 +305,15 @@ def _lifecycles(sources, items, scope):
def _stacks(sources, low, high, items, params, scope):
grouped = {}
events = ordered = 0
for _item, batch in _traces(sources, low, high, items, ("id",)):
refs = sources.events(cast(int, r["id"]) for r in batch)
for _item, batch in _traces(
sources,
low,
high,
items,
("id", *METRICS, "action", "address", "size", "stream"),
with_sources=True,
):
refs = sources.events_from_rows(_item, batch)
for source in refs.values():
sources.budget.remaining()
text = source.event.get("callstack")
Expand Down
124 changes: 87 additions & 37 deletions src/pt_snap_cli/core/dataset_sources.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Read-only, batched device/event source lookup over a validated dataset.

No address joins, foreign-event insertion, text-to-frame reconstruction, or retained
source cache. The caller owns the Context LRU and the entire operation's deadline.
cross-operation source cache. The caller owns the Context LRU and the entire operation's deadline.
"""

from __future__ import annotations
Expand Down Expand Up @@ -140,6 +140,11 @@ def __init__(
self.budget = budget
self.query_count = 0
self._starts = [s.start_event_id for s in self.device.slices]
# Lifetime is one budgeted operation. Each retained source was charged
# when fetched; reusing the same object performs no further database work.
self._events: dict[int, EventSource] = {}
self._frames_resolved: set[int] = set()
self._frame_schemas: dict[Path, bool] = {}

@property
def namespace(self) -> str:
Expand Down Expand Up @@ -203,12 +208,15 @@ def read(

def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]:
grouped: dict[int, list[int]] = defaultdict(list)
result: dict[int, EventSource] = {}
for event in sorted(set(event_ids)):
self.budget.remaining()
if type(event) is int and event in self._events:
result[event] = self._events[event]
continue
item = self.owner(event)
if item is not None:
grouped[item.index].append(event)
result: dict[int, EventSource] = {}
for index, ids in grouped.items():
item = self.device.slices[index]
path = self.dataset.root / item.file
Expand All @@ -224,34 +232,71 @@ def events(self, event_ids: Iterable[int]) -> dict[int, EventSource]:
rows = self.read(
path, sql + f" WHERE t.id IN ({placeholders}) AND t.id>=0", list(batch)
)
for row in rows:
event_id = cast(int, row["id"])
text = row.get("callstack")
captured = isinstance(text, str) and text != ""
local_id = cast(int | None, row.pop("callstackId", None))
# V2 IDs select text WITHIN the owning DB; they are provenance,
# not dataset grouping keys. Global interning equivalence uses
# the full captured text, never a shortened display prefix.
identity = (
"text:sha256:"
+ hashlib.sha256(
(text if isinstance(text, str) else "").encode("utf-8")
).hexdigest()
)
result[event_id] = EventSource(
row,
index,
(
f"{self.namespace}:{identity}"
if captured
else f"{self.namespace}:category:missing"
),
"captured" if captured else "missing",
local_stack_id=local_id,
)
self._attach_frames(path, ids, result)
result.update(self.events_from_rows(item, rows, with_frames=False))
frame_groups: dict[int, list[int]] = defaultdict(list)
for event, source in result.items():
frame_groups[source.slice_index].append(event)
for index, ids in frame_groups.items():
self._ensure_frames(self.device.slices[index], ids)
self.budget.remaining()
return {event: self._events[event] for event in result}

def events_from_rows(
self, item: SliceRecord, rows: list[dict[str, object]], *, with_frames: bool = True
) -> dict[int, EventSource]:
"""Resolve already charged full event rows without fetching them again.

Native rows must include joined callstack text and the local callstackId.
Frame reads still incur the same work budget as events() source lookup.
Sources remain cached until this resolver operation ends. Serialized
value accounting excludes Python object overhead and is not an RSS cap.
"""
result: dict[int, EventSource] = {}
for original in rows:
self.budget.remaining()
event_id = cast(int, original["id"])
if self.owner(event_id) != item:
raise QueryExecutionError("Source event does not belong to the selected shard.")
if event_id in self._events:
result[event_id] = self._events[event_id]
continue
row = dict(original)
event_id = cast(int, row["id"])
text = row.get("callstack")
captured = isinstance(text, str) and text != ""
local_id = cast(int | None, row.pop("callstackId", None))
# V2 IDs select text WITHIN the owning DB; they are provenance,
# not dataset grouping keys. Global interning equivalence uses
# the full captured text, never a shortened display prefix.
identity = (
"text:sha256:"
+ hashlib.sha256(
(text if isinstance(text, str) else "").encode("utf-8")
).hexdigest()
)
result[event_id] = EventSource(
row,
item.index,
(
f"{self.namespace}:{identity}"
if captured
else f"{self.namespace}:category:missing"
),
"captured" if captured else "missing",
local_stack_id=local_id,
)
fresh = {event: source for event, source in result.items() if event not in self._events}
self._events.update(fresh)

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Low] events_from_rows 把每个扫描行写入 self._events,并保留到整个操作结束。旧的 _stacks 每批结果处理后即释放。预算上限是序列化字节(注释也说明不是 RSS 上限),Python 对象开销会使实际内存高于计费字节。去除双读的取舍是合理的,但建议在注释中说明,或在多分片 v1 数据集上测一下峰值内存。

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已更新至 973843032d61a999ec9d00b544f8bb89d1ff49e5。

已在 events_from_rows 文档中明确:源对象保留到当前 resolver 操作结束,序列化值预算不包含 Python 对象开销,也不是 RSS 上限。本次未将其表述为峰值内存保证。

验证:相关回归 54/54、provenance 7/7,Ruff/Black/diff 与 type check(0 errors)通过。最终完整套件 1950/1950,wheel 构建/审计/安装验收通过。远端提交与 CI 另行补充。

if with_frames:
self._ensure_frames(item, list(result))
self.budget.remaining()
return result
return {event: self._events[event] for event in result}

def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None:
pending = [event for event in ids if event not in self._frames_resolved]
if pending:
self._attach_frames(self.dataset.root / item.file, pending, self._events)
self._frames_resolved.update(pending)

def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None:
# This is a NEW, explicit read contract, not a guessed original-producer
Expand All @@ -266,14 +311,19 @@ def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSou
("frameJson", "TEXT", 0),
],
}
for table, columns in expected.items():
kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table])
schema = self.read(path, f'PRAGMA table_info("{table}")')
if (
kind != [{"type": "table"}]
or [(r["name"], r["type"], r["pk"]) for r in schema] != columns
):
return # Unknown/absent schemas never imply precision.
if path not in self._frame_schemas:
self._frame_schemas[path] = True
for table, columns in expected.items():
kind = self.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table])
schema = self.read(path, f'PRAGMA table_info("{table}")')
if (
kind != [{"type": "table"}]
or [(r["name"], r["type"], r["pk"]) for r in schema] != columns
):
self._frame_schemas[path] = False
break # Unknown/absent schemas never imply precision.
if not self._frame_schemas[path]:
return
for start in range(0, len(ids), SOURCE_BATCH_SIZE):
batch = ids[start : start + SOURCE_BATCH_SIZE]
placeholders = ",".join("?" for _ in batch)
Expand Down
36 changes: 36 additions & 0 deletions tests/core/test_dataset_global.py
Original file line number Diff line number Diff line change
Expand Up @@ -547,3 +547,39 @@ def test_cli_api_report_global_parity_and_range_options(tmp_path):
result = json.loads(cli.stdout)
assert result["scope"]["requested_range"] == {"first_event_id": 4, "last_event_id": 15}
assert result["budget_scope"] == "report_composition" and result["timeout_s"] == 10


@pytest.mark.parametrize("native", [False, True])
@pytest.mark.parametrize("direction", ["ASC", "DESC"])
def test_callstack_sorted_event_page_matches_standalone(tmp_path, native, direction):
if native:
request = dataclasses.replace(source_options(tmp_path, multiple=True), set_focus=False)
importer = ImportService()
root = importer.import_snapshot(request).db_path
single = importer.import_snapshot(
dataclasses.replace(request, events_per_slice=None)
).db_path
else:
root = case(tmp_path)
single = standalone(root, tmp_path)
# Standalone includes synthetic negative boundary IDs; compare real events.
params = {
"min_id": 7 if native else 0,
"order_by": "callstack",
"order_dir": direction,
"offset": 1,
"limit": 3,
}
before = hashes(root)
with SnapshotAnalyzer(root) as dataset, SnapshotAnalyzer(single) as reference:
for device in ((0, 1) if native else (0,)):
actual = dataset.execute_query("event", params, device_id=device, exact_total=True)
expected = reference.execute_query("event", params, device_id=device, exact_total=True)
assert actual["rows"]
keys = list(expected["rows"][0])
assert projected(actual["rows"], keys) == expected["rows"]
assert all("callstackId" not in row for row in actual["rows"])
assert actual["total"] == expected["total"]
assert actual["has_more"] == expected["has_more"]
assert actual["truncated"] == expected["truncated"]
assert hashes(root) == before
Loading
Loading