From 7d4f10c13429aaa452f4ae128aeee4b0a7c23304 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:44:40 +0800 Subject: [PATCH 1/2] fix: reduce active-event attribution before bounded hydration --- benchmarks/active_at_event_capacity.py | 310 ++++++++++++++++++ docs/en/sharded-snapshotdb.md | 17 +- docs/zh/sharded-snapshotdb.md | 15 +- src/pt_snap_cli/core/dataset_active_sql.py | 200 +++++++++++ src/pt_snap_cli/core/dataset_attribution.py | 162 +++++---- .../statistical/active_blocks_at_event.yaml | 2 +- tests/core/test_dataset_active_sql.py | 192 +++++++++++ tests/core/test_dataset_attribution.py | 13 +- tests/core/test_dataset_realistic.py | 2 + tests/core/test_dataset_source_reuse.py | 91 ++--- tests/core/test_report_reuse.py | 21 +- 11 files changed, 910 insertions(+), 115 deletions(-) create mode 100644 benchmarks/active_at_event_capacity.py create mode 100644 src/pt_snap_cli/core/dataset_active_sql.py create mode 100644 tests/core/test_dataset_active_sql.py diff --git a/benchmarks/active_at_event_capacity.py b/benchmarks/active_at_event_capacity.py new file mode 100644 index 0000000..ef43b43 --- /dev/null +++ b/benchmarks/active_at_event_capacity.py @@ -0,0 +1,310 @@ +"""Synthetic text-attribution capacity/differential gate; no pickle or producer. + +Default: 1.5M events, three 500k-event shards, 200k active blocks. Uses private +TemporaryDirectory storage and deletes it on success/failure. Both published v1 +and native-v2 schemas are supported. Timing includes normal validation/hashing; +work bytes measure serialized fetched/output values, not process RSS. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sqlite3 +import tempfile +import time +from contextlib import closing +from dataclasses import asdict +from pathlib import Path + +from pt_snap_cli.core.dataset_contract import ACTION_NAMES, BLOCK_STATES +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.models import ImportMetadata +from pt_snap_cli.core.native_dataset_contract import NATIVE_FORMAT, cache_identity +from pt_snap_cli.core.query_service import QueryService + +SCHEMA = """CREATE TABLE dictionary ("table" TEXT,"column" TEXT,"key" TEXT,"value" TEXT); +CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY,action INTEGER,address INTEGER, +size INTEGER,stream INTEGER,allocated INTEGER,active INTEGER,reserved INTEGER,callstack TEXT); +CREATE TABLE block_0 (id INTEGER PRIMARY KEY,address INTEGER,size INTEGER, +requestedSize INTEGER,state INTEGER,allocEventId INTEGER,freeEventId INTEGER);""" + + +def digest(path): + with path.open("rb") as file: + value = hashlib.sha256() + for chunk in iter(lambda: file.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def inventory(root): + return {str(path.relative_to(root)): digest(path) for path in root.rglob("*") if path.is_file()} + + +def build_case(root: Path, capacity=500_000, active=200_000, groups=32, layout="v1"): + """Build a bounded deterministic dataset plus full-text standalone reference. + + Half the allocations are in each of the first two owners. Completed frees + are in the third, after the target. Native local stack IDs deliberately name + different texts in different shards; global grouping must use their full text. + """ + if not (2 <= active <= capacity <= 500_000 and active % 2 == 0 and 1 <= groups <= active): + raise ValueError("Require even 2<=active<=capacity<=500000 and 1<=groups<=active") + root.mkdir() + dataset = root / "dataset" + (dataset / "device_0").mkdir(parents=True) + half = active // 2 + + def occupied(count): + return 8 * (6 * (count // 3) + (count % 3) * (count % 3 + 1) // 2) + + metadata = asdict( + ImportMetadata( + 1, + 2, + "0" * 64, + 0, + "synthetic.sql", + None, + "synthetic-capacity-benchmark", + "1", + "2026-10-11T00:00:00+00:00", + ) + ) + records = [] + for index in range(3): + lo, hi = index * capacity, (index + 1) * capacity - 1 + file = f"device_0/slice_{index:05d}.db" + path = dataset / file + with closing(sqlite3.connect(path)) as conn, conn: + conn.executescript( + SCHEMA + if layout == "v1" + else SCHEMA.replace("callstack TEXT", "callstackId INTEGER") + ) + conn.executemany( + "INSERT INTO dictionary VALUES (?,?,?,?)", + [ + ("trace_entry_0", "action", str(i), name) + for i, name in enumerate( + ACTION_NAMES if layout == "v1" else (*ACTION_NAMES, "oom") + ) + ] + + [("block_0", "state", str(i), name) for i, name in BLOCK_STATES], + ) + if layout == "v2": + conn.execute("CREATE TABLE callstack (id INTEGER PRIMARY KEY,callstack TEXT)") + conn.executemany( + "INSERT INTO callstack VALUES (?,?)", + ((i + 1, f"stack-{(i + index) % groups:06d}") for i in range(groups)), + ) + conn.execute( + "CREATE TABLE pt_snap_block_reference (blockId INTEGER PRIMARY KEY,stream INTEGER,allocCallstack TEXT,freeCallstack TEXT)" + ) + columns = ",".join( + f'"{key}" {"INTEGER" if isinstance(value, int) or key == "requested_device" else "TEXT"}' + for key, value in metadata.items() + ) + conn.execute(f"CREATE TABLE pt_snap_metadata (id INTEGER PRIMARY KEY,{columns})") + conn.execute( + "INSERT INTO pt_snap_metadata VALUES (" + + ",".join("?" for _ in range(len(metadata) + 1)) + + ")", + [1, *metadata.values()], + ) + + def trace(lo=lo, hi=hi, index=index): + for event in range(lo, hi + 1): + position = event - lo + ordinal = position + (half if index == 1 else 0) + action = ( + 4 + if index < 2 and position < half + else 6 if index == 2 and position < active else 7 + ) + stack = ordinal % groups + live = ( + occupied(index * half + min(position + 1, half)) + if index < 2 + else occupied(active) - occupied(min(position + 1, active)) + ) + yield ( + event, + action, + ordinal * 32, + 8 * (1 + ordinal % 3), + 0, + live, + live, + occupied(active), + f"stack-{stack:06d}" if layout == "v1" else (stack - index) % groups + 1, + ) + + conn.executemany("INSERT INTO trace_entry_0 VALUES (?,?,?,?,?,?,?,?,?)", trace()) + count = half if index == 0 else active + conn.executemany( + "INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", + ( + ( + ordinal if ordinal < half else capacity + ordinal - half, + ordinal * 32, + 8 * (1 + ordinal % 3), + 8, + 1, + ordinal if ordinal < half else capacity + ordinal - half, + 2 * capacity + ordinal, + ) + for ordinal in range(count) + ), + ) + if layout == "v2": + conn.execute( + "INSERT INTO pt_snap_block_reference SELECT id,0,NULL,NULL FROM block_0" + ) + record = {"index": index, "startEventId": lo, "endEventId": hi, "file": file, "ready": True} + if layout == "v2": + record.update(startPosition=lo, endPosition=hi, sha256=digest(path)) + records.append(record) + manifest = { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "synthetic.sql", + "devices": { + "0": { + "eventCount": capacity * 3, + "sliceCount": 3, + "readySlices": [0, 1, 2], + "slices": records, + } + }, + } + if layout == "v2": + manifest.update( + format=NATIVE_FORMAT, + identity=cache_identity("0" * 64, None, capacity), + metadata=metadata, + omittedDevices=[], + ) + else: + manifest.update(eventsPerSlice=capacity, cacheHash="") + (dataset / "manifest.json").write_text(json.dumps(manifest)) + single = root / "standalone.db" + with closing(sqlite3.connect(single, uri=True)) as conn: + conn.executescript(SCHEMA) + for record in records: + conn.execute( + "ATTACH DATABASE ? AS src", [(dataset / record["file"]).as_uri() + "?mode=ro"] + ) + if record["index"] == 0: + conn.execute("INSERT INTO dictionary SELECT * FROM src.dictionary") + if layout == "v1": + conn.execute("INSERT INTO trace_entry_0 SELECT * FROM src.trace_entry_0") + else: + conn.execute( + "INSERT INTO trace_entry_0 SELECT t.id,t.action,t.address,t.size,t.stream,t.allocated,t.active,t.reserved,c.callstack FROM src.trace_entry_0 t LEFT JOIN src.callstack c ON c.id=t.callstackId" + ) + conn.execute("INSERT OR REPLACE INTO block_0 SELECT * FROM src.block_0") + conn.commit() + conn.execute("DETACH DATABASE src") + return dataset, single + + +def run_case(root, capacity, active, groups, layout, timeout): + dataset, single = build_case(root, capacity, active, groups, layout) + before = inventory(root) + results = [] + with closing(QueryService()) as service: + for template, params in ( + ("active_blocks_at_event", {"limit": 20}), + ("active_memory_callstack_at_event", {"top_n": 20}), + ): + params["event_id"] = 2 * capacity - 1 + budget = QueryBudget(timeout, time.monotonic()) + started = time.monotonic() + result = service.execute_query( + template, params, db_path=dataset, exact_total=True, _budget=budget + ) + elapsed = time.monotonic() - started + expected = service.execute_query( + template, params, db_path=single, exact_total=True, timeout_s=timeout + ) + fields = set(expected.rows[0]) if expected.rows else set() + assert [{key: row[key] for key in fields} for row in result.rows] == expected.rows + assert ( + result.total + == expected.total + == (active if template == "active_blocks_at_event" else groups) + ) + coverage = result.scope["source_coverage"] + assert ( + coverage["active_blocks"] + == coverage["allocation_events_resolved"] + == coverage["free_events_resolved"] + == active + ) + assert budget.work_rows < 150 + results.append( + { + "template": template, + "elapsed_s": elapsed, + "work_rows": budget.work_rows, + "work_bytes": budget.work_bytes, + "total": result.total, + "returned": result.returned, + } + ) + rejected = False + try: + service.execute_query( + "active_blocks_at_event", + {"event_id": 2 * capacity - 1, "limit": -1}, + db_path=dataset, + timeout_s=timeout, + ) + except QueryExecutionError as error: + if "work budget" not in str(error): + raise + rejected = True + if active > 100_000: + assert rejected + assert inventory(root) == before + return { + "layout": layout, + "events": 3 * capacity, + "active": active, + "queries": results, + "unbounded_output_rejected": rejected, + "standalone_differential": "passed", + "source_unchanged": True, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--capacity", type=int, default=500_000) + parser.add_argument("--active", type=int, default=200_000) + parser.add_argument("--groups", type=int, default=32) + parser.add_argument("--layout", choices=("v1", "v2"), default="v1") + parser.add_argument("--timeout", type=float, default=120) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="pt-snap-active-capacity-") as temporary: + print( + json.dumps( + run_case( + Path(temporary) / "case", + args.capacity, + args.active, + args.groups, + args.layout, + args.timeout, + ), + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53bf44a..3ee2a8a 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -230,8 +230,21 @@ restricted real interval, dataset fingerprint and boundary exclusion. Totals and pagination apply to this scope, **not** to an implicitly truncated full dataset. Caller parameter defaults in CLI `effective_params` are additionally constrained by `scope`. Point-event `active_blocks_at_event` and -`active_memory_callstack_at_event` now resolve alloc/free/full stack sources in -bounded batches before grouping/ranking, including original no-extension artifacts. +`active_memory_callstack_at_event` prove alloc/free/full-stack coverage before +pagination/ranking, including future completed-free owners and original no-extension +artifacts. Text-only v1/v2 queries use read-only SQL reduction when the required +owners fit SQLite's ATTACH limit: coverage and full-text groups stay in SQL, and +only returned block details are hydrated. Global top_n percentages still use all +included groups before max_rows. Ordered-frame inputs or larger owner sets retain +the budgeted source-batch fallback. SQL scans/sorts share the timeout; reduced +rows, hydrated sources and output still consume the work budget. A finite page +can therefore succeed where unbounded output exceeds the same budget. + +The opt-in synthetic gate `PYTHONPATH=src python benchmarks/active_at_event_capacity.py` +(and `--layout v2`) checks 1.5 million events / three 500000-event shards / 200000 +active blocks, page20 and top20, exact totals, standalone differential results, +read-only hashes and rejection of unbounded block output. It creates and removes +private temporary data; it is not producer/GUI or general RSS/performance evidence. See [point-event attribution](querying.md#dataset-point-event-attribution) for lifecycle identity, separate source/frame coverage, and the optional versioned ordered-frame reader interface. Dataset-global built-ins use the explicit P3 matrix diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index f89cfad..9465cc0 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -190,8 +190,19 @@ with SnapshotAnalyzer(Path("/capture/snapshot.pkl.msinsight"), device_id=0) as a selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、数据集指纹和边界排除。 总数和分页针对该 scope,**不是**隐式截断的整份数据集。CLI `effective_params` 的调用者 参数默认值另受 `scope` 约束。定点 `active_blocks_at_event` 和 -`active_memory_callstack_at_event` 现于分组/排名前批量解析 alloc/free/完整栈来源, -原版无扩展产物也适用。生命周期身份、独立来源/frame 覆盖及版本化有序 frame reader +`active_memory_callstack_at_event` 在分页/排名前验证全部 alloc/free/完整栈来源覆盖, +包括未来 completed-free 所在片,原版无扩展产物也适用。纯文本 v1/v2 查询在所需 owner +集合不超过 SQLite ATTACH 上限时使用只读 SQL 归约:全量覆盖与完整文本分组保留在 SQL +内,仅补充返回 block 的来源详情。全局 top_n 百分比分母仍是 max_rows 前的全部入选组。 +有序 frame 输入或更大的 owner 集合保留受预算约束的来源批次回退。SQL 扫描/排序共用 +超时;归约行、实际补充来源及输出仍计入工作预算,因此有限页可成功而无界输出仍超限。 + +可选合成验收 `PYTHONPATH=src python benchmarks/active_at_event_capacity.py`(以及 +`--layout v2`)检查 150 万事件 / 三个 500000 事件分片 / 20 万活跃 block、page20、 +top20、精确总数、单库差分、只读哈希和无界 block 输出硬失败。它创建并清理私有临时 +数据,不是 producer/GUI 验收或通用 RSS/性能保证。 + +生命周期身份、独立来源/frame 覆盖及版本化有序 frame reader 接口详见[定点归因](querying.md#数据集定点事件归因)。数据集全局内建查询遵循 P3 矩阵 及累计工作上限;任意 SQL/自定义 override 明确失败,不静默选首片/最新片。单库真实 事件 peak/gap 与全部 action 完整文本栈统计版本在相同合同中明确说明。 diff --git a/src/pt_snap_cli/core/dataset_active_sql.py b/src/pt_snap_cli/core/dataset_active_sql.py new file mode 100644 index 0000000..dd655c4 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_active_sql.py @@ -0,0 +1,200 @@ +"""Read-only SQL reduction for text-only point-event attribution. + +Coverage is computed over the full active set before ranking or pagination. Only +aggregate rows and the requested page cross into Python and incur its fetched +row/value budget. SQLite scans and temporary sorting remain deadline bounded, +not an RSS/temporary-disk promise. Ordered-frame inputs retain the source path. +""" + +from __future__ import annotations + +import hashlib +from typing import cast + +from pt_snap_cli.core.dataset_contract import SliceRecord +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources +from pt_snap_cli.core.dataset_lifecycle_sql import _attachment_limit, _proof, _required_slices +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver + + +def _active_cte( + sources: DatasetSourceResolver, + item: SliceRecord, + required: list[SliceRecord], + include_static: bool, +) -> str: + device = sources.device.device_id + text_cases = [] + for owner in required: + table = f'"s{owner.index}"."trace_entry_{device}"' + join = ( + f' LEFT JOIN "s{owner.index}".callstack cs ON cs.id=e.callstackId' + if sources.dataset.callstack_layout == "v2" + else "" + ) + text = "cs.callstack" if join else "e.callstack" + text_cases.append( + f"WHEN b.allocEventId BETWEEN {owner.start_event_id} AND {owner.end_event_id} " + f"THEN (SELECT {text} FROM {table} e{join} " + "WHERE e.id=b.allocEventId AND e.id>=0 AND e.action=4)" + ) + return f"""WITH active AS ( + SELECT * FROM "s{item.index}"."block_{device}" + WHERE size>=? AND (allocEventId=-1 OR allocEventId<=?) + AND (freeEventId=-1 OR freeEventId>?) + {'' if include_static else 'AND allocEventId!=-1'} + ), attributed AS ( + SELECT b.*, {_proof(sources, required, 'allocEventId', 4)} AS allocation_proved, + {_proof(sources, required, 'freeEventId', 6)} AS completion_proved, + CASE {' '.join(text_cases)} END AS source_text + FROM active b + ), classified AS ( + SELECT *, CASE WHEN allocEventId>=0 THEN 'dynamic_live_at_event' + WHEN freeEventId=-1 THEN 'static' ELSE 'preexisting_live_at_event' END AS category, + CASE WHEN allocEventId>=0 THEN + CASE WHEN typeof(source_text)='text' AND source_text!='' THEN 'captured' + ELSE 'missing' END + WHEN freeEventId=-1 THEN 'static' ELSE 'preexisting_live_at_event' END AS stack_kind + FROM attributed + ) """ + + +def _group_cte(cte: str, top_n: int) -> str: + return cte + f""", labelled AS ( + SELECT *, CASE stack_kind + WHEN 'captured' THEN source_text + WHEN 'static' THEN '[static] allocEventId=-1, freeEventId=-1' + WHEN 'preexisting_live_at_event' THEN '[preexisting live] allocEventId=-1' + ELSE '[missing callstack]' END AS callstack + FROM classified + ), grouped AS ( + SELECT category, stack_kind, callstack, COUNT(*) AS block_count, + SUM(size) AS size_bytes, SUM(requestedSize) AS requested_bytes, + MIN(CASE WHEN stack_kind='captured' THEN allocEventId END) AS stack_event_id + FROM labelled GROUP BY stack_kind, callstack + ), ranked AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY category ORDER BY + size_bytes DESC, block_count DESC, callstack, + stack_kind='captured') AS group_rank FROM grouped + ), included AS ( + SELECT * FROM ranked WHERE category!='dynamic_live_at_event' + OR {top_n}<0 OR group_rank<={top_n} + ) """ + + +def reduce_active( + sources: DatasetSourceResolver, + item: SliceRecord, + template: str, + params: dict[str, object], + max_rows: int | None, +) -> tuple[list[dict[str, object]], dict[str, object], int, bool, int] | None: + """Return reduced output/full coverage, or preserve the budgeted fallback. + + Required proof owners include future completed frees. Local v2 stack IDs are + used only to obtain their owning shard's full text, never as global identity. + A frame extension, even partially covered, must use ordered-frame attribution. + """ + if sources.dataset.ordered_frames_version == 1: + return None + required = _required_slices(sources, [item], _attachment_limit()) + if required is None: + return None + cte = _active_cte(sources, item, required, bool(params["include_static"])) + values = [params["min_size"], params["event_id"], params["event_id"]] + cap = max_rows if max_rows is not None and max_rows > 0 else -1 + with _AttachedSources(sources, required) as database: + # A valid block size is int64, but whole-set byte totals need not be. + # Sum four 16-bit components in SQLite (bounded even at SQLite's maximum + # physical table size), then reconstruct exact Python integers from the + # single charged summary row. This keeps finite pages/top_n independent + # of int64 SUM overflow in whole-set coverage totals. Group/denominator + # sums retain their existing SQLite int64 overflow behavior. + byte_fields = { + "active_bytes": "size", + "dynamic_bytes": "CASE WHEN allocEventId>=0 THEN size ELSE 0 END", + "captured_dynamic_bytes": "CASE WHEN stack_kind='captured' THEN size ELSE 0 END", + } + byte_sums = ", ".join( + f'COALESCE(SUM((({expression}) >> {16 * part}) & 65535),0) AS "{field}_{part}"' + for field, expression in byte_fields.items() + for part in range(4) + ) + coverage = database.read( + cte + f"""SELECT COUNT(*) AS active_blocks, {byte_sums}, + COALESCE(SUM(allocEventId>=0),0) AS dynamic_blocks, + COALESCE(SUM(allocEventId>=0 AND allocation_proved),0) AS allocation_events_resolved, + COALESCE(SUM(stack_kind='captured'),0) AS allocation_stacks_captured, + COALESCE(SUM(allocEventId<0),0) AS unknown_preexisting_blocks, + COALESCE(SUM(freeEventId>=0),0) AS free_references, + COALESCE(SUM(completion_proved),0) AS free_events_resolved + FROM classified""", + values, + )[0] + for field in byte_fields: + coverage[field] = sum( + cast(int, coverage.pop(f"{field}_{part}")) << (16 * part) for part in range(4) + ) + coverage.update( + range_complete=True, + event_position="after", + allocation_source_complete=( + coverage["dynamic_blocks"] == coverage["allocation_events_resolved"] + ), + ordered_frame_blocks=0, + ordered_frames_complete=False, + frame_degradation="text_only_or_uncovered", + ) + if template == "active_blocks_at_event": + limit = cast(int, params["limit"]) + if cap >= 0: + limit = min(limit, cap) if limit >= 0 else cap + offset = max(0, cast(int, params["offset"])) + rows = database.read( + cte + f'SELECT * FROM active ORDER BY "{params["order_by"]}" ' + f'{params["order_dir"]}, id ASC LIMIT ? OFFSET ?', + [*values, limit, offset], + ) + total, rank_full = cast(int, coverage["active_blocks"]), False + else: + top_n = cast(int, params["top_n"]) + grouped = _group_cte(cte, top_n) + summary = database.read( + grouped + """SELECT COUNT(*) AS total, + COALESCE(SUM(category='dynamic_live_at_event'),0) AS dynamic_groups + FROM grouped""", + values, + )[0] + rows = database.read( + grouped + """SELECT category, stack_kind, callstack, block_count, + size_bytes, requested_bytes, stack_event_id, + ROUND(size_bytes / 1073741824.0,6) AS size_gib, + ROUND(requested_bytes / 1073741824.0,6) AS requested_gib, + ROUND(size_bytes * 100.0 / NULLIF((SELECT SUM(size_bytes) FROM included),0),4) + AS percent_of_active_blocks + FROM included ORDER BY size_bytes DESC, block_count DESC, callstack, + stack_kind='captured' LIMIT ?""", + [*values, cap], + ) + for row in rows: + sources.budget.remaining() + kind = row["stack_kind"] + identity = ( + "text:sha256:" + + hashlib.sha256(cast(str, row["callstack"]).encode("utf-8")).hexdigest() + if kind == "captured" + else f"category:{kind}" + ) + row.update( + source_stack_id=f"{sources.namespace}:{identity}", + stack_id=f"{sources.namespace}:{identity}", + text_kind=kind, + frames=None, + frames_status="text_only", + ) + total = cast(int, summary["total"]) + dynamic = cast(int, summary["dynamic_groups"]) + rank_full = top_n >= 0 and dynamic > 0 and dynamic >= top_n + offset = 0 + coverage["source_queries"] = sources.query_count + return rows, coverage, total, rank_full, offset diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index 1d806f3..450ef18 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -7,6 +7,7 @@ from contextlib import closing, contextmanager from typing import cast +from pt_snap_cli.core.dataset_active_sql import reduce_active from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, EventSource from pt_snap_cli.core.errors import QueryExecutionError from pt_snap_cli.core.models import QueryResult @@ -29,6 +30,30 @@ def event_attribution( event_id = cast(int, params["event_id"]) item = sources.target(event_id, slice_index) path = sources.dataset.root / item.file + reduced = reduce_active(sources, item, template, params, max_rows) + if reduced is not None: + selected, coverage, total, rank_full, offset = reduced + if template == "active_blocks_at_event": + _hydrate_blocks(sources, selected) + coverage["source_queries"] = sources.query_count + scope = _scope(sources, item.index, event_id, coverage) + has_more = total > len(selected) + offset or rank_full + if exact_total: + has_more = total > len(selected) + offset + sources.budget.consume(selected) + return QueryResult( + total=total if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=has_more, + truncated=has_more or offset > 0, + total_is_exact=exact_total or not has_more and offset == 0, + timeout_s=sources.budget.timeout_s, + scope=scope, + ) blocks = sources.read( path, f'SELECT * FROM "block_{sources.device.device_id}" WHERE size>=? ' @@ -37,6 +62,80 @@ def event_attribution( + "ORDER BY id", [params["min_size"], event_id, event_id], ) + references, coverage = _hydrate_blocks(sources, blocks) + scope = _scope(sources, item.index, event_id, coverage) + if template == "active_blocks_at_event": + key = cast(str, params["order_by"]) + blocks.sort(key=lambda b: cast(int, b["id"])) + blocks.sort(key=lambda b: cast(int, b[key]), reverse=params["order_dir"] == "DESC") + all_rows = blocks + rank_full = False + limit = cast(int, params["limit"]) + offset = max(0, cast(int, params["offset"])) + selected = all_rows[offset : offset + limit] if limit >= 0 else all_rows[offset:] + else: + all_rows = _groups(sources, blocks, references) + top_n = cast(int, params["top_n"]) + dynamic = [row for row in all_rows if row["category"] == "dynamic_live_at_event"] + rank_full = top_n >= 0 and bool(dynamic) and len(dynamic) >= top_n + selected = [row for row in all_rows if row["category"] != "dynamic_live_at_event"] + ( + dynamic[:top_n] if top_n >= 0 else dynamic + ) + selected.sort(key=_rank) + denominator = sum(cast(int, row["size_bytes"]) for row in selected) + # Use the same SQLite arithmetic/ROUND implementation as standalone SQL. + # A Decimal conversion of a binary float changes some near-half values. + # This connection evaluates scalars only: no source reads or merge tables. + with _rounding_cursor(sources) as cursor: + for row in selected: + sources.budget.remaining() + row["percent_of_active_blocks"] = cursor.execute( + "SELECT ROUND(? * 100.0 / NULLIF(?, 0), 4)", + (row["size_bytes"], denominator), + ).fetchone()[0] + offset = 0 + has_more = len(selected) < len(all_rows) - offset or rank_full + if max_rows is not None and max_rows > 0: + has_more = has_more or len(selected) > max_rows + selected = selected[:max_rows] + if exact_total: + has_more = len(all_rows) > len(selected) + offset + sources.budget.consume(selected) + return QueryResult( + total=len(all_rows) if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=has_more, + truncated=has_more or offset > 0, + total_is_exact=exact_total or not has_more and offset == 0, + timeout_s=sources.budget.timeout_s, + scope=scope, + ) + + +def _scope( + sources: DatasetSourceResolver, slice_index: int, event_id: int, coverage: dict[str, object] +) -> dict[str, object]: + return { + "kind": "event", + "device_id": sources.device.device_id, + "slice_index": slice_index, + "first_event_id": event_id, + "last_event_id": event_id, + "db_path": str(sources.dataset.root / sources.device.slices[slice_index].file), + "dataset_path": str(sources.dataset.root), + "fingerprint": sources.dataset.fingerprint, + "boundary_events_included": False, + "source_coverage": coverage, + } + + +def _hydrate_blocks( + sources: DatasetSourceResolver, blocks: list[dict[str, object]] +) -> tuple[dict[int, EventSource], dict[str, object]]: references = sources.events( cast(int, block[field]) for block in blocks @@ -112,68 +211,7 @@ def event_attribution( ), "source_queries": sources.query_count, } - scope: dict[str, object] = { - "kind": "event", - "device_id": sources.device.device_id, - "slice_index": item.index, - "first_event_id": event_id, - "last_event_id": event_id, - "db_path": str(path), - "dataset_path": str(sources.dataset.root), - "fingerprint": sources.dataset.fingerprint, - "boundary_events_included": False, - "source_coverage": coverage, - } - if template == "active_blocks_at_event": - key = cast(str, params["order_by"]) - blocks.sort(key=lambda b: cast(int, b["id"])) - blocks.sort(key=lambda b: cast(int, b[key]), reverse=params["order_dir"] == "DESC") - all_rows = blocks - rank_full = False - limit = cast(int, params["limit"]) - offset = max(0, cast(int, params["offset"])) - selected = all_rows[offset : offset + limit] if limit >= 0 else all_rows[offset:] - else: - all_rows = _groups(sources, blocks, references) - top_n = cast(int, params["top_n"]) - dynamic = [row for row in all_rows if row["category"] == "dynamic_live_at_event"] - rank_full = top_n >= 0 and bool(dynamic) and len(dynamic) >= top_n - selected = [row for row in all_rows if row["category"] != "dynamic_live_at_event"] + ( - dynamic[:top_n] if top_n >= 0 else dynamic - ) - selected.sort(key=_rank) - denominator = sum(cast(int, row["size_bytes"]) for row in selected) - # Use the same SQLite arithmetic/ROUND implementation as standalone SQL. - # A Decimal conversion of a binary float changes some near-half values. - # This connection evaluates scalars only: no source reads or merge tables. - with _rounding_cursor(sources) as cursor: - for row in selected: - sources.budget.remaining() - row["percent_of_active_blocks"] = cursor.execute( - "SELECT ROUND(? * 100.0 / NULLIF(?, 0), 4)", - (row["size_bytes"], denominator), - ).fetchone()[0] - offset = 0 - has_more = len(selected) < len(all_rows) - offset or rank_full - if max_rows is not None and max_rows > 0: - has_more = has_more or len(selected) > max_rows - selected = selected[:max_rows] - if exact_total: - has_more = len(all_rows) > len(selected) + offset - sources.budget.consume(selected) - return QueryResult( - total=len(all_rows) if exact_total else len(selected), - returned=len(selected), - device_id=sources.device.device_id, - rows=selected, - template=template, - semantics_version=semantics_version, - has_more=has_more, - truncated=has_more or offset > 0, - total_is_exact=exact_total or not has_more and offset == 0, - timeout_s=sources.budget.timeout_s, - scope=scope, - ) + return references, coverage def _rank(row: dict[str, object]) -> tuple[int, int, str, str]: diff --git a/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml b/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml index c6bfd94..edb434d 100644 --- a/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml +++ b/src/pt_snap_cli/query/templates/statistical/active_blocks_at_event.yaml @@ -76,7 +76,7 @@ queries: LIMIT {{ limit|int }} {% if offset > 0 %}OFFSET {{ offset|int }}{% endif %} interpretation_limits: - - "Dataset point-event queries resolve alloc/free sources in batches before paging; scope.source_coverage describes all filtered blocks and is separate from row truncation. Stored state is a slice observation, not necessarily state at the target event." + - "Dataset point-event queries prove alloc/free coverage for all filtered blocks before paging, then hydrate returned details; scope.source_coverage is separate from row truncation. Stored state is a slice observation, not necessarily state at the target event." output_schema: - column: lifecycle_id type: str diff --git a/tests/core/test_dataset_active_sql.py b/tests/core/test_dataset_active_sql.py new file mode 100644 index 0000000..d372db7 --- /dev/null +++ b/tests/core/test_dataset_active_sql.py @@ -0,0 +1,192 @@ +"""Full-scope active reduction and bounded text/detail materialization.""" + +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from benchmarks.active_at_event_capacity import build_case, run_case +from pt_snap_cli.core import dataset_active_sql +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.query_service import QueryService +from tests.core.test_dataset_attribution import case +from tests.test_dataset_focus import hashes + + +def normalized(result): + value = dict(result.__dict__) + value["scope"] = dict(value["scope"]) + value["scope"]["source_coverage"] = { + k: v for k, v in value["scope"]["source_coverage"].items() if k != "source_queries" + } + return value + + +@pytest.mark.parametrize("layout", ["v1", "v2"]) +def test_small_capacity_gate_checks_exact_total_native_local_ids_and_future_frees(tmp_path, layout): + receipt = run_case(tmp_path / layout, 1000, 200, 32, layout, 10) + assert receipt["standalone_differential"] == "passed" + assert receipt["source_unchanged"] + + +@pytest.mark.parametrize( + "template,params,cap,exact", + [ + ("active_blocks_at_event", {"limit": 0}, None, False), + ("active_blocks_at_event", {"limit": -1}, 2, True), + ( + "active_blocks_at_event", + {"limit": 2, "offset": 1, "order_by": "freeEventId", "order_dir": "ASC"}, + 1, + False, + ), + ("active_blocks_at_event", {"limit": 2, "offset": 100}, None, True), + ("active_memory_callstack_at_event", {"top_n": -1}, None, True), + ("active_memory_callstack_at_event", {"top_n": 0}, None, False), + ("active_memory_callstack_at_event", {"top_n": 1}, 1, True), + ("active_memory_callstack_at_event", {"top_n": 5}, None, False), + ("active_memory_callstack_at_event", {"min_size": 999}, None, True), + ("active_memory_callstack_at_event", {"include_static": False}, None, False), + ], +) +def test_sql_matches_full_source_path_including_identity_coverage_and_completeness( + tmp_path, monkeypatch, template, params, cap, exact +): + root = case(tmp_path, devices=(0,)) + params = ( + {"event_id": 8, "stack_bytes": 0, **params} + if template != "active_blocks_at_event" + else {"event_id": 8, **params} + ) + before = hashes(root) + with closing(QueryService()) as service: + sql = service.execute_query(template, params, db_path=root, max_rows=cap, exact_total=exact) + monkeypatch.setattr(dataset_active_sql, "_attachment_limit", lambda: 0) + raw = service.execute_query(template, params, db_path=root, max_rows=cap, exact_total=exact) + assert normalized(sql) == normalized(raw) + assert hashes(root) == before + + +@pytest.mark.parametrize("template", ["active_blocks_at_event", "active_memory_callstack_at_event"]) +def test_missing_allocation_and_future_free_outside_page_keep_full_coverage(tmp_path, template): + root, _ = build_case(tmp_path / "case", 1000, 200) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as connection, connection: + connection.execute("UPDATE trace_entry_0 SET action=5 WHERE id=0") + with closing(sqlite3.connect(root / "device_0/slice_00002.db")) as connection, connection: + connection.execute("UPDATE trace_entry_0 SET action=5 WHERE id=2000") + before = hashes(root) + params = {"event_id": 1999, "min_size": 0} + params.update({"limit": 1} if template == "active_blocks_at_event" else {"top_n": 1}) + with closing(QueryService()) as service: + result = service.execute_query(template, params, db_path=root, max_rows=1, exact_total=True) + coverage = result.scope["source_coverage"] + assert coverage["active_blocks"] == coverage["free_references"] == 200 + assert coverage["allocation_events_resolved"] == coverage["free_events_resolved"] == 199 + assert not coverage["allocation_source_complete"] + assert result.returned == 1 + if template == "active_blocks_at_event": + assert result.rows[0]["id"] != 0 + assert result.rows[0]["free_source_status"] == "resolved" + assert hashes(root) == before + + +def test_zero_frame_coverage_still_uses_ordered_fallback(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + manifest = root / "manifest.json" + data = json.loads(manifest.read_text()) + data["extensions"] = {"ptSnapOrderedFrames": {"version": 1}} + manifest.write_text(json.dumps(data)) + monkeypatch.setattr( + dataset_active_sql, "_required_slices", lambda *args: pytest.fail("frame fallback") + ) + with closing(QueryService()) as service: + result = service.execute_query( + "active_memory_callstack_at_event", {"event_id": 8}, db_path=root + ) + assert not result.scope["source_coverage"]["ordered_frames_complete"] + + +@pytest.mark.parametrize("fault", ["timeout", "work_rows", "work_bytes"]) +def test_attached_reduction_failure_closes_connection_and_reusable_service( + tmp_path, monkeypatch, fault +): + root, _ = build_case(tmp_path / "case", 1000, 200) + before = hashes(root) + owners = [] + original = _AttachedSources.read + budget = QueryBudget(None, time.monotonic()) + + def fail(database, sql, values=None): + owners.append(database) + if "FROM included ORDER" in sql: + if fault == "timeout": + database.sources.budget.timeout_s = 0.0001 + database.sources.budget.started = time.monotonic() - 1 + elif fault == "work_rows": + database.sources.budget.max_work_rows = database.sources.budget.work_rows + else: + database.sources.budget.max_work_bytes = database.sources.budget.work_bytes + return original(database, sql, values) + + with closing(QueryService()) as service: + with monkeypatch.context() as patch: + patch.setattr(_AttachedSources, "read", fail) + with pytest.raises(QueryTimeoutError if fault == "timeout" else QueryExecutionError): + service.execute_query( + "active_memory_callstack_at_event", + {"event_id": 1999}, + db_path=root, + _budget=budget, + ) + assert owners + for owner in owners: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + owner.connection.execute("SELECT 1") + result = service.execute_query( + "active_memory_callstack_at_event", {"event_id": 1999}, db_path=root + ) + assert result.rows and result.scope["source_coverage"]["active_blocks"] == 200 + assert hashes(root) == before + + +@pytest.mark.slow +@pytest.mark.parametrize("layout", ["v1", "v2"]) +def test_default_export_capacity_with_two_hundred_thousand_active_blocks(tmp_path, layout): + receipt = run_case(tmp_path / layout, 500_000, 200_000, 32, layout, 120) + assert receipt["events"] == 1_500_000 and receipt["active"] == 200_000 + assert receipt["unbounded_output_rejected"] + assert [query["returned"] for query in receipt["queries"]] == [20, 20] + + +@pytest.mark.parametrize("template", ["active_blocks_at_event", "active_memory_callstack_at_event"]) +def test_coverage_totals_above_int64_remain_exact_for_bounded_results( + tmp_path, monkeypatch, template +): + from tests.core.test_dataset_realistic import rounding_case + + root, single = rounding_case(tmp_path, (1, 1)) + # Each group fits int64; the full-scope coverage sum does not. The low bit + # also detects a lossy floating-point TOTAL() replacement. + for path in [single, *root.glob("device_0/*.db")]: + with closing(sqlite3.connect(path)) as connection, connection: + connection.execute( + "UPDATE block_0 SET size=?,requestedSize=? WHERE id=0", (2**62 + 1, 2**62 + 1) + ) + connection.execute( + "UPDATE block_0 SET size=?,requestedSize=? WHERE id=1", (2**62, 2**62) + ) + params = {"event_id": 1} + params.update({"limit": 1} if template == "active_blocks_at_event" else {"top_n": 1}) + with closing(QueryService()) as service: + result = service.execute_query(template, params, db_path=root) + monkeypatch.setattr(dataset_active_sql, "_attachment_limit", lambda: 0) + raw = service.execute_query(template, params, db_path=root) + expected = service.execute_query(template, params, db_path=single) + assert normalized(result) == normalized(raw) + assert result.scope["source_coverage"]["active_bytes"] == 2**63 + 1 + fields = set(expected.rows[0]) + assert [{key: row[key] for key in fields} for row in result.rows] == expected.rows diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py index 92e7515..cd4da5f 100644 --- a/tests/core/test_dataset_attribution.py +++ b/tests/core/test_dataset_attribution.py @@ -608,7 +608,9 @@ def test_high_block_count_batches_distinct_sources_and_bounds_cache_queries(tmp_ "active_memory_callstack_at_event", {"event_id": 599, "top_n": -1} ) assert sum(r["block_count"] for r in high["rows"]) == 599 - assert high["scope"]["source_coverage"]["source_queries"] == 302 + assert ( + high["scope"]["source_coverage"]["source_queries"] == 303 + ) # Includes the bounded proof-owner probe. assert len(cache) <= 2 # A single large shard specifically crosses the parameter batch bound. single = make_dataset(tmp_path / "wide", devices=(0,), slices=1) @@ -646,6 +648,15 @@ def counted(self, path, sql, params=None): return original(self, path, sql, params) monkeypatch.setattr(DatasetSourceResolver, "read", counted) + from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources + + original_attached = _AttachedSources.read + + def counted_attached(self, sql, params=None): + values.append(self.sources.budget.remaining()) + return original_attached(self, sql, params) + + monkeypatch.setattr(_AttachedSources, "read", counted_attached) service.execute_query( "active_memory_callstack_at_event", {"event_id": 8}, diff --git a/tests/core/test_dataset_realistic.py b/tests/core/test_dataset_realistic.py index 2a38823..0d61ad4 100644 --- a/tests/core/test_dataset_realistic.py +++ b/tests/core/test_dataset_realistic.py @@ -220,6 +220,8 @@ def test_rounding_int64_aggregate_overflow_is_a_domain_error(tmp_path, same_stac @pytest.mark.parametrize("rounding_pass", [1, 2], ids=["gib", "percentages"]) def test_rounding_timeout_closes_scalar_connection_and_cursor(tmp_path, monkeypatch, rounding_pass): root, _ = rounding_case(tmp_path, (1, 127)) + # Exercise the retained ordered-frame/fallback scalar ownership path. + monkeypatch.setattr(dataset_attribution, "reduce_active", lambda *args: None) connections, cursors = [], [] connect = sqlite3.connect diff --git a/tests/core/test_dataset_source_reuse.py b/tests/core/test_dataset_source_reuse.py index e20baad..ff3db50 100644 --- a/tests/core/test_dataset_source_reuse.py +++ b/tests/core/test_dataset_source_reuse.py @@ -80,58 +80,65 @@ def test_target_and_distinct_events_reuse_only_identical_sources(tmp_path): assert first[0].stack_id == first[1].stack_id -@pytest.mark.parametrize("count, succeeds", [(49_999, True), (50_000, False)]) -def test_distinct_block_and_allocation_work_still_counts(tmp_path, count, succeeds): +@pytest.mark.parametrize("count", [49_999, 50_000]) +def test_active_reduction_does_not_materialize_blocks_and_allocation_events(tmp_path, count): root = dataset(tmp_path, count, blocks=True) budget = QueryBudget(None, time.monotonic()) - service = QueryService() - try: - if succeeds: - result = service.execute_query( - "active_memory_callstack_at_event", - params={"event_id": count - 1}, + with closing(QueryService()) as service: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": count - 1}, + db_path=root, + _budget=budget, + ) + assert result.returned == 1 + assert result.rows[0]["block_count"] == count + # Target, full coverage summary, group count, reduced group and enriched output. + assert budget.work_rows == 5 + + +def test_shared_long_text_reduction_is_small_but_full_sources_remain_budgeted(tmp_path): + root = dataset(tmp_path, 5_000, stack="x" * 16_384, blocks=True) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "active_memory_callstack_at_event", + params={"event_id": 4_999}, + db_path=root, + _budget=budget, + ) + assert result.rows[0]["block_count"] == 5_000 + assert budget.work_rows == 5 and budget.work_bytes < 100_000 + full = QueryBudget(None, time.monotonic()) + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( + "active_blocks_at_event", + params={"event_id": 4_999, "limit": -1}, db_path=root, - _budget=budget, + _budget=full, ) - assert result.returned == 1 - assert budget.work_rows == 2 * count + 1 - else: - with pytest.raises(QueryExecutionError, match="work budget exceeded"): - service.execute_query( - "active_memory_callstack_at_event", - params={"event_id": count - 1}, - db_path=root, - _budget=budget, - ) - assert budget.work_rows == 100_001 - finally: - service.close() + assert full.work_rows < full.max_work_rows + assert full.work_bytes > full.max_work_bytes -@pytest.mark.parametrize("count, succeeds", [(3_000, True), (5_000, False)]) -def test_long_distinct_source_payload_remains_budgeted(tmp_path, count, succeeds): - root = dataset(tmp_path, count, stack="x" * 16_384, blocks=True) - budget = QueryBudget(None, time.monotonic()) +def test_fallback_still_charges_distinct_blocks_and_sources(tmp_path, monkeypatch): + from pt_snap_cli.core import dataset_attribution + + monkeypatch.setattr(dataset_attribution, "reduce_active", lambda *args: None) + root = dataset(tmp_path, 100, blocks=True) with closing(QueryService()) as service: - if succeeds: - result = service.execute_query( + budget = QueryBudget(None, time.monotonic(), max_work_rows=201) + result = service.execute_query( + "active_memory_callstack_at_event", {"event_id": 99}, db_path=root, _budget=budget + ) + assert result.returned == 1 and budget.work_rows == 201 + with pytest.raises(QueryExecutionError, match="work budget exceeded"): + service.execute_query( "active_memory_callstack_at_event", - params={"event_id": count - 1}, + {"event_id": 99}, db_path=root, - _budget=budget, + _budget=QueryBudget(None, time.monotonic(), max_work_rows=200), ) - assert result.returned == 1 - assert budget.work_rows == 2 * count + 1 - else: - with pytest.raises(QueryExecutionError, match="work budget exceeded"): - service.execute_query( - "active_memory_callstack_at_event", - params={"event_id": count - 1}, - db_path=root, - _budget=budget, - ) - assert budget.work_rows < budget.max_work_rows - assert budget.work_bytes > budget.max_work_bytes def test_id_sorted_page_does_not_materialize_unneeded_stack_text(tmp_path): diff --git a/tests/core/test_report_reuse.py b/tests/core/test_report_reuse.py index 9a53565..0773a32 100644 --- a/tests/core/test_report_reuse.py +++ b/tests/core/test_report_reuse.py @@ -145,19 +145,30 @@ def test_report_projects_independent_earliest_peaks_and_same_event_counters( ) -def test_report_attribution_keeps_real_work_limit(tmp_path): +def test_report_attribution_reduces_static_blocks_but_keeps_shared_work_limit( + tmp_path, monkeypatch +): root = cycle_dataset(tmp_path, 60000) with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: conn.executemany( "INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", ((-100000 - index, 2000000 + index, 1, 1, 1, -1, -1) for index in range(100001)), ) - service = ReportService() - try: + with closing(ReportService()) as service: + result = service.peak_memory_report(root) + assert result.source_coverage["active_blocks"] == 100_002 + assert sum(row["block_count"] for row in result.callstack_groups) == 100_002 + consume = QueryBudget.consume + + def exhaust_after_gap(self, rows): + if rows and "peak_allocated" in rows[0]: + self.max_work_rows = self.work_rows + len(rows) + consume(self, rows) + + # Reduced attribution still spends the SAME remaining report budget. + monkeypatch.setattr(QueryBudget, "consume", exhaust_after_gap) with pytest.raises(QueryExecutionError, match="work budget"): service.peak_memory_report(root) - finally: - service.close() def test_report_deadline_is_not_restarted_for_attribution(tmp_path, monkeypatch): From 5077c4185ba5634be91b935c77d13ea6fc8f70f1 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:20:55 +0800 Subject: [PATCH 2/2] docs: clarify page hydration preserves full active coverage --- src/pt_snap_cli/core/dataset_attribution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index 450ef18..7745e68 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -34,6 +34,8 @@ def event_attribution( if reduced is not None: selected, coverage, total, rank_full, offset = reduced if template == "active_blocks_at_event": + # Hydrate only the returned page; discard its coverage so the SQL + # coverage for the full active set before pagination remains intact. _hydrate_blocks(sources, selected) coverage["source_queries"] = sources.query_count scope = _scope(sources, item.index, event_id, coverage)