From bcf030ebedbe569115143fd19501de9f9659aa69 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:47:01 +0800 Subject: [PATCH] perf: skip whole shards for ID-ordered deep pages --- docs/en/sharded-snapshotdb.md | 12 + docs/zh/sharded-snapshotdb.md | 8 + src/pt_snap_cli/core/dataset_global.py | 75 +++++- tests/core/test_dataset_global.py | 6 +- tests/core/test_dataset_id_pagination.py | 308 +++++++++++++++++++++++ 5 files changed, 401 insertions(+), 8 deletions(-) create mode 100644 tests/core/test_dataset_id_pagination.py diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 53bf44a..1cb5380 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -239,6 +239,18 @@ and cumulative work ceilings; arbitrary SQL/custom overrides fail explicitly, ne silently choosing the first/latest slice. Single-DB real-event peak/gap and all-action full-text stack-statistics versions are documented with the same contracts. +For dataset `event` and `allocation` queries ordered by `id` (ASC or DESC), deep +`offset` pagination uses per-shard matching counts to skip whole shards and a local +SQL OFFSET in the landing shard, then reads subsequent shards only to fill the +window. Sparse native IDs are counted as rows, never as ID distances. Only selected +event rows receive source text and ordered-frame hydration. Range counts still +cover the requested interval before non-range filters; totals, completeness flags, +and cumulative accounting for fetched counts, page rows, sources and output remain +unchanged. This reduces Python-fetched work, not SQLite scan work or full-dataset +validation cost. Other sort keys retain their existing global candidate strategy; +unlimited or large returned pages can still exceed the work budget. No cursor API +is added. + `core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable `ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))` addresses all matching files even for a cross-slice range, but does not execute it. diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index f89cfad..8d59004 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -196,6 +196,14 @@ selector。输出 `scope` 标识实际 DB/device/slice、受限真实区间、 及累计工作上限;任意 SQL/自定义 override 明确失败,不静默选首片/最新片。单库真实 事件 peak/gap 与全部 action 完整文本栈统计版本在相同合同中明确说明。 +数据集 `event` 和 `allocation` 以 `id` 排序(ASC 或 DESC)时,深 `offset` 分页按逐片 +匹配行数跳过整片,只在落点片使用局部 SQL OFFSET,再读取后续片补满窗口。原生稀疏 ID +按实际行数计数,不按 ID 差值推算。仅最终选中的 event 行解析来源文本和有序 frame。 +范围计数仍描述非范围筛选前的请求区间;总数、完整性标记,以及获取计数行、页内行、 +来源与输出的累计计费语义保持不变。这减少的是取入 Python 的工作量,不保证降低 SQLite +内部扫描量或完整数据集校验成本。其他排序键保持原有全局候选策略;无限或较大返回页 +仍可能超过工作预算。没有新增 cursor API。 + `core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset` (单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。 默认首次 inspect 完整校验最终化合同,并在校验前后对内容取哈希。进程级 LRU 最多保留 diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index cf03b6e..4c6101e 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -73,6 +73,7 @@ def _scope( def _range_statistics(sources, low, high, items, scope, predicate="1", values=None): """Range evidence is deliberately collected BEFORE non-range filters.""" count = matching = 0 + matching_by_shard = {} first = last = None for item in items: row = sources.read( @@ -85,6 +86,7 @@ def _range_statistics(sources, low, high, items, scope, predicate="1", values=No )[0] count += row["count"] matching += row["matching"] + matching_by_shard[item.index] = row["matching"] if row["first"] is not None: first = row["first"] if first is None else min(first, row["first"]) last = row["last"] if last is None else max(last, row["last"]) @@ -93,7 +95,7 @@ def _range_statistics(sources, low, high, items, scope, predicate="1", values=No "last_event_id": last, "real_event_count": count, } - return matching + return matching, matching_by_shard def _peaks(sources, low, high, items, template, scope): @@ -187,18 +189,76 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None: rows.sort(key=lambda r: (r[key] is not None, r[key]), reverse=descending) -def _events(sources, low, high, items, params, template, scope, candidate_limit): +def _id_event_page(sources, low, high, items, params, fields, columns, counts, offset, limit): + """Page ordered, disjoint shard ranges without fetching the skipped prefix. + + Counts describe matching rows, not ID distances: native IDs may be sparse. + Only the landing shard uses a local OFFSET; later shards fill the remaining + window. SQLite may still scan skipped rows, but only fetched work crosses the + Python budget boundary. Source text/frames are hydrated after final selection. + """ + rows = [] + direction = params["order_dir"] + ordered = reversed(items) if direction == "DESC" else iter(items) + predicate, values = _predicates(params, fields, "t.") + projection = ", ".join(f't."{column}"' for column in columns) + for item in ordered: + sources.budget.remaining() + if limit == 0: + break + count = counts[item.index] + if offset >= count: + offset -= count + continue + batch = sources.read( + sources.dataset.root / item.file, + f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" t ' + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {predicate} " + f"ORDER BY t.id {direction} LIMIT ? OFFSET ?", + [low, high, *values, limit, offset], + ) + rows.extend(batch) + offset = 0 + if limit >= 0: + limit -= len(batch) + return rows + + +def _events(sources, low, high, items, params, template, scope, offset, limit): fields = METRICS if template == "event": fields += ("id", "action", "address", "size", "stream") predicate, values = _predicates(params, fields) - total = _range_statistics(sources, low, high, items, scope, predicate, values) + total, counts = _range_statistics(sources, low, high, items, scope, predicate, values) rows = [] order = params["order_by"] direction = params["order_dir"] columns = ("id", *METRICS) if template == "event": columns = ("id", "action", "address", "size", "stream", *METRICS) + candidate_limit = offset + limit if limit >= 0 else -1 + if order == "id": + # Preserve the existing finite candidate-window int64 error, including + # empty matches, even though the optimized plan binds a smaller LIMIT. + if items and candidate_limit > 2**63 - 1: + raise QueryExecutionError( + "Query execution failed: Python int too large to convert to SQLite INTEGER" + ) + return ( + _id_event_page( + sources, + low, + high, + items, + params, + fields, + columns, + counts, + offset, + limit if limit >= 0 else -1, + ), + total, + ) for item in items: trace = f'"trace_entry_{sources.device.device_id}"' projection = ", ".join(f't."{column}"' for column in columns) @@ -489,12 +549,15 @@ def global_query( offset, limit = _window(params, max_rows) candidate_limit = offset + limit if limit >= 0 else -1 total_override = None + selected_offset = offset if template in ("memory_peak", "allocator_gap"): rows = _peaks(sources, low, high, items, template, scope) elif template in ("event", "allocation"): rows, total_override = _events( - sources, low, high, items, params, template, scope, candidate_limit + sources, low, high, items, params, template, scope, offset, limit ) + if params["order_by"] == "id": + selected_offset = 0 elif template == "callstack_analysis": rows = _stacks(sources, low, high, items, params, scope) elif template == "preexisting_live": @@ -567,7 +630,9 @@ def global_query( "No dataset-global merge implementation for this template." ) total = len(rows) if total_override is None else total_override - selected = rows[offset : offset + limit] if limit >= 0 else rows[offset:] + selected = ( + rows[selected_offset : selected_offset + limit] if limit >= 0 else rows[selected_offset:] + ) if template == "event": refs = sources.events(cast(int, r["id"]) for r in selected) detailed = ( diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 46d158d..7f89b6b 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -318,9 +318,9 @@ def test_work_limits_detached_batches_borrowed_lru_timeout_cleanup_and_report_de budget = QueryBudget(None, time.monotonic(), max_work_rows=4) with pytest.raises(QueryExecutionError, match="work budget"): service.execute_query("block", db_path=root, _budget=budget) - # The entire real range consumes 16 rows before final max_rows=1; - # its source lookup must use the SAME cumulative budget, not reset. - budget = QueryBudget(None, time.monotonic(), max_work_rows=16) + # Eight shard count rows plus one ID page row consume nine work rows; + # source hydration must use the SAME cumulative budget, not reset. + budget = QueryBudget(None, time.monotonic(), max_work_rows=9) with pytest.raises(QueryExecutionError, match="work budget"): service.execute_query( "event", db_path=root, max_rows=1, exact_total=True, _budget=budget diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py new file mode 100644 index 0000000..df4756c --- /dev/null +++ b/tests/core/test_dataset_id_pagination.py @@ -0,0 +1,308 @@ +"""ID pages skip counted matches without materializing the skipped prefix.""" + +import dataclasses +import json +import sqlite3 +import time +from contextlib import closing + +import pytest + +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.query_service import QueryService +from tests.core.test_dataset_attribution import case, standalone +from tests.core.test_dataset_import import source_options +from tests.core.test_dataset_source_reuse import framed_dataset +from tests.test_dataset_focus import hashes, make_dataset + + +@pytest.fixture(scope="module", params=["inline", "native"]) +def paired_dataset(tmp_path_factory, request): + directory = tmp_path_factory.mktemp(f"id-page-{request.param}") + if request.param == "inline": + root = case(directory, devices=(0,)) + single = standalone(root, directory) + else: + options = dataclasses.replace(source_options(directory), set_focus=False) + importer = ImportService() + root = importer.import_snapshot(options).db_path + single = importer.import_snapshot( + dataclasses.replace(options, events_per_slice=None) + ).db_path + resolved = DatasetResolver().inspect(root) + return root, single, resolved.device(0) + + +WINDOWS = [ + (0, -1, None), + (0, 0, None), + (0, 1, None), + (1, 0, None), + (1, 3, None), + (2, 3, None), + (3, 2, None), + (7, 3, None), + (8, -1, None), + (9, 1, None), + (15, 1, None), + (16, 3, None), + (1_000_000, 3, None), + (2, -1, 2), + (2, 3, 1), + (2, 0, 1), + (2, -1, 0), + (2, -1, -1), + (-3, 2, None), +] + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_id_windows_match_standalone_with_filters_and_sparse_ids( + paired_dataset, template, direction +): + root, single, device = paired_dataset + low, high = device.slices[0].start_event_id, device.slices[-1].end_event_id + filters = [ + {}, + {"min_allocated": 1}, + {"min_active": 1, "min_reserved": 1}, + {"min_id": low + 1, "max_id": high - 1}, + {"min_id": low + 1, "max_id": low + 1}, + ] + if template == "event": + filters += [{"action": 4}, {"action": 6}, {"action": 99}, {"stream": 1}] + before = hashes(root) + with closing(QueryService()) as service, closing(sqlite3.connect(single)) as conn: + for extra in filters: + range_low, range_high = extra.get("min_id", low), extra.get("max_id", high) + first, last, count = conn.execute( + "SELECT MIN(id), MAX(id), COUNT(*) FROM trace_entry_0 WHERE id>=? AND id<=?", + [range_low, range_high], + ).fetchone() + for offset, limit, max_rows in WINDOWS: + params = { + "min_id": low, + "max_id": high, + "order_by": "id", + "order_dir": direction, + "offset": offset, + "limit": limit, + **extra, + } + for exact in (False, True): + actual = service.execute_query( + template, params, root, max_rows=max_rows, exact_total=exact + ) + expected = service.execute_query( + template, params, single, max_rows=max_rows, exact_total=exact + ) + assert actual.rows == expected.rows, params + for field in ("total", "returned", "has_more", "truncated", "total_is_exact"): + assert getattr(actual, field) == getattr(expected, field), (params, field) + assert actual.scope["actual_range"] == { + "first_event_id": first, + "last_event_id": last, + "real_event_count": count, + } + assert actual.scope["boundary_events_included"] is False + assert hashes(root) == before + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_id_slice_pages_preserve_scope_and_exact_total(paired_dataset, template, direction): + root, single, device = paired_dataset + with closing(QueryService()) as service: + for item in (device.slices[0], device.slices[2], device.slices[-1]): + for offset in (0, 1, 2, 3): + params = { + "min_id": item.start_event_id, + "max_id": item.end_event_id, + "order_by": "id", + "order_dir": direction, + "offset": offset, + "limit": 2, + } + actual = service.execute_query( + template, params, root, slice_index=item.index, exact_total=True + ) + expected = service.execute_query(template, params, single, exact_total=True) + assert actual.rows == expected.rows + assert actual.total == expected.total + assert actual.has_more == expected.has_more + assert actual.truncated == expected.truncated + assert actual.scope["slice_indices"] == [item.index] + assert actual.scope["actual_range"]["real_event_count"] == expected.total + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_id_page_fetches_only_landing_and_following_matching_shards( + tmp_path, monkeypatch, direction +): + root = make_dataset(tmp_path / "filtered", devices=(0,), slices=4) + for member in root.glob("device_0/*.db"): + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET action=5 WHERE id IN (0,1,3,4,6,7)") + queries = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + if "ORDER BY t.id" in sql: + queries.append((path.name, values[-2:])) + return original(self, path, sql, values) + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + budget = QueryBudget(None, time.monotonic(), max_work_rows=13) + with closing(QueryService()) as service: + result = service.execute_query( + "event", + {"action": 5, "order_dir": direction, "offset": 1, "limit": 3}, + root, + exact_total=True, + _budget=budget, + ) + if direction == "ASC": + assert [row["id"] for row in result.rows] == [1, 3, 4] + assert queries == [ + ("slice_00000.db", [3, 1]), + ("slice_00001.db", [2, 0]), + ("slice_00002.db", [1, 0]), + ] + else: + assert [row["id"] for row in result.rows] == [6, 4, 3] + assert queries == [ + ("slice_00003.db", [3, 1]), + ("slice_00002.db", [2, 0]), + ("slice_00001.db", [1, 0]), + ] + assert result.total == 6 and result.has_more and result.truncated + assert budget.work_rows == 4 + 3 + 3 + 3 # counts + page + sources + output + assert result.scope["actual_range"]["real_event_count"] == 8 + + +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_id_page_hydrates_only_final_event_sources_and_ordered_frames( + tmp_path, monkeypatch, direction +): + root = framed_dataset(tmp_path, 40) + reads = [] + original = DatasetSourceResolver.read + + def observed(self, path, sql, values=None): + rows = original(self, path, sql, values) + reads.append((sql, values, len(rows))) + return rows + + monkeypatch.setattr(DatasetSourceResolver, "read", observed) + budget = QueryBudget(None, time.monotonic()) + with closing(QueryService()) as service: + result = service.execute_query( + "event", {"order_dir": direction, "offset": 20, "limit": 2}, root, _budget=budget + ) + ids = [20, 21] if direction == "ASC" else [19, 18] + assert [row["id"] for row in result.rows] == ids + assert all(row["frames"] == [{"name": "f"}] for row in result.rows) + for sql, values, _ in reads: + if " IN (" in sql: + assert set(values) == set(ids) + if "ORDER BY t.id" in sql: + assert "callstack" not in sql and "frame" not in sql + assert budget.work_rows == sum(count for _, _, count in reads) + 2 == 18 + assert result.scope["source_coverage"]["events_resolved"] == 2 + assert result.scope["source_coverage"]["ordered_frame_events"] == 2 + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +@pytest.mark.parametrize("offset,limit", [(2**63 - 1, 1), (2**63, 0), (1, 2**63 - 1)]) +def test_id_window_preserves_int64_overflow_errors(tmp_path, template, offset, limit): + root = make_dataset(tmp_path / "overflow", devices=(0,)) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="too large"): + service.execute_query(template, {"offset": offset, "limit": limit}, root) + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +def test_zero_page_fetches_only_charged_counts_and_keeps_more(tmp_path, template): + root = make_dataset(tmp_path / "empty-page", devices=(0,)) + budget = QueryBudget(None, time.monotonic(), max_work_rows=3) + with closing(QueryService()) as service: + result = service.execute_query( + template, {"offset": 2, "limit": 0}, root, exact_total=True, _budget=budget + ) + assert result.rows == [] and result.total == 6 + assert result.has_more and result.truncated and result.total_is_exact + assert budget.work_rows == 3 + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query( + template, + {"offset": 2, "limit": 0}, + root, + _budget=QueryBudget(None, time.monotonic(), max_work_rows=2), + ) + + +@pytest.fixture(scope="module") +def million_event_dataset(tmp_path_factory): + root = make_dataset( + tmp_path_factory.mktemp("million-id-page") / "dataset", devices=(0,), slices=4 + ) + manifest_path = root / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + per_shard = 250_003 + manifest["eventsPerSlice"] = per_shard + manifest["devices"]["0"]["eventCount"] = 4 * per_shard + for item in manifest["devices"]["0"]["slices"]: + low = item["index"] * per_shard + high = low + per_shard - 1 + item.update(startEventId=low, endEventId=high) + with closing(sqlite3.connect(root / item["file"])) as conn, conn: + conn.execute("DELETE FROM trace_entry_0 WHERE id>=0") + conn.executemany( + "INSERT INTO trace_entry_0 VALUES (?,4,100,8,0,8,8,16,'shared')", + ((event,) for event in range(low, high + 1)), + ) + manifest_path.write_text(json.dumps(manifest)) + return root + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +@pytest.mark.parametrize("direction", ["ASC", "DESC"]) +def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction): + work = [] + with closing(QueryService()) as service: + for offset in (0, 1_000_000): + budget = QueryBudget(None, time.monotonic(), max_work_rows=19) + result = service.execute_query( + template, + {"order_dir": direction, "offset": offset, "limit": 5}, + million_event_dataset, + exact_total=True, + _budget=budget, + ) + expected = ( + list(range(offset, offset + 5)) + if direction == "ASC" + else list(range(1_000_011 - offset, 1_000_006 - offset, -1)) + ) + assert [row["id"] for row in result.rows] == expected + assert result.total == 1_000_012 and result.returned == 5 + assert result.has_more and result.truncated and result.total_is_exact + assert result.scope["actual_range"]["real_event_count"] == 1_000_012 + work.append((budget.work_rows, budget.work_bytes)) + assert work[0][0] == work[1][0] == (19 if template == "event" else 14) + assert max(size for _, size in work) < 10_000 + + +@pytest.mark.parametrize("template", ["event", "allocation"]) +def test_negative_limit_keeps_existing_dataset_unbounded_normalization(tmp_path, template): + root = make_dataset(tmp_path / "negative-limit", devices=(0,)) + with closing(QueryService()) as service: + expected = service.execute_query(template, {"offset": 2, "limit": -1}, root) + actual = service.execute_query(template, {"offset": 2, "limit": -(2**100)}, root) + assert actual.rows == expected.rows + assert actual.returned == 4 and not actual.has_more + assert actual.truncated and not actual.total_is_exact