Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/en/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,18 @@ and cumulative work ceilings; arbitrary SQL/custom overrides fail explicitly, ne
silently choosing the first/latest slice. Single-DB real-event peak/gap and all-action
full-text stack-statistics versions are documented with the same contracts.

For dataset `event` and `allocation` queries ordered by `id` (ASC or DESC), deep
`offset` pagination uses per-shard matching counts to skip whole shards and a local
SQL OFFSET in the landing shard, then reads subsequent shards only to fill the
window. Sparse native IDs are counted as rows, never as ID distances. Only selected
event rows receive source text and ordered-frame hydration. Range counts still
cover the requested interval before non-range filters; totals, completeness flags,
and cumulative accounting for fetched counts, page rows, sources and output remain
unchanged. This reduces Python-fetched work, not SQLite scan work or full-dataset
validation cost. Other sort keys retain their existing global candidate strategy;
unlimited or large returned pages can still exceed the work budget. No cursor API
is added.

`core.dataset_resolver.DatasetResolver.inspect(path)` returns an immutable
`ResolvedDataset` (or `None` for a standalone DB). Its `paths(QueryScope(...))`
addresses all matching files even for a cross-slice range, but does not execute it.
Expand Down
8 changes: 8 additions & 0 deletions docs/zh/sharded-snapshotdb.md
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,14 @@ top20、精确总数、单库差分、只读哈希和无界 block 输出硬失
及累计工作上限;任意 SQL/自定义 override 明确失败,不静默选首片/最新片。单库真实
事件 peak/gap 与全部 action 完整文本栈统计版本在相同合同中明确说明。

数据集 `event` 和 `allocation` 以 `id` 排序(ASC 或 DESC)时,深 `offset` 分页按逐片
匹配行数跳过整片,只在落点片使用局部 SQL OFFSET,再读取后续片补满窗口。原生稀疏 ID
按实际行数计数,不按 ID 差值推算。仅最终选中的 event 行解析来源文本和有序 frame。
范围计数仍描述非范围筛选前的请求区间;总数、完整性标记,以及获取计数行、页内行、
来源与输出的累计计费语义保持不变。这减少的是取入 Python 的工作量,不保证降低 SQLite
内部扫描量或完整数据集校验成本。其他排序键保持原有全局候选策略;无限或较大返回页
仍可能超过工作预算。没有新增 cursor API。

`core.dataset_resolver.DatasetResolver.inspect(path)` 返回不可变 `ResolvedDataset`
(单库返回 `None`);其 `paths(QueryScope(...))` 可寻址跨片范围的全部文件,但不执行查询。
默认首次 inspect 完整校验最终化合同,并在校验前后对内容取哈希。进程级 LRU 最多保留
Expand Down
75 changes: 70 additions & 5 deletions src/pt_snap_cli/core/dataset_global.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def _scope(
def _range_statistics(sources, low, high, items, scope, predicate="1", values=None):
"""Range evidence is deliberately collected BEFORE non-range filters."""
count = matching = 0
matching_by_shard = {}
first = last = None
for item in items:
row = sources.read(
Expand All @@ -85,6 +86,7 @@ def _range_statistics(sources, low, high, items, scope, predicate="1", values=No
)[0]
count += row["count"]
matching += row["matching"]
matching_by_shard[item.index] = row["matching"]
if row["first"] is not None:
first = row["first"] if first is None else min(first, row["first"])
last = row["last"] if last is None else max(last, row["last"])
Expand All @@ -93,7 +95,7 @@ def _range_statistics(sources, low, high, items, scope, predicate="1", values=No
"last_event_id": last,
"real_event_count": count,
}
return matching
return matching, matching_by_shard


def _peaks(sources, low, high, items, template, scope):
Expand Down Expand Up @@ -187,18 +189,76 @@ def _sort(rows: list[dict[str, object]], key: str, descending: bool) -> None:
rows.sort(key=lambda r: (r[key] is not None, r[key]), reverse=descending)


def _events(sources, low, high, items, params, template, scope, candidate_limit):
def _id_event_page(sources, low, high, items, params, fields, columns, counts, offset, limit):
"""Page ordered, disjoint shard ranges without fetching the skipped prefix.

Counts describe matching rows, not ID distances: native IDs may be sparse.
Only the landing shard uses a local OFFSET; later shards fill the remaining
window. SQLite may still scan skipped rows, but only fetched work crosses the
Python budget boundary. Source text/frames are hydrated after final selection.
"""
rows = []
direction = params["order_dir"]
ordered = reversed(items) if direction == "DESC" else iter(items)
predicate, values = _predicates(params, fields, "t.")
projection = ", ".join(f't."{column}"' for column in columns)
for item in ordered:
sources.budget.remaining()
if limit == 0:
break
count = counts[item.index]
if offset >= count:
offset -= count
continue
batch = sources.read(
sources.dataset.root / item.file,
f'SELECT {projection} FROM "trace_entry_{sources.device.device_id}" t '
f"WHERE t.id>=0 AND t.id>=? AND t.id<=? AND {predicate} "
f"ORDER BY t.id {direction} LIMIT ? OFFSET ?",
[low, high, *values, limit, offset],
)
rows.extend(batch)
offset = 0
if limit >= 0:
limit -= len(batch)
return rows


def _events(sources, low, high, items, params, template, scope, offset, limit):
fields = METRICS
if template == "event":
fields += ("id", "action", "address", "size", "stream")
predicate, values = _predicates(params, fields)
total = _range_statistics(sources, low, high, items, scope, predicate, values)
total, counts = _range_statistics(sources, low, high, items, scope, predicate, values)
rows = []
order = params["order_by"]
direction = params["order_dir"]
columns = ("id", *METRICS)
if template == "event":
columns = ("id", "action", "address", "size", "stream", *METRICS)
candidate_limit = offset + limit if limit >= 0 else -1
if order == "id":
# Preserve the existing finite candidate-window int64 error, including
# empty matches, even though the optimized plan binds a smaller LIMIT.
if items and candidate_limit > 2**63 - 1:
raise QueryExecutionError(
"Query execution failed: Python int too large to convert to SQLite INTEGER"
)
return (
_id_event_page(
sources,
low,
high,
items,
params,
fields,
columns,
counts,
offset,
limit if limit >= 0 else -1,
),
total,
)
for item in items:
trace = f'"trace_entry_{sources.device.device_id}"'
projection = ", ".join(f't."{column}"' for column in columns)
Expand Down Expand Up @@ -489,12 +549,15 @@ def global_query(
offset, limit = _window(params, max_rows)
candidate_limit = offset + limit if limit >= 0 else -1
total_override = None
selected_offset = offset
if template in ("memory_peak", "allocator_gap"):
rows = _peaks(sources, low, high, items, template, scope)
elif template in ("event", "allocation"):
rows, total_override = _events(
sources, low, high, items, params, template, scope, candidate_limit
sources, low, high, items, params, template, scope, offset, limit
)
if params["order_by"] == "id":
selected_offset = 0
elif template == "callstack_analysis":
rows = _stacks(sources, low, high, items, params, scope)
elif template == "preexisting_live":
Expand Down Expand Up @@ -567,7 +630,9 @@ def global_query(
"No dataset-global merge implementation for this template."
)
total = len(rows) if total_override is None else total_override
selected = rows[offset : offset + limit] if limit >= 0 else rows[offset:]
selected = (
rows[selected_offset : selected_offset + limit] if limit >= 0 else rows[selected_offset:]
)
if template == "event":
refs = sources.events(cast(int, r["id"]) for r in selected)
detailed = (
Expand Down
6 changes: 3 additions & 3 deletions tests/core/test_dataset_global.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,9 @@ def test_work_limits_detached_batches_borrowed_lru_timeout_cleanup_and_report_de
budget = QueryBudget(None, time.monotonic(), max_work_rows=4)
with pytest.raises(QueryExecutionError, match="work budget"):
service.execute_query("block", db_path=root, _budget=budget)
# The entire real range consumes 16 rows before final max_rows=1;
# its source lookup must use the SAME cumulative budget, not reset.
budget = QueryBudget(None, time.monotonic(), max_work_rows=16)
# Eight shard count rows plus one ID page row consume nine work rows;
# source hydration must use the SAME cumulative budget, not reset.
budget = QueryBudget(None, time.monotonic(), max_work_rows=9)
with pytest.raises(QueryExecutionError, match="work budget"):
service.execute_query(
"event", db_path=root, max_rows=1, exact_total=True, _budget=budget
Expand Down
Loading
Loading