Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 155 additions & 0 deletions benchmarks/lifecycle_query_plans.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
"""Opt-in same-artifact ATTACH, scratch and automatic lifecycle-plan comparison.

Read only an already finalized dataset; never import pickle or run a model. Each
query gets a fresh cache/budget and may create bounded private scratch storage.
Admission is outside the timings. This is an internal reducer microbenchmark,
not a cold CLI/end-to-end measurement or a portable speed assertion.

PYTHONPATH=src python benchmarks/lifecycle_query_plans.py DATASET --runs 3
"""

from __future__ import annotations

import argparse
import hashlib
import json
import platform
import sqlite3
import statistics
import time
from contextlib import closing, nullcontext
from pathlib import Path
from unittest.mock import patch

from pt_snap_cli.core import dataset_lifecycle_sql
from pt_snap_cli.core.context_cache import ContextCache
from pt_snap_cli.core.dataset_global import global_query
from pt_snap_cli.core.dataset_lifecycle_scratch import _ScratchSources
from pt_snap_cli.core.dataset_resolver import DatasetResolver
from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget
from pt_snap_cli.query.registry import get_query


def measure(resolved, device, template, plan):
config = get_query(template)
budget = QueryBudget(
None,
time.monotonic(),
max_work_rows=128,
max_scratch_db_bytes=0 if plan == "attach" else 512 * 1024**2,
)
storage = []
original_build = _ScratchSources.build

def build(database, items, maximum):
original_build(database, items, maximum)
storage.append(
database.connection.execute("PRAGMA page_count").fetchone()[0]
* database.connection.execute("PRAGMA page_size").fetchone()[0]
)

forced = (
patch.object(dataset_lifecycle_sql, "_required_slices", return_value=None)
if plan == "scratch"
else nullcontext()
)
with closing(ContextCache()) as cache, forced, patch.object(_ScratchSources, "build", build):
source = DatasetSourceResolver(resolved, device, cache, budget)
started = time.perf_counter()
result = global_query(
source, template, config.validate_params({}), 5, True, None, config.semantics_version
)
elapsed = time.perf_counter() - started
coverage = {
key: value
for key, value in result.scope["source_coverage"].items()
if key != "source_queries"
}
signature = json.dumps(
[result.rows, result.total, result.total_is_exact, result.has_more, coverage],
sort_keys=True,
).encode()
return {
"template": template,
"plan": plan,
"seconds": elapsed,
"work_rows": budget.work_rows,
"work_bytes": budget.work_bytes,
"scratch_builds": len(storage),
"derived_db_bytes": storage,
"result_sha256": hashlib.sha256(signature).hexdigest(),
}


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("dataset", type=Path)
parser.add_argument("--device", type=int, default=0)
parser.add_argument("--runs", type=int, default=3)
parser.add_argument("--warmups", type=int, default=1)
parser.add_argument(
"--plans",
nargs="+",
choices=("attach", "scratch", "auto"),
default=["attach", "scratch", "auto"],
)
args = parser.parse_args()
if args.runs < 1 or args.warmups < 0:
parser.error("runs must be positive and warmups nonnegative")
resolved = DatasetResolver().inspect(args.dataset)
if resolved is None:
parser.error("expected a finalized dataset")
device = resolved.device(args.device)
capacity = dataset_lifecycle_sql._attachment_limit()
if capacity < 1 or ("attach" in args.plans and len(device.slices) > capacity):
parser.error("forced ATTACH needs the complete device within this SQLite attachment limit")
samples = []
signatures = {}
for template in ("block", "leak_detection", "freed_block_lifetime"):
for run in range(-args.warmups, args.runs):
# Alternate plan order to reduce systematic warm-cache/order bias.
for plan in args.plans if run % 2 == 0 else reversed(args.plans):
sample = measure(resolved, args.device, template, plan)
expected = signatures.setdefault(template, sample["result_sha256"])
if sample["result_sha256"] != expected:
raise AssertionError(f"Different lifecycle semantics: {template}/{plan}")
if run >= 0:
samples.append({"run": run, **sample})
resolved.require_unchanged()
print(
json.dumps(
{
"dataset": str(resolved.root),
"fingerprint": resolved.fingerprint,
"source_bytes": sum(
(resolved.root / item.file).stat().st_size for item in device.slices
),
"slices": len(device.slices),
"python": platform.python_version(),
"platform": platform.platform(),
"sqlite": sqlite3.sqlite_version,
"warmups_per_plan": args.warmups,
"runs_per_plan": args.runs,
"timing_scope": "admitted dataset; fresh cache/budget; page=5; exact total; includes routing probe",
"all_results_equal": True,
"sources_unchanged": True,
"median_seconds": {
template: {
plan: statistics.median(
sample["seconds"]
for sample in samples
if sample["template"] == template and sample["plan"] == plan
)
for plan in args.plans
}
for template in signatures
},
"samples": samples,
},
indent=2,
)
)


if __name__ == "__main__":
main()
14 changes: 11 additions & 3 deletions docs/en/querying.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,16 +177,24 @@ shard. Event/allocation pages apply SQL filters and ordering per shard, merge at
is measured before non-range filters. Ordinary text-stack statistics merge full
per-shard groups before global thresholds; averages use summed sizes and counts.
Canonical ordered-frame statistics retain the bounded source-reading path because
formatted text is not their identity. Block/leak/lifetime reductions use read-only
formatted text is not their identity. Small block/leak/lifetime reductions use read-only
attached shards for proof, conflicts and latest observations before pagination;
the attachment limit (normally ten) counts selected observation shards plus any
other shards owning their allocation/free references. Slice queries discover those
proof owners before filtering or paging. Above the limit, a query-private temporary
proof owners before filtering or paging. Above the attachment limit, or above 4096
selected block observations when scratch is enabled, a query-private temporary
SQLite database retains only distinct event references and one narrow state row per
lifecycle. Sources are attached read-only one at a time: reference ownership/actions
are proved first, then all observations are reduced and conflicts checked before
filters, totals or pagination. Only returned candidates hydrate complete source events.
The small attached-set path is unchanged.
Summary and page/lifetime buckets reuse the reduced table, avoiding two full
history/window reductions. Routing counts at most 4097 observations across the
selected members with bounded SQL probes; only scalar counts enter the fetched-work
budget. This observation threshold preserves the small read-only path and does not
predict every layout's performance. Setting the scratch budget to zero skips the
probe and retains ATTACH when the complete proof-owner set fits, otherwise the
original bounded source reader. Once scratch is selected, storage failure is an
error with cleanup, never a retry through a less bounded plan.

The temporary directory is private (0700), its derived database is created with 0600
permissions, and normal completion, errors and cancellation close connections and
Expand Down
11 changes: 8 additions & 3 deletions docs/zh/querying.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,17 @@ SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入
排序,每片最多取 `offset + limit` 个候选后全局归并,并单独统计匹配总数;实际范围证据在
非范围筛选前计算。普通文本栈统计先合并各片完整分组,再应用全局阈值,平均值由总大小
与总计数计算。规范化原始有序帧的身份不能用格式化文本替代,因此仍走受预算约束的来源
读取路径。block/leak/lifetime 通过只读附加分片先证明身份、检测冲突、选最新观察,再分页;
读取路径。小规模 block/leak/lifetime 通过只读附加分片先证明身份、检测冲突、选最新观察,再分页;
附加上限(通常十片)按选定观察分片与其分配/释放引用所属的其他分片合计。
slice 查询在筛选、分页前发现所需的证明来源分片。超限时使用查询专属的临时 SQLite
slice 查询在筛选、分页前发现所需的证明来源分片。超过附加上限,或启用 scratch 且选定
block 观察超过 4096 条时,使用查询专属的临时 SQLite
派生库,仅保留去重后的事件引用及每个生命周期的一条窄状态记录。源分片逐个只读挂载:
先证明引用的所属片及 action,再归约全部观察、检查冲突,之后才筛选、计数和分页。
仅返回候选补全完整来源事件。未超过附加上限的原路径不变。
仅返回候选补全完整来源事件。汇总和分页/时长桶复用已归约的表,避免两次完整历史与窗口
归约。路由探测通过有界 SQL 在选定分片合计最多统计 4097 条观察,仅把标量计数纳入取回
预算。此观察数阈值保留小输入的只读路径,不保证所有布局都有同样性能。scratch 预算为 0
时跳过探测;完整证明来源集合不超附加上限则保留 ATTACH,否则沿用原有有界来源读取。
一旦选定 scratch,存储失败便清理并报错,不重试约束更弱的执行路径。

临时目录权限为 0700,派生文件以 0600 创建;正常完成、错误和取消均关闭连接并删除目录。
强制杀进程或文件系统清理失败可能留下私有残余;这不是安全擦除。派生主库页数空间默认
Expand Down
2 changes: 1 addition & 1 deletion src/pt_snap_cli/core/dataset_lifecycle_scratch.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Private disk-backed lifecycle reduction when the read-only ATTACH set is too large.
"""Private disk-backed reduction for large lifecycle observation or ATTACH sets.

Only references and one narrow row per identity are retained. The derived main
DB has a page ceiling; this does NOT limit all SQLite journals/sort files or RSS.
Expand Down
50 changes: 38 additions & 12 deletions src/pt_snap_cli/core/dataset_lifecycle_sql.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
"""Read-only relational lifecycle reduction over a bounded set of attached shards.

No rows are copied to a merge database. SQLite performs proof, conflict checks,
latest-observation selection and page ranking; only aggregate/page rows cross the
Python boundary. The fetched-row/byte budget does not bound SQLite's internal
scans or temporary space. Summary and page/buckets each evaluate the full CTE;
all statements share the operation deadline. Larger required shard sets use a
private, size-limited derived SQLite database; source members remain read-only.
"""Bounded lifecycle reduction with one reusable state row per large-scope identity.

Small observation sets use read-only ATTACH/CTEs; larger sets reuse a private,
size-limited derived table for summary and page/buckets. Only aggregates/pages
cross the Python boundary. The fetched-row/byte budget does not bound SQLite's
internal scans or temporary space. All statements, including the bounded routing
probe, share the operation deadline. Source members always remain read-only.
"""

from __future__ import annotations
Expand All @@ -17,6 +16,12 @@
from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources
from pt_snap_cli.core.errors import QueryExecutionError

# A row threshold, not an ATTACH/shard-count heuristic. Same-artifact synthetic
# measurements show scratch's fixed storage cost dominates tiny inputs, while
# reduction reuse wins well before this conservative observation bound. Probe
# only this many + 1 observations across all members, never COUNT the whole scope.
_ATTACHED_OBSERVATION_LIMIT = 4096


def _attachment_limit() -> int:
# getlimit is unavailable on supported Python 3.10. Ten is SQLite's default.
Expand Down Expand Up @@ -75,6 +80,21 @@ def _proof(sources, items, field, action):
return f"CASE {cases} ELSE 0 END"


def _large_observation_set(sources, items):
remaining = _ATTACHED_OBSERVATION_LIMIT + 1
for item in items:
count = sources.read(
sources.dataset.root / item.file,
f'SELECT COUNT(*) AS count FROM (SELECT 1 FROM "block_{sources.device.device_id}" '
"LIMIT ?)",
[remaining],
)[0]["count"]
remaining -= count
if remaining == 0:
return True
return False


def _lifecycle_cte(sources, items, proof_items):
allocated = _proof(sources, proof_items, "allocEventId", 4)
completed = _proof(sources, proof_items, "freeEventId", 6)
Expand Down Expand Up @@ -136,15 +156,21 @@ def _filter(template, params):


def lifecycle_query(sources, items, template, params, scope, candidate_limit):
required = _required_slices(sources, items, _attachment_limit())
attachment_limit = _attachment_limit()
if attachment_limit < 1:
return None
required = _required_slices(sources, items, attachment_limit)
predicate, values = _filter(template, params)
if required is None:
scratch_enabled = sources.budget.max_scratch_db_bytes != 0
if scratch_enabled and (required is None or _large_observation_set(sources, items)):
from pt_snap_cli.core.dataset_lifecycle_scratch import scratch_lifecycles

if sources.budget.max_scratch_db_bytes == 0 or _attachment_limit() < 1:
return None
# Once selected, storage/deadline failures must fail the whole operation;
# falling back could hide exhaustion and redo unbounded expensive work.
execution = scratch_lifecycles(sources, items)
else:
if required is None:
return None
from contextlib import contextmanager

@contextmanager
Expand Down
Loading
Loading