From fb28bd26647b9b07b75b2e8f98b7481986b01236 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:44:43 +0800 Subject: [PATCH] feat: expose bounded query limits across API and CLI --- README.md | 6 + README_zh.md | 5 + docs/en/querying.md | 55 +++- docs/en/snapshot-analyzer-api.md | 42 +++ docs/zh/querying.md | 48 +++- docs/zh/snapshot-analyzer-api.md | 36 +++ src/pt_snap_cli/api.py | 12 +- src/pt_snap_cli/cli.py | 19 +- src/pt_snap_cli/cli_limits.py | 30 ++ src/pt_snap_cli/cli_reports.py | 18 ++ src/pt_snap_cli/core/__init__.py | 2 + .../core/dataset_lifecycle_scratch.py | 16 +- src/pt_snap_cli/core/dataset_sources.py | 42 ++- src/pt_snap_cli/core/dataset_support.py | 12 +- src/pt_snap_cli/core/query_limits.py | 95 +++++++ src/pt_snap_cli/core/query_service.py | 31 ++- src/pt_snap_cli/core/report_service.py | 17 +- tests/core/test_query_limits.py | 258 ++++++++++++++++++ tests/test_contract_cli_api.py | 45 +++ tests/test_query_limits.py | 192 +++++++++++++ 20 files changed, 945 insertions(+), 36 deletions(-) create mode 100644 src/pt_snap_cli/cli_limits.py create mode 100644 src/pt_snap_cli/core/query_limits.py create mode 100644 tests/core/test_query_limits.py create mode 100644 tests/test_query_limits.py diff --git a/README.md b/README.md index 8ab43cf..93bc3fe 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,12 @@ text-only databases require re-importing the original snapshot. `report memory-tree` also accepts `--json`. The HTML view has full-tree search and keyboard-accessible child lists, so tiny and zero-byte frames remain reachable. +Dataset work limits are configurable with `--max-work-rows`, `--max-work-bytes`, +`--max-scratch-db-bytes`, environment variables, or immutable Python `QueryLimits`. +Defaults remain 100000 work rows / 64 MiB work bytes / 512 MiB scratch pages; see +[public dataset limits](docs/en/querying.md#public-dataset-limits) for precedence, +deep-pagination costs and all-or-error semantics. + ## Agent Skills For long diagnostic stacks, opt into `stack_bytes` on `event` or diff --git a/README_zh.md b/README_zh.md index 9da42a1..ec00160 100644 --- a/README_zh.md +++ b/README_zh.md @@ -104,6 +104,11 @@ included bytes 为分母。解读有上限的分组结果前,请参阅 `report memory-tree` 同样支持 `--json`。HTML 支持全树搜索和可键盘操作的子节点列表, 极小占用及零字节帧也能进入查看。 +数据集工作预算可通过 `--max-work-rows`、`--max-work-bytes`、 +`--max-scratch-db-bytes`、环境变量或不可变 Python `QueryLimits` 配置。 +默认仍为 100000 工作行 / 64 MiB 工作字节 / 512 MiB scratch 页;优先级、 +深分页成本和整次失败语义见[公开数据集预算](docs/zh/querying.md#公开数据集预算)。 + ## Agent Skills 诊断长调用栈时,可在 `event` 或 `active_memory_callstack_at_event` 上显式设置 diff --git a/docs/en/querying.md b/docs/en/querying.md index 47091a3..52a975f 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -149,7 +149,7 @@ marked, never guessed by address/local stack ID and never asserted as proved lea Dataset reads detach batches before foreign-owner lookups, using the borrowed bounded Context LRU. One diminishing deadline includes all shards, sources, grouping and exact totals; reports explicitly share **one** budget across peak selection and attribution. Cumulative -fetched/output work is limited to **100000 rows / 64 MiB serialized values**, including +fetched/output work defaults to **100000 rows / 64 MiB serialized values**, including repeated source reads. Exceeding either fails the entire operation, never an alleged complete partial result. Python materialization/sort are bounded by this work ceiling, **not** by max_rows. This is not a process RSS limit: Python/SQLite, one batch and individual @@ -192,9 +192,10 @@ The temporary directory is private (0700), its derived database is created with permissions, and normal completion, errors and cancellation close connections and remove the directory. A killed process or failed filesystem cleanup can leave private residue; this is not secure erasure. Its main database page ceiling defaults to -512 MiB, configurable through the internal `QueryBudget.max_scratch_db_bytes` field from +512 MiB, configurable with `--max-scratch-db-bytes`, +`PT_SNAP_MAX_SCRATCH_DB_BYTES` or public `QueryLimits.max_scratch_db_bytes` from 4096 bytes to 4 GiB; zero disables this path and retains the original budgeted source -reader. This is not a new CLI option. Exceeding the page ceiling or running out of +reader. See [public dataset limits](#public-dataset-limits). Exceeding the page ceiling or running out of storage rejects the entire operation. The ceiling does **not** bound rollback journals, SQLite sorting files, total temporary space or RSS. Main/source page caches request 2 MiB each; these are cache targets, not memory guarantees. Internal SQL scans and @@ -210,6 +211,54 @@ or deadline. The limit counts rows fetched into Python and serialized output, not internal SQL scans, SQLite sorting space or total process memory. +## Public dataset limits + +`query`, `report peak-memory` and `report memory-tree` accept the same options: + +| CLI option | Environment variable | Default / valid values | +| --- | --- | --- | +| `--max-work-rows` | `PT_SNAP_MAX_WORK_ROWS` | `100000`; positive integer | +| `--max-work-bytes` | `PT_SNAP_MAX_WORK_BYTES` | `67108864` (64 MiB); positive integer bytes | +| `--max-scratch-db-bytes` | `PT_SNAP_MAX_SCRATCH_DB_BYTES` | `536870912` (512 MiB); `0` disables scratch, otherwise `4096` through `4294967296` (4 GiB) | + +Each explicit flag overrides its environment variable, then the built-in default. +Environment values are decimal integers without units; unset or blank uses the default. +Booleans, fractions, negative values and zero work limits are errors; there is no +unlimited work-budget setting. Scratch zero **does not disable the work budget**. +Non-page-aligned scratch limits are rounded down to whole 4096-byte pages by SQLite. +Invalid configuration fails before database access, including on standalone commands. +These limits govern dataset execution; they do not add a new work cap to standalone +SQLite queries or make a dataset-unsupported template supported. + +```bash +pt-snap query '' --template-use event \ + --params '{"offset":100000,"limit":5}' --max-work-rows 300000 \ + --max-work-bytes 134217728 --exact-total --json +pt-snap report peak-memory '' --max-work-rows 300000 \ + --max-scratch-db-bytes 1073741824 --timeout 30 --json +``` + +Use the [immutable Python `QueryLimits` API](snapshot-analyzer-api.md#dataset-query-limits) +for the same settings in `SnapshotAnalyzer`, `QueryService` and `ReportService`. +A successful dataset query/report returns actual effective limits in +`scope.work_budget`: `max_rows`, `max_bytes`, `max_scratch_db_bytes`, and units. +Template `dataset_support` reports effective `work_limit_rows`, `work_limit_bytes` +and `max_scratch_db_bytes`, including environment or analyzer defaults; query +`--template-info` also honors explicit limit flags. Limits in an operation's scope +are authoritative for that operation, even if the environment later changes. +Budget and scratch exhaustion errors identify the configured limit and reject the +whole operation without a partial global result. Work exhaustion also reports +consumed rows/bytes. Peak-report steps share one cumulative budget and deadline; +each independent query/report starts fresh counters, even with reused connections. + +Raising limits explicitly permits more work and may increase memory/disk costs; it +is not a performance optimization or an RSS/total-temporary-space guarantee. +On event/allocation pagination paths that fetch `offset + limit` candidates +per shard before global sorting and slicing, that full candidate work is charged. A small output can therefore exceed +the work ceiling; `-n`, `limit`, `stack_bytes` and `timeout` do not raise it. Prefer +narrower ranges when suitable, or deliberately size the limits for the operation. +Raising a budget does not optimize its pagination plan or silently change default limits. + ## Dataset point-event attribution A complete compatibility-v1 or native-v2 directory/manifest supports `event` diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index 728a526..e1b2a0d 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -31,6 +31,48 @@ and `device_id` is its default explicit query-device override. When `db_path` is omitted, the database uses the same resolution order as the CLI: `PT_SNAP_DB_PATH`, the nearest `.pt-snap/focus.json`, then legacy global config. +## Dataset query limits + +Import the frozen `QueryLimits` configuration alongside `SnapshotAnalyzer` (or +from `pt_snap_cli.core` for direct services). It contains only three immutable +limits, never shared mutable counters: + +```python +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer + +limits = QueryLimits.resolve(max_work_rows=300_000, max_work_bytes=128 * 1024**2) +with SnapshotAnalyzer(Path("/path/to/dataset"), query_limits=limits) as analyzer: + result = analyzer.execute_query("event", {"offset":100_000, "limit":5}) + print(result["scope"]["work_budget"]) + # One-call override; subsequent calls still use the constructor's limits. + other = analyzer.execute_query("event", query_limits=QueryLimits(max_work_rows=400_000)) +``` + +`QueryLimits()` always uses built-in defaults: `max_work_rows=100000`, +`max_work_bytes=67108864`, `max_scratch_db_bytes=536870912`. +`QueryLimits.resolve(...)` resolves each omitted field from its corresponding +`PT_SNAP_MAX_WORK_ROWS`, `PT_SNAP_MAX_WORK_BYTES`, or `PT_SNAP_MAX_SCRATCH_DB_BYTES` +environment value, then the default. Explicit fields ignore even invalid matching +environment values. Work limits must be positive integers (not booleans/floats); +scratch accepts integer zero to disable or 4096 through 4294967296 bytes. No +unlimited work setting exists. Invalid values raise core `InvalidParameterError`. + +For each operation, a `query_limits` argument replaces the entire constructor +configuration; constructor configuration replaces environment/default resolution. +When neither is supplied, the environment is read afresh for each operation. +Use `QueryLimits.resolve(...)` if a partial override should retain other environment +settings. `get_template_info(..., query_limits=...)` can inspect a one-call override; +`list_capabilities()` reflects the analyzer's normal effective defaults. + +`QueryService` and `ReportService` likewise accept constructor `query_limits`. +`execute_query`, `peak_memory_report`, `event_attribution`, and `memory_tree_report` +accept per-call `query_limits`. Peak-report steps share one cumulative counter and +deadline; independent operations start fresh counters. Passing mutable internal +`QueryBudget` objects is not the public configuration API. These limits only bound +dataset work, not standalone query materialization, RSS, or all SQLite temporary +space. See [configuration and failure semantics](querying.md#public-dataset-limits), +including deep-pagination costs and the unchanged default caps. + ## Connection Lifetime and Cache Ownership An analyzer reuses read-only SQLite connections for queries and overviews. Use diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 7975d55..aeaa81f 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -128,7 +128,7 @@ exactness 标记诚实。`has_more/truncated` 描述窗口,不表示来源/范 独立。未证明历史身份按片局部单独表达,不按地址/局部栈 ID 猜合并,不断言为已证明泄漏。 数据集读取先分离批次再查外片来源,使用借用的有界 Context LRU。所有片、来源、分组、 -精确总数共用递减 deadline;报告的峰值选择与归因明确共用**一个**预算。累计取回/输出工作上限为 +精确总数共用递减 deadline;报告的峰值选择与归因明确共用**一个**预算。累计取回/输出工作默认上限为 **100000 行 / 64 MiB 序列化值**,含重复来源读取;超限整体失败,不返回冒充完整的部分 结论。物化/排序由工作上限约束,**不由 max_rows 约束**;不是进程 RSS 上限,Python/ SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入:每次对 manifest/全部成员 @@ -157,8 +157,9 @@ slice 查询在筛选、分页前发现所需的证明来源分片。超限时 临时目录权限为 0700,派生文件以 0600 创建;正常完成、错误和取消均关闭连接并删除目录。 强制杀进程或文件系统清理失败可能留下私有残余;这不是安全擦除。派生主库页数空间默认 -上限为 512 MiB,可通过内部 `QueryBudget.max_scratch_db_bytes` 字段设为 4096 字节至 -4 GiB;0 禁用该路径、沿用原有受预算约束的来源读取。这不是新增 CLI 选项。页数超限或 +上限为 512 MiB,可通过 `--max-scratch-db-bytes`、`PT_SNAP_MAX_SCRATCH_DB_BYTES` +或公开 `QueryLimits.max_scratch_db_bytes` 设为 4096 字节至 4 GiB;0 禁用该路径、 +沿用原有受预算约束的来源读取。详见[公开数据集预算](#公开数据集预算)。页数超限或 磁盘满会拒绝整个查询。该上限**不包括**回滚 journal、SQLite 排序文件、全部临时空间或 RSS。主库和源库各请求 2 MiB page cache,仅为缓存目标,不保证内存上限。SQL 内部扫描 和临时计算不算 Python 取回行;全部阶段、汇总/分页或时长分桶及来源补全共用递减 deadline。 @@ -169,6 +170,47 @@ RSS。主库和源库各请求 2 MiB page cache,仅为缓存目标,不保证 或 deadline。预算统计进入 Python 的实际取回行和序列化输出,不限制 SQL 内部扫描、SQLite 排序空间或整个进程内存。 +## 公开数据集预算 + +`query`、`report peak-memory` 与 `report memory-tree` 接受同一组选项: + +| CLI 选项 | 环境变量 | 默认值 / 合法范围 | +| --- | --- | --- | +| `--max-work-rows` | `PT_SNAP_MAX_WORK_ROWS` | `100000`;正整数 | +| `--max-work-bytes` | `PT_SNAP_MAX_WORK_BYTES` | `67108864`(64 MiB);正整数字节数 | +| `--max-scratch-db-bytes` | `PT_SNAP_MAX_SCRATCH_DB_BYTES` | `536870912`(512 MiB);`0` 禁用 scratch,否则为 `4096` 至 `4294967296`(4 GiB) | + +逐字段按显式 CLI flag、对应环境变量、内建默认值依次解析。环境变量只接受十进制整数, +不接受单位后缀;未设置或空白时使用默认值。布尔值、小数、负数和零工作预算都非法; +不存在无限工作预算。scratch 为零**不会禁用工作预算**。非整页 scratch 值由 SQLite +向下取整到 4096 字节页。非法配置在访问数据库前失败,单库命令也执行配置校验。 +这些预算约束数据集执行,不给单库 SQLite 查询新增工作上限,也不让不支持数据集的模板 +变得可用。 + +```bash +pt-snap query '' --template-use event \ + --params '{"offset":100000,"limit":5}' --max-work-rows 300000 \ + --max-work-bytes 134217728 --exact-total --json +pt-snap report peak-memory '' --max-work-rows 300000 \ + --max-scratch-db-bytes 1073741824 --timeout 30 --json +``` + +`SnapshotAnalyzer`、`QueryService` 和 `ReportService` 共用 +[不可变 Python `QueryLimits` API](snapshot-analyzer-api.md#数据集查询预算)。成功的数据集 +查询/报告在 `scope.work_budget` 回显实际 `max_rows`、`max_bytes`、 +`max_scratch_db_bytes` 及单位。模板 `dataset_support` 中的 `work_limit_rows`、 +`work_limit_bytes` 与 `max_scratch_db_bytes` 反映当前环境或 analyzer 默认配置; +`query --template-info` 也接受显式预算 flag。单次结果的 scope 是该次有效预算的依据, +不受后续环境变更影响。工作/scratch 超限报错包含实际配置值,整次失败且不返回部分全局 +结论;工作超限还报告已消耗的行数/字节数。峰值报告全部步骤共用累计预算与 deadline; +独立查询/报告每次创建新计数器,即使底层复用连接。 + +显式提高预算允许更多工作,可能增加内存/磁盘开销;这不是性能优化,也不保证 RSS 或 +全部临时空间上限。对先从每片取 `offset + limit` 个候选、全局排序后再切页的 +event/allocation 分页路径,全部候选工作仍计费,因此输出少也可能超预算。`-n`、`limit`、`stack_bytes`、`timeout` +不会提高工作预算。适用时优先缩小范围,或明确为该操作选择更大容量。提高预算本身 +不会优化分页计划,也不暗中放宽默认预算。 + ## 数据集定点事件归因 完整兼容 v1 或原生 v2 目录/manifest 支持 `event` 寻址,以及接受真实 `event_id` 的 diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index 08fa5b1..2cc5bc7 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -28,6 +28,42 @@ analyzer = SnapshotAnalyzer( 显式设备覆盖。省略 `db_path` 时,数据库解析顺序与 CLI 相同:`PT_SNAP_DB_PATH`、 最近的 `.pt-snap/focus.json`,最后是 legacy 全局配置。 +## 数据集查询预算 + +可从 `pt_snap_cli.api` 与 `SnapshotAnalyzer` 一起导入冻结的 `QueryLimits`,直接调用 +服务时也可从 `pt_snap_cli.core` 导入。它只包含三个不可变上限,不含共享可变计数器: + +```python +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer + +limits = QueryLimits.resolve(max_work_rows=300_000, max_work_bytes=128 * 1024**2) +with SnapshotAnalyzer(Path("/path/to/dataset"), query_limits=limits) as analyzer: + result = analyzer.execute_query("event", {"offset":100_000, "limit":5}) + print(result["scope"]["work_budget"]) + # 只覆盖本次调用;后续仍使用构造时的配置。 + other = analyzer.execute_query("event", query_limits=QueryLimits(max_work_rows=400_000)) +``` + +`QueryLimits()` 始终使用内建默认值:`max_work_rows=100000`、 +`max_work_bytes=67108864`、`max_scratch_db_bytes=536870912`。 +`QueryLimits.resolve(...)` 对省略的字段读取相应 `PT_SNAP_MAX_WORK_ROWS`、 +`PT_SNAP_MAX_WORK_BYTES`、`PT_SNAP_MAX_SCRATCH_DB_BYTES` 环境变量,再回退默认值。 +显式字段覆盖对应环境变量,即使该环境值非法。工作上限必须是正整数,不接受布尔值或 +浮点数;scratch 接受整数零(禁用)或 4096 至 4294967296 字节。没有无限工作预算; +非法值抛出 core `InvalidParameterError`。 + +每次操作的 `query_limits` 替换整个构造配置;构造配置又覆盖环境/默认值解析。两处均 +省略时,每次操作重新读取环境。若只覆盖部分字段、其余继续读取环境,应调用 +`QueryLimits.resolve(...)`。`get_template_info(..., query_limits=...)` 可查看单次覆盖, +`list_capabilities()` 则反映该 analyzer 的常规有效默认配置。 + +`QueryService`、`ReportService` 构造函数也接受 `query_limits`;`execute_query`、 +`peak_memory_report`、`event_attribution`、`memory_tree_report` 接受单次覆盖。 +峰值报告步骤共用累计计数器和 deadline,独立操作使用全新计数器。可变内部 +`QueryBudget` 不是公开配置接口。预算只约束数据集工作,不约束单库结果物化、RSS 或 +所有 SQLite 临时空间。深分页成本、失败语义及保持不变的默认上限见 +[公开配置说明](querying.md#公开数据集预算)。 + ## 连接生命周期与缓存所有权 analyzer 会为查询和概览复用只读 SQLite 连接。使用 `with` 可确定性释放连接, diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index 1e462ef..fe7d9a1 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -24,6 +24,7 @@ TemplateNotFoundError, ) from pt_snap_cli.core.context_cache import ContextCache +from pt_snap_cli.core.query_limits import QueryLimits class TemplateSummaryPayload(TypedDict): @@ -70,6 +71,7 @@ def __init__( device_id: int | None = None, *, context_cache: ContextCache | None = None, + query_limits: QueryLimits | None = None, ) -> None: self._config: Config = Config() self._db_path: Path | None = db_path @@ -86,7 +88,7 @@ def __init__( self._owns_context_cache: bool = context_cache is None self._closed: bool = False self._query_service: QueryService = QueryService( - self._focus_service, context_cache=self._context_cache + self._focus_service, context_cache=self._context_cache, query_limits=query_limits ) self._metadata_service: ImportMetadataService = ImportMetadataService() self._capability_service: CapabilityService = CapabilityService( @@ -195,10 +197,12 @@ def list_templates(self, category: str | None = None) -> list[TemplateSummaryPay for template in self._query_service.list_templates(category) ] - def get_template_info(self, name: str) -> dict[str, object] | None: + def get_template_info( + self, name: str, *, query_limits: QueryLimits | None = None + ) -> dict[str, object] | None: self._ensure_open() try: - info = self._query_service.get_template_info(name) + info = self._query_service.get_template_info(name, query_limits=query_limits) except TemplateNotFoundError: return None @@ -214,6 +218,7 @@ def execute_query( exact_total: bool = False, timeout_s: float | None = None, slice_index: int | None = None, + query_limits: QueryLimits | None = None, ) -> QueryResultPayload: self._ensure_open() try: @@ -226,6 +231,7 @@ def execute_query( exact_total=exact_total, timeout_s=timeout_s, slice_index=slice_index, + query_limits=query_limits, ) except FocusNotConfiguredError as exc: raise RuntimeError("No database configured. Call set_focus() first.") from exc diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index cfa20e1..43e5dd7 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -25,6 +25,7 @@ ClickAbort = Abort from pt_snap_cli import __version__ +from pt_snap_cli.cli_limits import MaxScratchDBBytes, MaxWorkBytes, MaxWorkRows from pt_snap_cli.cli_output import ( JSON_MODE as _JSON_MODE, ) @@ -95,6 +96,7 @@ ) from pt_snap_cli.core.error_codes import DATABASE_NOT_FOUND, ERROR, INVALID_PARAMETER from pt_snap_cli.core.errors import InvalidParameterError +from pt_snap_cli.core.query_limits import QueryLimits from pt_snap_cli.query.config import OUTPUT_COLUMN_OPTIONAL from pt_snap_cli.query.executor import reported_sql_limit from pt_snap_cli.query.registry import discover_categories, get_query @@ -601,11 +603,23 @@ def query_database( help="Query execution timeout in seconds (separate from -n; <= 0 disables; default: PT_SNAP_QUERY_TIMEOUT or unlimited)", ), ] = None, + max_work_rows: MaxWorkRows = None, + max_work_bytes: MaxWorkBytes = None, + max_scratch_db_bytes: MaxScratchDBBytes = None, json_output: Annotated[bool, _json_flag()] = False, ) -> None: """Execute queries on the memory snapshot database.""" focus_service = _focus_service() query_service = _query_service() + try: + query_limits = QueryLimits.resolve( + max_work_rows=max_work_rows, + max_work_bytes=max_work_bytes, + max_scratch_db_bytes=max_scratch_db_bytes, + ) + except InvalidParameterError as exc: + query_service.close() + _error_from_exc(exc) if category is not None and not list_templates and not template_use and not template_info: list_templates = True @@ -653,7 +667,7 @@ def query_database( if template_info: try: - info = query_service.get_template_info(template_info) + info = query_service.get_template_info(template_info, query_limits=query_limits) except TemplateNotFoundError as e: _error_from_exc(e) @@ -756,6 +770,7 @@ def query_database( exact_total=exact_total, timeout_s=timeout, slice_index=slice_index, + query_limits=query_limits, ) if json_output: resolved = focus_service.resolve_focus( @@ -888,7 +903,7 @@ def show_capabilities( service = CapabilityService() try: catalog = service.catalog() - except SkillCatalogError as e: + except (SkillCatalogError, InvalidParameterError) as e: _error_from_exc(e) if json_output: _emit_json(json_success(**service.catalog_to_dict(catalog))) diff --git a/src/pt_snap_cli/cli_limits.py b/src/pt_snap_cli/cli_limits.py new file mode 100644 index 0000000..27aa0a7 --- /dev/null +++ b/src/pt_snap_cli/cli_limits.py @@ -0,0 +1,30 @@ +"""Shared query/report options; resolution and validation stay in core.""" + +from typing import Annotated + +import typer + +MaxWorkRows = Annotated[ + int | None, + typer.Option( + "--max-work-rows", + rich_help_panel="Dataset limits", + help="Dataset cumulative fetched/output rows, positive integer (env: PT_SNAP_MAX_WORK_ROWS; default: 100000)", + ), +] +MaxWorkBytes = Annotated[ + int | None, + typer.Option( + "--max-work-bytes", + rich_help_panel="Dataset limits", + help="Dataset cumulative serialized work bytes, positive integer (env: PT_SNAP_MAX_WORK_BYTES; default: 67108864); not RSS", + ), +] +MaxScratchDBBytes = Annotated[ + int | None, + typer.Option( + "--max-scratch-db-bytes", + rich_help_panel="Dataset limits", + help="Dataset derived DB page bytes: 0 disables, otherwise 4096..4294967296 (env: PT_SNAP_MAX_SCRATCH_DB_BYTES; default: 536870912)", + ), +] diff --git a/src/pt_snap_cli/cli_reports.py b/src/pt_snap_cli/cli_reports.py index 9c16af6..838e36d 100644 --- a/src/pt_snap_cli/cli_reports.py +++ b/src/pt_snap_cli/cli_reports.py @@ -9,6 +9,7 @@ import typer +from pt_snap_cli.cli_limits import MaxScratchDBBytes, MaxWorkBytes, MaxWorkRows from pt_snap_cli.cli_output import ( error as _error, ) @@ -37,6 +38,7 @@ ) from pt_snap_cli.core.error_codes import DATABASE_NOT_FOUND, INVALID_PARAMETER from pt_snap_cli.core.errors import PtSnapCoreError +from pt_snap_cli.core.query_limits import QueryLimits from pt_snap_cli.memory_tree_render import render_memory_tree report_app = typer.Typer(help="Generate memory analysis reports") @@ -62,6 +64,9 @@ def report_memory_tree( timeout: Annotated[ float | None, typer.Option("--timeout", help="Query deadline in seconds; <=0 disables") ] = None, + max_work_rows: MaxWorkRows = None, + max_work_bytes: MaxWorkBytes = None, + max_scratch_db_bytes: MaxScratchDBBytes = None, json_output: Annotated[bool, _json_flag()] = False, ) -> None: """Decompose event-time active memory by ordered allocation frames.""" @@ -76,6 +81,11 @@ def report_memory_tree( include_static=include_static, min_size=min_size, timeout_s=timeout, + query_limits=QueryLimits.resolve( + max_work_rows=max_work_rows, + max_work_bytes=max_work_bytes, + max_scratch_db_bytes=max_scratch_db_bytes, + ), ) if json_output: typer.echo(json.dumps({"event_id": event_id, **asdict(result)}, indent=2)) @@ -131,6 +141,9 @@ def report_peak_memory( float | None, typer.Option("--timeout", help="One report-wide deadline in seconds; <=0 disables"), ] = None, + max_work_rows: MaxWorkRows = None, + max_work_bytes: MaxWorkBytes = None, + max_scratch_db_bytes: MaxScratchDBBytes = None, json_output: Annotated[bool, _json_flag()] = False, ) -> None: """Generate a peak memory attribution report.""" @@ -147,6 +160,11 @@ def report_peak_memory( start_id=start_id, end_id=end_id, timeout_s=timeout, + query_limits=QueryLimits.resolve( + max_work_rows=max_work_rows, + max_work_bytes=max_work_bytes, + max_scratch_db_bytes=max_scratch_db_bytes, + ), ) except ValueError as e: _error(str(e), code=INVALID_PARAMETER, hint="Use --metric active, allocated, or reserved.") diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index a5b523e..807155b 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -58,6 +58,7 @@ TemplateSummary, ) from pt_snap_cli.core.overview_service import OverviewService +from pt_snap_cli.core.query_limits import QueryLimits from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService from pt_snap_cli.core.skill_service import SkillService @@ -116,6 +117,7 @@ "FocusService", "ImportMetadataService", "QueryService", + "QueryLimits", "ReportService", "ImportService", "SplitService", diff --git a/src/pt_snap_cli/core/dataset_lifecycle_scratch.py b/src/pt_snap_cli/core/dataset_lifecycle_scratch.py index e354af3..6a86289 100644 --- a/src/pt_snap_cli/core/dataset_lifecycle_scratch.py +++ b/src/pt_snap_cli/core/dataset_lifecycle_scratch.py @@ -17,9 +17,10 @@ from pt_snap_cli.core.dataset_files import require_readonly_member from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.core.query_limits import MAX_SCRATCH_DB_BYTES, MIN_SCRATCH_DB_BYTES -_PAGE_SIZE = 4096 -_MAX_SCRATCH_BYTES = 4 * 1024 * 1024 * 1024 +_PAGE_SIZE = MIN_SCRATCH_DB_BYTES +_MAX_SCRATCH_BYTES = MAX_SCRATCH_DB_BYTES _CACHE_KIB = 2048 _COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") @@ -38,7 +39,9 @@ def execute(self, sql, values=()): ) or "database or disk is full" in str(exc): raise QueryExecutionError( "Dataset lifecycle scratch space exhausted (derived database page limit " - "or disk full); no partial global result." + "or disk full; " + f"max_scratch_db_bytes={self.sources.budget.max_scratch_db_bytes}); " + "no partial global result." ) from exc raise QueryExecutionError(str(exc)) from exc @@ -172,7 +175,7 @@ def scratch_lifecycles(sources, items): maximum = sources.budget.max_scratch_db_bytes if type(maximum) is not int or not _PAGE_SIZE <= maximum <= _MAX_SCRATCH_BYTES: raise QueryExecutionError( - "Dataset lifecycle scratch limit must be 0 (disabled) or between 4096 bytes " + f"Dataset lifecycle scratch limit {maximum!r} must be 0 (disabled) or between 4096 bytes " "and 4 GiB; this bounds derived database pages, not total temporary space or RSS." ) try: @@ -188,4 +191,7 @@ def scratch_lifecycles(sources, items): database.__exit__(None, None, None) except (OSError, sqlite3.Error) as exc: sources.budget.remaining() - raise QueryExecutionError(f"Dataset lifecycle scratch storage failed: {exc}") from exc + raise QueryExecutionError( + "Dataset lifecycle scratch storage failed " + f"(max_scratch_db_bytes={maximum}); no partial global result: {exc}" + ) from exc diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 4dc49ec..68a4d0a 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -29,6 +29,12 @@ QueryExecutionError, QueryTimeoutError, ) +from pt_snap_cli.core.query_limits import ( + DEFAULT_MAX_SCRATCH_DB_BYTES, + DEFAULT_MAX_WORK_BYTES, + DEFAULT_MAX_WORK_ROWS, + QueryLimits, +) from pt_snap_cli.query.executor import QueryExecutionError as ExecutorError from pt_snap_cli.query.executor import QueryExecutor from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorTimeout @@ -64,13 +70,34 @@ def _bounded_depth(value: object) -> bool: class QueryBudget: timeout_s: float | None started: float - max_work_rows: int = 100_000 - max_work_bytes: int = 64 * 1024 * 1024 + max_work_rows: int = DEFAULT_MAX_WORK_ROWS + max_work_bytes: int = DEFAULT_MAX_WORK_BYTES work_rows: int = 0 work_bytes: int = 0 # Per-operation derived database pages, not total SQLite temp space or RSS. - # Zero retains the bounded in-Python fallback. Local/internal API only. - max_scratch_db_bytes: int = 512 * 1024 * 1024 + # Zero retains the bounded in-Python fallback. + max_scratch_db_bytes: int = DEFAULT_MAX_SCRATCH_DB_BYTES + + @classmethod + def from_limits( + cls, limits: QueryLimits, timeout_s: float | None, started: float + ) -> QueryBudget: + return cls( + timeout_s, + started, + max_work_rows=limits.max_work_rows, + max_work_bytes=limits.max_work_bytes, + max_scratch_db_bytes=limits.max_scratch_db_bytes, + ) + + def describe(self) -> dict[str, object]: + return { + "max_rows": self.max_work_rows, + "max_bytes": self.max_work_bytes, + "max_scratch_db_bytes": self.max_scratch_db_bytes, + "unit": "cumulative_fetched_and_output_serialized_values; not_RSS", + "scratch_unit": "derived_database_pages; not_total_temporary_space_or_RSS", + } def consume(self, rows: list[dict[str, object]]) -> None: """Bound cumulative fetched rows/serialized values, not merely output. @@ -86,8 +113,11 @@ def consume(self, rows: list[dict[str, object]]) -> None: ) if self.work_rows > self.max_work_rows or self.work_bytes > self.max_work_bytes: raise QueryExecutionError( - "Dataset query work budget exceeded (100000 fetched rows / 64 MiB serialized " - "values by default); no partial global result. Narrow the scope or select a member." + "Dataset query work budget exceeded " + f"(max_work_rows={self.max_work_rows}, max_work_bytes={self.max_work_bytes}, " + f"max_scratch_db_bytes={self.max_scratch_db_bytes}; " + f"consumed_rows={self.work_rows}, consumed_bytes={self.work_bytes}); " + "no partial global result. Narrow the scope or explicitly raise query limits." ) def remaining(self) -> float | None: diff --git a/src/pt_snap_cli/core/dataset_support.py b/src/pt_snap_cli/core/dataset_support.py index 6d268dc..0c60053 100644 --- a/src/pt_snap_cli/core/dataset_support.py +++ b/src/pt_snap_cli/core/dataset_support.py @@ -1,5 +1,7 @@ """One shared, explicit per-template dataset execution support matrix.""" +from pt_snap_cli.core.query_limits import QueryLimits + DATASET_SUPPORT: dict[str, str] = { "event": "real_events_global_filter_sort_page", "allocation": "real_events_global_filter_sort_page", @@ -15,14 +17,18 @@ } -def dataset_support(name: str, packaged: bool) -> dict[str, object]: +def dataset_support( + name: str, packaged: bool, limits: QueryLimits | None = None +) -> dict[str, object]: + limits = limits if limits is not None else QueryLimits() semantics = DATASET_SUPPORT.get(name) if packaged else None return { "supported": semantics is not None, "semantics": semantics, "device_scope": "one_selected_device; never sum device peaks", "custom_sql": "unsupported; select an explicit standalone member", - "work_limit_rows": 100_000, - "work_limit_bytes": 64 * 1024 * 1024, + "work_limit_rows": limits.max_work_rows, + "work_limit_bytes": limits.max_work_bytes, + "max_scratch_db_bytes": limits.max_scratch_db_bytes, "budget_scope": "one_query_or_explicit_report_composition", } diff --git a/src/pt_snap_cli/core/query_limits.py b/src/pt_snap_cli/core/query_limits.py new file mode 100644 index 0000000..273458e --- /dev/null +++ b/src/pt_snap_cli/core/query_limits.py @@ -0,0 +1,95 @@ +"""Immutable public dataset limits, separate from one operation's mutable counters.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass + +from pt_snap_cli.core.errors import InvalidParameterError + +DEFAULT_MAX_WORK_ROWS = 100_000 +DEFAULT_MAX_WORK_BYTES = 64 * 1024 * 1024 +DEFAULT_MAX_SCRATCH_DB_BYTES = 512 * 1024 * 1024 +MIN_SCRATCH_DB_BYTES = 4096 +MAX_SCRATCH_DB_BYTES = 4 * 1024**3 + + +@dataclass(frozen=True) +class QueryLimits: + """Dataset work/storage ceilings; never an unlimited-work or RSS setting. + + Rows and bytes must be positive integers. Scratch zero disables the derived + disk database; otherwise its page ceiling must be between 4096 and 4 GiB. + This object has no counters or deadline and can safely be reused. + """ + + max_work_rows: int = DEFAULT_MAX_WORK_ROWS + max_work_bytes: int = DEFAULT_MAX_WORK_BYTES + max_scratch_db_bytes: int = DEFAULT_MAX_SCRATCH_DB_BYTES + + def __post_init__(self) -> None: + for name, value in ( + ("max_work_rows", self.max_work_rows), + ("max_work_bytes", self.max_work_bytes), + ): + if type(value) is not int or value <= 0: + raise InvalidParameterError(f"{name} must be a positive integer; got {value!r}.") + value = self.max_scratch_db_bytes + if type(value) is not int or not ( + value == 0 or MIN_SCRATCH_DB_BYTES <= value <= MAX_SCRATCH_DB_BYTES + ): + raise InvalidParameterError( + "max_scratch_db_bytes must be 0 (disabled) or an integer between " + f"4096 and {MAX_SCRATCH_DB_BYTES} (4 GiB); got {value!r}." + ) + + @classmethod + def resolve( + cls, + *, + max_work_rows: int | None = None, + max_work_bytes: int | None = None, + max_scratch_db_bytes: int | None = None, + ) -> QueryLimits: + """Resolve each field: explicit value > environment > built-in default. + + Environment values are base-10 integers in bytes/rows, without suffixes. + Missing or blank values use the default; invalid values fail before I/O. + An explicit field ignores even an invalid corresponding environment value. + """ + return cls( + max_work_rows=_resolve_integer( + max_work_rows, "PT_SNAP_MAX_WORK_ROWS", DEFAULT_MAX_WORK_ROWS + ), + max_work_bytes=_resolve_integer( + max_work_bytes, "PT_SNAP_MAX_WORK_BYTES", DEFAULT_MAX_WORK_BYTES + ), + max_scratch_db_bytes=_resolve_integer( + max_scratch_db_bytes, + "PT_SNAP_MAX_SCRATCH_DB_BYTES", + DEFAULT_MAX_SCRATCH_DB_BYTES, + ), + ) + + +def resolve_query_limits(explicit: QueryLimits | None) -> QueryLimits: + if explicit is None: + return QueryLimits.resolve() + if not isinstance(explicit, QueryLimits): + raise InvalidParameterError("query_limits must be a QueryLimits instance or None.") + return explicit + + +def _resolve_integer(explicit: int | None, name: str, default: int) -> int: + if explicit is not None: + return explicit + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return default + if re.fullmatch(r"[+-]?[0-9]+", raw.strip()) is None: + raise InvalidParameterError(f"{name} must be an integer, got {raw!r}.") + try: + return int(raw) + except ValueError as exc: + raise InvalidParameterError(f"{name} must be an integer, got {raw!r}.") from exc diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 92e8476..2b5049e 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -36,6 +36,7 @@ TemplateParameter, TemplateSummary, ) +from pt_snap_cli.core.query_limits import QueryLimits, resolve_query_limits from pt_snap_cli.core.stack_summary import summarize_stacks from pt_snap_cli.query.executor import QueryExecutionError as ExecutorQueryExecutionError from pt_snap_cli.query.executor import QueryExecutor @@ -91,7 +92,11 @@ def __init__( focus_service: FocusService | None = None, *, context_cache: ContextCache | None = None, + query_limits: QueryLimits | None = None, ) -> None: + if query_limits is not None: + _ = resolve_query_limits(query_limits) + self._query_limits: QueryLimits | None = query_limits self._focus_service = focus_service or FocusService() # Cache Context instances across calls so long-lived SnapshotAnalyzer # owners skip the per-call schema validation and @@ -150,7 +155,9 @@ def list_templates( ) return summaries - def get_template_info(self, name: str) -> TemplateInfo: + def get_template_info( + self, name: str, *, query_limits: QueryLimits | None = None + ) -> TemplateInfo: template = get_query(name) if template is None: raise TemplateNotFoundError(f"Template '{name}' not found") @@ -183,6 +190,7 @@ def get_template_info(self, name: str) -> TemplateInfo: name, is_packaged_query(name) and not (self._executor is not None and self._executor.has_runtime_template(name)), + self.resolve_limits(query_limits), ), semantics_version=info.get("semantics_version"), interpretation_limits=( @@ -232,14 +240,11 @@ def execute_query( exact_total: bool = False, timeout_s: float | None = None, slice_index: int | None = None, + query_limits: QueryLimits | None = None, _budget: QueryBudget | None = None, _resolved: _ResolvedQuery | None = None, ) -> QueryResult: - budget = ( - _budget - if _budget is not None - else QueryBudget(resolve_query_timeout(timeout_s), time.monotonic()) - ) + budget = _budget if _budget is not None else self._new_budget(timeout_s, query_limits) timeout_s, started = budget.timeout_s, budget.started budget.remaining() resolution = _resolved or self._resolve_query(db_path, device_id, start_dir, budget) @@ -301,6 +306,8 @@ def execute_query( # Inspection establishes the baseline. Reject changed contents # after each composed step, before any report can be returned. dataset.require_unchanged(budget) + if result.scope is not None: + result.scope["work_budget"] = budget.describe() return result else: if slice_index is not None: @@ -371,6 +378,18 @@ def execute_query( scope=scope, ) + def resolve_limits(self, explicit: QueryLimits | None = None) -> QueryLimits: + """Per-call limits override instance limits, then environment/defaults.""" + return resolve_query_limits(explicit if explicit is not None else self._query_limits) + + def _new_budget( + self, timeout_s: float | None = None, query_limits: QueryLimits | None = None + ) -> QueryBudget: + """Create fresh counters and a deadline for one operation/composition.""" + return QueryBudget.from_limits( + self.resolve_limits(query_limits), resolve_query_timeout(timeout_s), time.monotonic() + ) + def _resolve_query( self, db_path: Path | str | None, diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index b319d73..d099b3d 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -1,13 +1,13 @@ from __future__ import annotations -import time from pathlib import Path from typing import Literal, cast from pt_snap_cli.core.dataset_sources import QueryBudget from pt_snap_cli.core.focus_service import FocusService from pt_snap_cli.core.models import PeakMemoryReport, QueryResult -from pt_snap_cli.core.query_service import QueryService, _ResolvedQuery, resolve_query_timeout +from pt_snap_cli.core.query_limits import QueryLimits +from pt_snap_cli.core.query_service import QueryService, _ResolvedQuery PeakMetric = Literal["active", "allocated", "reserved"] @@ -34,9 +34,11 @@ class ReportService: - def __init__(self, focus_service: FocusService | None = None) -> None: + def __init__( + self, focus_service: FocusService | None = None, *, query_limits: QueryLimits | None = None + ) -> None: self._focus_service = focus_service or FocusService() - self._query_service = QueryService(self._focus_service) + self._query_service = QueryService(self._focus_service, query_limits=query_limits) def close(self) -> None: """Release connections owned by this report service; allow later reuse.""" @@ -52,6 +54,7 @@ def memory_tree_report( *, start_dir: Path | None = None, timeout_s: float | None = None, + query_limits: QueryLimits | None = None, ) -> QueryResult: """Return frame occupancy through the shared standalone query contract.""" return self._query_service.execute_query( @@ -61,6 +64,7 @@ def memory_tree_report( device_id=device_id, start_dir=start_dir, timeout_s=timeout_s, + query_limits=query_limits, ) def event_attribution( @@ -74,6 +78,7 @@ def event_attribution( stack_bytes: int = -1, start_dir: Path | None = None, timeout_s: float | None = None, + query_limits: QueryLimits | None = None, _budget: QueryBudget | None = None, _resolved: _ResolvedQuery | None = None, ) -> QueryResult: @@ -95,6 +100,7 @@ def event_attribution( device_id=device_id, start_dir=start_dir, timeout_s=timeout_s, + query_limits=query_limits, _budget=_budget, _resolved=_resolved, ) @@ -112,13 +118,14 @@ def peak_memory_report( start_id: int | None = None, end_id: int | None = None, timeout_s: float | None = None, + query_limits: QueryLimits | None = None, ) -> PeakMemoryReport: if metric not in _EVENT_ID_BY_METRIC: raise ValueError( f"Invalid metric '{metric}'. Must be one of: active, allocated, reserved" ) - budget = QueryBudget(resolve_query_timeout(timeout_s), time.monotonic()) + budget = self._query_service._new_budget(timeout_s, query_limits) resolution = self._query_service._resolve_query(db_path, device_id, start_dir, budget) range_params = {"start_id": start_id, "end_id": end_id} gap_result = self._query_service.execute_query( diff --git a/tests/core/test_query_limits.py b/tests/core/test_query_limits.py new file mode 100644 index 0000000..a326d9b --- /dev/null +++ b/tests/core/test_query_limits.py @@ -0,0 +1,258 @@ +"""Public immutable limits resolve once per operation, without weakening defaults.""" + +import dataclasses +import sqlite3 +from contextlib import closing + +import pytest + +from pt_snap_cli.core import QueryLimits, QueryService, ReportService +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError, QueryTimeoutError +from pt_snap_cli.core.query_limits import resolve_query_limits +from tests.core.test_dataset_attribution import case +from tests.core.test_dataset_lifecycle_scratch import limited_connections, many_owners +from tests.core.test_dataset_sql_pushdown import large_dataset +from tests.test_dataset_focus import make_dataset + +ENV_NAMES = ( + "PT_SNAP_MAX_WORK_ROWS", + "PT_SNAP_MAX_WORK_BYTES", + "PT_SNAP_MAX_SCRATCH_DB_BYTES", +) + + +@pytest.fixture(autouse=True) +def isolate_limits(monkeypatch): + for name in (*ENV_NAMES, "PT_SNAP_QUERY_TIMEOUT"): + monkeypatch.delenv(name, raising=False) + + +def test_defaults_are_immutable_and_do_not_read_environment(monkeypatch): + monkeypatch.setenv(ENV_NAMES[0], "1") + limits = QueryLimits() + assert dataclasses.asdict(limits) == { + "max_work_rows": 100_000, + "max_work_bytes": 64 * 1024**2, + "max_scratch_db_bytes": 512 * 1024**2, + } + with pytest.raises(dataclasses.FrozenInstanceError): + limits.max_work_rows = 1 + assert not hasattr(limits, "work_rows") + assert QueryLimits.resolve().max_work_rows == 1 + + +@pytest.mark.parametrize("field", ["max_work_rows", "max_work_bytes"]) +@pytest.mark.parametrize("value", [True, False, 1.0, -1, 0, "10"]) +def test_work_limits_require_positive_non_boolean_integers(field, value): + with pytest.raises(InvalidParameterError, match=field): + QueryLimits(**{field: value}) + + +@pytest.mark.parametrize("value", [True, False, 4096.0, -1, 1, 4095, 4 * 1024**3 + 1, "0"]) +def test_scratch_limits_are_validated_before_execution(value): + with pytest.raises(InvalidParameterError, match="max_scratch_db_bytes"): + QueryLimits(max_scratch_db_bytes=value) + + +@pytest.mark.parametrize("value", [0, 4096, 4097, 4 * 1024**3]) +def test_scratch_disabled_and_boundary_values(value): + assert QueryLimits(max_scratch_db_bytes=value).max_scratch_db_bytes == value + + +@pytest.mark.parametrize("name", ENV_NAMES) +@pytest.mark.parametrize("value", ["false", "1.5", "1e6", "unlimited", "64MiB", "1_000"]) +def test_invalid_environment_is_a_domain_error(name, value, monkeypatch): + monkeypatch.setenv(name, value) + with pytest.raises(InvalidParameterError, match=name): + QueryLimits.resolve() + + +def test_per_field_explicit_environment_default_precedence(monkeypatch): + monkeypatch.setenv(ENV_NAMES[0], "invalid but overridden") + monkeypatch.setenv(ENV_NAMES[1], " 123456 ") + monkeypatch.setenv(ENV_NAMES[2], " ") + assert QueryLimits.resolve(max_work_rows=200_000) == QueryLimits(200_000, 123456) + assert resolve_query_limits(QueryLimits(55)) == QueryLimits(55) + with pytest.raises(InvalidParameterError, match="QueryLimits"): + resolve_query_limits(QueryBudget(None, 0)) + + +def test_new_operations_have_fresh_counters_and_effective_metadata(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "reuse", devices=(0,)) + monkeypatch.setenv(ENV_NAMES[0], "1") + limits = QueryLimits(100) + with closing(QueryService(query_limits=limits)) as service: + first = service.execute_query("event", db_path=root) + second = service.execute_query("event", db_path=root) + assert first == second + assert first.scope["work_budget"]["max_rows"] == 100 + info = service.get_template_info("event").dataset_support + assert info["work_limit_rows"] == 100 + assert info["max_scratch_db_bytes"] == limits.max_scratch_db_bytes + with pytest.raises(QueryExecutionError, match="max_work_rows=1"): + service.execute_query("event", db_path=root, query_limits=QueryLimits(1)) + assert service.execute_query("event", db_path=root) == first + assert ( + service.get_template_info("event", query_limits=QueryLimits(200)).dataset_support[ + "work_limit_rows" + ] + == 200 + ) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="max_work_rows=1"): + service.execute_query("event", db_path=root) + monkeypatch.setenv(ENV_NAMES[0], "100") + assert service.execute_query("event", db_path=root).rows == first.rows + + +def test_byte_budget_counts_long_source_text_before_display_truncation(tmp_path): + root = make_dataset(tmp_path / "long-text", devices=(0,), slices=1) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("UPDATE trace_entry_0 SET callstack=?", ("训练.py:42:run\n" * 1000,)) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="max_work_bytes=1024"): + service.execute_query( + "event", + {"stack_bytes": 0}, + db_path=root, + query_limits=QueryLimits(max_work_bytes=1024), + ) + result = service.execute_query( + "event", + {"stack_bytes": 0}, + db_path=root, + query_limits=QueryLimits(max_work_bytes=1024 * 1024), + ) + assert result.returned == 2 + assert result.scope["work_budget"]["max_bytes"] == 1024 * 1024 + + +def test_report_limits_are_shared_cumulatively_and_reset_per_report(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + original = QueryService.execute_query + stages = [] + + def tracked(self, template, *args, **kwargs): + budget = kwargs["_budget"] + before = budget.work_rows + result = original(self, template, *args, **kwargs) + stages.append((budget, budget.work_rows - before)) + return result + + monkeypatch.setattr(QueryService, "execute_query", tracked) + with closing(ReportService(query_limits=QueryLimits(1000))) as report_service: + expected = report_service.peak_memory_report(root) + assert len(stages) == 2 and stages[0][0] is stages[1][0] + assert all(count > 0 for _, count in stages) + maximum = max(count for _, count in stages) + assert sum(count for _, count in stages) > maximum + with pytest.raises(QueryExecutionError, match=f"max_work_rows={maximum}"): + report_service.peak_memory_report(root, query_limits=QueryLimits(maximum)) + actual = report_service.peak_memory_report(root) + assert actual == expected + assert actual.scope["work_budget"]["max_rows"] == 1000 + + +def test_report_deadline_is_not_restarted_with_explicit_limits(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + original = QueryService.execute_query + + def expire_after_peak(self, template, *args, **kwargs): + result = original(self, template, *args, **kwargs) + if template == "allocator_gap": + kwargs["_budget"].started -= 100 + return result + + monkeypatch.setattr(QueryService, "execute_query", expire_after_peak) + with closing(ReportService()) as service: + with pytest.raises(QueryTimeoutError): + service.peak_memory_report(root, query_limits=QueryLimits(1000), timeout_s=1) + + +def test_public_scratch_failure_reports_effective_limit_and_zero_keeps_work_cap( + tmp_path, monkeypatch +): + root = many_owners(tmp_path) + limited_connections(monkeypatch, 1) + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="max_scratch_db_bytes=4096"): + service.execute_query( + "block", db_path=root, query_limits=QueryLimits(1000, max_scratch_db_bytes=4096) + ) + expected = service.execute_query("block", db_path=root) + disabled = service.execute_query( + "block", db_path=root, query_limits=QueryLimits(max_scratch_db_bytes=0) + ) + assert disabled.rows == expected.rows + assert disabled.scope["work_budget"]["max_scratch_db_bytes"] == 0 + with pytest.raises(QueryExecutionError, match="max_work_rows=1"): + service.execute_query( + "block", db_path=root, query_limits=QueryLimits(1, max_scratch_db_bytes=0) + ) + + +@pytest.fixture(scope="module") +def large_limits_dataset(tmp_path_factory): + return large_dataset(tmp_path_factory.mktemp("public-query-limits")) + + +@pytest.mark.parametrize( + "template,params,returned", + [ + ("event", {"offset": 100_000, "limit": 5, "order_by": "allocated", "order_dir": "ASC"}, 5), + ("leak_detection", {"limit": -1}, 120_000), + ], +) +def test_explicitly_raised_limits_enable_deep_pages_and_large_results( + large_limits_dataset, template, params, returned +): + with closing(QueryService()) as service: + with pytest.raises(QueryExecutionError, match="work budget"): + service.execute_query(template, params, db_path=large_limits_dataset) + result = service.execute_query( + template, + params, + db_path=large_limits_dataset, + exact_total=True, + query_limits=QueryLimits(max_work_rows=1_000_000, max_work_bytes=512 * 1024**2), + ) + assert result.total == 120_000 and result.returned == returned + assert result.total_is_exact + if template == "event": + assert [row["id"] for row in result.rows] == list(range(100_000, 100_005)) + + +def test_report_resolves_environment_once_for_all_steps(tmp_path, monkeypatch): + root = case(tmp_path, devices=(0,)) + original = QueryService.execute_query + monkeypatch.setenv(ENV_NAMES[0], "1000") + + def change_environment(self, template, *args, **kwargs): + result = original(self, template, *args, **kwargs) + if template == "allocator_gap": + monkeypatch.setenv(ENV_NAMES[0], "invalid between report steps") + return result + + monkeypatch.setattr(QueryService, "execute_query", change_environment) + with closing(ReportService()) as service: + result = service.peak_memory_report(root) + assert result.scope["work_budget"]["max_rows"] == 1000 + with pytest.raises(InvalidParameterError, match=ENV_NAMES[0]): + service.peak_memory_report(root) + + +@pytest.mark.parametrize("constructor", [QueryService, ReportService]) +def test_service_constructor_rejects_mutable_budget_or_wrong_config(constructor): + with pytest.raises(InvalidParameterError, match="QueryLimits"): + constructor(query_limits=QueryBudget(None, 0)) + + +def test_excessively_long_environment_integer_is_a_domain_error(monkeypatch): + import sys + + if not hasattr(sys, "get_int_max_str_digits") or not sys.get_int_max_str_digits(): + pytest.skip("Interpreter has no configured integer-string conversion limit") + monkeypatch.setenv(ENV_NAMES[0], "9" * (sys.get_int_max_str_digits() + 1)) + with pytest.raises(InvalidParameterError, match=ENV_NAMES[0]): + QueryLimits.resolve() diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index ef0c9d1..1f8034b 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -1072,3 +1072,48 @@ def test_overview_error_contract_maps_four_domain_failures(tmp_path: Path) -> No assert unconfigured_error["code"] == "FOCUS_NOT_CONFIGURED" with pytest.raises(RuntimeError, match="No database configured"): SnapshotAnalyzer().get_database_overview() + + +def test_explicit_dataset_limits_and_failures_match_cli_api(tmp_path): + from pt_snap_cli.api import QueryLimits + + root = make_dataset(tmp_path / "limits-contract", devices=(0,)) + limits = QueryLimits(100, 123456, 0) + with SnapshotAnalyzer(root) as analyzer: + api = analyzer.execute_query("event", query_limits=limits) + with pytest.raises(QueryExecutionError, match="max_work_rows=1") as error: + analyzer.execute_query("event", query_limits=QueryLimits(1)) + cli = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + "event", + "--max-work-rows", + "100", + "--max-work-bytes", + "123456", + "--max-scratch-db-bytes", + "0", + "--json", + ], + ) + assert cli.exit_code == 0, cli.output + payload = json.loads(cli.stdout) + assert payload["scope"] == api["scope"] + assert payload["rows"] == api["rows"] + failed = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + "event", + "--max-work-rows", + "1", + "--json", + ], + ) + assert failed.exit_code == 1 and failed.stdout == "" + assert json.loads(failed.stderr)["error"]["message"] == str(error.value) diff --git a/tests/test_query_limits.py b/tests/test_query_limits.py new file mode 100644 index 0000000..9c0ce09 --- /dev/null +++ b/tests/test_query_limits.py @@ -0,0 +1,192 @@ +"""Query, API, report and capability adapters expose the same public limits.""" + +import json +import re +from contextlib import closing + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.errors import InvalidParameterError, QueryExecutionError +from pt_snap_cli.core.report_service import ReportService +from tests.core.test_dataset_attribution import case +from tests.test_dataset_focus import make_dataset + +runner = CliRunner() +ENV_NAMES = ( + "PT_SNAP_MAX_WORK_ROWS", + "PT_SNAP_MAX_WORK_BYTES", + "PT_SNAP_MAX_SCRATCH_DB_BYTES", +) + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in (*ENV_NAMES, "PT_SNAP_DB_PATH", "PT_SNAP_QUERY_TIMEOUT"): + monkeypatch.delenv(name, raising=False) + + +def test_analyzer_constructor_per_call_environment_and_catalog_precedence(tmp_path, monkeypatch): + root = make_dataset(tmp_path / "public", devices=(0,)) + monkeypatch.setenv(ENV_NAMES[0], "1") + with SnapshotAnalyzer(root, query_limits=QueryLimits(100)) as analyzer: + expected = analyzer.execute_query("event") + assert analyzer.execute_query("event") == expected + assert expected["scope"]["work_budget"]["max_rows"] == 100 + assert analyzer.get_template_info("event")["dataset_support"]["work_limit_rows"] == 100 + assert all( + t["dataset_support"]["work_limit_rows"] == 100 + for t in analyzer.list_capabilities()["templates"] + ) + with pytest.raises(QueryExecutionError, match="max_work_rows=1"): + analyzer.execute_query("event", query_limits=QueryLimits(1)) + with pytest.raises(InvalidParameterError, match="QueryLimits"): + analyzer.execute_query("event", query_limits={"max_work_rows": 1}) + assert analyzer.execute_query("event") == expected + with SnapshotAnalyzer(root) as analyzer: + with pytest.raises(QueryExecutionError, match="max_work_rows=1"): + analyzer.execute_query("event") + assert analyzer.execute_query("event", query_limits=QueryLimits(100)) == expected + + +@pytest.mark.parametrize( + "command", [["query"], ["report", "peak-memory"], ["report", "memory-tree"]] +) +def test_query_and_report_help_show_all_three_flags(command): + result = runner.invoke(app, [*command, "--help"]) + assert result.exit_code == 0 + text = re.sub(r"\x1b\[[0-9;]*m", "", result.output) + for flag in ("--max-work-rows", "--max-work-bytes", "--max-scratch-db-bytes"): + assert flag in text + + +@pytest.mark.parametrize( + "command", + [ + ["query", "--template-use", "event"], + ["report", "peak-memory"], + ["report", "memory-tree", "--event-id", "0"], + ], +) +@pytest.mark.parametrize( + "flag,value", + [("--max-work-rows", "0"), ("--max-work-bytes", "-1"), ("--max-scratch-db-bytes", "4095")], +) +def test_cli_invalid_values_fail_as_json_before_database_io(command, flag, value): + result = runner.invoke(app, [*command, flag, value, "--json"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "INVALID_PARAMETER" + assert "Traceback" not in result.stderr + + +@pytest.mark.parametrize( + "command", [["query", "--template-use", "event"], ["report", "peak-memory"]] +) +def test_cli_environment_is_validated_and_explicit_flags_override_it( + tmp_path, monkeypatch, command +): + root = case(tmp_path, devices=(0,)) + monkeypatch.setenv(ENV_NAMES[0], "invalid") + failed = runner.invoke(app, [*command, str(root), "--json"]) + assert failed.exit_code == 1 + assert json.loads(failed.stderr)["error"]["code"] == "INVALID_PARAMETER" + monkeypatch.setenv(ENV_NAMES[1], "123456") + passed = runner.invoke(app, [*command, str(root), "--max-work-rows", "1000", "--json"]) + assert passed.exit_code == 0, passed.output + budget = json.loads(passed.stdout)["scope"]["work_budget"] + assert budget["max_rows"] == 1000 and budget["max_bytes"] == 123456 + assert budget["max_scratch_db_bytes"] == QueryLimits().max_scratch_db_bytes + + +def test_cli_report_and_direct_report_use_same_limits(tmp_path): + root = case(tmp_path, devices=(0,)) + limits = QueryLimits(1000, 123456, 0) + with closing(ReportService()) as service: + expected = service.peak_memory_report(root, query_limits=limits) + attribution = service.event_attribution(4, db_path=root, query_limits=limits) + result = runner.invoke( + app, + [ + "report", + "peak-memory", + str(root), + "--max-work-rows", + "1000", + "--max-work-bytes", + "123456", + "--max-scratch-db-bytes", + "0", + "--json", + ], + ) + assert result.exit_code == 0, result.output + actual = json.loads(result.stdout) + assert actual["scope"] == expected.scope + assert actual["callstack_groups"] == expected.callstack_groups + assert attribution.scope["work_budget"] == expected.scope["work_budget"] + + +def test_template_info_and_capabilities_use_effective_environment_values(monkeypatch): + monkeypatch.setenv(ENV_NAMES[0], "200000") + monkeypatch.setenv(ENV_NAMES[1], "123456") + monkeypatch.setenv(ENV_NAMES[2], "4096") + result = runner.invoke(app, ["capabilities", "--json"]) + assert result.exit_code == 0, result.output + for template in json.loads(result.stdout)["templates"]: + assert template["dataset_support"]["work_limit_rows"] == 200000 + assert template["dataset_support"]["work_limit_bytes"] == 123456 + assert template["dataset_support"]["max_scratch_db_bytes"] == 4096 + info = runner.invoke( + app, ["query", "--template-info", "event", "--max-work-rows", "300000", "--json"] + ) + assert info.exit_code == 0, info.output + assert json.loads(info.stdout)["dataset_support"]["work_limit_rows"] == 300000 + + +def test_invalid_limit_environment_is_json_error_for_capabilities(monkeypatch): + monkeypatch.setenv(ENV_NAMES[0], "true") + result = runner.invoke(app, ["capabilities", "--json"]) + assert result.exit_code == 1 and result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "INVALID_PARAMETER" + + +def test_memory_tree_forwards_limits_without_capping_standalone_output(tmp_path, monkeypatch): + from pt_snap_cli.core.query_service import QueryService + from tests.frame_tree_helpers import create_frame_tree_db + + path = create_frame_tree_db(tmp_path / "frames.db") + observed = [] + original = QueryService._new_budget + + def capture(self, timeout_s=None, query_limits=None): + budget = original(self, timeout_s, query_limits) + observed.append(budget) + return budget + + monkeypatch.setattr(QueryService, "_new_budget", capture) + monkeypatch.setenv(ENV_NAMES[0], "1") + with closing(ReportService()) as service: + expected = service.memory_tree_report(8, db_path=path, query_limits=QueryLimits(2)) + result = runner.invoke( + app, + [ + "report", + "memory-tree", + str(path), + "--event-id", + "8", + "--max-work-rows", + "2", + "--json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["rows"] == expected.rows + assert expected.returned > 2 + assert len(observed) == 2 and observed[0] is not observed[1] + assert all(budget.max_work_rows == 2 for budget in observed)