From 0d7d15b49c39eff26d2f5a737e6403448e517da7 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:04:17 +0800 Subject: [PATCH] fix(benchmark): distinguish unresolved command usage from zero Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../research-usage-comparison.tsx | 10 +++--- docs/reference/trajectory-assessment.md | 16 ++++++++++ .../trajectory_assessment_types.ts | 5 ++- .../capabilities/trajectory_loopx_requests.ts | 19 ++++++++--- .../trajectory_usage_comparison.ts | 12 ++++--- .../test_trajectory_input_capture.py | 27 ++++++++++++---- .../trajectory_usage_comparison.test.ts | 32 +++++++++++++++++-- 7 files changed, 98 insertions(+), 23 deletions(-) diff --git a/apps/presentation/dashboard/src/features/personal-workspace/research-usage-comparison.tsx b/apps/presentation/dashboard/src/features/personal-workspace/research-usage-comparison.tsx index bbe91524d5..3fe0bf74ef 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/research-usage-comparison.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/research-usage-comparison.tsx @@ -1,6 +1,7 @@ import {useEffect, useRef, useState} from "react"; import {CircleHelp, TriangleAlert} from "lucide-react"; import type {TrajectorySolverUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js"; +import {hasObservedTrajectoryLoopXRequestUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_loopx_requests.js"; import {trajectoryUsageComparisonReason, type TrajectoryUsageCapture, type TrajectoryUsageComparison} from "../../../../../../loopx/control_plane/capabilities/trajectory_usage_comparison.js"; import {ContextHelp} from "./context-help"; @@ -12,19 +13,20 @@ const percent = (n: number | null, zh: boolean) => n === null ? "—" export function LoopXRequestFootprint({usage, zh}: {usage: TrajectorySolverUsage; zh: boolean}) { const v = usage.loopx_requests; if (!v) return null; // Historical projections keep their missing observation. + const measured = v.status === "observed" && hasObservedTrajectoryLoopXRequestUsage(v); return

{zh ? "LoopX 命令请求 · 整次用量" : "LoopX command requests · whole-request usage"}

{zh ? "按调用、原生响应回执和唯一返回的记录顺序，保守关联发出简单 LoopX 命令的请求。数值包含该请求读取的任务历史；与控制文本输入归因、工具返回字节是重叠视角，不能相加。" : "Conservative association by ordered call, native response receipt and unique return identifies requests issuing simple LoopX commands. Their counters include task history. This overlaps control-text attribution and tool-return bytes; never add these views."}

-

{zh ? "只识别已记录的简单调用；复合脚本及未匹配返回保留未归类。同次请求出现其他调用时标为混合调用；其余也不证明纯控制。未知来源不等于没有控制，整次用量不等于可省成本。模型耗时和实扣费用未采集。" : "Only recorded simple calls are recognized; compound scripts and unmatched returns remain unclassified. A request with other calls is marked mixed; the remainder does not establish pure control. Unknown sources do not mean no control, and whole-request usage is not avoidable cost. Model latency and actual charges were not captured."}

+

{zh ? "复合脚本或未匹配返回可能无法归属。若没有请求被识别且仍有此类调用，用量未知。已识别请求的用量只覆盖该子集，不表示纯控制用量或可省成本；同次请求含其他调用时标为混合。模型耗时和实扣费用未采集。" : "Compound scripts and unmatched returns may prevent association. Usage is unknown when no requests are recognized and calls remain unresolved. Reported usage covers only the recognized subset; it does not measure pure control or savings. Requests with other calls are marked mixed. Model latency and actual charges were not captured."}

{v.status === "unavailable" ?

{zh ? "请求关联不可用 · 用量未知" : "Request association unavailable · usage unknown"}

: <> -

{percent(v.input_fraction, zh)}{v.recognized_requests} / {v.total_requests} {zh ? "次请求已识别 · 归属未全" : "requests recognized · incomplete attribution"}

-
+

{measured ? percent(v.input_fraction, zh) : zh ? "未知" : "Unknown"}{v.recognized_requests} / {v.total_requests} {zh ? "次请求已识别" : "requests recognized"}{v.unclassified_calls || v.unmatched_calls ? (zh ? " · 归属未全" : " · incomplete attribution") : null}

+ {measured ?
{zh ? "整次输入（含缓存）" : "Whole input (includes cache)"}
{count(v.tokens.input_tokens, zh)} tokens
{zh ? "其中未缓存" : "Of which uncached"}
{count(v.tokens.input_tokens - v.tokens.cached_input_tokens, zh)} tokens
{zh ? "整次输出（含推理）" : "Whole output (includes reasoning)"}
{count(v.tokens.output_tokens, zh)} tokens
-
+
: null}
{zh ? "请求关联明细" : "Request association details"}
{zh ? "方法" : "Method"}
{v.method}
{zh ? "含其他调用的请求" : "Requests with other calls"}
{v.mixed_call_requests}
diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 235e300d6e..113a80e01a 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -1245,6 +1245,19 @@ once. Duplicate response receipts remain deduplicated by the existing request reconciler. Late/unmatched calls and unclassified commands stay explicit. Python only transports the original line positions, literal argv and return metadata; neither command arguments nor returned text enter this projection. +Code Mode can wrap several operations in one custom tool call. A command +completion carrying only its own identity and the Turn does not link the +command to an issuing call or response. These completions do not classify the +whole call or assign its returned text to LoopX. + +When no requests are recognized and any calls are unclassified or unmatched, +the command-request input shares are unknown (`null`). The saved `tokens` +object still sums the recognized subset, so an empty sum is zero. Research and +the comparison reader withhold command token values and deltas in that case, +including for older packs that saved zero shares. They retain the counts and +do not rewrite those packs. When some requests are recognized, their sums and +shares describe only that subset. Without unresolved calls, a measured empty +subset can show zero. Keep three overlapping observations separate: @@ -1302,6 +1315,9 @@ disable it; no ledger migration, source rewrite or runtime activation is needed. 返回保留，不将其认作纯控制、全部开销或可省成本。输入已含缓存、输出已含推理， 压缩镜像不重复计数；模型耗时与实扣未知。对照沿同一 TS owner 校验原 Goal、案例、 修订、窗口、方法及独立 Session，并保留原 run/pack/来源和时间，不据此核准科学配对。 +Code Mode 的命令完成记录若缺少外层调用或响应 ID，不能据此归属请求或混合返回文本。 +识别数为零且仍有未分类或未匹配调用时，占比为 `null`；保存的 `tokens` 仅是已识别 +子集之和。研究页与对照会隐藏此时的命令用量和差值，旧包同样按此读取但不改写。 不平均百分比；归因缺失/部分、方法变化、来源失效或零分母保持未知。比例下降但绝对 输入增加会明确提示。浏览零模型调用，不重采集、不重写旧包或账本；旧包缺字段不补零。 既有 GET reader 在注册 Goal/来源命名空间中校验两个显式 pack，调用同一 builtin； diff --git a/loopx/control_plane/capabilities/trajectory_assessment_types.ts b/loopx/control_plane/capabilities/trajectory_assessment_types.ts index bd7b12a270..706a3fa0df 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_types.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_types.ts @@ -106,7 +106,10 @@ export interface TrajectorySolverUsage { export type TrajectoryLoopXRequests = {method: 'codex_ordered_loopx_requests_v1'} & ( {status: 'observed'; total_requests: number; recognized_requests: number; mixed_call_requests: number; - unmatched_calls: number; unclassified_calls: number; tokens: TrajectoryRequestTokens; + unmatched_calls: number; unclassified_calls: number; + /** Sums of the recognized subset; an empty subset does not measure unresolved calls. */ + tokens: TrajectoryRequestTokens; + /** Unknown when no requests are recognized and any calls are unclassified or unmatched. */ input_fraction: number | null; uncached_input_fraction: number | null; /** Neither ordered tool records nor token receipts measure generation latency or money. */ model_seconds: null; charged_cost: null; diff --git a/loopx/control_plane/capabilities/trajectory_loopx_requests.ts b/loopx/control_plane/capabilities/trajectory_loopx_requests.ts index 181e3dbdc5..4722eabede 100644 --- a/loopx/control_plane/capabilities/trajectory_loopx_requests.ts +++ b/loopx/control_plane/capabilities/trajectory_loopx_requests.ts @@ -10,6 +10,13 @@ type Call = {line: number; operation: string | null; response?: string; returned const keys: Array = ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens']; +/** Empty recognized sums describe coverage, not usage, if any calls are unresolved. + * Also used when reading frozen captures that stored zero before this distinction. */ +export function hasObservedTrajectoryLoopXRequestUsage(value: Pick, + 'recognized_requests' | 'unmatched_calls' | 'unclassified_calls'>): boolean { + return value.recognized_requests > 0 || (value.unmatched_calls === 0 && value.unclassified_calls === 0); +} + export function summarizeTrajectoryLoopXRequests(rows: Row[], tools: unknown, coverage: TrajectoryRequestCoverage | undefined): TrajectoryLoopXRequests | undefined { if (tools === undefined) return undefined; @@ -74,12 +81,14 @@ export function summarizeTrajectoryLoopXRequests(rows: Row[], tools: unknown, other_calls: group.other, tokens: {...usage}}); } const all = coverage.all_requests, uncached = all.input_tokens - all.cached_input_tokens; - return {method, status: 'observed', total_requests: seen.size, recognized_requests: requests.length, - mixed_call_requests: requests.filter(r => r.other_calls > 0).length, + const recognition = {recognized_requests: requests.length, unmatched_calls: [...calls.values()].filter(c => c.operation && !c.returned).length, - unclassified_calls: [...calls.values()].filter(c => !c.operation).length, - tokens, input_fraction: all.input_tokens > 0 ? tokens.input_tokens / all.input_tokens : null, - uncached_input_fraction: uncached > 0 ? (tokens.input_tokens - tokens.cached_input_tokens) / uncached : null, + unclassified_calls: [...calls.values()].filter(c => !c.operation).length}; + const measured = hasObservedTrajectoryLoopXRequestUsage(recognition); + return {method, status: 'observed', total_requests: seen.size, ...recognition, + mixed_call_requests: requests.filter(r => r.other_calls > 0).length, + tokens, input_fraction: measured && all.input_tokens > 0 ? tokens.input_tokens / all.input_tokens : null, + uncached_input_fraction: measured && uncached > 0 ? (tokens.input_tokens - tokens.cached_input_tokens) / uncached : null, model_seconds: null, charged_cost: null, requests}; } catch (error) { return {method, status: 'unavailable', reason: error instanceof Error ? error.message : 'Request association unavailable.'}; diff --git a/loopx/control_plane/capabilities/trajectory_usage_comparison.ts b/loopx/control_plane/capabilities/trajectory_usage_comparison.ts index 93be79042f..1df2e111b6 100644 --- a/loopx/control_plane/capabilities/trajectory_usage_comparison.ts +++ b/loopx/control_plane/capabilities/trajectory_usage_comparison.ts @@ -1,6 +1,7 @@ /** Descriptive comparison of two frozen captures. No arm inference, study * qualification, cross-run pooling, tokenization, billing or scientific score. */ import type {TrajectoryInput, TrajectoryLoopXInput, TrajectoryLoopXRequests, TrajectorySolverUsage} from './trajectory_assessment_types.ts'; +import {hasObservedTrajectoryLoopXRequestUsage} from './trajectory_loopx_requests.ts'; type ComparisonProjection = Pick & { loopx_requests?: Omit, 'requests'> | Extract; @@ -20,6 +21,7 @@ function values(capture: TrajectoryUsageCapture) { const input = usage.loopx_input; const attributed = input && input.status !== 'unavailable' ? input : null; const requests = usage.loopx_requests?.status === 'observed' ? usage.loopx_requests : null; + const measured = requests && hasObservedTrajectoryLoopXRequestUsage(requests) ? requests : null; return {pack_id: capture.pack_id, title: capture.title, total: {input: all.input_tokens, cached: all.cached_input_tokens, uncached: all.input_tokens - all.cached_input_tokens, output: all.output_tokens, requests: coverage.unique_requests}, @@ -28,11 +30,11 @@ function values(capture: TrajectoryUsageCapture) { /** Uncovered requests are not zero control input. */ unattributed_requests: coverage.unique_requests - (attributed?.attributed_requests ?? 0), status: attributed?.status ?? 'unavailable', method: input?.method ?? null}, - commands: {input: requests?.tokens.input_tokens ?? null, - cached: requests?.tokens.cached_input_tokens ?? null, - uncached: requests ? requests.tokens.input_tokens - requests.tokens.cached_input_tokens : null, - output: requests?.tokens.output_tokens ?? null, fraction: requests?.input_fraction ?? null, - uncached_fraction: requests?.uncached_input_fraction ?? null, + commands: {input: measured?.tokens.input_tokens ?? null, + cached: measured?.tokens.cached_input_tokens ?? null, + uncached: measured ? measured.tokens.input_tokens - measured.tokens.cached_input_tokens : null, + output: measured?.tokens.output_tokens ?? null, fraction: measured?.input_fraction ?? null, + uncached_fraction: measured?.uncached_input_fraction ?? null, recognized_requests: requests?.recognized_requests ?? null, mixed_call_requests: requests?.mixed_call_requests ?? null, unclassified_calls: requests?.unclassified_calls ?? null, unmatched_calls: requests?.unmatched_calls ?? null, model_seconds: null, charged_cost: null}, diff --git a/tests/capabilities/test_trajectory_input_capture.py b/tests/capabilities/test_trajectory_input_capture.py index cc8f2a3ab5..8658daf708 100644 --- a/tests/capabilities/test_trajectory_input_capture.py +++ b/tests/capabilities/test_trajectory_input_capture.py @@ -135,17 +135,26 @@ def test_tool_inventory_is_opt_in_separate_and_retains_native_counter_identity(t assert path.read_bytes() == raw -@pytest.mark.parametrize("direct_call", [False, True]) -def test_real_cli_registers_recorded_component_without_model_execution(tmp_path, direct_call): +@pytest.mark.parametrize("call_kind", ["absent", "direct", "compound"]) +def test_real_cli_registers_recorded_component_without_model_execution(tmp_path, call_kind): from loopx.capabilities.benchmark_toolkit.trajectory_assessment_store import inspect_pack source, rows = native_trace(tmp_path) - if direct_call: + if call_kind != "absent": call = {"type": "custom_tool_call", "call_id": "tool", "name": "exec", "input": 'text(await tools.exec_command({cmd: "loopx status"}))'} reply = {"type": "custom_tool_call_output", "call_id": "tool", "output": "saved"} items = [{"type": "response_item", "payload": item} for item in (call, reply)] - (tmp_path / "usage.jsonl").write_text("\n".join(map(json.dumps, rows[:2] + [items[0], rows[2], items[1]] + rows[3:]))) + native_command = [] + if call_kind == "compound": + call["input"] = 'text(await Promise.all([tools.exec_command({cmd: "loopx status"}), tools.exec_command({cmd: "pwd"})]));' + # Same Turn and matching time window, but the command's ID is not the parent tool call ID. + native_command = [{"type": "event_msg", "timestamp": rows[2]["timestamp"], "payload": { + "type": "item_completed", "thread_id": "session", "turn_id": "turn", + "item": {"type": "CommandExecution", "id": "exec-child", "command": ["bash", "-lc", "loopx status"]}, + }}] + usage_rows = rows[:2] + [items[0], rows[2], *native_command, items[1]] + rows[3:] + (tmp_path / "usage.jsonl").write_text("\n".join(map(json.dumps, usage_rows))) evidence = "Synthetic fixed negative result." (tmp_path / "evidence.txt").write_text(evidence) data = {"schema_version": "trajectory_assessment_input_v1", @@ -172,14 +181,20 @@ def test_real_cli_registers_recorded_component_without_model_execution(tmp_path, inventory = view["solver_usage"]["projection"]["tool_context"] footprint = view["solver_usage"]["projection"]["loopx_requests"] assert footprint["status"] == "observed" - if direct_call: + if call_kind == "direct": assert inventory["operations"] == [{"operation": "status", "calls": 1, "bytes": 5}] assert inventory["loopx_tool_output_fraction"] == 1.0 assert footprint["recognized_requests"] == 1 assert footprint["tokens"]["input_tokens"] == 100 assert footprint["input_fraction"] == 1.0 # Whole request, distinct from the packet estimate above. + elif call_kind == "compound": + assert view["solver_usage"]["projection"]["request_coverage"]["all_requests"]["input_tokens"] == 100 + assert footprint["recognized_requests"] == 0 and footprint["unclassified_calls"] == 1 + assert footprint["input_fraction"] is None and footprint["uncached_input_fraction"] is None + assert inventory["loopx_tool_output_bytes"] == 0 # The mixed return is still unassigned. + assert "exec-child" not in json.dumps(view) else: assert inventory["loopx_tool_output_fraction"] is None - assert footprint["recognized_requests"] == 0 + assert footprint["recognized_requests"] == 0 and footprint["input_fraction"] == 0 assert view["job"] is None and view["result"] is None assert "hello world" not in json.dumps(view) diff --git a/tests/control_plane_ts/trajectory_usage_comparison.test.ts b/tests/control_plane_ts/trajectory_usage_comparison.test.ts index cea4c504eb..72000b4816 100644 --- a/tests/control_plane_ts/trajectory_usage_comparison.test.ts +++ b/tests/control_plane_ts/trajectory_usage_comparison.test.ts @@ -70,12 +70,12 @@ test('multiple calls in one request count once; duplicates do not count twice; m assert.equal(d.summarize().request_coverage?.status, 'verified'); }); -test('a late return across another response is unassociated; zero observed matches is distinct from missing observations', () => { +test('a late return across another response is unassociated and cannot measure a zero footprint', () => { const c = capture(); c.tools[1].line = 14; c.tools.sort((a,b) => a.line-b.line); const v = c.summarize().loopx_requests!; assert.equal(v.status, 'observed'); if (v.status !== 'observed') return; - assert.equal(v.recognized_requests, 0); assert.equal(v.input_fraction, 0); assert.equal(v.unmatched_calls, 1); + assert.equal(v.recognized_requests, 0); assert.equal(v.input_fraction, null); assert.equal(v.unmatched_calls, 1); assert.equal(c.summarize({tool_context: undefined}).loopx_requests, undefined); const mirrored = capture(); for (const row of [...mirrored.events, ...mirrored.tools]) if (row.line >= 4) row.line += 2; @@ -94,6 +94,34 @@ test('a late return across another response is unassociated; zero observed match assert.equal(compact.unmatched_calls, 1); }); +test('unclassified calls with no matches withhold command deltas, including for saved zeros', () => { + const opaque = capture('opaque'); + opaque.tools[0].argv = null; + const usage = opaque.summarize(); + const footprint = usage.loopx_requests!; + assert.equal(footprint.status, 'observed'); if (footprint.status !== 'observed') return; + assert.equal(footprint.recognized_requests, 0); + assert.equal(footprint.unclassified_calls, 2); + assert.equal(footprint.tokens.input_tokens, 0, 'The saved sum still describes the empty recognized subset'); + assert.equal(footprint.input_fraction, null); + assert.equal(footprint.uncached_input_fraction, null); + const left = pack('opaque', [100, 900]), right = pack('direct', [100, 100]); + for (const observed of [footprint, {...footprint, input_fraction: 0, uncached_input_fraction: 0}]) { + left.solver_usage!.projection = {...usage, loopx_requests: observed}; + const compared = compareTrajectoryUsage(left, right); + assert.equal(compared.status, 'observed'); if (compared.status !== 'observed') return; + assert.equal(compared.left.commands.input, null); + assert.equal(compared.left.commands.fraction, null); + assert.equal(compared.left.commands.unclassified_calls, 2); + assert.equal(compared.delta.command_input, null); + assert.equal(compared.delta.command_uncached_percentage_points, null); + assert.equal(compared.delta.total_input, 800, 'The verified native totals still compare'); + } + const empty = opaque.summarize({tool_context: []}).loopx_requests!; + assert.equal(empty.status, 'observed'); + if (empty.status === 'observed') assert.equal(empty.input_fraction, 0, 'Zero is valid without unresolved calls'); +}); + test('comparison uses sums, preserves partial attribution, and reports a falling share with rising absolute input', () => { const left = pack('left', [100, 900], [10, 90]), right = pack('right', [100, 100], [10, 10]); const v = compareTrajectoryUsage(left, right);