Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {useEffect, useRef, useState} from "react";
import {CircleHelp, TriangleAlert} from "lucide-react";
import type {TrajectorySolverUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js";
import {hasObservedTrajectoryLoopXRequestUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_loopx_requests.js";
import {trajectoryUsageComparisonReason, type TrajectoryUsageCapture, type TrajectoryUsageComparison} from "../../../../../../loopx/control_plane/capabilities/trajectory_usage_comparison.js";
import {ContextHelp} from "./context-help";

Expand All @@ -12,19 +13,20 @@ const percent = (n: number | null, zh: boolean) => n === null ? "—"
export function LoopXRequestFootprint({usage, zh}: {usage: TrajectorySolverUsage; zh: boolean}) {
const v = usage.loopx_requests;
if (!v) return null; // Historical projections keep their missing observation.
const measured = v.status === "observed" && hasObservedTrajectoryLoopXRequestUsage(v);
return <section className="research-request-coverage" aria-label={zh ? "LoopX 命令请求" : "Requests issuing LoopX commands"}>
<div className="research-usage-heading"><h4>{zh ? "LoopX 命令请求 · 整次用量" : "LoopX command requests · whole-request usage"}</h4>
<ContextHelp icon={v.status === "unavailable" ? TriangleAlert : CircleHelp} label={zh ? "整次请求与控制文本的区别" : "Whole requests versus control text"}>
<p>{zh ? "按调用、原生响应回执和唯一返回的记录顺序,保守关联发出简单 LoopX 命令的请求。数值包含该请求读取的任务历史;与控制文本输入归因、工具返回字节是重叠视角,不能相加。" : "Conservative association by ordered call, native response receipt and unique return identifies requests issuing simple LoopX commands. Their counters include task history. This overlaps control-text attribution and tool-return bytes; never add these views."}</p>
<p>{zh ? "只识别已记录的简单调用;复合脚本及未匹配返回保留未归类。同次请求出现其他调用时标为混合调用;其余也不证明纯控制。未知来源不等于没有控制,整次用量不等于可省成本。模型耗时和实扣费用未采集。" : "Only recorded simple calls are recognized; compound scripts and unmatched returns remain unclassified. A request with other calls is marked mixed; the remainder does not establish pure control. Unknown sources do not mean no control, and whole-request usage is not avoidable cost. Model latency and actual charges were not captured."}</p>
<p>{zh ? "复合脚本或未匹配返回可能无法归属。若没有请求被识别且仍有此类调用,用量未知。已识别请求的用量只覆盖该子集,不表示纯控制用量或可省成本;同次请求含其他调用时标为混合。模型耗时和实扣费用未采集。" : "Compound scripts and unmatched returns may prevent association. Usage is unknown when no requests are recognized and calls remain unresolved. Reported usage covers only the recognized subset; it does not measure pure control or savings. Requests with other calls are marked mixed. Model latency and actual charges were not captured."}</p>
</ContextHelp></div>
{v.status === "unavailable" ? <p role="status">{zh ? "请求关联不可用 · 用量未知" : "Request association unavailable · usage unknown"}</p> : <>
<p className="research-input-share-value"><strong>{percent(v.input_fraction, zh)}</strong><span>{v.recognized_requests} / {v.total_requests} {zh ? "次请求已识别 · 归属未全" : "requests recognized · incomplete attribution"}</span></p>
<dl className="research-request-values">
<p className="research-input-share-value"><strong>{measured ? percent(v.input_fraction, zh) : zh ? "未知" : "Unknown"}</strong><span>{v.recognized_requests} / {v.total_requests} {zh ? "次请求已识别" : "requests recognized"}{v.unclassified_calls || v.unmatched_calls ? (zh ? " · 归属未全" : " · incomplete attribution") : null}</span></p>
{measured ? <dl className="research-request-values">
<div><dt>{zh ? "整次输入(含缓存)" : "Whole input (includes cache)"}</dt><dd>{count(v.tokens.input_tokens, zh)} tokens</dd></div>
<div><dt>{zh ? "其中未缓存" : "Of which uncached"}</dt><dd>{count(v.tokens.input_tokens - v.tokens.cached_input_tokens, zh)} tokens</dd></div>
<div><dt>{zh ? "整次输出(含推理)" : "Whole output (includes reasoning)"}</dt><dd>{count(v.tokens.output_tokens, zh)} tokens</dd></div>
</dl>
</dl> : null}
<details className="research-usage-detail"><summary>{zh ? "请求关联明细" : "Request association details"}</summary>
<dl className="research-usage-basis"><dt>{zh ? "方法" : "Method"}</dt><dd>{v.method}</dd>
<dt>{zh ? "含其他调用的请求" : "Requests with other calls"}</dt><dd>{v.mixed_call_requests}</dd>
Expand Down
16 changes: 16 additions & 0 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -1245,6 +1245,19 @@ once. Duplicate response receipts remain deduplicated by the existing request
reconciler. Late/unmatched calls and unclassified commands stay explicit.
Python only transports the original line positions, literal argv and return
metadata; neither command arguments nor returned text enter this projection.
Code Mode can wrap several operations in one custom tool call. A command
completion carrying only its own identity and the Turn does not link the
command to an issuing call or response. These completions do not classify the
whole call or assign its returned text to LoopX.

When no requests are recognized and any calls are unclassified or unmatched,
the command-request input shares are unknown (`null`). The saved `tokens`
object still sums the recognized subset, so an empty sum is zero. Research and
the comparison reader withhold command token values and deltas in that case,
including for older packs that saved zero shares. They retain the counts and
do not rewrite those packs. When some requests are recognized, their sums and
shares describe only that subset. Without unresolved calls, a measured empty
subset can show zero.

Keep three overlapping observations separate:

Expand Down Expand Up @@ -1302,6 +1315,9 @@ disable it; no ledger migration, source rewrite or runtime activation is needed.
返回保留,不将其认作纯控制、全部开销或可省成本。输入已含缓存、输出已含推理,
压缩镜像不重复计数;模型耗时与实扣未知。对照沿同一 TS owner 校验原 Goal、案例、
修订、窗口、方法及独立 Session,并保留原 run/pack/来源和时间,不据此核准科学配对。
Code Mode 的命令完成记录若缺少外层调用或响应 ID,不能据此归属请求或混合返回文本。
识别数为零且仍有未分类或未匹配调用时,占比为 `null`;保存的 `tokens` 仅是已识别
子集之和。研究页与对照会隐藏此时的命令用量和差值,旧包同样按此读取但不改写。
不平均百分比;归因缺失/部分、方法变化、来源失效或零分母保持未知。比例下降但绝对
输入增加会明确提示。浏览零模型调用,不重采集、不重写旧包或账本;旧包缺字段不补零。
既有 GET reader 在注册 Goal/来源命名空间中校验两个显式 pack,调用同一 builtin;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,10 @@ export interface TrajectorySolverUsage {

export type TrajectoryLoopXRequests = {method: 'codex_ordered_loopx_requests_v1'} & (
{status: 'observed'; total_requests: number; recognized_requests: number; mixed_call_requests: number;
unmatched_calls: number; unclassified_calls: number; tokens: TrajectoryRequestTokens;
unmatched_calls: number; unclassified_calls: number;
/** Sums of the recognized subset; an empty subset does not measure unresolved calls. */
tokens: TrajectoryRequestTokens;
/** Unknown when no requests are recognized and any calls are unclassified or unmatched. */
input_fraction: number | null; uncached_input_fraction: number | null;
/** Neither ordered tool records nor token receipts measure generation latency or money. */
model_seconds: null; charged_cost: null;
Expand Down
19 changes: 14 additions & 5 deletions loopx/control_plane/capabilities/trajectory_loopx_requests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ type Call = {line: number; operation: string | null; response?: string; returned
const keys: Array<keyof TrajectoryRequestTokens> = ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens',
'output_tokens', 'reasoning_output_tokens', 'total_tokens'];

/** Empty recognized sums describe coverage, not usage, if any calls are unresolved.
* Also used when reading frozen captures that stored zero before this distinction. */
export function hasObservedTrajectoryLoopXRequestUsage(value: Pick<Extract<TrajectoryLoopXRequests, {status: 'observed'}>,
'recognized_requests' | 'unmatched_calls' | 'unclassified_calls'>): boolean {
return value.recognized_requests > 0 || (value.unmatched_calls === 0 && value.unclassified_calls === 0);
}

export function summarizeTrajectoryLoopXRequests(rows: Row[], tools: unknown,
coverage: TrajectoryRequestCoverage | undefined): TrajectoryLoopXRequests | undefined {
if (tools === undefined) return undefined;
Expand Down Expand Up @@ -74,12 +81,14 @@ export function summarizeTrajectoryLoopXRequests(rows: Row[], tools: unknown,
other_calls: group.other, tokens: {...usage}});
}
const all = coverage.all_requests, uncached = all.input_tokens - all.cached_input_tokens;
return {method, status: 'observed', total_requests: seen.size, recognized_requests: requests.length,
mixed_call_requests: requests.filter(r => r.other_calls > 0).length,
const recognition = {recognized_requests: requests.length,
unmatched_calls: [...calls.values()].filter(c => c.operation && !c.returned).length,
unclassified_calls: [...calls.values()].filter(c => !c.operation).length,
tokens, input_fraction: all.input_tokens > 0 ? tokens.input_tokens / all.input_tokens : null,
uncached_input_fraction: uncached > 0 ? (tokens.input_tokens - tokens.cached_input_tokens) / uncached : null,
unclassified_calls: [...calls.values()].filter(c => !c.operation).length};
const measured = hasObservedTrajectoryLoopXRequestUsage(recognition);
return {method, status: 'observed', total_requests: seen.size, ...recognition,
mixed_call_requests: requests.filter(r => r.other_calls > 0).length,
tokens, input_fraction: measured && all.input_tokens > 0 ? tokens.input_tokens / all.input_tokens : null,
uncached_input_fraction: measured && uncached > 0 ? (tokens.input_tokens - tokens.cached_input_tokens) / uncached : null,
model_seconds: null, charged_cost: null, requests};
} catch (error) {
return {method, status: 'unavailable', reason: error instanceof Error ? error.message : 'Request association unavailable.'};
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/** Descriptive comparison of two frozen captures. No arm inference, study
* qualification, cross-run pooling, tokenization, billing or scientific score. */
import type {TrajectoryInput, TrajectoryLoopXInput, TrajectoryLoopXRequests, TrajectorySolverUsage} from './trajectory_assessment_types.ts';
import {hasObservedTrajectoryLoopXRequestUsage} from './trajectory_loopx_requests.ts';

type ComparisonProjection = Pick<TrajectorySolverUsage, 'session_id' | 'method' | 'start' | 'end' | 'request_coverage' | 'loopx_input'> & {
loopx_requests?: Omit<Extract<TrajectoryLoopXRequests, {status: 'observed'}>, 'requests'> | Extract<TrajectoryLoopXRequests, {status: 'unavailable'}>;
Expand All @@ -20,6 +21,7 @@ function values(capture: TrajectoryUsageCapture) {
const input = usage.loopx_input;
const attributed = input && input.status !== 'unavailable' ? input : null;
const requests = usage.loopx_requests?.status === 'observed' ? usage.loopx_requests : null;
const measured = requests && hasObservedTrajectoryLoopXRequestUsage(requests) ? requests : null;
return {pack_id: capture.pack_id, title: capture.title,
total: {input: all.input_tokens, cached: all.cached_input_tokens,
uncached: all.input_tokens - all.cached_input_tokens, output: all.output_tokens, requests: coverage.unique_requests},
Expand All @@ -28,11 +30,11 @@ function values(capture: TrajectoryUsageCapture) {
/** Uncovered requests are not zero control input. */
unattributed_requests: coverage.unique_requests - (attributed?.attributed_requests ?? 0),
status: attributed?.status ?? 'unavailable', method: input?.method ?? null},
commands: {input: requests?.tokens.input_tokens ?? null,
cached: requests?.tokens.cached_input_tokens ?? null,
uncached: requests ? requests.tokens.input_tokens - requests.tokens.cached_input_tokens : null,
output: requests?.tokens.output_tokens ?? null, fraction: requests?.input_fraction ?? null,
uncached_fraction: requests?.uncached_input_fraction ?? null,
commands: {input: measured?.tokens.input_tokens ?? null,
cached: measured?.tokens.cached_input_tokens ?? null,
uncached: measured ? measured.tokens.input_tokens - measured.tokens.cached_input_tokens : null,
output: measured?.tokens.output_tokens ?? null, fraction: measured?.input_fraction ?? null,
uncached_fraction: measured?.uncached_input_fraction ?? null,
recognized_requests: requests?.recognized_requests ?? null, mixed_call_requests: requests?.mixed_call_requests ?? null,
unclassified_calls: requests?.unclassified_calls ?? null, unmatched_calls: requests?.unmatched_calls ?? null,
model_seconds: null, charged_cost: null},
Expand Down
27 changes: 21 additions & 6 deletions tests/capabilities/test_trajectory_input_capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,17 +135,26 @@ def test_tool_inventory_is_opt_in_separate_and_retains_native_counter_identity(t
assert path.read_bytes() == raw


@pytest.mark.parametrize("direct_call", [False, True])
def test_real_cli_registers_recorded_component_without_model_execution(tmp_path, direct_call):
@pytest.mark.parametrize("call_kind", ["absent", "direct", "compound"])
def test_real_cli_registers_recorded_component_without_model_execution(tmp_path, call_kind):
from loopx.capabilities.benchmark_toolkit.trajectory_assessment_store import inspect_pack

source, rows = native_trace(tmp_path)
if direct_call:
if call_kind != "absent":
call = {"type": "custom_tool_call", "call_id": "tool", "name": "exec",
"input": 'text(await tools.exec_command({cmd: "loopx status"}))'}
reply = {"type": "custom_tool_call_output", "call_id": "tool", "output": "saved"}
items = [{"type": "response_item", "payload": item} for item in (call, reply)]
(tmp_path / "usage.jsonl").write_text("\n".join(map(json.dumps, rows[:2] + [items[0], rows[2], items[1]] + rows[3:])))
native_command = []
if call_kind == "compound":
call["input"] = 'text(await Promise.all([tools.exec_command({cmd: "loopx status"}), tools.exec_command({cmd: "pwd"})]));'
# Same Turn and matching time window, but the command's ID is not the parent tool call ID.
native_command = [{"type": "event_msg", "timestamp": rows[2]["timestamp"], "payload": {
"type": "item_completed", "thread_id": "session", "turn_id": "turn",
"item": {"type": "CommandExecution", "id": "exec-child", "command": ["bash", "-lc", "loopx status"]},
}}]
usage_rows = rows[:2] + [items[0], rows[2], *native_command, items[1]] + rows[3:]
(tmp_path / "usage.jsonl").write_text("\n".join(map(json.dumps, usage_rows)))
evidence = "Synthetic fixed negative result."
(tmp_path / "evidence.txt").write_text(evidence)
data = {"schema_version": "trajectory_assessment_input_v1",
Expand All @@ -172,14 +181,20 @@ def test_real_cli_registers_recorded_component_without_model_execution(tmp_path,
inventory = view["solver_usage"]["projection"]["tool_context"]
footprint = view["solver_usage"]["projection"]["loopx_requests"]
assert footprint["status"] == "observed"
if direct_call:
if call_kind == "direct":
assert inventory["operations"] == [{"operation": "status", "calls": 1, "bytes": 5}]
assert inventory["loopx_tool_output_fraction"] == 1.0
assert footprint["recognized_requests"] == 1
assert footprint["tokens"]["input_tokens"] == 100
assert footprint["input_fraction"] == 1.0 # Whole request, distinct from the packet estimate above.
elif call_kind == "compound":
assert view["solver_usage"]["projection"]["request_coverage"]["all_requests"]["input_tokens"] == 100
assert footprint["recognized_requests"] == 0 and footprint["unclassified_calls"] == 1
assert footprint["input_fraction"] is None and footprint["uncached_input_fraction"] is None
assert inventory["loopx_tool_output_bytes"] == 0 # The mixed return is still unassigned.
assert "exec-child" not in json.dumps(view)
else:
assert inventory["loopx_tool_output_fraction"] is None
assert footprint["recognized_requests"] == 0
assert footprint["recognized_requests"] == 0 and footprint["input_fraction"] == 0
assert view["job"] is None and view["result"] is None
assert "hello world" not in json.dumps(view)
Loading
Loading