Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type Pack = ResearchProgressPack & {source_count: number; analysis_key: string;
continuation_unavailable?: {base_pack_id: string};
continuation?: TrajectoryContinuation & {base_pack_id: string; base_title: string; base_available: boolean; base_episode_count: number}};
type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]; unreadable_pack_ids?: string[]}; pack: Pack | null};
type Citation = AssessmentCitation & {artifact: string; text: string; applicability: string};
type Citation = AssessmentCitation & {evidence_kind?: "usage_counters"; artifact: string; text: string; applicability: string};
const errorText = (error: unknown) => error instanceof Error ? error.message : String(error);
const locationFor = (goal: string) => {
const query = new URLSearchParams(window.location.search);
Expand Down Expand Up @@ -175,18 +175,20 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean
} catch (e) {if (current === generation.current) setError(errorText(e));}
finally {if (current === generation.current) setCancelPending(false);}
};
const readCitation = async (c: AssessmentCitation, origin?: HTMLButtonElement) => {
const readEvidence = async (selection: Record<string, string>, route: string, origin?: HTMLButtonElement) => {
const p = pack;
if (!p) return;
if (origin) sourceTrigger.current = origin;
const current = ++citeGeneration.current; setCitation(null); setCitationBusy(true); setCitationError("");
const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, output_language: outputLanguage, source_id: c.source_id, first_line: String(c.first_line), last_line: String(c.last_line)});
try {const value = await requestJson<Citation>(sourceReadApiUrl(readSourceUrl, `${endpoint}/citation?${query}`)); if (current === citeGeneration.current) setCitation(value);}
const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, output_language: outputLanguage, ...selection});
try {const value = await requestJson<Citation>(sourceReadApiUrl(readSourceUrl, `${endpoint}/${route}?${query}`)); if (current === citeGeneration.current) setCitation(value);}
catch (e) {if (current === citeGeneration.current) setCitationError(errorText(e));}
finally {if (current === citeGeneration.current) setCitationBusy(false);}
};
const readCitation = (c: AssessmentCitation, origin?: HTMLButtonElement) => readEvidence({source_id: c.source_id, first_line: String(c.first_line), last_line: String(c.last_line)}, "citation", origin);
const readUsageTurn = (turnId: string, origin: HTMLButtonElement) => readEvidence({turn_id: turnId}, "usage-turn", origin);
const closeCitation = () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); requestAnimationFrame(() => sourceTrigger.current?.focus());};
return {readOnly: Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true, citation, citationBusy, citationError, sourcePanel, readCitation, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel,
return {readOnly: Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true, citation, citationBusy, citationError, sourcePanel, readCitation, readUsageTurn, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel,
close: () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); navigate(false); requestAnimationFrame(() => trigger.current?.focus());}, select: (id: string) => navigate(location.open, id, true)};
}
type Controller = ReturnType<typeof useResearchAnalysis>;
Expand Down Expand Up @@ -279,7 +281,7 @@ export function ResearchAnalysisStatus({analysis: a}: {analysis: Controller}) {
export function ResearchProgressSummary({analysis: a, includeReport = true}: {analysis: Controller; includeReport?: boolean}) {
const packs = a.snapshot?.catalog.packs ?? [];
if (!a.pack && !packs.length) return null;
return <ResearchProgress pack={a.pack} packs={packs} selectedId={a.location.packId ?? a.snapshot?.catalog.default_pack_id ?? ""} zh={a.zh} analysisOpen={a.location.open} includeReport={includeReport} onSelect={a.select} onCitation={a.readCitation}/>;
return <ResearchProgress pack={a.pack} packs={packs} selectedId={a.location.packId ?? a.snapshot?.catalog.default_pack_id ?? ""} zh={a.zh} analysisOpen={a.location.open} includeReport={includeReport} onSelect={a.select} onCitation={a.readCitation} onUsageTurn={a.readUsageTurn}/>;
}

export function ResearchReportSummary({analysis: a}: {analysis: Controller}) {
Expand Down Expand Up @@ -373,6 +375,7 @@ export function ResearchCitationPanel({analysis: a}: {analysis: Controller}) {
{a.citationBusy ? <p role="status">{zh ? "正在读取引用原文…" : "Reading cited source…"}</p> : null}
{a.citationError ? <p className="analysis-error" role="alert">{a.citationError}</p> : null}
{a.citation ? <><p>{a.citation.artifact} · {zh ? "行" : "lines"} {a.citation.first_line}–{a.citation.last_line}</p>
{a.citation.evidence_kind === "usage_counters" ? <p>{zh ? "计数事件:前一累计观测、本轮开始、末次计数及完成(如有)。仅显示计数字段;提示词、工具内容与账户信息不传入浏览器。" : "Counter events: preceding cumulative observation, Turn start, latest counters and completion if present. Only counter fields are shown; prompts, tool content and account metadata stay outside the browser."}</p> : null}
<CitationContent key={`${a.pack?.analysis_key}:${a.citation.source_id}:${a.citation.first_line}:${a.citation.last_line}`} citation={a.citation} zh={zh}/></> : null}
</section>;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,14 @@
.research-usage-basis { grid-template-columns: 1fr; gap: 4px; }
.research-progress-basis dd, .research-usage-basis dd { margin-bottom: 8px; }
}

.research-turn-costs { border-top: 1px solid var(--pw-line); padding-top: 12px; margin-top: 12px; }
.research-turn-costs > p { max-width: 72ch; color: var(--pw-muted); font-size: 12px; }
.research-turn-cost-list { list-style: none; padding: 0; margin: 12px 0; }
.research-turn-cost-list > li { padding: 12px 0; border-top: 1px solid var(--pw-line); }
.research-turn-cost-list time { display: block; font-size: 11px; color: var(--pw-muted); overflow-wrap: anywhere; }
.research-turn-cost-list dl { display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 12px; margin: 8px 0; }
.research-turn-cost-list dt { font-size: 11px; color: var(--pw-muted); }
.research-turn-cost-list dd { margin: 0; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; }
.research-turn-costs nav { display: flex; align-items: center; gap: 12px; }
@media (max-width: 600px) { .research-turn-cost-list dl { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type {AssessmentCitation, AssessmentVerdict, TrajectoryAssessment, Trajec
import type {TrajectoryNativeOutcomes} from "../../../../../../loopx/control_plane/capabilities/trajectory_native_outcomes.js";
import {ContextHelp} from "./context-help";
import {ResearchSynthesis} from "./goal-research-synthesis";
import {ResearchTurnCosts} from "./research-turn-costs";
import {ToolContextInventory} from "./goal-tool-context";
import "./goal-research-progress.css";

Expand Down Expand Up @@ -253,7 +254,7 @@ function ResearchMetrics({pack: p, zh, onCitation}: {pack: ResearchProgressPack
</div>;
}

export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCitation, analysisOpen = false, includeReport = true}: {pack: ResearchProgressPack | null; packs: Pick<ResearchProgressPack, "pack_id" | "title">[]; selectedId: string; zh: boolean; analysisOpen?: boolean; includeReport?: boolean; onSelect: (id: string) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) {
export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCitation, onUsageTurn, analysisOpen = false, includeReport = true}: {pack: ResearchProgressPack | null; packs: Pick<ResearchProgressPack, "pack_id" | "title">[]; selectedId: string; zh: boolean; analysisOpen?: boolean; includeReport?: boolean; onSelect: (id: string) => void; onUsageTurn?: (turnId: string, origin: HTMLButtonElement) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) {
const measured = p?.solver_usage, usage = measured?.projection, result = p?.result;
const historical = p?.applicability === "historical";
const resultHelp = historical ? (result ? (zh ? "历史分析 · 当前判断未核验" : "Historical analysis · Current judgments unverified")
Expand Down Expand Up @@ -334,6 +335,7 @@ export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCi
</> : null}
</div> : null}
</div>
{usage && onUsageTurn ? <ResearchTurnCosts key={p.pack_id} usage={usage} zh={zh} onRead={onUsageTurn}/> : null}
{usage && measured ? <details className="research-usage-detail"><summary>{zh ? "计数明细与来源" : "Counter details and source"}</summary>
<dl className="research-usage-basis">
<dt>{zh ? "捕获时间(UTC)" : "Capture window (UTC)"}</dt><dd><time dateTime={usage.start}>{stamp(usage.start)}</time> — <time dateTime={usage.end}>{stamp(usage.end)}</time></dd>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import {useId, useState} from "react";
import type {TrajectorySolverUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js";

/** Presentation of existing Turn counters; never estimates missing usage or attributes it to control work. */
export function ResearchTurnCosts({usage, zh, onRead}: {usage: TrajectorySolverUsage; zh: boolean;
onRead: (turnId: string, origin: HTMLButtonElement) => void}) {
const id = useId();
const [sort, setSort] = useState("input");
const [page, setPage] = useState(0);
const rows: Array<Omit<TrajectorySolverUsage["turns"][number], "usage"> & {usage: TrajectorySolverUsage["turns"][number]["usage"] | null; ordinal: number; incomplete: boolean}> = usage.turns.map((turn, index) => ({...turn, ordinal: index + 1, incomplete: false}));
const tail = usage.incomplete_turn;
if (tail) rows.push({...tail, end: tail.observed_through,
elapsed_seconds: Math.max(0, (Date.parse(tail.observed_through) - Date.parse(tail.start)) / 1000),
ordinal: rows.length + 1, incomplete: true, usage: tail.usage});
const ordered = [...rows].sort((a, b) => {
const value = (row: typeof a) => sort === "elapsed" ? row.elapsed_seconds
: sort === "time" ? row.ordinal : row.usage?.input_tokens ?? -1;
return (sort === "time" ? value(a) - value(b) : value(b) - value(a)) || a.ordinal - b.ordinal;
});
const size = 8, pages = Math.max(1, Math.ceil(rows.length / size)), current = Math.min(page, pages - 1);
const number = (n: number | undefined | null) => n == null ? "—" : n.toLocaleString(zh ? "zh-CN" : "en");
return <details className="research-turn-costs">
<summary>{zh ? "定位耗时与输入最多的轮次" : "Find time and input hotspots"}</summary>
<p>{zh ? "仅比较当前捕获内的轮次。耗时含工具、等待与编排,不是可节省时间;轮次输入不等于 LoopX 控制输入。金额未采集。" : "Compares Turns within this capture only. Elapsed time includes tools, waiting and orchestration, not avoidable time. Turn input is not LoopX control input. Dollar cost is unmeasured."}</p>
<label htmlFor={id}>{zh ? "排序" : "Sort by"}</label>{" "}
<select id={id} value={sort} onChange={event => {setSort(event.target.value); setPage(0);}}>
<option value="input">{zh ? "输入最多" : "Most input"}</option>
<option value="elapsed">{zh ? "耗时最长" : "Longest elapsed"}</option>
<option value="time">{zh ? "时间顺序" : "Chronological"}</option>
</select>
{rows.length ? <ol className="research-turn-cost-list">
{ordered.slice(current * size, (current + 1) * size).map(turn => <li key={turn.turn_id}>
<div><strong>{zh ? `第 ${turn.ordinal} 轮` : `Turn ${turn.ordinal}`}</strong>
{turn.incomplete ? <span className="research-usage-scope">{zh ? " · 未完成(仅已观测)" : " · Incomplete (observed only)"}</span> : null}
<time dateTime={turn.start}>{turn.start.replace("T", " ")}</time>
</div>
<dl>
<div><dt>{zh ? "耗时" : "Elapsed"}</dt><dd>{(turn.elapsed_seconds / 60).toLocaleString(zh ? "zh-CN" : "en", {maximumFractionDigits: 1})} {zh ? "分钟" : "min"}</dd></div>
<div><dt>{zh ? "输入" : "Input"}</dt><dd>{number(turn.usage?.input_tokens)}</dd></div>
<div><dt>{zh ? "其中缓存" : "Cached subset"}</dt><dd>{number(turn.usage?.cached_input_tokens)}</dd></div>
<div><dt>{zh ? "未缓存" : "Uncached"}</dt><dd>{number(turn.usage?.uncached_input_tokens)}</dd></div>
<div><dt>{zh ? "输出" : "Output"}</dt><dd>{number(turn.usage?.output_tokens)}</dd></div>
</dl>
<button type="button" aria-label={zh ? `第 ${turn.ordinal} 轮:查看计数证据` : `Turn ${turn.ordinal}: read counter evidence`}
onClick={event => onRead(turn.turn_id, event.currentTarget)}>{zh ? "计数证据" : "Counter evidence"}</button>
</li>)}
</ol> : <p>{zh ? "此捕获没有可读轮次。" : "No captured Turns are available."}</p>}
{pages > 1 ? <nav aria-label={zh ? "轮次分页" : "Turn pages"}>
<button type="button" disabled={current === 0} onClick={() => setPage(current - 1)}>{zh ? "上一页" : "Previous"}</button>
<span aria-live="polite">{current + 1} / {pages}</span>
<button type="button" disabled={current + 1 === pages} onClick={() => setPage(current + 1)}>{zh ? "下一页" : "Next"}</button>
</nav> : null}
</details>;
}
Binary file added docs/images/research-turn-costs-desktop.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/images/research-turn-costs-mobile.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added docs/images/research-turn-costs-overview.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
40 changes: 40 additions & 0 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -644,6 +644,46 @@ journey or qualify the model assessor.

## Measured solver session cost (optional)

### Find expensive Turns and inspect their evidence

In a Goal's **Research** view, select a registered capture with solver usage.
The first screen keeps aggregate cost and coverage together; expand **Sources
and usage**, then **Find time and input hotspots** to inspect individual Turns.
No analysis execution or model call is needed for this path.

![Research entry with measured totals and unknown attribution](../images/research-turn-costs-overview.png)

Choose **Most input**, **Longest elapsed** or **Chronological**. Each page shows
at most eight Turns with elapsed time, input, its cached subset, uncached input
and output. Cached input is already included in input. An unfinished Turn is
labelled observed-only; a dash means unavailable, never zero. The following
synthetic example puts the largest input Turn first, even though another Turn
took longest.

![Turn input hotspot and cached versus uncached counters](../images/research-turn-costs-desktop.png)

**Counter evidence** opens the existing source reader. It shows the preceding
cumulative counter observation, this Turn's start, its latest counters and its
completion when present, with original line numbers and capture digest. These
are filtered counter/lifecycle fields, not a contiguous transcript: prompts,
tool content and account metadata are excluded. Escape closes the reader and
returns focus to the selected Turn. Sorting, paging and evidence reading also
work in the narrow layout.

![Narrow Turn cost reader](../images/research-turn-costs-mobile.png)

The read endpoint accepts only the registered Goal, pack, analysis key and Turn
identifier. It verifies the complete source digest before returning evidence;
a missing or changed source stays unavailable rather than being substituted.

This inspection compares Turns only within the selected capture. Turn input
is **not** control-layer input; the separate measured LoopX input share retains
its own coverage rules. Elapsed time includes tools, waiting and orchestration.
Different source revisions, capture windows and incomplete runs are not matched
experiments. These counters do not establish waste, monetary cost, avoidable
time or a causal benefit from LoopX. The screenshots use synthetic data from the
retained `research-turn-costs` browser scenario, not a live benchmark.

### Native outcome readback (optional, separate from analyzer judgments)

The trusted provider/Agent may add `--native-outcomes-json native-outcomes.json`
Expand Down
3 changes: 2 additions & 1 deletion examples/personal-workspace-browser-smoke.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {reportDownloadScenario} from "./personal-workspace-browser/report-downlo
import { selectScenarioShard } from "./personal-workspace-browser/shard.mjs";
import {completedHistoryIsolationScenario} from "./personal-workspace-browser/completed-history-isolation.mjs";
import {researchWorkerReadbackScenario} from "./personal-workspace-browser/research-worker-readback.mjs";
import {researchTurnCostsScenario} from "./personal-workspace-browser/research-turn-costs.mjs";
import {researchToolContextScenario} from "./personal-workspace-browser/research-tool-context.mjs";
import { zcodeGoalScenario } from "./personal-workspace-browser/zcode-goal.mjs";
import {goalWorkspaceCreateScenario} from "./personal-workspace-browser/goal-workspace-create.mjs";
Expand Down Expand Up @@ -133,7 +134,7 @@ scenarioCatalog.push(attentionUserActionScenario);
scenarioCatalog.push(researchResultsScenario);
scenarioCatalog.push(researchHistoryScenario);
scenarioCatalog.push(researchMeasurementsScenario);
scenarioCatalog.push(researchToolContextScenario);
scenarioCatalog.push(researchToolContextScenario, researchTurnCostsScenario);
scenarioCatalog.push(researchNativeReadingScenario);
scenarioCatalog.push(researchFindingReaderScenario);
scenarioCatalog.push(researchRecordSearchScenario);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ def main():
config=analysis.get("config", fixture_config),
usage_jsonl=analysis.get("usage_jsonl"),
usage_session_id=analysis.get("usage_session_id"),
include_incomplete_usage=analysis.get("include_incomplete_usage", False),
include_tool_context=analysis.get("include_tool_context", False),
native_outcomes_json=analysis.get("native_outcomes_json"))
for language, output in analysis["outputs"].items():
Expand Down Expand Up @@ -145,7 +146,7 @@ def do_GET(self):
metrics["history_reads"] += 1
if path in {"/healthz", "/status.json", "/api/chat/explore-results",
"/api/chat/capabilities", "/api/chat/endpoints", "/api/chat/sessions", "/api/chat/goal-configuration",
"/api/chat/research-analysis", "/api/chat/research-analysis/citation",
"/api/chat/research-analysis", "/api/chat/research-analysis/citation", "/api/chat/research-analysis/usage-turn",
"/api/actions", "/api/chat/goals/contexts", "/api/chat/goal-results",
"/api/chat/sidebar-goal-order", "/api/chat/usage-statistics", "/api/chat/todo/detail",
"/api/chat/delivery-review", "/api/chat/goal-acceptance", "/api/chat/projects",
Expand Down
2 changes: 1 addition & 1 deletion examples/personal-workspace-browser/research-history.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1762,7 +1762,7 @@ print(json.dumps([
await progress.locator(`.research-progress-scope[data-pack-id="${"d".repeat(64)}"]`).waitFor();

comparisonFixture = true;
await progress.locator('select').selectOption('a'.repeat(64));
await progress.locator(':scope > header select').selectOption('a'.repeat(64));
await progress.locator(`.research-progress-scope[data-pack-id="${"a".repeat(64)}"]`).waitFor();
await page.reload({waitUntil: "networkidle"});
const comparison = progress.locator('[data-metric="goal_alignment"]');
Expand Down
Loading
Loading