diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx index 7c6c492504..1ba9888b4c 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-analysis.tsx @@ -23,7 +23,7 @@ type Pack = ResearchProgressPack & {source_count: number; analysis_key: string; continuation_unavailable?: {base_pack_id: string}; continuation?: TrajectoryContinuation & {base_pack_id: string; base_title: string; base_available: boolean; base_episode_count: number}}; type Snapshot = {ok: true; execution_enabled?: boolean; catalog: {availability: string; reason_code: string | null; default_pack_id: string | null; packs: Pack[]; unreadable_pack_ids?: string[]}; pack: Pack | null}; -type Citation = AssessmentCitation & {artifact: string; text: string; applicability: string}; +type Citation = AssessmentCitation & {evidence_kind?: "usage_counters"; artifact: string; text: string; applicability: string}; const errorText = (error: unknown) => error instanceof Error ? error.message : String(error); const locationFor = (goal: string) => { const query = new URLSearchParams(window.location.search); @@ -175,18 +175,20 @@ export function useResearchAnalysis(goalId: string, active: boolean, zh: boolean } catch (e) {if (current === generation.current) setError(errorText(e));} finally {if (current === generation.current) setCancelPending(false);} }; - const readCitation = async (c: AssessmentCitation, origin?: HTMLButtonElement) => { + const readEvidence = async (selection: Record, route: string, origin?: HTMLButtonElement) => { const p = pack; if (!p) return; if (origin) sourceTrigger.current = origin; const current = ++citeGeneration.current; setCitation(null); setCitationBusy(true); setCitationError(""); - const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, output_language: outputLanguage, source_id: c.source_id, first_line: String(c.first_line), last_line: String(c.last_line)}); - try {const value = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}/citation?${query}`)); if (current === citeGeneration.current) setCitation(value);} + const query = new URLSearchParams({goal_id: goalId, pack_id: p.pack_id, analysis_key: p.analysis_key, output_language: outputLanguage, ...selection}); + try {const value = await requestJson(sourceReadApiUrl(readSourceUrl, `${endpoint}/${route}?${query}`)); if (current === citeGeneration.current) setCitation(value);} catch (e) {if (current === citeGeneration.current) setCitationError(errorText(e));} finally {if (current === citeGeneration.current) setCitationBusy(false);} }; + const readCitation = (c: AssessmentCitation, origin?: HTMLButtonElement) => readEvidence({source_id: c.source_id, first_line: String(c.first_line), last_line: String(c.last_line)}, "citation", origin); + const readUsageTurn = (turnId: string, origin: HTMLButtonElement) => readEvidence({turn_id: turnId}, "usage-turn", origin); const closeCitation = () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); requestAnimationFrame(() => sourceTrigger.current?.focus());}; - return {readOnly: Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true, citation, citationBusy, citationError, sourcePanel, readCitation, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel, + return {readOnly: Boolean(readSourceUrl) && currentSnapshot?.execution_enabled !== true, citation, citationBusy, citationError, sourcePanel, readCitation, readUsageTurn, closeCitation, goalId, zh, location, snapshot: currentSnapshot, pack, readError, error, loading, pending, cancelPending, recordedRunning, panel, trigger, read, show, execute, cancel, close: () => {citeGeneration.current++; setCitation(null); setCitationError(""); setCitationBusy(false); navigate(false); requestAnimationFrame(() => trigger.current?.focus());}, select: (id: string) => navigate(location.open, id, true)}; } type Controller = ReturnType; @@ -279,7 +281,7 @@ export function ResearchAnalysisStatus({analysis: a}: {analysis: Controller}) { export function ResearchProgressSummary({analysis: a, includeReport = true}: {analysis: Controller; includeReport?: boolean}) { const packs = a.snapshot?.catalog.packs ?? []; if (!a.pack && !packs.length) return null; - return ; + return ; } export function ResearchReportSummary({analysis: a}: {analysis: Controller}) { @@ -373,6 +375,7 @@ export function ResearchCitationPanel({analysis: a}: {analysis: Controller}) { {a.citationBusy ?

{zh ? "正在读取引用原文…" : "Reading cited source…"}

: null} {a.citationError ?

{a.citationError}

: null} {a.citation ? <>

{a.citation.artifact} · {zh ? "行" : "lines"} {a.citation.first_line}–{a.citation.last_line}

+ {a.citation.evidence_kind === "usage_counters" ?

{zh ? "计数事件：前一累计观测、本轮开始、末次计数及完成（如有）。仅显示计数字段；提示词、工具内容与账户信息不传入浏览器。" : "Counter events: preceding cumulative observation, Turn start, latest counters and completion if present. Only counter fields are shown; prompts, tool content and account metadata stay outside the browser."}

: null} : null} ; } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css index b0dff8caf6..8e618a113d 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css @@ -147,3 +147,14 @@ .research-usage-basis { grid-template-columns: 1fr; gap: 4px; } .research-progress-basis dd, .research-usage-basis dd { margin-bottom: 8px; } } + +.research-turn-costs { border-top: 1px solid var(--pw-line); padding-top: 12px; margin-top: 12px; } +.research-turn-costs > p { max-width: 72ch; color: var(--pw-muted); font-size: 12px; } +.research-turn-cost-list { list-style: none; padding: 0; margin: 12px 0; } +.research-turn-cost-list > li { padding: 12px 0; border-top: 1px solid var(--pw-line); } +.research-turn-cost-list time { display: block; font-size: 11px; color: var(--pw-muted); overflow-wrap: anywhere; } +.research-turn-cost-list dl { display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 12px; margin: 8px 0; } +.research-turn-cost-list dt { font-size: 11px; color: var(--pw-muted); } +.research-turn-cost-list dd { margin: 0; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.research-turn-costs nav { display: flex; align-items: center; gap: 12px; } +@media (max-width: 600px) { .research-turn-cost-list dl { grid-template-columns: repeat(2, minmax(0, 1fr)); } } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx index 80e117af02..17220c65c9 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx @@ -5,6 +5,7 @@ import type {AssessmentCitation, AssessmentVerdict, TrajectoryAssessment, Trajec import type {TrajectoryNativeOutcomes} from "../../../../../../loopx/control_plane/capabilities/trajectory_native_outcomes.js"; import {ContextHelp} from "./context-help"; import {ResearchSynthesis} from "./goal-research-synthesis"; +import {ResearchTurnCosts} from "./research-turn-costs"; import {ToolContextInventory} from "./goal-tool-context"; import "./goal-research-progress.css"; @@ -253,7 +254,7 @@ function ResearchMetrics({pack: p, zh, onCitation}: {pack: ResearchProgressPack ; } -export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCitation, analysisOpen = false, includeReport = true}: {pack: ResearchProgressPack | null; packs: Pick[]; selectedId: string; zh: boolean; analysisOpen?: boolean; includeReport?: boolean; onSelect: (id: string) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) { +export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCitation, onUsageTurn, analysisOpen = false, includeReport = true}: {pack: ResearchProgressPack | null; packs: Pick[]; selectedId: string; zh: boolean; analysisOpen?: boolean; includeReport?: boolean; onSelect: (id: string) => void; onUsageTurn?: (turnId: string, origin: HTMLButtonElement) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) { const measured = p?.solver_usage, usage = measured?.projection, result = p?.result; const historical = p?.applicability === "historical"; const resultHelp = historical ? (result ? (zh ? "历史分析 · 当前判断未核验" : "Historical analysis · Current judgments unverified") @@ -334,6 +335,7 @@ export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCi : null} : null} + {usage && onUsageTurn ? : null} {usage && measured ?
{zh ? "计数明细与来源" : "Counter details and source"}
{zh ? "捕获时间（UTC）" : "Capture window (UTC)"}
—
diff --git a/apps/presentation/dashboard/src/features/personal-workspace/research-turn-costs.tsx b/apps/presentation/dashboard/src/features/personal-workspace/research-turn-costs.tsx new file mode 100644 index 0000000000..f542aa341d --- /dev/null +++ b/apps/presentation/dashboard/src/features/personal-workspace/research-turn-costs.tsx @@ -0,0 +1,54 @@ +import {useId, useState} from "react"; +import type {TrajectorySolverUsage} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js"; + +/** Presentation of existing Turn counters; never estimates missing usage or attributes it to control work. */ +export function ResearchTurnCosts({usage, zh, onRead}: {usage: TrajectorySolverUsage; zh: boolean; + onRead: (turnId: string, origin: HTMLButtonElement) => void}) { + const id = useId(); + const [sort, setSort] = useState("input"); + const [page, setPage] = useState(0); + const rows: Array & {usage: TrajectorySolverUsage["turns"][number]["usage"] | null; ordinal: number; incomplete: boolean}> = usage.turns.map((turn, index) => ({...turn, ordinal: index + 1, incomplete: false})); + const tail = usage.incomplete_turn; + if (tail) rows.push({...tail, end: tail.observed_through, + elapsed_seconds: Math.max(0, (Date.parse(tail.observed_through) - Date.parse(tail.start)) / 1000), + ordinal: rows.length + 1, incomplete: true, usage: tail.usage}); + const ordered = [...rows].sort((a, b) => { + const value = (row: typeof a) => sort === "elapsed" ? row.elapsed_seconds + : sort === "time" ? row.ordinal : row.usage?.input_tokens ?? -1; + return (sort === "time" ? value(a) - value(b) : value(b) - value(a)) || a.ordinal - b.ordinal; + }); + const size = 8, pages = Math.max(1, Math.ceil(rows.length / size)), current = Math.min(page, pages - 1); + const number = (n: number | undefined | null) => n == null ? "—" : n.toLocaleString(zh ? "zh-CN" : "en"); + return
+ {zh ? "定位耗时与输入最多的轮次" : "Find time and input hotspots"} +

{zh ? "仅比较当前捕获内的轮次。耗时含工具、等待与编排，不是可节省时间；轮次输入不等于 LoopX 控制输入。金额未采集。" : "Compares Turns within this capture only. Elapsed time includes tools, waiting and orchestration, not avoidable time. Turn input is not LoopX control input. Dollar cost is unmeasured."}

+ {" "} + + {rows.length ?
    + {ordered.slice(current * size, (current + 1) * size).map(turn =>
  1. +
    {zh ? `第 ${turn.ordinal} 轮` : `Turn ${turn.ordinal}`} + {turn.incomplete ? {zh ? " · 未完成（仅已观测）" : " · Incomplete (observed only)"} : null} + +
    +
    +
    {zh ? "耗时" : "Elapsed"}
    {(turn.elapsed_seconds / 60).toLocaleString(zh ? "zh-CN" : "en", {maximumFractionDigits: 1})} {zh ? "分钟" : "min"}
    +
    {zh ? "输入" : "Input"}
    {number(turn.usage?.input_tokens)}
    +
    {zh ? "其中缓存" : "Cached subset"}
    {number(turn.usage?.cached_input_tokens)}
    +
    {zh ? "未缓存" : "Uncached"}
    {number(turn.usage?.uncached_input_tokens)}
    +
    {zh ? "输出" : "Output"}
    {number(turn.usage?.output_tokens)}
    +
    + +
    2. )} +
:

{zh ? "此捕获没有可读轮次。" : "No captured Turns are available."}

} + {pages > 1 ? : null} +
; +} diff --git a/docs/images/research-turn-costs-desktop.png b/docs/images/research-turn-costs-desktop.png new file mode 100644 index 0000000000..960502ad18 Binary files /dev/null and b/docs/images/research-turn-costs-desktop.png differ diff --git a/docs/images/research-turn-costs-mobile.png b/docs/images/research-turn-costs-mobile.png new file mode 100644 index 0000000000..d59c0902d3 Binary files /dev/null and b/docs/images/research-turn-costs-mobile.png differ diff --git a/docs/images/research-turn-costs-overview.png b/docs/images/research-turn-costs-overview.png new file mode 100644 index 0000000000..b94cfcaf85 Binary files /dev/null and b/docs/images/research-turn-costs-overview.png differ diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index e4b39714e0..542d31ee87 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -644,6 +644,46 @@ journey or qualify the model assessor. ## Measured solver session cost (optional) +### Find expensive Turns and inspect their evidence + +In a Goal's **Research** view, select a registered capture with solver usage. +The first screen keeps aggregate cost and coverage together; expand **Sources +and usage**, then **Find time and input hotspots** to inspect individual Turns. +No analysis execution or model call is needed for this path. + +![Research entry with measured totals and unknown attribution](../images/research-turn-costs-overview.png) + +Choose **Most input**, **Longest elapsed** or **Chronological**. Each page shows +at most eight Turns with elapsed time, input, its cached subset, uncached input +and output. Cached input is already included in input. An unfinished Turn is +labelled observed-only; a dash means unavailable, never zero. The following +synthetic example puts the largest input Turn first, even though another Turn +took longest. + +![Turn input hotspot and cached versus uncached counters](../images/research-turn-costs-desktop.png) + +**Counter evidence** opens the existing source reader. It shows the preceding +cumulative counter observation, this Turn's start, its latest counters and its +completion when present, with original line numbers and capture digest. These +are filtered counter/lifecycle fields, not a contiguous transcript: prompts, +tool content and account metadata are excluded. Escape closes the reader and +returns focus to the selected Turn. Sorting, paging and evidence reading also +work in the narrow layout. + +![Narrow Turn cost reader](../images/research-turn-costs-mobile.png) + +The read endpoint accepts only the registered Goal, pack, analysis key and Turn +identifier. It verifies the complete source digest before returning evidence; +a missing or changed source stays unavailable rather than being substituted. + +This inspection compares Turns only within the selected capture. Turn input +is **not** control-layer input; the separate measured LoopX input share retains +its own coverage rules. Elapsed time includes tools, waiting and orchestration. +Different source revisions, capture windows and incomplete runs are not matched +experiments. These counters do not establish waste, monetary cost, avoidable +time or a causal benefit from LoopX. The screenshots use synthetic data from the +retained `research-turn-costs` browser scenario, not a live benchmark. + ### Native outcome readback (optional, separate from analyzer judgments) The trusted provider/Agent may add `--native-outcomes-json native-outcomes.json` diff --git a/examples/personal-workspace-browser-smoke.mjs b/examples/personal-workspace-browser-smoke.mjs index 120acecbca..4978957dbb 100644 --- a/examples/personal-workspace-browser-smoke.mjs +++ b/examples/personal-workspace-browser-smoke.mjs @@ -4,6 +4,7 @@ import {reportDownloadScenario} from "./personal-workspace-browser/report-downlo import { selectScenarioShard } from "./personal-workspace-browser/shard.mjs"; import {completedHistoryIsolationScenario} from "./personal-workspace-browser/completed-history-isolation.mjs"; import {researchWorkerReadbackScenario} from "./personal-workspace-browser/research-worker-readback.mjs"; +import {researchTurnCostsScenario} from "./personal-workspace-browser/research-turn-costs.mjs"; import {researchToolContextScenario} from "./personal-workspace-browser/research-tool-context.mjs"; import { zcodeGoalScenario } from "./personal-workspace-browser/zcode-goal.mjs"; import {goalWorkspaceCreateScenario} from "./personal-workspace-browser/goal-workspace-create.mjs"; @@ -133,7 +134,7 @@ scenarioCatalog.push(attentionUserActionScenario); scenarioCatalog.push(researchResultsScenario); scenarioCatalog.push(researchHistoryScenario); scenarioCatalog.push(researchMeasurementsScenario); -scenarioCatalog.push(researchToolContextScenario); +scenarioCatalog.push(researchToolContextScenario, researchTurnCostsScenario); scenarioCatalog.push(researchNativeReadingScenario); scenarioCatalog.push(researchFindingReaderScenario); scenarioCatalog.push(researchRecordSearchScenario); diff --git a/examples/personal-workspace-browser/research-history-backend.py b/examples/personal-workspace-browser/research-history-backend.py index b4ad045ff1..186c37c2d0 100644 --- a/examples/personal-workspace-browser/research-history-backend.py +++ b/examples/personal-workspace-browser/research-history-backend.py @@ -105,6 +105,7 @@ def main(): config=analysis.get("config", fixture_config), usage_jsonl=analysis.get("usage_jsonl"), usage_session_id=analysis.get("usage_session_id"), + include_incomplete_usage=analysis.get("include_incomplete_usage", False), include_tool_context=analysis.get("include_tool_context", False), native_outcomes_json=analysis.get("native_outcomes_json")) for language, output in analysis["outputs"].items(): @@ -145,7 +146,7 @@ def do_GET(self): metrics["history_reads"] += 1 if path in {"/healthz", "/status.json", "/api/chat/explore-results", "/api/chat/capabilities", "/api/chat/endpoints", "/api/chat/sessions", "/api/chat/goal-configuration", - "/api/chat/research-analysis", "/api/chat/research-analysis/citation", + "/api/chat/research-analysis", "/api/chat/research-analysis/citation", "/api/chat/research-analysis/usage-turn", "/api/actions", "/api/chat/goals/contexts", "/api/chat/goal-results", "/api/chat/sidebar-goal-order", "/api/chat/usage-statistics", "/api/chat/todo/detail", "/api/chat/delivery-review", "/api/chat/goal-acceptance", "/api/chat/projects", diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index c519a55b42..9e0c47fea1 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -1762,7 +1762,7 @@ print(json.dumps([ await progress.locator(`.research-progress-scope[data-pack-id="${"d".repeat(64)}"]`).waitFor(); comparisonFixture = true; - await progress.locator('select').selectOption('a'.repeat(64)); + await progress.locator(':scope > header select').selectOption('a'.repeat(64)); await progress.locator(`.research-progress-scope[data-pack-id="${"a".repeat(64)}"]`).waitFor(); await page.reload({waitUntil: "networkidle"}); const comparison = progress.locator('[data-metric="goal_alignment"]'); diff --git a/examples/personal-workspace-browser/research-native-reading.mjs b/examples/personal-workspace-browser/research-native-reading.mjs index 302bad1e44..ab0168c969 100644 --- a/examples/personal-workspace-browser/research-native-reading.mjs +++ b/examples/personal-workspace-browser/research-native-reading.mjs @@ -100,7 +100,7 @@ export const researchNativeReadingScenario = { await page.addInitScript(() => {if (!localStorage.getItem("loopx-pw-locale")) localStorage.setItem("loopx-pw-locale", "zh-CN");}); await page.goto(`${owner.base}/chat/?goalId=${fixture.goal_id}&view=research&statusUrl=%2Fstatus.json`, {waitUntil: "domcontentloaded"}); const native = page.getByRole("region", {name: "原生测量", exact: true}); - const selector = page.locator(".research-progress select"); + const selector = page.locator(".research-progress > header select"); await native.waitFor(); const pausedClock = await page.addStyleTag({content: "*, *::before, *::after {animation-play-state: paused !important;}"}); const tabs = page.getByRole("navigation", {name: "Goal 视图", exact: true}); diff --git a/examples/personal-workspace-browser/research-tool-context.mjs b/examples/personal-workspace-browser/research-tool-context.mjs index 22c6b3fd86..7bb9ba2f78 100644 --- a/examples/personal-workspace-browser/research-tool-context.mjs +++ b/examples/personal-workspace-browser/research-tool-context.mjs @@ -56,7 +56,7 @@ Path(sys.argv[1]).write_text(json.dumps(dict(goal_id=goal,display_name='Syntheti page.on("pageerror", error => errors.push(error.message)); await page.goto(`${owner.base}/chat/?goalId=synthetic-context-reading&view=research&research_goal=synthetic-context-reading&statusUrl=%2Fstatus.json`, {waitUntil: "networkidle"}); const progress = page.locator('.research-progress'); - const selector = progress.locator('select'); + const selector = progress.locator(':scope > header select'); await selector.selectOption({label: 'Without tool inventory'}); await progress.locator('.research-usage').waitFor({state: 'attached'}); await page.screenshot({path: resolve(outputDir, 'research-tool-context-before.png'), animations: 'disabled'}); diff --git a/examples/personal-workspace-browser/research-turn-costs.mjs b/examples/personal-workspace-browser/research-turn-costs.mjs new file mode 100644 index 0000000000..aa18f98ac2 --- /dev/null +++ b/examples/personal-workspace-browser/research-turn-costs.mjs @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict'; +import {execFileSync} from 'node:child_process'; +import {mkdtemp, rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {resolve} from 'node:path'; +import {resolveTestPython} from '../../scripts/test-python.mjs'; +import {outputDir, repoRoot} from './fixture.mjs'; +import {startHistoryOwner} from './research-history.mjs'; + +export const researchTurnCostsScenario = { + id: 'research-turn-costs', + async run({browser}) { + const temporary = await mkdtemp(resolve(tmpdir(), 'loopx-turn-costs-')); + let owner, page; + try { + const fixture = resolve(temporary, 'capture.json'); + execFileSync(resolveTestPython(), ['-c', ` +import hashlib,json,sys +from datetime import datetime,timedelta,timezone +from pathlib import Path +from loopx.capabilities.explore.result_log import build_explore_node_event +goal='synthetic-cost-study' +text='Compare observed costs; no causal benefit is established.' +pack=dict(schema_version='trajectory_assessment_input_v1',scope=dict(goal_id=goal,case_id='synthetic',run_id='fixed',revision='fixture',window='one captured session'),objective=text, + selection=dict(rule='synthetic fixture',population_episodes=1,exhaustive=True), + sources=[dict(id='facts',artifact='facts.txt',artifact_sha256=hashlib.sha256(text.encode()).hexdigest(),first_line=1,text=text)], + episodes=[dict(id='episode',question=text,source_ids=['facts'],eligible_metrics=['evidence_adoption'])]) +rows=[];seconds=0;totals=dict(input_tokens=0,cached_input_tokens=0,output_tokens=0) +def append(kind,payload): + rows.append(dict(type=kind,timestamp=(datetime(2026,1,1,tzinfo=timezone.utc)+timedelta(seconds=seconds)).isoformat(),payload=payload)) +append('session_meta',dict(id='session')) +for i in range(10): + seconds+=1;append('event_msg',dict(type='task_started',turn_id=f'turn-{i+1}')) + seconds+=1;append('response_item',dict(type='message',role='user',content='SENSITIVE_TEXT_NOT_FOR_BROWSER')) + counts=dict(input_tokens=5000 if i==2 else 100,cached_input_tokens=3000 if i==2 else 60,output_tokens=10) + totals={key:totals[key]+counts[key] for key in totals} + seconds+=120 if i==6 else 1 + append('event_msg',dict(type='token_count',info=dict(total_token_usage=totals.copy(),last_token_usage=counts))) + seconds+=1;append('event_msg',dict(type='task_complete',turn_id=f'turn-{i+1}')) +seconds+=1;append('event_msg',dict(type='task_started',turn_id='unfinished')) +analysis=dict(title='Observed session',input=pack,artifacts={'facts.txt':text,'usage.jsonl':'\\n'.join(map(json.dumps,rows))},outputs={},usage_jsonl='usage.jsonl',usage_session_id='session',include_incomplete_usage=True) +Path(sys.argv[1]).write_text(json.dumps(dict(goal_id=goal,display_name='Synthetic cost study',events=[build_explore_node_event(goal_id=goal,node_id='question',title=text,recorded_at='2026-01-01T00:00:00Z')],analyses=[analysis]))) +`, fixture], {cwd: repoRoot}); + owner = await startHistoryOwner(fixture); + page = await browser.newPage({viewport: {width: 1440, height: 1000}}); + const errors = [], writes = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('request', request => {if (request.method() === 'POST') writes.push(request.url());}); + await page.goto(`${owner.base}/chat/?goalId=synthetic-cost-study&view=research&research_goal=synthetic-cost-study&statusUrl=%2Fstatus.json`, {waitUntil: 'networkidle'}); + const progress = page.locator('.research-progress'); + await progress.locator('.research-usage').waitFor({state: 'attached'}); + await page.screenshot({path: resolve(outputDir, 'research-turn-costs-overview.png'), animations: 'disabled'}); + await progress.locator('.research-progress-basis > summary').click(); + const costs = progress.locator('.research-turn-costs'); + await costs.locator('summary').focus(); await page.keyboard.press('Enter'); + const items = costs.locator('li'); + assert.equal(await items.count(), 8); + assert.match(await items.first().innerText(), /第 3 轮/); + assert.match(await items.first().innerText(), /5,000/); + assert.match(await items.first().innerText(), /3,000/); + assert.match(await items.first().innerText(), /2,000/); + await costs.getByRole('combobox').selectOption('elapsed'); + assert.match(await items.first().innerText(), /第 7 轮/); + await items.first().getByRole('button').click(); + const source = page.getByRole('region', {name: '引用原文', exact: true}); + await source.locator('pre').waitFor(); + assert.match(await source.innerText(), /turn-7/); + assert.match(await source.innerText(), /token_count/); + assert.ok(!(await source.innerText()).includes('SENSITIVE_TEXT_NOT_FOR_BROWSER')); + await source.focus(); await page.keyboard.press('Escape'); + assert.equal(await items.first().getByRole('button').evaluate(element => element === document.activeElement), true); + await costs.getByRole('combobox').selectOption('time'); + await costs.getByRole('button', {name: '下一页', exact: true}).click(); + assert.equal(await items.count(), 3); + assert.match(await items.last().innerText(), /未完成/); + assert.match(await items.last().innerText(), /—/); + await items.last().getByRole('button').click(); + await source.locator('pre').waitFor(); + assert.match(await source.innerText(), /unfinished/); + await source.getByRole('button', {name: '关闭原文', exact: true}).click(); + await costs.getByRole('combobox').selectOption('input'); + for (const width of [1440, 390]) { + await page.setViewportSize({width, height: 1000}); + await costs.locator('summary').evaluate((element, offset) => {element.style.scrollMarginTop = `${offset}px`; element.scrollIntoView({block: 'start'});}, width < 600 ? 190 : 125); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + await page.screenshot({path: resolve(outputDir, `research-turn-costs-${width}.png`), animations: 'disabled'}); + } + assert.deepEqual(errors, []); + assert.deepEqual(writes, [], 'Cost inspection never starts analysis or mutates Goal state'); + return {coverageEntries: [], note: 'Real registered capture, typed reduction, HTTP and packaged UI: sort, pagination, incomplete/unknown counters, exact source evidence, keyboard return and mobile. No model or live benchmark.'}; + } catch (error) { + if (page && !page.isClosed()) await page.screenshot({path: resolve(outputDir, 'research-turn-costs-failed.png')}); + throw error; + } finally { + if (page) await page.close(); + if (owner) await owner.close(); + await rm(temporary, {recursive: true, force: true}); + } + }, +}; diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py index 0ba0b28b63..5fadb9403e 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py @@ -144,6 +144,67 @@ def _source_digest(path: Path) -> str: return hashlib.file_digest(stream, "sha256").hexdigest() +def _usage_event(line_number: int, event: dict) -> dict | None: + payload = event.get("payload") or {} + if not isinstance(payload, dict): + raise ValueError("Usage capture contains a non-object payload") + if event.get("type") == "session_meta": + payload = {k: payload[k] for k in ("id", "forked_from_id") if k in payload} + elif event.get("type") == "token_usage_record": + payload = _request_usage_record(payload) + elif event.get("type") == "loopx_input_token_attribution": + # Optional host measurement, not a tokenizer or prose classifier. + # Preserve only binding/counts; raw prompts and packet text stay private. + payload = { + key: payload[key] if isinstance(payload[key], str) else None + for key in ( + "method", "thread_id", "session_id", "turn_id", + "root_turn_id", "response_id", + ) if key in payload + } | { + key: payload[key] if type(payload[key]) is int else None + for key in ( + "input_tokens", "loopx_input_tokens", "loopx_cached_input_tokens", + ) if key in payload + } + elif event.get("type") == "compacted": + payload = { + "compaction_response_id": payload.get("compaction_response_id") + if isinstance(payload.get("compaction_response_id"), str) + else None, + "latest_token_usage_record": _request_usage_record( + payload.get("latest_token_usage_record") + ), + } + elif event.get("type") == "event_msg" and payload.get("type") in { + "task_started", + "task_complete", + "token_count", + }: + payload = { + k: payload[k] for k in ("type", "turn_id", "info") if k in payload + } + if isinstance(payload.get("info"), dict): + payload["info"] = { + name: { + key: value[key] + for key in ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + ) + if key in value + } + if isinstance(value, dict) + else value + for name in ("total_token_usage", "last_token_usage") + if (value := payload["info"].get(name)) is not None + } + else: + return None + return {"line": line_number, "type": event["type"], "timestamp": event.get("timestamp"), "payload": payload} + + def _usage_capture( source_root: Path, artifact: str, @@ -166,71 +227,9 @@ def _usage_capture( if include_tool_context: from .trajectory_tool_context import project_tool_context tool_context.extend(project_tool_context([event], first_line=line_number)) - payload = event.get("payload") or {} - if not isinstance(payload, dict): - raise ValueError("Usage capture contains a non-object payload") - if event.get("type") == "session_meta": - payload = {k: payload[k] for k in ("id", "forked_from_id") if k in payload} - elif event.get("type") == "token_usage_record": - payload = _request_usage_record(payload) - elif event.get("type") == "loopx_input_token_attribution": - # Optional host measurement, not a tokenizer or prose classifier. - # Preserve only binding/counts; raw prompts and packet text stay private. - payload = { - key: payload[key] if isinstance(payload[key], str) else None - for key in ( - "method", "thread_id", "session_id", "turn_id", - "root_turn_id", "response_id", - ) if key in payload - } | { - key: payload[key] if type(payload[key]) is int else None - for key in ( - "input_tokens", "loopx_input_tokens", "loopx_cached_input_tokens", - ) if key in payload - } - elif event.get("type") == "compacted": - payload = { - "compaction_response_id": payload.get("compaction_response_id") - if isinstance(payload.get("compaction_response_id"), str) - else None, - "latest_token_usage_record": _request_usage_record( - payload.get("latest_token_usage_record") - ), - } - elif event.get("type") == "event_msg" and payload.get("type") in { - "task_started", - "task_complete", - "token_count", - }: - payload = { - k: payload[k] for k in ("type", "turn_id", "info") if k in payload - } - if isinstance(payload.get("info"), dict): - payload["info"] = { - name: { - key: value[key] - for key in ( - "input_tokens", - "cached_input_tokens", - "output_tokens", - ) - if key in value - } - if isinstance(value, dict) - else value - for name in ("total_token_usage", "last_token_usage") - if (value := payload["info"].get(name)) is not None - } - else: - continue - events.append( - { - "line": line_number, - "type": event["type"], - "timestamp": event.get("timestamp"), - "payload": payload, - } - ) + observed = _usage_event(line_number, event) + if observed is not None: + events.append(observed) observation = None input_artifacts = [] if loopx_input_json is not None: @@ -739,3 +738,60 @@ def read_citation( "applicability": "historical" if requested in native_citations and view["native_outcomes"]["applicability"] == "historical" else view["applicability"], } + + +def read_usage_turn(*, runtime_root: Path, owner_goal_id: str, pack_id: str, + analysis_key: str, turn_id: str, output_language: str | None = None) -> dict: + """Read bounded counter evidence for an existing captured Turn, never raw model/tool text.""" + _, record = _resolve(runtime_root, owner_goal_id, pack_id) + identity = _prepared(record, output_language) if output_language is not None else record + if analysis_key != identity["analysis_key"]: + raise PackUnavailable("analysis_key_mismatch") + captured = record.get("solver_usage") + if not captured: + raise PackUnavailable("usage_not_captured") + projection = captured["projection"] + turns = [*projection["turns"]] + if projection.get("incomplete_turn"): + turns.append(projection["incomplete_turn"]) + matches = [turn for turn in turns if turn["turn_id"] == turn_id] + if len(matches) != 1: + raise PackUnavailable("usage_turn_not_captured") + turn = matches[0] + first, last = turn["first_line"], turn["last_line"] + digest = hashlib.sha256() + baseline = start = counter = end = None + try: + path = _source_path(Path(record["source_root"]), captured["artifact"]) + for line, event in _usage_rows(path, digest): + if not isinstance(event, dict): + raise ValueError("invalid usage event") + if line > last: + continue # Still hash the entire source before disclosing any evidence. + observed = _usage_event(line, event) + if not observed or observed["type"] != "event_msg": + continue + kind = observed["payload"].get("type") + if line < first: + if kind == "token_count" and observed["payload"].get("info"): + baseline = observed + elif kind == "task_started": + start = observed + elif kind == "token_count" and observed["payload"].get("info"): + counter = observed + elif kind == "task_complete": + end = observed + if digest.hexdigest() != captured["artifact_sha256"]: + raise PackUnavailable("source_changed") + except PackUnavailable: + raise + except (OSError, ValueError, TypeError, KeyError): + raise PackUnavailable("source_unavailable") from None + events = [event for event in (baseline, start, counter, end) if event is not None] + if not events: + raise PackUnavailable("usage_evidence_unavailable") + return {"source_id": "solver_usage", "artifact": captured["artifact"], + "first_line": events[0]["line"], "last_line": events[-1]["line"], + "applicability": "current", "evidence_kind": "usage_counters", + "text": json.dumps({"turn_id": turn_id, "artifact_sha256": captured["artifact_sha256"], + "events": events}, ensure_ascii=False, indent=2)} diff --git a/loopx/chat_server.py b/loopx/chat_server.py index 9b9f46f571..d6ec5d89c1 100644 --- a/loopx/chat_server.py +++ b/loopx/chat_server.py @@ -1705,6 +1705,7 @@ def do_GET(self) -> None: "/api/chat/explore-results": self._explore_results, RESEARCH_ANALYSIS_PATH: self._research_analysis_read, RESEARCH_ANALYSIS_PATH + "/citation": self._research_analysis_read, + RESEARCH_ANALYSIS_PATH + "/usage-turn": self._research_analysis_read, CHAT_SESSIONS_PATH: self._list_sessions, CHAT_ACTIONS_PATH: self._action_list, CHAT_GOAL_CONTEXTS_PATH: self._goal_contexts, diff --git a/loopx/presentation/research_analysis_api.py b/loopx/presentation/research_analysis_api.py index 64a4d72e0c..f90a51dc6c 100644 --- a/loopx/presentation/research_analysis_api.py +++ b/loopx/presentation/research_analysis_api.py @@ -63,9 +63,12 @@ def _research_analysis_scope(self, goal_id, *, cancel=False): def _research_analysis_read(self): try: query = self._research_analysis_query() + usage_turn = urlparse(self.path).path.endswith("/usage-turn") citation = urlparse(self.path).path.endswith("/citation") required = {"goal_id", "pack_id", "analysis_key", "source_id", "first_line", "last_line"} if citation else {"goal_id"} - allowed = (required if citation else {"goal_id", "pack_id"}) | {"output_language"} + if usage_turn: + required = {"goal_id", "pack_id", "analysis_key", "turn_id"} + allowed = (required if citation or usage_turn else {"goal_id", "pack_id"}) | {"output_language"} if not required <= query.keys() or query.keys() - allowed or any(len(v) != 1 for v in query.values()): raise ValueError("Use the registered Goal and opaque pack identifiers") values = {key: value[0] for key, value in query.items()} @@ -76,7 +79,11 @@ def _research_analysis_read(self): runtime = self._research_analysis_scope(goal_id) if runtime is None: return - if citation: + if usage_turn: + self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"]) + result = store.read_usage_turn(runtime_root=runtime, owner_goal_id=goal_id, + output_language=language, **values) + elif citation: self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"]) first = int(values.pop("first_line")) last = int(values.pop("last_line")) diff --git a/loopx/presentation/runtime_workspace.py b/loopx/presentation/runtime_workspace.py index 9390ea338f..eaadbb7626 100644 --- a/loopx/presentation/runtime_workspace.py +++ b/loopx/presentation/runtime_workspace.py @@ -181,7 +181,7 @@ def do_GET(self): self._json(self.server.reader.findings(goal_id, cursor, projector=_finding_rows)) else: self._json({"ok": False, "error": "Unknown Explore view."}, 400) - elif parsed.path in {RESEARCH_ANALYSIS_PATH, RESEARCH_ANALYSIS_PATH + "/citation"}: + elif parsed.path in {RESEARCH_ANALYSIS_PATH, RESEARCH_ANALYSIS_PATH + "/citation", RESEARCH_ANALYSIS_PATH + "/usage-turn"}: if self.server.analysis_execution_enabled: self._research_analysis_read() else: diff --git a/tests/capabilities/test_trajectory_assessment_store.py b/tests/capabilities/test_trajectory_assessment_store.py index 24e4b65ea4..10b81eb0c9 100644 --- a/tests/capabilities/test_trajectory_assessment_store.py +++ b/tests/capabilities/test_trajectory_assessment_store.py @@ -1368,3 +1368,42 @@ def observe(command, params, **kwargs): assert path.read_bytes() == original assert view["job"] is None and view["result"] is None assert "loopx_input" not in inspect_pack(**scope, pack_id=old["pack_id"]).get("solver_usage", {}).get("projection", {}) + + +@pytest.mark.parametrize("incomplete", [False, True]) +def test_turn_counter_evidence_preserves_baseline_and_excludes_content(pack, incomplete): + from loopx.capabilities.benchmark_toolkit.trajectory_assessment_store import read_usage_turn + + scope, _, old, input_path, sources, _ = pack + rows = [json.loads(line) for line in _usage_fixture().splitlines()] + for row in rows: + row["payload"]["private_prompt"] = "MUST_NOT_REACH_BROWSER" + if incomplete: + rows.pop() + capture = sources / "usage.jsonl" + capture.write_text("\n".join(map(json.dumps, rows)) + "\n") + registered = register_pack(**scope, input_path=input_path, source_root=sources, + title=old["title"], usage_jsonl="usage.jsonl", usage_session_id="solver-session", + include_incomplete_usage=incomplete) + operation = dict(**scope, pack_id=registered["pack_id"], analysis_key=registered["analysis_key"]) + before = {path: path.read_bytes() for path in scope["runtime_root"].rglob("*") if path.is_file()} + evidence = read_usage_turn(**operation, turn_id="b") + events = json.loads(evidence["text"])["events"] + assert [event["line"] for event in events] == ([3, 5, 6] if incomplete else [3, 5, 6, 7]) + assert events[0]["payload"]["info"]["total_token_usage"]["input_tokens"] == 100 + assert events[2]["payload"]["info"]["total_token_usage"]["input_tokens"] == 140 + assert "MUST_NOT_REACH_BROWSER" not in json.dumps(evidence) + assert evidence["evidence_kind"] == "usage_counters" + assert not list(scope["runtime_root"].rglob("receipt.json")) + assert all(path.read_bytes() == content for path, content in before.items()) + for changes, reason in [({"turn_id": "other"}, "usage_turn_not_captured"), + ({"analysis_key": "0" * 64}, "analysis_key_mismatch"), + ({"owner_goal_id": "other-owner"}, "pack_not_registered")]: + with pytest.raises(PackUnavailable, match=reason): + read_usage_turn(**(operation | {"turn_id": "b"} | changes)) + capture.write_text(capture.read_text() + "{}\n") + with pytest.raises(PackUnavailable, match="source_changed"): + read_usage_turn(**operation, turn_id="b") + capture.unlink() + with pytest.raises(PackUnavailable, match="source_unavailable"): + read_usage_turn(**operation, turn_id="b") diff --git a/tests/presentation/test_research_analysis_api.py b/tests/presentation/test_research_analysis_api.py index d2aedad164..8933315f17 100644 --- a/tests/presentation/test_research_analysis_api.py +++ b/tests/presentation/test_research_analysis_api.py @@ -794,3 +794,29 @@ def test_synthesis_only_citation_uses_existing_reader_and_does_not_execute(works read(suffix="/citation", **query, **{**citation,"last_line":3}) assert w["count"].read_text().splitlines() == ["started"] assert json.loads(path.read_text()) == data + + +def test_usage_turn_http_reads_registered_counter_evidence_without_analysis(workspace): + from tests.capabilities.test_trajectory_assessment_store import _usage_fixture + + w = workspace + capture = w["sources"] / "usage.jsonl" + capture.write_text(_usage_fixture()) + registered = register_pack(runtime_root=w["tmp_path"] / "runtime", owner_goal_id="research-owner", + input_path=w["tmp_path"] / "input.json", source_root=w["sources"], title="Captured Turn costs", + usage_jsonl="usage.jsonl", usage_session_id="solver-session") + query = dict(pack_id=registered["pack_id"], analysis_key=registered["analysis_key"], turn_id="b") + result = w["request"](suffix="/usage-turn", **query) + assert result["evidence_kind"] == "usage_counters" + assert [event["line"] for event in json.loads(result["text"])["events"]] == [3, 5, 6, 7] + assert not w["count"].exists(), "Reading costs must not invoke the analysis provider" + for invalid in [dict(query, turn_id="unknown"), dict(query, first_line="1"), + dict(query, goal="other-owner"), dict(query, analysis_key="0" * 64)]: + with pytest.raises(HTTPError) as error: + w["request"](suffix="/usage-turn", **invalid) + assert error.value.code in {400, 409} + capture.write_text(capture.read_text() + "{}\n") + with pytest.raises(HTTPError) as error: + w["request"](suffix="/usage-turn", **query) + assert error.value.code == 409 + assert not w["count"].exists()