From ce68f353f8d9181ce315c77547ba307250844c85 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:00:43 +0300 Subject: [PATCH 01/36] fix(llm): key streamed tool calls by id when the provider sends no index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI-compatible providers that emit one whole tool call per SSE event, with an `id` but no `index`, had every call folded onto slot 0: the parser substituted the call's position within its own event for the missing index, and the stream consumer keyed its accumulator on that number alone. Two parallel calls arrived as one, with the second call's name and arguments concatenated onto the first — a tool the registry has never heard of, or worse, the right tool run with spliced arguments. The parser now reports `index` only when the provider actually sent one, and the consumer resolves slot identity across events: provider index first, then call id, then position among the slots already open (which is what an id-less continuation delta means). Provider indexes still decide the output order, so a stream that opens slot 1 before slot 0 comes out in the provider's order; unindexed slots queue up behind in arrival order. Fixes #103 Co-Authored-By: Claude Opus 5 (1M context) --- .../openai/openai-stream-consumer.test.ts | 193 ++++++++++++++++++ .../provider/openai/openai-stream-consumer.ts | 93 ++++++++- src/llm/provider/openai/parse-sse-chunk.ts | 13 +- 3 files changed, 285 insertions(+), 14 deletions(-) create mode 100644 src/llm/provider/openai/openai-stream-consumer.test.ts diff --git a/src/llm/provider/openai/openai-stream-consumer.test.ts b/src/llm/provider/openai/openai-stream-consumer.test.ts new file mode 100644 index 00000000..a0a8b6b4 --- /dev/null +++ b/src/llm/provider/openai/openai-stream-consumer.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest"; + +import { createOpenAiStreamConsumer } from "./openai-stream-consumer.js"; +import type { StreamFinalResult } from "../completion-types.js"; + +function sseFrame(payload: Record): string { + return `data: ${JSON.stringify(payload)}\n\n`; +} + +function toolCallFrame( + toolCalls: Array>, +): string { + return sseFrame({ + model: "test-model", + choices: [{ index: 0, delta: { tool_calls: toolCalls }, finish_reason: null }], + }); +} + +function bodyOf(text: string): ReadableStream { + const bytes = new TextEncoder().encode(text); + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +/** Drain the consumer and return the final result it returns on completion. */ +async function drain(body: string): Promise { + const consumer = createOpenAiStreamConsumer("delta_reasoning"); + const iterator = consumer.consume(bodyOf(body), undefined); + for (;;) { + const step = await iterator.next(); + if (step.done) return step.value as StreamFinalResult; + } +} + +const DONE = sseFrame({ + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], +}) + "data: [DONE]\n\n"; + +describe("openai stream consumer tool-call assembly", () => { + it("keeps unindexed calls with distinct ids apart", async () => { + const result = await drain( + toolCallFrame([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + ]) + + toolCallFrame([ + { + id: "call_b", + type: "function", + function: { name: "os__fs__grep", arguments: '{"pattern":"b"}' }, + }, + ]) + + DONE, + ); + + expect(result.toolCalls).toEqual([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + { + id: "call_b", + type: "function", + function: { name: "os__fs__grep", arguments: '{"pattern":"b"}' }, + }, + ]); + }); + + it("reassembles fragments of one unindexed call that repeats its id", async () => { + const result = await drain( + toolCallFrame([ + { id: "call_a", type: "function", function: { name: "os__fs__read" } }, + ]) + + toolCallFrame([{ id: "call_a", function: { arguments: '{"path":' } }]) + + toolCallFrame([{ id: "call_a", function: { arguments: '"a.txt"}' } }]) + + DONE, + ); + + expect(result.toolCalls).toEqual([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + ]); + }); + + it("still folds id-less continuation deltas into the open call", async () => { + const result = await drain( + toolCallFrame([ + { id: "call_a", type: "function", function: { name: "os__fs__read" } }, + ]) + + toolCallFrame([{ function: { arguments: '{"path":"a.txt"}' } }]) + + DONE, + ); + + expect(result.toolCalls).toEqual([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + ]); + }); + + it("accumulates indexed parallel calls in index order, whatever the arrival order", async () => { + const result = await drain( + toolCallFrame([ + { + index: 1, + id: "call_b", + type: "function", + function: { name: "os__fs__grep", arguments: '{"pattern"' }, + }, + ]) + + toolCallFrame([ + { + index: 0, + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path"' }, + }, + ]) + + toolCallFrame([{ index: 1, function: { arguments: ':"b"}' } }]) + + toolCallFrame([{ index: 0, function: { arguments: ':"a.txt"}' } }]) + + DONE, + ); + + expect(result.toolCalls).toEqual([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + { + id: "call_b", + type: "function", + function: { name: "os__fs__grep", arguments: '{"pattern":"b"}' }, + }, + ]); + }); + + it("merges an id-only delta into the slot the provider opened by index", async () => { + const result = await drain( + toolCallFrame([ + { + index: 0, + id: "call_a", + type: "function", + function: { name: "os__fs__read" }, + }, + ]) + + toolCallFrame([{ id: "call_a", function: { arguments: '{"path":"a.txt"}' } }]) + + DONE, + ); + + expect(result.toolCalls).toEqual([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + ]); + }); + + it("keeps two unindexed calls in a single event apart", async () => { + const result = await drain( + toolCallFrame([ + { + id: "call_a", + type: "function", + function: { name: "os__fs__read", arguments: '{"path":"a.txt"}' }, + }, + { + id: "call_b", + type: "function", + function: { name: "os__fs__grep", arguments: '{"pattern":"b"}' }, + }, + ]) + DONE, + ); + + expect(result.toolCalls).toHaveLength(2); + expect(result.toolCalls?.map((call) => call.id)).toEqual(["call_a", "call_b"]); + }); +}); diff --git a/src/llm/provider/openai/openai-stream-consumer.ts b/src/llm/provider/openai/openai-stream-consumer.ts index c917bccf..dc03d35f 100644 --- a/src/llm/provider/openai/openai-stream-consumer.ts +++ b/src/llm/provider/openai/openai-stream-consumer.ts @@ -13,6 +13,8 @@ import { import { extractPartialReplyTextFromToolArguments } from "./tool-arguments-stream-parser.js"; type MutableToolCall = { + /** Position in the final array. See `orderFor`. */ + order: number; id?: string; type?: "function"; function: { @@ -21,6 +23,32 @@ type MutableToolCall = { }; }; +/** + * Cross-event state for assembling streamed tool calls. + * + * A delta belongs to one call ("slot"), and the provider says which in one + * of two ways — or in neither: + * + * - `index` — OpenAI's own scheme. Authoritative whenever it is present. + * - `id` — one whole call per event and no index at all, as several + * OpenAI-compatible providers do. Keying those by array position folds + * every call onto slot 0, which merges distinct calls into one (#103). + * - neither — a continuation of a call already opened, matched by + * position against the slots that existed before this event. + */ +type ToolCallAccumulator = { + /** Slots by resolved key: `#`, `@`, or `~` for neither. */ + slots: Map; + /** Slot key per call id, so a later id-only delta finds its own slot. */ + keyById: Map; + /** Next `order` to hand out for a slot the provider did not index. */ + nextOrder: number; +}; + +function createToolCallAccumulator(): ToolCallAccumulator { + return { slots: new Map(), keyById: new Map(), nextOrder: 0 }; +} + export function createOpenAiStreamConsumer( reasoningFormat: ReasoningFormat, ): StreamConsumer { @@ -45,7 +73,7 @@ export function createOpenAiStreamConsumer( // NOT be conflated with either, since a still-open tool call's // arguments may be mid-stream. let terminalObserved = false; - const toolCalls = new Map(); + const toolCalls = createToolCallAccumulator(); try { while (true) { if (signal?.aborted) break; @@ -130,14 +158,28 @@ export function createOpenAiStreamConsumer( } function applyToolCallDeltas( - toolCalls: Map, + accumulator: ToolCallAccumulator, deltas: readonly OpenAiToolCallDelta[], ): void { + // Snapshot before the loop: positional fallback matches against the slots + // that were open when the event arrived, never against ones it creates. + const openKeys = [...accumulator.slots.keys()]; + let position = 0; for (const delta of deltas) { - const current = toolCalls.get(delta.index) ?? { - function: { name: "", arguments: "" }, - }; - if (delta.id) current.id = delta.id; + const key = resolveSlotKey(accumulator, delta, position, openKeys); + position += 1; + let current = accumulator.slots.get(key); + if (!current) { + current = { + order: orderFor(accumulator, delta), + function: { name: "", arguments: "" }, + }; + accumulator.slots.set(key, current); + } + if (delta.id) { + current.id = delta.id; + accumulator.keyById.set(delta.id, key); + } if (delta.type) current.type = delta.type; if (delta.function?.name) { current.function.name = mergeToolName( @@ -148,10 +190,39 @@ function applyToolCallDeltas( if (delta.function?.arguments) { current.function.arguments += delta.function.arguments; } - toolCalls.set(delta.index, current); } } +function resolveSlotKey( + accumulator: ToolCallAccumulator, + delta: OpenAiToolCallDelta, + position: number, + openKeys: readonly string[], +): string { + if (typeof delta.index === "number") return `#${delta.index}`; + if (delta.id !== undefined) { + return accumulator.keyById.get(delta.id) ?? `@${delta.id}`; + } + return openKeys[position] ?? `~${accumulator.slots.size}`; +} + +/** + * Provider indexes double as the output position, so a stream that opens + * slot 1 before slot 0 still ends up in the provider's order. Slots the + * provider never indexed queue up behind the highest index seen so far, in + * arrival order. + */ +function orderFor( + accumulator: ToolCallAccumulator, + delta: OpenAiToolCallDelta, +): number { + if (typeof delta.index === "number") { + accumulator.nextOrder = Math.max(accumulator.nextOrder, delta.index + 1); + return delta.index; + } + return accumulator.nextOrder++; +} + /** * Fold a streamed `function.name` fragment into what we have so far. * @@ -191,12 +262,12 @@ function buildFinalResult(args: { finishReason: string | null; modelId: string | null; usage?: CompletionUsage; - toolCalls: ReadonlyMap; + toolCalls: ToolCallAccumulator; terminalObserved: boolean; }): StreamFinalResult { - const sortedToolCalls = [...args.toolCalls.entries()] - .sort(([a], [b]) => a - b) - .map(([, call]) => toOpenAiToolCall(call)) + const sortedToolCalls = [...args.toolCalls.slots.values()] + .sort((a, b) => a.order - b.order) + .map((call) => toOpenAiToolCall(call)) .filter((call): call is OpenAiToolCall => call !== null); return { content: args.content, diff --git a/src/llm/provider/openai/parse-sse-chunk.ts b/src/llm/provider/openai/parse-sse-chunk.ts index 69d6f33e..4aca111a 100644 --- a/src/llm/provider/openai/parse-sse-chunk.ts +++ b/src/llm/provider/openai/parse-sse-chunk.ts @@ -2,7 +2,14 @@ import type { ReasoningExtractor } from "./reasoning-extractor.js"; import { extractPartialReplyTextFromToolArguments } from "./tool-arguments-stream-parser.js"; export interface OpenAiToolCallDelta { - index: number; + /** + * The provider's own slot number, when it sends one. Omitted otherwise: + * some OpenAI-compatible providers emit one whole call per event with no + * `index` at all, and inventing a position here makes every one of them + * look like slot 0. Resolving identity needs state across events, so the + * stream consumer owns it. + */ + index?: number; id?: string; type?: "function"; function?: { @@ -95,8 +102,8 @@ export function parseOpenAiSseEvent( finishReason, modelId, usage, - toolCallDeltas: toolCalls.map((toolCall, fallbackIndex) => ({ - index: typeof toolCall.index === "number" ? toolCall.index : fallbackIndex, + toolCallDeltas: toolCalls.map((toolCall) => ({ + ...(typeof toolCall.index === "number" ? { index: toolCall.index } : {}), ...(typeof toolCall.id === "string" ? { id: toolCall.id } : {}), ...(toolCall.type === "function" ? { type: "function" as const } : {}), ...(toolCall.function From 53b54c196c8acd772ef7e8a75ff6d0cc4d562488 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:04:12 +0300 Subject: [PATCH 02/36] fix(cli): default NODE_ENV to production so the TUI stops leaking to OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every non-SEA way of starting the CLI leaves NODE_ENV unset, so react-reconciler resolves to its development build — the one carrying React 19's Component Performance Track, eight performance.measure() call sites fired on every commit. A TUI commits continuously and nothing drains Node's global performance entry buffer, so entries accumulate for the life of the process: Node warns at one million, V8 aborts at its ~4 GB ceiling around the eleven-hour mark, and an unattended agent goes silent with no JS stack. scripts/bundle-sea.ts already defines the constant away for the SEA binary. This gives the source/dist path the same footing via a side-effect import that must stay first in cli/index.ts — ES module bodies run after their dependency graph is evaluated, so setting the variable anywhere inside index.ts would land after ink had already required the reconciler and chosen a build. An explicit NODE_ENV is still honoured. Measured over 50 Ink rerenders: 206 measure entries before, 0 after. Fixes #307 Co-Authored-By: Claude Opus 5 (1M context) --- src/cli/index.ts | 4 +++ src/cli/node-env-bootstrap.test.ts | 49 ++++++++++++++++++++++++++++++ src/cli/node-env-bootstrap.ts | 48 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 src/cli/node-env-bootstrap.test.ts create mode 100644 src/cli/node-env-bootstrap.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index 7b71f344..3bff7fd1 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,4 +1,8 @@ #!/usr/bin/env node +// Must stay first: it defaults NODE_ENV to production before `ink` +// pulls in react-reconciler, which picks its build at require time. +// See src/cli/node-env-bootstrap.ts. +import "./node-env-bootstrap.js"; import { isSea } from "node:sea"; import { argv, exit } from "node:process"; import { runAgentCommand } from "./run-agent.js"; diff --git a/src/cli/node-env-bootstrap.test.ts b/src/cli/node-env-bootstrap.test.ts new file mode 100644 index 00000000..442b89a8 --- /dev/null +++ b/src/cli/node-env-bootstrap.test.ts @@ -0,0 +1,49 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { applyProductionNodeEnvDefault } from "./node-env-bootstrap.js"; + +describe("applyProductionNodeEnvDefault", () => { + it("fills in an unset NODE_ENV", () => { + const env: NodeJS.ProcessEnv = {}; + expect(applyProductionNodeEnvDefault(env)).toBe(true); + expect(env.NODE_ENV).toBe("production"); + }); + + it("fills in an empty NODE_ENV", () => { + const env: NodeJS.ProcessEnv = { NODE_ENV: "" }; + expect(applyProductionNodeEnvDefault(env)).toBe(true); + expect(env.NODE_ENV).toBe("production"); + }); + + it("leaves an explicit NODE_ENV alone", () => { + for (const value of ["development", "test", "staging"]) { + const env: NodeJS.ProcessEnv = { NODE_ENV: value }; + expect(applyProductionNodeEnvDefault(env)).toBe(false); + expect(env.NODE_ENV).toBe(value); + } + }); +}); + +describe("cli entry import order", () => { + /** + * The whole fix is the *position* of this import. ES modules evaluate + * their dependency graph before the importing module's own body, so + * anything that lands after `ink` in the import list runs too late to + * decide which react-reconciler build was loaded. An import sorter or a + * casual reorder would silently reintroduce the heap-OOM leak, and the + * symptom takes eleven hours to appear — hence a test on the source. + */ + it("imports the NODE_ENV bootstrap before every other module", () => { + const entry = readFileSync( + fileURLToPath(new URL("./index.ts", import.meta.url)), + "utf8", + ); + const firstImport = entry + .split("\n") + .find((line) => line.startsWith("import ")); + + expect(firstImport).toBe('import "./node-env-bootstrap.js";'); + }); +}); diff --git a/src/cli/node-env-bootstrap.ts b/src/cli/node-env-bootstrap.ts new file mode 100644 index 00000000..b793d407 --- /dev/null +++ b/src/cli/node-env-bootstrap.ts @@ -0,0 +1,48 @@ +/** + * Default `NODE_ENV` to `production` before anything else loads. + * + * React ships two builds behind a runtime + * `process.env.NODE_ENV === "production" ? prod : dev` switch. + * `scripts/bundle-sea.ts` defines the constant away for the SEA binary, + * so a released install always gets the production reconciler. Every + * other way of starting the CLI — `node dist/cli/index.js`, `npm run + * cli`, `npm run dev:cli`, a `npx`/`npm link` checkout — leaves NODE_ENV + * unset, which is how `react-reconciler` resolves to its *development* + * build on machines whose shell does not export it. That is every + * machine. + * + * The development reconciler carries React 19's Component Performance + * Track: eight `performance.measure()` call sites, fired on every + * commit. The production build has none. A TUI commits continuously — + * spinners, timers, streaming deltas — and nothing in Node ever drains + * the global performance entry buffer, so the entries accumulate for the + * life of the process at roughly 35-40 per second. Node warns at one + * million (`MaxPerformanceEntryBufferExceededWarning`), and V8 aborts at + * its ~4 GB ceiling: `FATAL ERROR: Ineffective mark-compacts near heap + * limit`, a native abort with no JS stack, around the eleven-hour mark. + * An unattended agent simply goes silent, and takes its tmux server with + * it if that was the only session. + * + * An explicit NODE_ENV is always honoured — this only fills in the + * blank. + * + * **This module must stay the first import in `src/cli/index.ts`.** ES + * module bodies run after their whole dependency graph is evaluated, so + * assigning `process.env.NODE_ENV` from inside `index.ts` would land + * long after `ink` had already required the reconciler and chosen a + * build. Only a side-effect import placed ahead of the others runs early + * enough. `src/cli/node-env-bootstrap.test.ts` guards the ordering. + */ + +/** + * Set `env.NODE_ENV` to `production` when it carries no value. + * + * @returns whether the default was applied. + */ +export function applyProductionNodeEnvDefault(env: NodeJS.ProcessEnv): boolean { + if (env.NODE_ENV !== undefined && env.NODE_ENV !== "") return false; + env.NODE_ENV = "production"; + return true; +} + +applyProductionNodeEnvDefault(process.env); From 4309b959eb09534da639e8c88ebce20880240c53 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:07:53 +0300 Subject: [PATCH 03/36] fix(install): say why an unpublished platform has no binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit darwin-x64 is commented out of the release matrix, so an Intel Mac resolved a slug the installer knows how to name but the release does not carry, then died on "download failed: " with a 404 behind it and no hint that the binary had never existed. Probe the asset before downloading and, on a literal 404, explain: no build for this slug, where to see what is published, and how to build from source. Intel Macs additionally get told the Apple Silicon build will not run for them, with the tracking issue. Only a 404 counts. Offline, a proxy, a 5xx, or no curl at all falls through to the real download, which reports failures as before. curl's own exit code is unusable here — a 404 that arrives after -L follows the release redirect surfaces as 56, not 22 — so the status is read directly. Whether to publish the darwin-x64 build is left as the maintainers' call. Fixes #300 Co-Authored-By: Claude Opus 5 (1M context) --- scripts/install.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 58398b0b..98d5fff7 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -375,6 +375,48 @@ else SHA_URL="${BASE}/releases/latest/download/${TAR_NAME}.sha256" fi +# Preflight: is there a build for this platform at all? +# +# Not every slug the installer can *name* is one the release matrix +# *publishes* — darwin-x64 is commented out of .github/workflows/release.yml, +# so an Intel Mac used to reach `download` and die on a bare +# "download failed: " with a 404 behind it and no hint that the binary +# had never existed. Say so instead, and say what to do about it. +# +# Only a literal 404 counts as "not published". Everything else — offline +# (000), a proxy, a 5xx, no curl at all — falls through to the real +# download, which reports failures as it always has. curl's own exit code +# is no good here: a 404 after -L follows the release redirect surfaces as +# 56, not 22, so read the status directly. +asset_missing() { + have curl || return 1 + _am_code="$(curl -sIL --retry 2 -o /dev/null -w '%{http_code}' "$1" 2>/dev/null || echo 000)" + [ "$_am_code" = "404" ] +} + +no_build_published() { + echo "no published build for ${SLUG}." >&2 + echo >&2 + if [ "$SLUG" = "darwin-x64" ]; then + echo "atomic-agent does not publish a macOS Intel binary yet, and the Apple" >&2 + echo "Silicon build will not run on this machine. Tracking:" >&2 + echo " https://github.com/${REPO}/issues/300" >&2 + else + echo "${TAR_NAME} is not attached to this release. See what is published:" >&2 + echo " https://github.com/${REPO}/releases" >&2 + fi + echo >&2 + echo "to run atomic-agent here, build it from source (needs Node 25.7+):" >&2 + echo " git clone https://github.com/${REPO}.git" >&2 + echo " cd atomic-agent && npm install && npm run build" >&2 + echo " node dist/cli/index.js" >&2 + exit 1 +} + +if asset_missing "$TAR_URL"; then + no_build_published +fi + TMPDIR="${TMPDIR:-/tmp}" WORK="$(mktemp -d "$TMPDIR/atomic-agent-install.XXXXXX")" TMP_BIN="" From 6a5691c89bf68ce564e310790283d428f530b754 Mon Sep 17 00:00:00 2001 From: Valerii Date: Wed, 2 Sep 2026 13:20:06 +0300 Subject: [PATCH 04/36] fix(tools): route source files from read_document to os.fs.read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models repeatedly picked os.fs.read_document for `.py`/`.ts` sources, got a generic "unsupported extension (override with `format`)" error, guessed the invalid `format: "text"`, and only then found os.fs.read. Nothing in the prompt or the error told them which reader owns source code, so the recovery was left to chance. Two deterministic nudges: - The stable-prefix summaries now split the two readers explicitly: os.fs.read is "the default for source code and text files", read_document is document extraction and "NOT for source code or text files: use os.fs.read". Both stay one line — the frequent-tool block ships on every turn, so the added wording is ~25 tokens. - detectFormat keeps a set of source-like extensions. They still are not mapped to `plain` (os.fs.read is the better tool: offset/limit pagination, no extractor in the way), but they now reject with a message that names os.fs.read first and spells out the accepted override `format: "plain"` instead of the bare word `format`. Unknown extensions carry no such signal, so their message offers both paths — and also names `format: "plain"`. Document formats and the plain-text extension set are untouched. --- src/prompt/build-prompt.test.ts | 22 ++++ src/prompt/default-tool-descriptors-a.ts | 10 +- .../os/read-document/read-document.test.ts | 106 ++++++++++++++++++ src/tools/os/read-document/read-document.ts | 30 ++++- 4 files changed, 164 insertions(+), 4 deletions(-) diff --git a/src/prompt/build-prompt.test.ts b/src/prompt/build-prompt.test.ts index 066e2b07..44b32285 100644 --- a/src/prompt/build-prompt.test.ts +++ b/src/prompt/build-prompt.test.ts @@ -5,6 +5,7 @@ import { QWEN_THINK_PROFILE, } from "../llm/model-profile.js"; import { buildPrompt } from "./build-prompt.js"; +import { DEFAULT_TOOL_DESCRIPTORS } from "./tool-descriptors.js"; import { createEmptySessionState } from "../session/session-state.js"; import type { SessionState } from "../session/session-state.js"; import type { @@ -232,6 +233,27 @@ describe("buildPrompt", () => { expect(prompt.stablePrefix).toContain("os.fs.read_document"); }); + it("frequent-tool summaries send source files to os.fs.read, not read_document", () => { + // Issue #113: the stable prefix is where a model decides between the + // two readers, and it ships on every turn. Pinning the wording here + // (against the real descriptors, not the stub TOOLS above) keeps a + // future summary edit from quietly dropping the routing hint that + // stops models bouncing off read_document's unsupported-extension + // error on `.py` / `.ts` files. + const prompt = buildPrompt({ + session: mkSession(), + toolDescriptors: DEFAULT_TOOL_DESCRIPTORS, + capabilities: CAPS, + skillCatalog: SKILLS, + }); + expect(prompt.stablePrefix).toContain( + "the default for source code and text files", + ); + expect(prompt.stablePrefix).toContain( + "NOT for source code or text files: use os.fs.read", + ); + }); + it("stable prefix changes deterministically when a skill is removed from the catalog (skills.disabled)", () => { // Pins the contract for the `skills.disabled` denylist: the // `SkillRegistry` filters disabled skills out of `list()`, the diff --git a/src/prompt/default-tool-descriptors-a.ts b/src/prompt/default-tool-descriptors-a.ts index 007a151c..8346977c 100644 --- a/src/prompt/default-tool-descriptors-a.ts +++ b/src/prompt/default-tool-descriptors-a.ts @@ -47,7 +47,8 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ }, { name: "os.fs.read", - summary: "Read a UTF-8 file; use offset/limit for ranges, lineNumbers for 'LINE|'.", + summary: + "Read a UTF-8 file — the default for source code and text files; use offset/limit for ranges, lineNumbers for 'LINE|'.", argsSchema: "{ path: string, maxBytes?: number, offset?: number /* 1-based; neg=from end */, limit?: number, lineNumbers?: boolean }", }, @@ -96,8 +97,13 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ argsSchema: "{ path: string, oldString: string, newString: string, replaceAll?: boolean }", }, { + // Models reach for read_document on `.py` / `.ts` source files, hit the + // unsupported-extension error and burn a step guessing `format`. The + // summary therefore names the sibling tool explicitly: source and text + // go to os.fs.read, this one is for document extraction only. name: "os.fs.read_document", - summary: "Extract plain text from PDF, Office, ODF, etc. (markers in output). Read-only.", + summary: + "Extract plain text from documents — PDF, Office, ODF, RTF (markers in output). NOT for source code or text files: use os.fs.read. Read-only.", argsSchema: "{ path: string, format?: string, maxBytes?: number, maxPages?: number, pagesFrom?: number, pagesTo?: number, sheets?: (string | number)[], pageSeparators?: boolean, includeTables?: boolean }", }, diff --git a/src/tools/os/read-document/read-document.test.ts b/src/tools/os/read-document/read-document.test.ts index 6d210cdc..407d8824 100644 --- a/src/tools/os/read-document/read-document.test.ts +++ b/src/tools/os/read-document/read-document.test.ts @@ -104,6 +104,112 @@ describe("os.fs.read_document dispatcher", () => { ); }); + // Issue #113: a `.py` handed to read_document used to produce a generic + // "unsupported extension (override with `format`)" error, which sent + // models into `format: "text"` — not a known format — instead of over to + // os.fs.read. These pin both halves of the recovery hint. + it.each(["py", "ts", "rs"])( + "points .%s source files at os.fs.read and names format: \"plain\"", + async (ext) => { + const tool = buildOsFsReadDocumentTool({}); + const path = join(dir, `module.${ext}`); + await writeFile(path, Buffer.from("print(1)\n")); + + const error = await tool + .run({ path }, makeCtx(dir)) + .then(() => undefined) + .catch((e: unknown) => e as Error); + + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain(`".${ext}" is a source/text file`); + expect(message).toContain("use os.fs.read instead"); + expect(message).toContain('format: "plain"'); + // The ambiguous bare-`format` phrasing is what produced the bad + // `format: "text"` guesses; it must not come back. + expect(message).not.toMatch(/override with `format`/); + }, + ); + + it("offers os.fs.read and format: \"plain\" for an unknown binary extension", async () => { + const tool = buildOsFsReadDocumentTool({}); + const path = join(dir, "blob.qzx"); + await writeFile(path, Buffer.from([0x00, 0x01, 0x02])); + + const error = await tool + .run({ path }, makeCtx(dir)) + .then(() => undefined) + .catch((e: unknown) => e as Error); + + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain('unsupported extension ".qzx"'); + expect(message).toContain("os.fs.read"); + expect(message).toContain('format: "plain"'); + }); + + it('reads a source file when format: "plain" is passed explicitly', async () => { + const tool = buildOsFsReadDocumentTool({ + extractors: { + plain: fakeExtractor({ format: "plain", text: "print(1)" }), + }, + }); + const path = join(dir, "script.py"); + await writeFile(path, Buffer.from("print(1)\n")); + + const result = await tool.run({ path, format: "plain" }, makeCtx(dir)); + + expect(result.details.format).toBe("plain"); + expect(result.summary).toContain("print(1)"); + }); + + it("keeps document extensions routed to their own extractors", async () => { + // Guards the other half of issue #113: the new source-file branch must + // not have moved any real document format into the reject path. + const seen: string[] = []; + const tool = buildOsFsReadDocumentTool({ + extractors: { + pdf: fakeExtractor({ format: "pdf", text: "p" }, () => seen.push("pdf")), + docx: fakeExtractor({ format: "docx", text: "d" }, () => seen.push("docx")), + xlsx: fakeExtractor({ format: "xlsx", text: "x" }, () => seen.push("xlsx")), + rtf: fakeExtractor({ format: "rtf", text: "r" }, () => seen.push("rtf")), + odt: fakeExtractor({ format: "odt", text: "o" }, () => seen.push("odt")), + pptx: fakeExtractor({ format: "pptx", text: "s" }, () => seen.push("pptx")), + plain: fakeExtractor({ format: "plain", text: "t" }, () => seen.push("plain")), + }, + }); + for (const name of [ + "a.pdf", + "a.docx", + "a.xlsx", + "a.rtf", + "a.odt", + "a.pptx", + "a.txt", + "a.md", + "a.csv", + "a.json", + "a.yaml", + ]) { + const path = join(dir, name); + await writeFile(path, Buffer.from("x")); + await tool.run({ path }, makeCtx(dir)); + } + expect(seen).toEqual([ + "pdf", + "docx", + "xlsx", + "rtf", + "odt", + "pptx", + "plain", + "plain", + "plain", + "plain", + "plain", + ]); + }); + it("rejects unknown format overrides", async () => { const tool = buildOsFsReadDocumentTool({}); const path = join(dir, "any.txt"); diff --git a/src/tools/os/read-document/read-document.ts b/src/tools/os/read-document/read-document.ts index b18471b2..5c18aaa0 100644 --- a/src/tools/os/read-document/read-document.ts +++ b/src/tools/os/read-document/read-document.ts @@ -40,7 +40,7 @@ export function buildOsFsReadDocumentTool( return { name: "os.fs.read_document", description: - "Extract plain text (with light structure markers) from PDF, DOCX, DOC (legacy), XLSX, RTF, ODT, PPTX, and plain-text files. Auto-detects format by extension; override with `format`. Read-only, no approval required.", + 'Extract plain text (with light structure markers) from PDF, DOCX, DOC (legacy), XLSX, RTF, ODT, PPTX, and plain-text files. NOT for source code or other UTF-8 text files — read those with os.fs.read. Auto-detects format by extension; override with `format` (the plain-text value is `format: "plain"`). Read-only, no approval required.', readonly: true, async run(rawArgs, ctx) { const args = await parseArgs(rawArgs, ctx.workingDir); @@ -197,6 +197,24 @@ function parseSheetsArg( return out; } +/** + * Extensions that are almost certainly source code or other line-oriented + * text. They are deliberately NOT mapped to `plain`: `os.fs.read` is the + * right tool for them (offset/limit pagination, `lineNumbers`, no document + * extractor in the way). The set exists only so the rejection can say which + * tool to reach for next — without that, models retry `read_document` with + * an invented `format: "text"` and burn another step before discovering + * `os.fs.read` (issue #113). + */ +const SOURCE_LIKE_EXTENSIONS: ReadonlySet = new Set([ + "bash", "c", "cc", "cfg", "cjs", "clj", "conf", "cpp", "cs", "css", "cxx", + "dart", "env", "erl", "ex", "exs", "fish", "go", "gradle", "h", "hh", "hpp", + "hs", "ini", "ipynb", "java", "js", "jsonc", "jsx", "kt", "kts", "less", + "lua", "m", "mjs", "mm", "php", "pl", "pm", "properties", "proto", "ps1", + "py", "pyi", "r", "rb", "rs", "sass", "scala", "scss", "sh", "sql", "svelte", + "swift", "tf", "toml", "ts", "tsv", "tsx", "vue", "zsh", +]); + /** * Resolve extension → canonical format. `.html/.xml/.json/.csv` currently * fall through to `plain` — it's the safest default until we need format- @@ -238,8 +256,16 @@ function detectFormat(absolute: string, override: unknown): DocumentFormat { case "yml": return "plain"; default: + // Two shapes on purpose: for a source-like extension the answer is + // almost always "wrong tool", so name os.fs.read first and mention the + // override second; for anything else the extension carries no signal, + // so offer both. Either way the message spells out the accepted value + // `format: "plain"` — the bare word `format` invited `format: "text"`, + // which is not a known format and costs another failed step. throw new Error( - `os.fs.read_document: unsupported extension ".${ext}" (override with \`format\`)`, + SOURCE_LIKE_EXTENSIONS.has(ext) + ? `os.fs.read_document: ".${ext}" is a source/text file — use os.fs.read instead; if you intended document extraction, retry with \`format: "plain"\`` + : `os.fs.read_document: unsupported extension ".${ext}" — use os.fs.read for source or text files, or retry with an explicit format (e.g. \`format: "plain"\`)`, ); } } From 315cf823dc5d07a160f102693e4bed822b678201 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:25:59 +0300 Subject: [PATCH 05/36] fix(openai): retry a stream that dies before its first chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cloud provider that answers 2xx, thinks for a long time (reasoning models, cold routes) and then drops the SSE socket before emitting a single delta failed the whole turn instantly with "Turn failed [transport]: terminated". Nobody owned that window. `openAiStartStream` retries the open, but its budget is spent the moment headers arrive; failures raised while reading the body are undici's bare `Error: terminated` and never pass through the HTTP client at all; and `createFallbackStreamer` only helps when a second provider is configured, which the reporter had not. `OpenAiProvider.completeStream` now reopens the stream while nothing has been yielded to its caller. That is the same argument `openAiStartStream` already makes for retrying the open: with zero chunks emitted, a replay is unobservable, so it cannot duplicate output. `canReopenStream` spells out the guards and their order — committed (any yielded chunk, reasoning or a bare role preamble, ends retrying for good), cancellation checked before the error is inspected, `OpenAiHttpError` rejected so the open's budget is not squared to 9 requests, `isNetworkError` so a consumer bug is not replayed, and the shared `OPENAI_MAX_ATTEMPTS` budget with `openAiRetryBackoff`'s existing pacing. The dead body is cancelled before reopening so a retry does not leak a socket. Resuming a stream that has already emitted output is deliberately left out: restarting would duplicate text or need prefill continuation, and non-deterministic sampling rules out a prefix dedupe. That is a maintainer design call, not a defect fix. Reported in Discord #feedback-and-bugs by thegreatteacher. --- AGENTS.md | 3 +- src/llm/provider/openai/openai-http.ts | 32 +- src/llm/provider/openai/openai-provider.ts | 138 +++++++-- .../openai/openai-stream-retry.test.ts | 277 ++++++++++++++++++ 4 files changed, 425 insertions(+), 25 deletions(-) create mode 100644 src/llm/provider/openai/openai-stream-retry.test.ts diff --git a/AGENTS.md b/AGENTS.md index dcbf3718..4c97fdd8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1787,10 +1787,11 @@ Deliberately out of scope: an opt-in whole-disk / drive index (the issue sketche ## LLM reliability policy -Two narrow retry layers sit between the agent loop and `llama-server`. Both are deliberately bounded and never replay already-executed tool calls: +Three narrow retry layers sit between the agent loop and the model server. All are deliberately bounded and never replay already-executed tool calls, and none of them ever replays output a caller has already seen: 1. **Parser retry (step-executor).** If the first `parseToolCall` on a completion throws, the executor calls the unary `llmComplete` exactly once more with the same prompt/slot and re-parses. A `parse_retry` event is emitted for observability. If the second attempt also fails, the original error (with a raw-output preview) is thrown. The streaming path always falls back to unary for the retry so partial SSE deltas are not double-emitted. 2. **Transport retry (LlamaServerClient).** `complete()` and the initial pre-body fetch of `completeStream()` are wrapped in a bounded retry governed by `llama.completionRetries` (default 3) and `llama.completionRetryBackoffMs` (default 150ms, exponential with ±20% jitter). Retries fire **only** for network errors (`LlamaServerError.status === null`) and HTTP 5xx. Grammar/validation 4xx and abort signals short-circuit immediately. Once the SSE body starts streaming, no further retries happen — the conversation state on the server is considered indeterminate. +3. **Pre-first-chunk stream retry (`OpenAiProvider.completeStream`).** Closes the one window nobody owned: a cloud provider answers 2xx, thinks for a long time (reasoning models, cold routes), then drops the socket **before emitting a single delta**. `openAiStartStream`'s budget is already spent by then, and undici surfaces the death as a bare `Error: terminated` from the body reader, so the whole turn used to fail instantly with `Turn failed [transport]: terminated`. `completeStream` now reopens the stream while `committed` is still false — i.e. while not one chunk has been yielded to its caller, which is exactly when a replay is unobservable. The guards, in order: **committed** (any yielded chunk, reasoning or even a bare `role` preamble, ends retrying forever), **cancellation** (`request.signal.aborted` is checked before the error is inspected — an Esc is not a network failure), **`OpenAiHttpError`** (came from the *open*, already spent `OPENAI_MAX_ATTEMPTS`; retrying here would square the budget to 9 requests), **shape** (`isNetworkError`, so a consumer bug is not replayed three times), and the shared `OPENAI_MAX_ATTEMPTS` budget with `openAiRetryBackoff`'s pacing. The dead response body is cancelled before reopening so a retry does not leak a socket. Resuming a stream that has *already* emitted output is deliberately out of scope — it would either duplicate text or need prefill continuation, and non-deterministic sampling rules out a prefix dedupe. Pinned by [src/llm/provider/openai/openai-stream-retry.test.ts](src/llm/provider/openai/openai-stream-retry.test.ts). ### Failure taxonomy diff --git a/src/llm/provider/openai/openai-http.ts b/src/llm/provider/openai/openai-http.ts index c25b9c55..7b5a1170 100644 --- a/src/llm/provider/openai/openai-http.ts +++ b/src/llm/provider/openai/openai-http.ts @@ -110,8 +110,13 @@ const OPENAI_ERROR_DETAIL_MAX_LEN = 300; * client's defaults (`localModels.completionRetries` = 3, 150ms base). * Deliberately not config-driven yet: the knob can follow once the * fallback work settles where such settings live for cloud providers. + * + * Exported because it is the budget for *one* streaming completion, not + * just for one HTTP call: `OpenAiProvider.completeStream` reopens a + * stream that died before its first chunk, and that reopen has to come + * out of this same budget rather than multiply it. */ -const OPENAI_MAX_ATTEMPTS = 3; +export const OPENAI_MAX_ATTEMPTS = 3; const OPENAI_BACKOFF_BASE_MS = 150; /** * Ceiling on how long a provider's `retry-after` can stall one attempt. @@ -172,7 +177,12 @@ export async function openAiPostJson( * connection errors, 429s, 5xxs — happen entirely inside the retry * loop, before the caller has consumed a single chunk, so retrying here * can never duplicate output. Once this resolves, the stream is live - * and failures downstream are not retryable at this layer. + * and failures downstream are not retryable at this layer — the caller + * owns that window. `OpenAiProvider.completeStream` extends the same + * "nothing emitted yet, so a replay is free" argument a little further + * by reopening when the body dies before its first chunk; a failure that + * escapes *this* function has already spent `OPENAI_MAX_ATTEMPTS` and + * must not be retried again there. */ export async function openAiStartStream( deps: OpenAiHttpDeps, @@ -189,6 +199,24 @@ export async function openAiStartStream( }); } +/** + * Wait exactly as long as `runOpenAiWithRetry` would wait before its + * next try — same exponential base, same ±20% jitter, same abort-aware + * sleep. Exported so the one retry that lives *outside* this file + * (`OpenAiProvider.completeStream` reopening a stream that died before + * its first chunk) reuses this client's pacing instead of inventing a + * second set of magic numbers. `attemptNumber` is the 1-based attempt + * that just failed. + */ +export async function openAiRetryBackoff( + attemptNumber: number, + signal?: AbortSignal, +): Promise { + // `null` as the error: a body-read death carries no `retry-after`, so + // only the plain backoff applies. + await sleep(resolveWaitMs(null, attemptNumber), signal); +} + export async function openAiFetch( deps: OpenAiHttpDeps, path: string, diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index e20ae8e1..f1341124 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -22,9 +22,13 @@ import { buildOpenAiHeaders, openAiGetJson, openAiPostJson, + openAiRetryBackoff, openAiStartStream, + OpenAiHttpError, + OPENAI_MAX_ATTEMPTS, type OpenAiHttpDeps, } from "./openai-http.js"; +import { isNetworkError } from "../../reliability/network-error.js"; import { normaliseOpenAiChatResponse } from "./openai-normalise-response.js"; import { normalizeOpenAiBaseUrl } from "./normalize-openai-base-url.js"; import { describeImageViaOpenAi } from "./openai-describe-image.js"; @@ -122,30 +126,60 @@ export class OpenAiProvider implements LlmProvider { request: CompletionRequest, ): AsyncGenerator { const body = buildOpenAiChatBody(request, this.defaultChatModel, true, this.extraBody); - // Opening the stream (connect + status check) happens inside the - // client's bounded retry, strictly before the first chunk exists. - // From here on the stream is live and failures are terminal. - const res = await openAiStartStream( - this.http, - `${this.apiPathPrefix}/chat/completions`, - body, - request, - ); + const path = `${this.apiPathPrefix}/chat/completions`; let accumulated = ""; let accumulatedReasoning = ""; - let streamFinal: StreamFinalResult | void; - const stream = this.streamConsumer.consume(res.body, request.signal); - while (true) { - const next = await stream.next(); - if (next.done) { - streamFinal = next.value; - break; - } - const chunk = next.value; - if (chunk.delta) accumulated += chunk.delta; - if (chunk.reasoningDelta) accumulatedReasoning += chunk.reasoningDelta; - if (!chunk.done) { - yield chunk; + let streamFinal: StreamFinalResult | void = undefined; + // Flipped the instant the first chunk leaves this generator. Before + // that the caller has seen nothing, so throwing the half-opened + // stream away and starting over is invisible to everyone — the same + // argument `openAiStartStream` makes for retrying the open. After + // it, the stream is COMMITTED: a restart would replay the completion + // from the top and duplicate text the user already read, and because + // sampling is non-deterministic no prefix dedupe can repair that. + let committed = false; + + // The window this loop exists for: a provider answers 2xx, thinks for + // a long time (reasoning models, cold routes), then drops the socket + // without ever emitting a delta. `openAiStartStream` has already + // returned by then, so its retry is spent, and undici surfaces the + // death as a bare `Error: terminated` from the body reader — which + // used to fail the whole turn ("Turn failed [transport]: terminated") + // even though not one byte of output existed. + attempts: for (let attempt = 1; ; attempt += 1) { + let res: (Response & { body: NonNullable }) | undefined; + try { + // Opening the stream (connect + status check) happens inside the + // client's bounded retry, strictly before the first chunk exists. + res = await openAiStartStream(this.http, path, body, request); + // A reopen starts from an empty transcript: whatever the dead + // attempt accumulated was never yielded and must not be mixed + // into the fresh one. + accumulated = ""; + accumulatedReasoning = ""; + streamFinal = undefined; + const stream = this.streamConsumer.consume(res.body, request.signal); + while (true) { + const next = await stream.next(); + if (next.done) { + streamFinal = next.value; + break attempts; + } + const chunk = next.value; + if (chunk.delta) accumulated += chunk.delta; + if (chunk.reasoningDelta) accumulatedReasoning += chunk.reasoningDelta; + if (!chunk.done) { + committed = true; + yield chunk; + } + } + } catch (err) { + if (!canReopenStream(err, request.signal, committed, attempt)) throw err; + // Hand the dead socket back before opening a new one, or the + // retry leaks a connection out of undici's pool for the rest of + // the process. + await discardResponseBody(res); + await openAiRetryBackoff(attempt, request.signal); } } const final = completionFromStreamFinal( @@ -258,6 +292,66 @@ function completionFromStreamFinal( }; } +/** + * May a failure raised between "2xx headers received" and "first chunk + * handed to our caller" be recovered by reopening the stream? + * + * The order of these guards is the contract, not a stylistic choice: + * + * 1. **Committed.** Once a chunk has been yielded, nothing below matters. + * This is deliberately the strictest reading of "output": a chunk that + * carries only the provider's opening `role` delta commits the stream + * just as a text delta does. We cannot know what a downstream consumer + * did with it, and being wrong here means duplicating a user's reply. + * Reasoning deltas are output for the same reason — the TUI renders + * them live. + * 2. **Cancellation.** A user pressing Esc is not a network failure, and + * an abort reaches us in several disguises (`AbortError`, a raw + * `Error: aborted`, or a custom `fetchImpl`'s own shape). The signal + * is the only reliable oracle, so it is consulted before the error is + * inspected at all — the ordering `network-error.ts` documents. + * 3. **`OpenAiHttpError`.** The failure came from *opening* the stream, + * which already ran inside `runOpenAiWithRetry` and already spent the + * whole `OPENAI_MAX_ATTEMPTS` budget on 429s/5xx/connect errors. + * Retrying it here would silently square the budget (3 × 3) and delay + * a real, actionable message — a bad API key would be tried nine + * times. Only untyped body-read deaths get past this guard. + * 4. **Shape.** Anything that is not a recognisable transport death — a + * bug in a stream consumer, a parse error — is a real error. Replaying + * it would just hide it behind three identical failures. + * 5. **Budget.** One streaming completion gets `OPENAI_MAX_ATTEMPTS` + * total, shared with the open, not a fresh budget per layer. + */ +function canReopenStream( + err: unknown, + signal: AbortSignal | undefined, + committed: boolean, + attempt: number, +): boolean { + if (committed) return false; + if (signal?.aborted) return false; + if (err instanceof OpenAiHttpError) return false; + if (!isNetworkError(err)) return false; + return attempt < OPENAI_MAX_ATTEMPTS; +} + +/** + * Release a response whose body died mid-read, so the reopen does not + * leak the socket. By the time we get here the stream consumer's + * `finally` has released its reader lock, but the body itself still owns + * the connection until it is cancelled. A body that refuses to cancel + * (already errored, still locked) is not worth failing the turn over — + * we are on our way to a fresh request either way. + */ +async function discardResponseBody(res: Response | undefined): Promise { + if (!res?.body) return; + try { + await res.body.cancel(); + } catch { + // Nothing left to release. + } +} + function applyToolCallTerminationSafety( result: CompletionResult, terminalObserved: boolean, diff --git a/src/llm/provider/openai/openai-stream-retry.test.ts b/src/llm/provider/openai/openai-stream-retry.test.ts new file mode 100644 index 00000000..42be7a95 --- /dev/null +++ b/src/llm/provider/openai/openai-stream-retry.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { CompletionResult, StreamChunk } from "../completion-types.js"; +import { OpenAiHttpError } from "./openai-http.js"; +import { OpenAiProvider } from "./openai-provider.js"; + +/** One SSE event, framed the way an OpenAI-compatible provider sends it. */ +function frame(obj: Record): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +function contentFrame(content: string): string { + return frame({ + model: "qwen-test", + choices: [ + { index: 0, delta: { role: "assistant", content }, finish_reason: null }, + ], + }); +} + +/** The clean tail of a completion: finish_reason, usage, `[DONE]`. */ +const STOP_FRAMES = + frame({ + model: "qwen-test", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }) + "data: [DONE]\n\n"; + +/** + * A part of a fake response body: bytes to write, an `Error` that kills + * the stream at that point, or a side effect to run first (used to abort + * the caller's signal at a precise moment). + */ +type BodyPart = string | Error | (() => void); + +/** + * A streaming `Response` that plays `parts` one `pull()` at a time. + * + * Pull-driven on purpose: `ReadableStreamDefaultController.error()` + * resets the queue, so a stream built by enqueueing everything up front + * and then erroring would drop the deltas it had already queued — which + * makes "the socket dies *after* a delta was delivered" untestable, and + * that is the case that must NOT be retried. + */ +function streamingResponse(parts: readonly BodyPart[]): Response { + const encoder = new TextEncoder(); + const queue = [...parts]; + const body = new ReadableStream({ + pull(controller) { + while (queue.length > 0) { + const part = queue.shift(); + if (typeof part === "function") { + part(); + continue; + } + if (part instanceof Error) { + controller.error(part); + return; + } + if (typeof part === "string") { + controller.enqueue(encoder.encode(part)); + return; + } + } + controller.close(); + }, + }); + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function provider(fetchImpl: unknown): OpenAiProvider { + return new OpenAiProvider({ + id: "test", + baseUrl: "https://example.invalid", + apiKey: "", + defaultChatModel: "qwen-test", + fetchImpl: fetchImpl as typeof fetch, + }); +} + +/** Drain a completion stream, keeping every chunk the caller was handed. */ +async function drain( + stream: AsyncGenerator, +): Promise<{ chunks: StreamChunk[]; result: CompletionResult }> { + const chunks: StreamChunk[] = []; + for (;;) { + const next = await stream.next(); + if (next.done) return { chunks, result: next.value }; + chunks.push(next.value); + } +} + +/** Drain until the stream throws; returns what was yielded before it did. */ +async function drainToError( + stream: AsyncGenerator, +): Promise<{ chunks: StreamChunk[]; error: unknown }> { + const chunks: StreamChunk[] = []; + try { + for (;;) { + const next = await stream.next(); + if (next.done) return { chunks, error: null }; + chunks.push(next.value); + } + } catch (err) { + return { chunks, error: err }; + } +} + +/** + * The window these tests pin: the provider answered 2xx, so + * `openAiStartStream`'s retry is spent, but the socket died before a + * single chunk reached the caller. Nothing was emitted downstream, so + * reopening is invisible — and once anything HAS been emitted, it is not. + */ +describe("OpenAiProvider stream transport retry (pre-first-chunk)", () => { + it("reopens when the body dies before the first delta, without duplicating output", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(streamingResponse([new Error("terminated")])) + .mockResolvedValueOnce( + streamingResponse([contentFrame("hello world"), STOP_FRAMES]), + ); + + const { chunks, result } = await drain( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(result.content).toBe("hello world"); + // Exactly one delivery of the text — a replayed stream must not stack. + expect(chunks.map((c) => c.delta).join("")).toBe("hello world"); + }); + + it("does not retry once a delta has been yielded to the caller", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + streamingResponse([contentFrame("part one"), new Error("terminated")]), + ) + .mockResolvedValueOnce( + streamingResponse([contentFrame("part one"), STOP_FRAMES]), + ); + + const { chunks, error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect((error as Error).message).toBe("terminated"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(chunks.map((c) => c.delta).join("")).toBe("part one"); + }); + + it("treats a reasoning delta as output and refuses to retry after it", async () => { + const reasoningFrame = frame({ + model: "qwen-test", + choices: [ + { + index: 0, + // `reasoning` is the field the default `delta_reasoning` format reads. + delta: { role: "assistant", reasoning: "thinking…" }, + finish_reason: null, + }, + ], + }); + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + streamingResponse([reasoningFrame, new Error("terminated")]), + ) + .mockResolvedValueOnce( + streamingResponse([contentFrame("hello"), STOP_FRAMES]), + ); + + const { chunks, error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect((error as Error).message).toBe("terminated"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(chunks.map((c) => c.reasoningDelta).join("")).toBe("thinking…"); + }); + + it("does not retry when the caller cancelled, even though the error looks like a drop", async () => { + // The consumer is parked inside `reader.read()` while the gate is + // closed, which is what makes the ordering deterministic: the abort + // lands while a read is in flight, so the failure genuinely reaches + // the retry decision as `Error: terminated` with an aborted signal — + // the exact collision the cancellation guard exists for. + const controller = new AbortController(); + let openGate: () => void = () => {}; + const gate = new Promise((resolve) => { + openGate = resolve; + }); + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + new Response( + new ReadableStream({ + async pull(streamController) { + await gate; + streamController.error(new Error("terminated")); + }, + }), + { + status: 200, + headers: { "content-type": "text/event-stream" }, + }, + ), + ) + .mockResolvedValueOnce( + streamingResponse([contentFrame("hello"), STOP_FRAMES]), + ); + + const pending = drainToError( + provider(fetchImpl).completeStream({ + prompt: "hi", + signal: controller.signal, + }), + ); + await new Promise((resolve) => setTimeout(resolve, 20)); + controller.abort(); + openGate(); + const { error } = await pending; + + expect((error as Error).message).toBe("terminated"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("does not retry a failure that is not a transport death", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(streamingResponse([new Error("consumer bug")])) + .mockResolvedValueOnce( + streamingResponse([contentFrame("hello"), STOP_FRAMES]), + ); + + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect((error as Error).message).toBe("consumer bug"); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("does not re-retry an open failure that already spent the HTTP budget", async () => { + // 500s are retried by `runOpenAiWithRetry` itself. If this layer + // retried the resulting `OpenAiHttpError` too, the budget would be + // squared (3 × 3 = 9 requests) instead of shared. + const fetchImpl = vi.fn( + async () => new Response("upstream exploded", { status: 500 }), + ); + + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect(error).toBeInstanceOf(OpenAiHttpError); + expect((error as OpenAiHttpError).status).toBe(500); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it("gives one streaming completion the shared attempt budget, then gives up", async () => { + const fetchImpl = vi.fn(async () => + streamingResponse([new Error("terminated")]), + ); + + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect((error as Error).message).toBe("terminated"); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); +}); From 84b6537e2df7224c0dbbe77aa4bedd57d2cee368 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:31:52 +0300 Subject: [PATCH 06/36] feat(loop-detector): detect overlapping re-reads of an unchanged file `ToolLoopTracker` hashes canonical arguments, so two reads of the same unchanged file with shifted `offset`/`limit` produce different signatures and neither the repeat counter nor the no-progress streak ever fires -- even when the second read returns only lines the first one already showed. `os.fs.read` is also (correctly) excluded from the wandering detector, because scanning many files is legitimate work. The result is that an overlapping re-read loop burns steps until the exact-repeat protection eventually catches a verbatim duplicate. Add a read-coverage detector beside the argument-hashing machinery rather than another way to hash arguments, because the question it answers is about the RESULT: did this read show the model a line it had not already seen? - `os.fs.read` now publishes `details.readCoverage`: the canonical (symlink-resolved) path, a hash of the bytes it actually read, and the line range it actually returned. Requested offset/limit cannot answer any of the three -- they are clamped, may be negative, and byte-mode reads carry no range at all. The hash is over content, so a same-size in-place replacement with an untouched mtime is caught. - `ToolLoopTracker.checkReadRepeat` / `recordRead` keep a per-file merged set of the lines read at the current content hash. A returned range fully inside that set (or an empty return) is no progress; new lines reset the streak; a changed hash discards the coverage entirely. - The batch gate raises a warn-only `read_repeat` signal after the call, since the read has already executed and there is nothing to veto. The notice, the `loop_detected` event and the log line carry the path, the range and the fingerprint transition -- line numbers only, never file content. Failed reads record nothing, a truncated read banks only the prefix it returned, and a scan over many distinct files never produces a signal. Fixes #114 --- AGENTS.md | 6 +- src/agent/agent-loop.ts | 67 +++- src/agent/batch-executor.ts | 72 ++++ src/agent/index.ts | 10 + src/agent/loop-detector.ts | 185 +++++++++- src/agent/read-coverage.test.ts | 477 ++++++++++++++++++++++++++ src/agent/read-coverage.ts | 149 ++++++++ src/cli/trace-formatter.ts | 11 +- src/tools/os/fs-read-coverage.test.ts | 196 +++++++++++ src/tools/os/fs-read-coverage.ts | 109 ++++++ src/tools/os/fs-read.ts | 80 ++++- src/tracing/trace/trace-event.ts | 21 +- src/tracing/trace/trace-recorder.ts | 1 + 13 files changed, 1362 insertions(+), 22 deletions(-) create mode 100644 src/agent/read-coverage.test.ts create mode 100644 src/agent/read-coverage.ts create mode 100644 src/tools/os/fs-read-coverage.test.ts create mode 100644 src/tools/os/fs-read-coverage.ts diff --git a/AGENTS.md b/AGENTS.md index dcbf3718..780d7884 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,13 +124,15 @@ The runtime guards against "stuck" turns where the model re-emits the same tool **Wandering detector (distinct-spread).** `getRepeatCount` / `getNoProgressStreak` only catch the *same* signature repeating. A model probing endless **distinct** URLs / queries / pages on one tool (e.g. guessing 8 different `os.web.fetch` URLs, or firing 21 search POSTs that differ only in volatile result fields) is a different failure mode. For wandering-prone tools (`isWanderingProneTool`: `os.web.fetch`, `os.http.request`, `browser.*`), `check()` computes `effectiveSpread` — the count of distinct completed `argsHash`es for that tool in the window, plus one when the prospective call introduces a new signature. Crossing `loopWanderingThreshold` ⇒ a `wandering` warn whose notice is an **actionable redirect** (`formatWanderingRedirect`: "stop probing URLs, run a web search or reply best-effort") rather than the repeat advisory. Crossing `loopWanderingEscalation` ⇒ `isWanderingEscalated()` returns true and the gate raises a `breaker` signal — the unique call **is** vetoed and the turn ends gracefully. Bulk reads over distinct files (`os.fs.read`) are deliberately **not** wandering-prone — scanning many files is legitimate work. +**Read-coverage detector (semantic progress on `os.fs.read`).** Argument hashing cannot see that `offset: 40, limit: 30` and `offset: 90, limit: 30` returned text the model already has, and reads are (correctly) not wandering-prone, so an overlapping re-read of one unchanged file used to be invisible. `os.fs.read` therefore publishes a `details.readCoverage` block — canonical (symlink-resolved) path, a hash of the bytes it actually read, and the line range it actually returned (see [src/tools/os/fs-read-coverage.ts](src/tools/os/fs-read-coverage.ts)) — and after each call the gate folds that range into a per-file coverage set (`checkReadRepeat` / `recordRead`, [src/agent/read-coverage.ts](src/agent/read-coverage.ts)). A read whose returned range is already fully covered at the same content hash makes no progress; newly covered lines do; a changed hash discards the file's coverage, so a same-size in-place edit with an untouched mtime resets it. Crossing `READ_REPEAT_WARNING_THRESHOLD` (2 consecutive no-progress reads) ⇒ a `read_repeat` **warn** (`formatReadRepeatNotice`, line numbers only — never file content). Warn-only by construction: the read has already executed, so there is nothing to veto. Failed reads record nothing, a truncated read banks only the prefix it returned, and a scan over many distinct files never signals. + **Volatile-stripping in `hashToolOutcome`.** Before hashing a generic (non-shell) result's `details`, `stripVolatile` recursively drops `VOLATILE_RESULT_KEYS` (`timestamp`, `ts`, `date`, `time`, `timeTotal`, `timeTotalSeconds`, `durationMs`, `sizeDownload`, `requestId`/`request_id`, `id`, `traceId`/`trace_id`, `sentAt`, `createdAt`, `deliveredAt`). Without this, per-call timings/sizes (e.g. `timeTotalSeconds`, `sizeDownload` on `os.http.request`) make every result hash unique, so a repeated dead/identical endpoint never registers as a no-progress streak. Mirrors OpenClaw's `stripVolatileSendIds`. **Breaker → graceful reply.** When `breakerVetoStreak` reaches `loopBreakerVetoStreak` (consecutive-veto path) **or** a wandering loop crosses `loopWanderingEscalation`, the gate raises a `breaker` signal. `AgentLoop` then ends the turn with a forced synthetic `reply` (`formatForcedLoopReply`) recorded as a normal `assistant_reply` turn — `reason: "reply"`, session stays `pending`. **No `loop_failed`, no `ModelError`.** `hashToolOutcome` keys results on error details / shell exit codes / volatile-stripped summary+details so two genuinely different results break the streak. -**Trace.** `loop_detected` events carry `level` (`warn` | `critical` | `breaker`) and `detector` (`generic_repeat` | `no_progress` | `wandering`) — see [src/tracing/trace/trace-event.ts](src/tracing/trace/trace-event.ts). +**Trace.** `loop_detected` events carry `level` (`warn` | `critical` | `breaker`) and `detector` (`generic_repeat` | `no_progress` | `wandering` | `test_repeat` | `read_repeat`) — see [src/tracing/trace/trace-event.ts](src/tracing/trace/trace-event.ts). A `read_repeat` event also carries `read` (resolved path, returned range, and the fingerprint on either side of the read), which is enough to audit why it fired without recording a line of the file. -Pinned by [src/agent/loop-detector.test.ts](src/agent/loop-detector.test.ts), [src/agent/batch-executor.test.ts](src/agent/batch-executor.test.ts) (veto single call / siblings survive / terminal never vetoed / breaker escalation), and [src/agent/agent-loop.test.ts](src/agent/agent-loop.test.ts) ("ends the turn with a graceful reply (not loop_failed) when the breaker trips"). +Pinned by [src/agent/loop-detector.test.ts](src/agent/loop-detector.test.ts), [src/agent/read-coverage.test.ts](src/agent/read-coverage.test.ts) (containment / partial overlap / pagination / symlink identity / same-size replacement / truncation / mutation / multi-file scan), [src/agent/batch-executor.test.ts](src/agent/batch-executor.test.ts) (veto single call / siblings survive / terminal never vetoed / breaker escalation), and [src/agent/agent-loop.test.ts](src/agent/agent-loop.test.ts) ("ends the turn with a graceful reply (not loop_failed) when the breaker trips"). ### Out of scope (deferred) diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index 3bdf8f3c..07dd69bd 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -43,7 +43,9 @@ import { executeStep } from "./step-executor.js"; import type { LlmStreamParams, StepEvent } from "./step-executor.js"; import { ToolLoopTracker, + READ_REPEAT_WARNING_THRESHOLD, TEST_REPEAT_WARNING_THRESHOLD, + formatReadRepeatNotice, formatRepeatNotice, formatTestRepeatNotice, formatWanderingRedirect, @@ -349,7 +351,25 @@ export type AgentLoopEvent = /** Graduated severity from the `ToolLoopTracker`. */ level?: "warn" | "critical" | "breaker"; /** Which sub-detector fired. */ - detector?: "generic_repeat" | "no_progress" | "wandering" | "test_repeat"; + detector?: + | "generic_repeat" + | "no_progress" + | "wandering" + | "test_repeat" + | "read_repeat"; + /** + * `read_repeat` only: the resolved file, the range that read + * returned, and the fingerprint on either side of it (equal ⇒ the + * content did not change, which is what makes the read redundant). + * Line numbers and a path — never file content. + */ + read?: { + path: string; + startLine: number; + endLine: number; + previousFingerprint: string; + fingerprint: string; + }; } | { type: "loop_completed"; @@ -797,9 +817,13 @@ export class AgentLoop { // re-injected on every subsequent identical step. for (const sig of loopSignals) { if (sig.kind !== "warn") continue; - // The test-repeat detector has its own floor: the 2nd - // equivalent run is already conclusive, so it must not wait - // for the generic warning threshold (default 3). + // Two detectors carry their own floor because their signal is + // conclusive earlier than a byte-identical repeat is. A 2nd + // test run against an unchanged workspace cannot produce new + // evidence; a 2nd consecutive read of an unchanged file that + // returned nothing new cannot produce new text. Waiting for + // the generic threshold (default 3) would burn another step in + // both cases. const emit = sig.detector === "test_repeat" ? loopTracker.shouldEmitWarning( @@ -807,7 +831,13 @@ export class AgentLoop { sig.count, TEST_REPEAT_WARNING_THRESHOLD, ) - : loopTracker.shouldEmitWarning(sig.warningKey, sig.count); + : sig.detector === "read_repeat" + ? loopTracker.shouldEmitWarning( + sig.warningKey, + sig.count, + READ_REPEAT_WARNING_THRESHOLD, + ) + : loopTracker.shouldEmitWarning(sig.warningKey, sig.count); if (!emit) { continue; } @@ -816,7 +846,9 @@ export class AgentLoop { ? formatWanderingRedirect(sig.tool, sig.count) : sig.detector === "test_repeat" ? formatTestRepeatNotice(sig) - : formatRepeatNotice(sig); + : sig.detector === "read_repeat" && sig.read !== undefined + ? formatReadRepeatNotice({ count: sig.count, ...sig.read }) + : formatRepeatNotice(sig); this.deps.onEvent?.({ type: "loop_detected", tool: sig.tool, @@ -824,12 +856,35 @@ export class AgentLoop { stepIndex: i, level: "warn", detector: sig.detector, + ...(sig.read !== undefined + ? { + read: { + path: sig.read.path, + startLine: sig.read.startLine, + endLine: sig.read.endLine, + previousFingerprint: sig.read.previousFingerprint, + fingerprint: sig.read.fingerprint, + }, + } + : {}), }); this.deps.logger?.warn("no-progress loop detected", { sessionId: state.id, stepIndex: i, tool: sig.tool, count: sig.count, + detector: sig.detector, + // Path, range and fingerprints only — enough to reconstruct + // WHY the detector fired without putting a line of the file + // into the log. + ...(sig.read !== undefined + ? { + path: sig.read.path, + range: `${sig.read.startLine}-${sig.read.endLine}`, + fingerprint: sig.read.fingerprint, + previousFingerprint: sig.read.previousFingerprint, + } + : {}), }); } state = await refreshMemoryContext(this.deps, state, options); diff --git a/src/agent/batch-executor.ts b/src/agent/batch-executor.ts index ed56bdb5..ff12ee82 100644 --- a/src/agent/batch-executor.ts +++ b/src/agent/batch-executor.ts @@ -19,6 +19,7 @@ import { type ToolLoopTracker, } from "./loop-detector.js"; import { classifyTestCommand } from "./test-command-key.js"; +import { classifyReadResult } from "./read-coverage.js"; import { fingerprintWorkspace } from "./workspace-fingerprint.js"; /** @@ -47,6 +48,27 @@ export interface BatchLoopSignal { * reproduced. */ previousSummary?: string; + /** + * `read_repeat` only: what the redundant read landed on. Feeds both + * the notice (path, ranges) and the `loop_detected` event (path, + * range, fingerprint transition). Line numbers and a path — never any + * file content. + */ + read?: { + /** Canonical (symlink-resolved) path of the file read. */ + path: string; + /** Range this read returned; `0`/`0` when it returned nothing. */ + startLine: number; + endLine: number; + /** Lines visible in the read window. */ + totalLines: number; + /** Compact list of lines already read this turn, e.g. `"1-40, 88-120"`. */ + covered: string; + /** Content fingerprint this read saw. */ + fingerprint: string; + /** Fingerprint of the previous read; equal ⇒ the content is unchanged. */ + previousFingerprint: string; + }; } /** @@ -356,6 +378,7 @@ export async function executeBatch( // (args + result) entry. Terminal verbs are not tracked. if (ctx.tracker && input.resourceClass !== "terminal") { ctx.tracker.recordOutcome(input.call.tool, input.call.args, compressed); + observeReadCoverage(input, compressed, ctx.tracker, loopSignals); } ctx.onCallFinished?.({ batchIndex: input.batchIndex, @@ -615,6 +638,55 @@ function runSyncLoopGate( return { proceed: true }; } +/** + * Read-coverage gate (issue #114, companion of #118). Runs AFTER the call + * completed, because the facts it needs — which file the read resolved + * to, which version of it was read, and which lines came back — are + * properties of the result, not of the arguments. Requested + * `offset`/`limit` are clamped and can be negative, so they cannot + * answer any of the three. + * + * Warn-only, like the test-repeat detector: the read has already + * happened, so there is nothing to block, and a scan over many distinct + * files never produces a signal at all (each file's coverage grows, and + * only a read that returns nothing new counts). Non-read tools and + * failed reads return `null` from `classifyReadResult` and leave no + * trace here. + */ +function observeReadCoverage( + input: BatchCallInput, + result: CompressedToolResult, + tracker: ToolLoopTracker, + loopSignals: BatchLoopSignal[], +): void { + const observation = classifyReadResult(input.call.tool, result); + if (observation === null) return; + const repeat = tracker.checkReadRepeat(observation); + tracker.recordRead(observation); + if (!repeat.repeat) return; + loopSignals.push({ + kind: "warn", + tool: input.call.tool, + count: repeat.count, + detector: "read_repeat", + // Keyed by file VERSION: editing the file starts a fresh warn bucket, + // so a nudge about the old content is never suppressed for the new. + warningKey: `read_repeat:${observation.path}:${observation.contentHash}`, + read: { + path: observation.path, + startLine: observation.span?.start ?? 0, + endLine: observation.span?.end ?? 0, + totalLines: observation.totalLines, + covered: repeat.covered, + fingerprint: observation.contentHash, + // `checkReadRepeat` only reports a repeat when it has seen this + // file before, so the previous fingerprint is always present here; + // the fallback keeps the type honest without a non-null assertion. + previousFingerprint: repeat.previousFingerprint ?? observation.contentHash, + }, + }); +} + /** * Helper: turn a parsed `ToolCallPayload[]` into the `BatchCallInput[]` * shape `executeBatch` expects, computing each call's resource class. diff --git a/src/agent/index.ts b/src/agent/index.ts index 48543830..e6d618e5 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -18,6 +18,7 @@ export { isLoopVetoResult, hashToolCall, hashToolOutcome, + formatReadRepeatNotice, formatRepeatNotice, formatTestRepeatNotice, formatVetoInstruction, @@ -27,13 +28,22 @@ export { LOOP_VETO_DENIED_REASON, LOOP_WARNING_BUCKET_SIZE, TEST_REPEAT_WARNING_THRESHOLD, + READ_REPEAT_WARNING_THRESHOLD, } from "./loop-detector.js"; export type { ToolLoopTrackerOptions, LoopCheckVerdict, LoopCheckLevel, TestRepeatCheck, + ReadRepeatCheck, } from "./loop-detector.js"; +export { + classifyReadResult, + describeCoverage, + mergeRange, + newlyCoveredCount, +} from "./read-coverage.js"; +export type { LineRange, ReadObservation } from "./read-coverage.js"; export { classifyTestCommand } from "./test-command-key.js"; export type { RecognizedTestCommand } from "./test-command-key.js"; export { diff --git a/src/agent/loop-detector.ts b/src/agent/loop-detector.ts index 7cf87085..5db8390f 100644 --- a/src/agent/loop-detector.ts +++ b/src/agent/loop-detector.ts @@ -1,5 +1,12 @@ import { createHash } from "node:crypto"; import type { CompressedToolResult } from "../compressor/result-compressor.js"; +import { + describeCoverage, + mergeRange, + newlyCoveredCount, + type LineRange, + type ReadObservation, +} from "./read-coverage.js"; /** * Synthetic tool name used for batched-step diagnostics. A multi-call @@ -36,6 +43,45 @@ export type LoopCheckLevel = "ok" | "warn" | "critical"; */ export const TEST_REPEAT_WARNING_THRESHOLD = 2; +/** + * No-progress read count at which the read-coverage detector (issue #114) + * warns. A single redundant re-read is ordinary behaviour — the model + * re-opens a file to re-orient itself, or widens a window it half + * remembers — so the floor is the SECOND consecutive read of one + * unchanged file that returned nothing new. Warn-only, like the + * test-repeat detector: nothing is ever vetoed on this signal. + */ +export const READ_REPEAT_WARNING_THRESHOLD = 2; + +/** + * Cap on files tracked for read coverage in one turn. A wide scan (a + * grep-driven sweep over hundreds of files) must not grow the tracker + * without bound, and the interesting file is always a recently read one, + * so the least-recently-read entry is evicted first. Eviction can only + * cost a detection, never cause a false one. + */ +const MAX_TRACKED_READ_FILES = 200; + +/** + * Verdict of `ToolLoopTracker.checkReadRepeat`: did this read show the + * model any line it had not already seen this turn? + */ +export interface ReadRepeatCheck { + /** True when the read returned no line the turn had not already seen. */ + repeat: boolean; + /** Consecutive no-progress reads of this file version; ≥1 when `repeat`. */ + count: number; + /** Compact list of lines already read, e.g. `"1-40, 88-120"`. */ + covered: string; + /** + * Fingerprint of the version this file was last read at, when it was + * read before. Equal to the observation's own hash for a `repeat` — + * that equality IS the "unchanged content" half of the verdict, so the + * event carries both sides and a trace reader can check it. + */ + previousFingerprint?: string; +} + /** * Verdict of `ToolLoopTracker.checkTestRepeat`: is this recognized test * command an equivalent re-run against an unchanged workspace? @@ -79,7 +125,12 @@ export interface LoopCheckVerdict { * distinct-args spread (wandering). */ count: number; - detector: "generic_repeat" | "no_progress" | "wandering" | "test_repeat"; + detector: + | "generic_repeat" + | "no_progress" + | "wandering" + | "test_repeat" + | "read_repeat"; /** Stable key for warn de-duplication and breaker signalling. */ warningKey: string; tool: string; @@ -163,6 +214,17 @@ export class ToolLoopTracker { * right `testRuns` entry without re-classifying the command. */ private readonly pendingTestKeys = new Map(); + /** + * Read-coverage detector state (issue #114): canonical file path → the + * content fingerprint that path was last read at, the merged set of + * lines read at THAT fingerprint, and how many reads in a row have + * returned nothing outside it. Insertion order doubles as a + * least-recently-read order for eviction (see `MAX_TRACKED_READ_FILES`). + */ + private readonly readCoverage = new Map< + string, + { contentHash: string; covered: LineRange[]; noProgress: number } + >(); constructor(options: ToolLoopTrackerOptions = {}) { this.warningThreshold = Math.max(2, options.warningThreshold ?? 3); @@ -355,6 +417,77 @@ export class ToolLoopTracker { this.pendingTestKeys.set(hashToolCall(tool, args), key); } + /** + * Classify a completed read against the coverage recorded for its file + * (issue #114). Pure — call BEFORE `recordRead`. + * + * Unlike the other detectors this one is post-hoc by necessity: which + * lines a read returns, and which version of the file it saw, are facts + * about the RESULT. There is nothing to gate at dispatch time, which is + * also why the signal is warn-only — the read has already happened, so + * blocking it would cost the model information without saving anything. + * + * No progress means: the file's content is byte-identical to what it + * was when this turn last read it, and every line this read returned + * was already returned earlier in the turn. A read that returned no + * lines at all (an offset past the end) also counts — it cannot have + * shown anything new — but only once the file has been seen at this + * version, so the first such read is never flagged. + */ + checkReadRepeat(observation: ReadObservation): ReadRepeatCheck { + const prev = this.readCoverage.get(observation.path); + if (prev === undefined) return { repeat: false, count: 0, covered: "" }; + const previousFingerprint = prev.contentHash; + if (prev.contentHash !== observation.contentHash) { + return { repeat: false, count: 0, covered: "", previousFingerprint }; + } + const fresh = + observation.span === null + ? 0 + : newlyCoveredCount(prev.covered, observation.span); + if (fresh > 0) { + return { repeat: false, count: 0, covered: "", previousFingerprint }; + } + return { + repeat: true, + count: prev.noProgress + 1, + covered: describeCoverage(prev.covered), + previousFingerprint, + }; + } + + /** + * Fold a completed read into its file's coverage. Call AFTER + * `checkReadRepeat`. + * + * A different content fingerprint discards the previous coverage + * outright: the lines the turn read before belong to a version of the + * file that no longer exists, so counting them again would mark a + * genuinely new read as no progress. That reset is also what makes an + * edit-then-re-read cycle free of false warnings. + */ + recordRead(observation: ReadObservation): void { + const prev = this.readCoverage.get(observation.path); + const sameVersion = + prev !== undefined && prev.contentHash === observation.contentHash; + const covered = sameVersion ? prev.covered : []; + const fresh = + observation.span === null ? 0 : newlyCoveredCount(covered, observation.span); + // Re-insert rather than mutate in place so the map's iteration order + // stays "least recently read first" for eviction. + this.readCoverage.delete(observation.path); + this.readCoverage.set(observation.path, { + contentHash: observation.contentHash, + covered: + observation.span === null ? covered : mergeRange(covered, observation.span), + noProgress: sameVersion && fresh === 0 ? prev.noProgress + 1 : 0, + }); + if (this.readCoverage.size > MAX_TRACKED_READ_FILES) { + const oldest = this.readCoverage.keys().next(); + if (!oldest.done) this.readCoverage.delete(oldest.value); + } + } + /** * Attach a completed run's summary to its pending test-key entry so * the next equivalent-run warning can quote the previous result. @@ -745,6 +878,56 @@ export function formatTestRepeatNotice(verdict: { return lines.join("\n"); } +/** + * Notice injected when the same unchanged file was read again without + * reaching a new line (issue #114, warn-only). + * + * Deliberately concrete about WHAT was already read — the last returned + * range and the covered line set — because the failure mode this catches + * is the model not realising its shifted `offset`/`limit` landed inside + * text it already has. Line numbers and the path only: no file content + * appears here, in the event, or in the log line. + */ +export function formatReadRepeatNotice(verdict: { + count: number; + path: string; + startLine: number; + endLine: number; + totalLines: number; + covered: string; +}): string { + const label = sanitizeReadPath(verdict.path); + const returned = + verdict.startLine === 0 + ? "returned no lines at all" + : `returned lines ${verdict.startLine}-${verdict.endLine}`; + const lines = [ + `You read ${label} ${verdict.count} times in a row without reaching a line you had not already read this turn. The last read ${returned}, and the file's content has not changed since the previous read.`, + ]; + if (verdict.covered.length > 0) { + lines.push( + `Already read this turn: lines ${verdict.covered}${verdict.totalLines > 0 ? ` (of ${verdict.totalLines} readable lines)` : ""}.`, + ); + } + lines.push( + "Re-reading a covered range returns the same text. Read a range you have not covered, open a different file, or act on what you already have. If the repeat was intentional, continue — this is a warning, nothing was blocked.", + ); + return lines.join("\n"); +} + +/** + * Path label for the read-repeat notice. `sanitizeLoopTarget` keeps the + * HEAD of an over-long label, which is exactly wrong for a path — the + * identifying part of `/very/long/prefix/src/agent/loop-detector.ts` is + * its tail — so a long path is elided from the left instead. + */ +function sanitizeReadPath(raw: string): string { + const cleaned = raw.replace(/[`\r\n]+/g, " ").trim(); + if (cleaned.length === 0) return "that file"; + const label = cleaned.length > 80 ? `…${cleaned.slice(-77)}` : cleaned; + return `\`${label}\``; +} + /** * Compact a previous-result summary for inline quoting in a notice: * whitespace collapsed to one line, length-capped. Returns `undefined` diff --git a/src/agent/read-coverage.test.ts b/src/agent/read-coverage.test.ts new file mode 100644 index 00000000..02ff3df0 --- /dev/null +++ b/src/agent/read-coverage.test.ts @@ -0,0 +1,477 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtemp, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { compressToolResult } from "../compressor/result-compressor.js"; +import { ToolRegistry } from "../tools/tool-registry.js"; +import { osFsReadTool } from "../tools/os/fs-read.js"; +import { READ_COVERAGE_DETAIL_KEY } from "../tools/os/fs-read-coverage.js"; +import { + classifyReadResult, + describeCoverage, + mergeRange, + newlyCoveredCount, + type LineRange, +} from "./read-coverage.js"; +import { + formatReadRepeatNotice, + READ_REPEAT_WARNING_THRESHOLD, + ToolLoopTracker, +} from "./loop-detector.js"; +import { + executeBatch, + toBatchInputs, + type BatchLoopSignal, +} from "./batch-executor.js"; + +// ---------------------------------------------------------------- algebra + +describe("read coverage range algebra", () => { + it("counts only lines outside the covered set", () => { + const covered: LineRange[] = [{ start: 10, end: 20 }]; + expect(newlyCoveredCount(covered, { start: 12, end: 18 })).toBe(0); + expect(newlyCoveredCount(covered, { start: 10, end: 20 })).toBe(0); + expect(newlyCoveredCount(covered, { start: 15, end: 25 })).toBe(5); + expect(newlyCoveredCount(covered, { start: 5, end: 12 })).toBe(5); + expect(newlyCoveredCount(covered, { start: 30, end: 32 })).toBe(3); + }); + + it("merges adjacent ranges so pagination collapses to one interval", () => { + let covered = mergeRange([], { start: 1, end: 40 }); + covered = mergeRange(covered, { start: 41, end: 80 }); + expect(covered).toEqual([{ start: 1, end: 80 }]); + // The seam between two pages must not read as uncovered. + expect(newlyCoveredCount(covered, { start: 38, end: 45 })).toBe(0); + }); + + it("keeps disjoint ranges separate and sorted", () => { + let covered = mergeRange([], { start: 50, end: 60 }); + covered = mergeRange(covered, { start: 1, end: 10 }); + covered = mergeRange(covered, { start: 20, end: 30 }); + expect(covered).toEqual([ + { start: 1, end: 10 }, + { start: 20, end: 30 }, + { start: 50, end: 60 }, + ]); + covered = mergeRange(covered, { start: 5, end: 55 }); + expect(covered).toEqual([{ start: 1, end: 60 }]); + }); + + it("describes coverage as line numbers only", () => { + expect(describeCoverage([])).toBe(""); + expect( + describeCoverage([ + { start: 1, end: 40 }, + { start: 88, end: 88 }, + ]), + ).toBe("1-40, 88"); + const many: LineRange[] = Array.from({ length: 6 }, (_, i) => ({ + start: i * 10 + 1, + end: i * 10 + 2, + })); + expect(describeCoverage(many)).toContain("… (2 more)"); + }); +}); + +// ------------------------------------------------------------ observation + +describe("classifyReadResult", () => { + const detail = { + path: "/tmp/a.ts", + contentHash: "hash1", + startLine: 3, + endLine: 9, + totalLines: 40, + }; + + it("extracts the observation from a successful read", () => { + const result = compressToolResult({ + tool: "os.fs.read", + status: "ok", + output: "body", + details: { [READ_COVERAGE_DETAIL_KEY]: detail }, + }); + expect(classifyReadResult("os.fs.read", result)).toEqual({ + path: "/tmp/a.ts", + contentHash: "hash1", + span: { start: 3, end: 9 }, + totalLines: 40, + }); + }); + + it("ignores other tools even when they carry a coverage detail", () => { + const result = compressToolResult({ + tool: "os.fs.grep", + status: "ok", + output: "body", + details: { [READ_COVERAGE_DETAIL_KEY]: detail }, + }); + expect(classifyReadResult("os.fs.grep", result)).toBeNull(); + }); + + it("records nothing for a failed read", () => { + const result = compressToolResult({ + tool: "os.fs.read", + status: "error", + output: "ENOENT", + details: { errorName: "Error", [READ_COVERAGE_DETAIL_KEY]: detail }, + }); + expect(classifyReadResult("os.fs.read", result)).toBeNull(); + }); + + it("maps an empty return to a null span", () => { + const result = compressToolResult({ + tool: "os.fs.read", + status: "ok", + output: "", + details: { + [READ_COVERAGE_DETAIL_KEY]: { ...detail, startLine: 0, endLine: 0 }, + }, + }); + expect(classifyReadResult("os.fs.read", result)?.span).toBeNull(); + }); +}); + +// ---------------------------------------------------------------- tracker + +function observe( + path: string, + contentHash: string, + span: LineRange | null, + totalLines = 200, +): Parameters[0] { + return { path, contentHash, span, totalLines }; +} + +describe("ToolLoopTracker read-coverage detector", () => { + it("does not flag the first read of a file", () => { + const tracker = new ToolLoopTracker(); + const obs = observe("/a.ts", "v1", { start: 1, end: 50 }); + expect(tracker.checkReadRepeat(obs).repeat).toBe(false); + tracker.recordRead(obs); + }); + + it("flags a re-read fully contained in what was already read", () => { + const tracker = new ToolLoopTracker(); + const first = observe("/a.ts", "v1", { start: 1, end: 100 }); + tracker.recordRead(first); + // Different offset/limit, so the argument hash differs and the generic + // detectors stay silent — this is the case issue #114 is about. + const contained = observe("/a.ts", "v1", { start: 20, end: 60 }); + const check = tracker.checkReadRepeat(contained); + expect(check.repeat).toBe(true); + expect(check.count).toBe(1); + expect(check.covered).toBe("1-100"); + expect(check.previousFingerprint).toBe("v1"); + }); + + it("counts consecutive no-progress reads and reaches the warn floor", () => { + const tracker = new ToolLoopTracker(); + tracker.recordRead(observe("/a.ts", "v1", { start: 1, end: 100 })); + const again = observe("/a.ts", "v1", { start: 10, end: 20 }); + expect(tracker.checkReadRepeat(again).count).toBe(1); + tracker.recordRead(again); + const third = observe("/a.ts", "v1", { start: 30, end: 40 }); + const check = tracker.checkReadRepeat(third); + expect(check.count).toBe(READ_REPEAT_WARNING_THRESHOLD); + expect(check.repeat).toBe(true); + }); + + it("treats a partial overlap as progress and only banks the new lines", () => { + const tracker = new ToolLoopTracker(); + tracker.recordRead(observe("/a.ts", "v1", { start: 1, end: 50 })); + const overlapping = observe("/a.ts", "v1", { start: 40, end: 80 }); + expect(tracker.checkReadRepeat(overlapping).repeat).toBe(false); + tracker.recordRead(overlapping); + // 1-80 is now covered: a read inside it is no longer progress. + expect( + tracker.checkReadRepeat(observe("/a.ts", "v1", { start: 60, end: 75 })) + .repeat, + ).toBe(true); + expect( + tracker.checkReadRepeat(observe("/a.ts", "v1", { start: 81, end: 90 })) + .repeat, + ).toBe(false); + }); + + it("treats plain pagination as progress on every page", () => { + const tracker = new ToolLoopTracker(); + for (let page = 0; page < 6; page += 1) { + const span = { start: page * 40 + 1, end: page * 40 + 40 }; + expect(tracker.checkReadRepeat(observe("/a.ts", "v1", span)).repeat).toBe( + false, + ); + tracker.recordRead(observe("/a.ts", "v1", span)); + } + }); + + it("resets the streak when a read finally reaches new lines", () => { + const tracker = new ToolLoopTracker(); + tracker.recordRead(observe("/a.ts", "v1", { start: 1, end: 50 })); + tracker.recordRead(observe("/a.ts", "v1", { start: 10, end: 20 })); + tracker.recordRead(observe("/a.ts", "v1", { start: 30, end: 40 })); + tracker.recordRead(observe("/a.ts", "v1", { start: 51, end: 60 })); + const check = tracker.checkReadRepeat(observe("/a.ts", "v1", { start: 1, end: 5 })); + expect(check.count).toBe(1); + }); + + it("resets coverage when the content changes", () => { + const tracker = new ToolLoopTracker(); + tracker.recordRead(observe("/a.ts", "v1", { start: 1, end: 100 })); + tracker.recordRead(observe("/a.ts", "v1", { start: 10, end: 20 })); + const afterEdit = observe("/a.ts", "v2", { start: 10, end: 20 }); + const check = tracker.checkReadRepeat(afterEdit); + expect(check.repeat).toBe(false); + expect(check.previousFingerprint).toBe("v1"); + tracker.recordRead(afterEdit); + // The pre-edit coverage is gone: 1-100 is worth reading again. + expect( + tracker.checkReadRepeat(observe("/a.ts", "v2", { start: 1, end: 100 })) + .repeat, + ).toBe(false); + }); + + it("keeps files independent, so a multi-file scan never trips", () => { + const tracker = new ToolLoopTracker(); + for (let i = 0; i < 50; i += 1) { + const obs = observe(`/file-${i}.ts`, `v${i}`, { start: 1, end: 120 }); + expect(tracker.checkReadRepeat(obs).repeat).toBe(false); + tracker.recordRead(obs); + } + // The same range in a different file is a different question. + expect( + tracker.checkReadRepeat(observe("/other.ts", "v0", { start: 1, end: 120 })) + .repeat, + ).toBe(false); + }); + + it("flags a read that returned nothing, but never the first one", () => { + const tracker = new ToolLoopTracker(); + const past = observe("/a.ts", "v1", null, 40); + expect(tracker.checkReadRepeat(past).repeat).toBe(false); + tracker.recordRead(past); + expect(tracker.checkReadRepeat(observe("/a.ts", "v1", null, 40)).repeat).toBe( + true, + ); + }); + + it("bounds the number of tracked files", () => { + const tracker = new ToolLoopTracker(); + const first = observe("/file-0.ts", "v0", { start: 1, end: 10 }); + tracker.recordRead(first); + for (let i = 1; i <= 250; i += 1) { + tracker.recordRead(observe(`/file-${i}.ts`, `v${i}`, { start: 1, end: 10 })); + } + // The oldest entry has been evicted, so its re-read reads as fresh — + // a missed detection, which is the safe direction. + expect(tracker.checkReadRepeat(first).repeat).toBe(false); + }); +}); + +describe("formatReadRepeatNotice", () => { + const base = { + count: 2, + path: "/repo/src/agent/loop-detector.ts", + startLine: 20, + endLine: 60, + totalLines: 900, + covered: "1-120", + }; + + it("names the file, the range and the coverage without any content", () => { + const notice = formatReadRepeatNotice(base); + expect(notice).toContain("loop-detector.ts"); + expect(notice).toContain("lines 20-60"); + expect(notice).toContain("1-120"); + expect(notice).toContain("900"); + expect(notice).toContain("nothing was blocked"); + }); + + it("elides a long path from the left so the basename survives", () => { + const notice = formatReadRepeatNotice({ + ...base, + path: `/${"deep/".repeat(40)}target.ts`, + }); + expect(notice).toContain("target.ts"); + expect(notice).toContain("…"); + }); + + it("words an empty return honestly", () => { + const notice = formatReadRepeatNotice({ ...base, startLine: 0, endLine: 0 }); + expect(notice).toContain("returned no lines at all"); + }); +}); + +// -------------------------------------------------------------- end-to-end + +function batchCtx(workingDir: string, tracker: ToolLoopTracker) { + return { + workingDir, + sessionId: "s1", + stepIndex: 0, + signal: new AbortController().signal, + tracker, + }; +} + +function readRegistry(): ToolRegistry { + const registry = new ToolRegistry(); + registry.register(osFsReadTool); + return registry; +} + +/** Run one `os.fs.read` through the real batch executor and gate. */ +async function runRead( + dir: string, + tracker: ToolLoopTracker, + args: Record, +): Promise { + const outcome = await executeBatch( + toBatchInputs([{ tool: "os.fs.read", args }]), + readRegistry(), + batchCtx(dir, tracker), + ); + expect(outcome.results[0]?.compressed?.status).toBe("ok"); + return outcome.loopSignals; +} + +describe("read-coverage detection end to end", () => { + let dir: string; + let tracker: ToolLoopTracker; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "atomic-read-loop-")); + tracker = new ToolLoopTracker(); + const body = Array.from({ length: 200 }, (_, i) => `line ${i + 1}`).join("\n"); + await writeFile(join(dir, "src.ts"), `${body}\n`, "utf8"); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("raises a read_repeat warn for a shifted re-read of the same file", async () => { + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + // Both re-reads use offsets the first (byte-mode) read never + // mentioned, so every argument hash differs and only the coverage + // detector can see that nothing new came back. + expect( + await runRead(dir, tracker, { path: "src.ts", offset: 40, limit: 30 }), + ).toHaveLength(1); + const signals = await runRead(dir, tracker, { + path: "src.ts", + offset: 90, + limit: 30, + }); + expect(signals).toHaveLength(1); + const signal = signals[0]!; + expect(signal.kind).toBe("warn"); + expect(signal.detector).toBe("read_repeat"); + expect(signal.count).toBe(READ_REPEAT_WARNING_THRESHOLD); + expect(signal.read?.startLine).toBe(90); + expect(signal.read?.endLine).toBe(119); + expect(signal.read?.covered).toBe("1-200"); + // The fingerprint transition proves the content stood still. + expect(signal.read?.previousFingerprint).toBe(signal.read?.fingerprint); + // Path and numbers only: no line of the file leaks into the signal. + expect(JSON.stringify(signal)).not.toContain("line 90"); + }); + + it("sees a symlink and its target as one file", async () => { + await symlink(join(dir, "src.ts"), join(dir, "alias.ts")); + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + const signals = await runRead(dir, tracker, { + path: "alias.ts", + offset: 10, + limit: 5, + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.detector).toBe("read_repeat"); + }); + + it("stays silent while paginating a long file", async () => { + for (let page = 0; page < 4; page += 1) { + const signals = await runRead(dir, tracker, { + path: "src.ts", + offset: page * 50 + 1, + limit: 50, + }); + expect(signals).toEqual([]); + } + }); + + it("stays silent while scanning many distinct files", async () => { + for (let i = 0; i < 12; i += 1) { + await writeFile(join(dir, `f${i}.ts`), `unique body ${i}\n`, "utf8"); + } + for (let i = 0; i < 12; i += 1) { + expect(await runRead(dir, tracker, { path: `f${i}.ts` })).toEqual([]); + } + }); + + it("goes quiet again after a same-size edit with a pinned mtime", async () => { + const file = join(dir, "small.ts"); + const pinned = new Date(1_700_000_000_000); + await writeFile(file, "aaa\nbbb\nccc\n", "utf8"); + await utimes(file, pinned, pinned); + + expect(await runRead(dir, tracker, { path: "small.ts" })).toEqual([]); + expect( + await runRead(dir, tracker, { path: "small.ts", offset: 2, limit: 2 }), + ).toHaveLength(1); + + await writeFile(file, "xxx\nyyy\nzzz\n", "utf8"); + await utimes(file, pinned, pinned); + // Same size, same mtime, different bytes: the edit must reset coverage. + expect( + await runRead(dir, tracker, { path: "small.ts", offset: 1, limit: 3 }), + ).toEqual([]); + }); + + it("credits a truncated read with only the prefix it returned", async () => { + // The file is ~1.5 KB; a 400-byte cap returns a few dozen lines. The + // banked coverage must be that prefix, never the whole 200-line file. + expect( + await runRead(dir, tracker, { path: "src.ts", maxBytes: 400 }), + ).toEqual([]); + const signals = await runRead(dir, tracker, { + path: "src.ts", + maxBytes: 400, + offset: 5, + limit: 5, + }); + expect(signals).toHaveLength(1); + const read = signals[0]!.read!; + expect(read.totalLines).toBeGreaterThan(0); + expect(read.totalLines).toBeLessThan(200); + expect(read.covered).toBe(`1-${read.totalLines}`); + }); + + it("treats a widened byte window as a new version, not a repeat", async () => { + // A larger `maxBytes` reads a longer prefix, so the fingerprint moves + // and coverage resets. That is a deliberately missed detection (the + // model may genuinely be reaching for content the cap hid), never a + // false one. + expect( + await runRead(dir, tracker, { path: "src.ts", maxBytes: 400 }), + ).toEqual([]); + expect( + await runRead(dir, tracker, { path: "src.ts", maxBytes: 4000, offset: 1, limit: 5 }), + ).toEqual([]); + }); + + it("records nothing for a failed read", async () => { + const outcome = await executeBatch( + toBatchInputs([ + { tool: "os.fs.read", args: { path: "missing.ts" } }, + { tool: "os.fs.read", args: { path: "missing.ts" } }, + ]), + readRegistry(), + batchCtx(dir, tracker), + ); + expect(outcome.results[0]?.compressed?.status).toBe("error"); + expect( + outcome.loopSignals.filter((s) => s.detector === "read_repeat"), + ).toEqual([]); + }); +}); diff --git a/src/agent/read-coverage.ts b/src/agent/read-coverage.ts new file mode 100644 index 00000000..5d759f19 --- /dev/null +++ b/src/agent/read-coverage.ts @@ -0,0 +1,149 @@ +import type { CompressedToolResult } from "../compressor/result-compressor.js"; +import { parseReadCoverage } from "../tools/os/fs-read-coverage.js"; + +/** + * Semantic progress signal for file reads (issue #114, companion of #118). + * + * `ToolLoopTracker` hashes canonical arguments, so two reads of the same + * unchanged file with shifted `offset`/`limit` are different signatures + * and neither the repeat counter nor the no-progress streak ever fires — + * even when the second read returns lines the first one already showed. + * `os.fs.read` is also (correctly) excluded from the wandering detector, + * because scanning many files is legitimate work. + * + * What is missing is not a fourth way to hash arguments but a different + * question: did this read show the model a line it had not already seen? + * That is answered by the RESULT — the resolved file, the version of its + * content, and the range that actually came back — which is why this + * detector sits beside the argument-hashing machinery and consumes + * `os.fs.read` results, and why it only ever produces the same kind of + * warn signal the tracker's other detectors do. + * + * Coverage is per file VERSION: a content change starts a fresh, empty + * coverage set, so re-reading a file after editing it is progress. + */ + +/** 1-based inclusive line range. */ +export interface LineRange { + start: number; + end: number; +} + +/** What one successful `os.fs.read` observed, for coverage bookkeeping. */ +export interface ReadObservation { + /** Canonical (symlink-resolved) path — the file's identity. */ + path: string; + /** Fingerprint of the content this read looked at. */ + contentHash: string; + /** Range returned, or `null` when the read returned no lines at all. */ + span: LineRange | null; + /** Lines visible in the read window (see `ReadCoverageDetail`). */ + totalLines: number; +} + +/** + * Extract a read observation from a completed tool result, or `null` when + * the result carries no usable read semantics. + * + * Returns `null` for every non-`os.fs.read` tool and for every failed + * read: an error result has no returned range, and inventing one (say, + * the requested offset/limit) would credit coverage for lines the model + * never saw and could then suppress a later, genuinely useful read. A + * truncated read is NOT skipped — it returned real lines — but it only + * ever contributes the range it actually returned, never the whole file. + */ +export function classifyReadResult( + tool: string, + result: CompressedToolResult, +): ReadObservation | null { + if (tool !== "os.fs.read") return null; + if (result.status !== "ok") return null; + const detail = parseReadCoverage(result.details); + if (detail === null) return null; + return { + path: detail.path, + contentHash: detail.contentHash, + span: + detail.startLine === 0 + ? null + : { start: detail.startLine, end: detail.endLine }, + totalLines: detail.totalLines, + }; +} + +/** + * How many lines of `span` are not already in `covered`. + * + * `covered` must be sorted, disjoint and non-adjacent (the shape + * `mergeRange` maintains). Zero means the read was fully contained in + * what the turn had already seen — the no-progress case the issue is + * about, which a plain "same start line?" check misses whenever the model + * shifts the offset. + */ +export function newlyCoveredCount( + covered: readonly LineRange[], + span: LineRange, +): number { + let fresh = span.end - span.start + 1; + for (const range of covered) { + if (range.end < span.start) continue; + if (range.start > span.end) break; + fresh -= Math.min(range.end, span.end) - Math.max(range.start, span.start) + 1; + } + return Math.max(0, fresh); +} + +/** + * Fold `span` into `covered`, returning a new sorted, disjoint list. + * + * Adjacent ranges are merged (`1-40` + `41-80` → `1-80`) so ordinary + * pagination collapses to one interval instead of growing the list by one + * entry per page — and so a later read of `40-45` is correctly seen as + * fully covered rather than falling into a seam between two intervals. + */ +export function mergeRange( + covered: readonly LineRange[], + span: LineRange, +): LineRange[] { + const merged: LineRange[] = []; + let current = { ...span }; + let inserted = false; + for (const range of covered) { + if (range.end + 1 < current.start) { + merged.push(range); + continue; + } + if (range.start > current.end + 1) { + if (!inserted) { + merged.push(current); + inserted = true; + } + merged.push(range); + continue; + } + current = { + start: Math.min(current.start, range.start), + end: Math.max(current.end, range.end), + }; + } + if (!inserted) merged.push(current); + return merged.sort((a, b) => a.start - b.start); +} + +/** + * Compact human description of a coverage set (`"1-40, 88-120"`) for the + * notice text. Line numbers only — no file content ever reaches a + * message, an event, or a log line from this detector. + */ +export function describeCoverage( + covered: readonly LineRange[], + maxRanges = 4, +): string { + if (covered.length === 0) return ""; + const shown = covered + .slice(0, maxRanges) + .map((range) => (range.start === range.end ? `${range.start}` : `${range.start}-${range.end}`)); + return covered.length > maxRanges + ? `${shown.join(", ")}, … (${covered.length - maxRanges} more)` + : shown.join(", "); +} diff --git a/src/cli/trace-formatter.ts b/src/cli/trace-formatter.ts index 8b68fbd8..925c1560 100644 --- a/src/cli/trace-formatter.ts +++ b/src/cli/trace-formatter.ts @@ -81,7 +81,16 @@ function formatTraceEvent(event: TraceEvent, raw: boolean): string { case "parse_retry": return `${head} step=${event.stepIndex} attempt=${event.attempt} reason=${event.reason}`; case "loop_detected": - return `${head} step=${event.stepIndex} tool=${event.tool} count=${event.count}`; + return `${head} step=${event.stepIndex} tool=${event.tool} count=${event.count}${ + event.detector !== undefined ? ` detector=${event.detector}` : "" + }${ + // Read-repeat lines are unreadable without the file and range the + // detector was talking about; the fingerprints show that the + // content did not move between the two reads. + event.read !== undefined + ? ` path=${event.read.path} lines=${event.read.startLine}-${event.read.endLine} fingerprint=${event.read.previousFingerprint}→${event.read.fingerprint}` + : "" + }`; case "error": return `${head} message=${event.message}`; case "trace_truncated": diff --git a/src/tools/os/fs-read-coverage.test.ts b/src/tools/os/fs-read-coverage.test.ts new file mode 100644 index 00000000..4acf1674 --- /dev/null +++ b/src/tools/os/fs-read-coverage.test.ts @@ -0,0 +1,196 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + mkdtemp, + realpath, + rm, + stat, + symlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ToolContext } from "../tool-registry.js"; +import { osFsReadTool } from "./fs-read.js"; +import { parseReadCoverage, type ReadCoverageDetail } from "./fs-read-coverage.js"; + +function makeCtx(workingDir: string): ToolContext { + return { + workingDir, + sessionId: "test-session", + stepIndex: 0, + signal: new AbortController().signal, + }; +} + +async function readCoverageOf( + dir: string, + args: Record, +): Promise { + const result = await osFsReadTool.run(args, makeCtx(dir)); + expect(result.status).toBe("ok"); + const coverage = parseReadCoverage(result.details); + expect(coverage).not.toBeNull(); + return coverage!; +} + +describe("os.fs.read coverage detail", () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "atomic-read-coverage-")); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it("reports the whole prefix as the returned range in byte mode", async () => { + await writeFile(join(dir, "a.txt"), "one\ntwo\nthree\n", "utf8"); + const coverage = await readCoverageOf(dir, { path: "a.txt" }); + expect(coverage.startLine).toBe(1); + expect(coverage.endLine).toBe(3); + expect(coverage.totalLines).toBe(3); + }); + + it("keeps the byte-mode top-level details free of line fields", async () => { + // Byte-mode results have never carried `totalLines` / `startLine`, and + // consumers tell the two modes apart by exactly that. The coverage + // detail must not smuggle those fields up to the top level. + await writeFile(join(dir, "a.txt"), "one\ntwo\n", "utf8"); + const result = await osFsReadTool.run({ path: "a.txt" }, makeCtx(dir)); + expect(result.details.totalLines).toBeUndefined(); + expect(result.details.startLine).toBeUndefined(); + expect(result.details.endLine).toBeUndefined(); + }); + + it("reports the returned range, not the requested one, in line mode", async () => { + await writeFile(join(dir, "a.txt"), "1\n2\n3\n4\n5\n", "utf8"); + // limit 99 is clamped to the end of the file; the coverage detail must + // describe what came back (2-5), not what was asked for (2-100). + const coverage = await readCoverageOf(dir, { + path: "a.txt", + offset: 2, + limit: 99, + }); + expect(coverage.startLine).toBe(2); + expect(coverage.endLine).toBe(5); + expect(coverage.totalLines).toBe(5); + }); + + it("resolves a negative offset to the range it actually returned", async () => { + await writeFile(join(dir, "a.txt"), "1\n2\n3\n4\n5\n", "utf8"); + const coverage = await readCoverageOf(dir, { path: "a.txt", offset: -2 }); + expect(coverage.startLine).toBe(4); + expect(coverage.endLine).toBe(5); + }); + + it("gives a symlink and its target the same identity", async () => { + await writeFile(join(dir, "real.txt"), "alpha\nbeta\n", "utf8"); + await symlink(join(dir, "real.txt"), join(dir, "link.txt")); + const direct = await readCoverageOf(dir, { path: "real.txt" }); + const viaLink = await osFsReadTool.run( + { path: "link.txt" }, + makeCtx(dir), + ); + const linked = parseReadCoverage(viaLink.details)!; + expect(linked.path).toBe(direct.path); + expect(linked.path).toBe(await realpath(join(dir, "real.txt"))); + // The path shown to the model and the UI is still the one asked for. + expect(viaLink.details.path).toBe(join(dir, "link.txt")); + }); + + it("changes the fingerprint on a same-size replacement with an unchanged mtime", async () => { + const file = join(dir, "a.txt"); + // A fixed timestamp on both sides of the edit: `utimes` has + // millisecond resolution, so pinning it explicitly is the only way to + // get a byte-identical mtime before and after. + const pinned = new Date(1_700_000_000_000); + await writeFile(file, "aaaa\nbbbb\n", "utf8"); + await utimes(file, pinned, pinned); + const before = await stat(file); + const first = await readCoverageOf(dir, { path: "a.txt" }); + + await writeFile(file, "xxxx\nyyyy\n", "utf8"); + // Force size AND mtime back to their pre-edit values: a fingerprint + // built from stat metadata would call this file unchanged. + await utimes(file, pinned, pinned); + const after = await stat(file); + expect(after.size).toBe(before.size); + expect(after.mtimeMs).toBe(before.mtimeMs); + + const second = await readCoverageOf(dir, { path: "a.txt" }); + expect(second.contentHash).not.toBe(first.contentHash); + }); + + it("keeps the fingerprint stable across reads of different ranges", async () => { + await writeFile(join(dir, "a.txt"), "1\n2\n3\n4\n5\n6\n", "utf8"); + const head = await readCoverageOf(dir, { path: "a.txt", offset: 1, limit: 2 }); + const tail = await readCoverageOf(dir, { path: "a.txt", offset: 5, limit: 2 }); + expect(tail.contentHash).toBe(head.contentHash); + }); + + it("credits a byte-truncated read with only the lines it returned", async () => { + await writeFile(join(dir, "big.txt"), "aaaa\nbbbb\ncccc\ndddd\n", "utf8"); + // 10 bytes reaches into line 3 only; lines 3 and 4 were never returned + // in full and must not appear as covered. + const result = await osFsReadTool.run( + { path: "big.txt", maxBytes: 10 }, + makeCtx(dir), + ); + expect(result.details.truncated).toBe(true); + const coverage = parseReadCoverage(result.details)!; + expect(coverage.startLine).toBe(1); + expect(coverage.endLine).toBeLessThan(4); + expect(coverage.totalLines).toBeLessThan(4); + }); + + it("reports an empty span for a read past the end of the file", async () => { + await writeFile(join(dir, "a.txt"), "1\n2\n", "utf8"); + const coverage = await readCoverageOf(dir, { + path: "a.txt", + offset: 99, + limit: 5, + }); + expect(coverage.startLine).toBe(0); + expect(coverage.endLine).toBe(0); + expect(coverage.totalLines).toBe(2); + }); + + it("reports an empty span for an empty file", async () => { + await writeFile(join(dir, "empty.txt"), "", "utf8"); + const coverage = await readCoverageOf(dir, { path: "empty.txt" }); + expect(coverage.startLine).toBe(0); + expect(coverage.endLine).toBe(0); + expect(coverage.totalLines).toBe(0); + }); +}); + +describe("parseReadCoverage", () => { + const valid = { + path: "/tmp/a.ts", + contentHash: "abc123", + startLine: 2, + endLine: 4, + totalLines: 9, + }; + + it("accepts a well-formed detail", () => { + expect(parseReadCoverage({ readCoverage: valid })).toEqual(valid); + }); + + it("returns null when the detail is absent (older or replayed results)", () => { + expect(parseReadCoverage({ path: "/tmp/a.ts" })).toBeNull(); + }); + + it.each([ + ["missing path", { ...valid, path: "" }], + ["missing hash", { ...valid, contentHash: 1 }], + ["inverted range", { ...valid, startLine: 8, endLine: 4 }], + ["half-empty range", { ...valid, startLine: 0, endLine: 4 }], + ["negative line", { ...valid, startLine: -1, endLine: 4 }], + ["fractional line", { ...valid, startLine: 1.5 }], + ])("rejects a malformed detail (%s)", (_label, detail) => { + expect(parseReadCoverage({ readCoverage: detail })).toBeNull(); + }); +}); diff --git a/src/tools/os/fs-read-coverage.ts b/src/tools/os/fs-read-coverage.ts new file mode 100644 index 00000000..2178401d --- /dev/null +++ b/src/tools/os/fs-read-coverage.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; + +/** + * Wire shape shared by `os.fs.read` and the agent-side read-coverage + * detector (issue #114). + * + * The detector needs three things the raw arguments cannot give it: WHICH + * file the read actually landed on (a symlink and its target are the same + * file), WHICH version of that file was read, and WHICH lines came back. + * Requested `offset`/`limit` answer none of them — they are clamped, + * negative offsets count from the end, and a byte-mode read carries no + * range at all. So the tool reports the resolved facts here and the + * detector consumes them instead of re-deriving anything. + * + * This module is a leaf on purpose: the tool and the agent both import it, + * and it pulls in nothing but `node:crypto`. + */ + +/** `details` key under which `os.fs.read` publishes `ReadCoverageDetail`. */ +export const READ_COVERAGE_DETAIL_KEY = "readCoverage"; + +export interface ReadCoverageDetail { + /** + * Canonical (symlink-resolved) absolute path. Two reads through + * different links to one file share this identity; `details.path` keeps + * the path the caller asked for. + */ + path: string; + /** + * Digest of the bytes the read actually looked at — never the file's + * size or mtime, so a same-size in-place replacement is caught and a + * touched-but-unchanged file is not. + * + * `os.fs.read` always reads the prefix `[0, min(size, maxBytes))`, in + * both byte and line mode, so this digest covers exactly the region any + * range of this call can return. Content beyond the byte cap is + * unobservable through this call and deliberately not fingerprinted: a + * change out there cannot alter what a repeated read returns. A caller + * that raises `maxBytes` reads a longer prefix and therefore gets a + * different digest, which the detector reads as a new version and + * resets coverage for — a missed repeat, never a false one. + */ + contentHash: string; + /** + * 1-based inclusive line range actually returned. `0`/`0` when the read + * returned no lines at all (empty file, or an offset past the end). + */ + startLine: number; + endLine: number; + /** + * Lines visible in the read window (the byte-capped prefix), i.e. the + * largest `endLine` any range of this call could have returned. + */ + totalLines: number; +} + +/** Digest of the bytes a read looked at. Short — this is an identity, not a checksum. */ +export function hashReadContent(buffer: Buffer): string { + return createHash("sha1").update(buffer).digest("hex").slice(0, 16); +} + +/** + * Split read text into lines the way `os.fs.read` numbers them: a + * trailing newline produces an empty final element, which is dropped so + * line numbers line up with typical editor line counts. + */ +export function splitReadLines(text: string): string[] { + const lines = text.split(/\r?\n/); + if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop(); + return lines; +} + +/** + * Read the coverage detail back out of a compressed tool result. + * + * Defensive by design: the detector must degrade to "no observation" + * (and therefore stay silent) rather than throw or invent a range when a + * result predates this field, comes from a replayed trace, or is + * malformed. A range is only accepted when it is coherent — a + * `startLine > endLine`, a negative line, or a non-integer would produce + * bogus coverage, so all of them are rejected outright. + */ +export function parseReadCoverage( + details: Record, +): ReadCoverageDetail | null { + const raw = details[READ_COVERAGE_DETAIL_KEY]; + if (raw === null || typeof raw !== "object") return null; + const record = raw as Record; + const path = record.path; + const contentHash = record.contentHash; + if (typeof path !== "string" || path.length === 0) return null; + if (typeof contentHash !== "string" || contentHash.length === 0) return null; + const startLine = asLineNumber(record.startLine); + const endLine = asLineNumber(record.endLine); + const totalLines = asLineNumber(record.totalLines); + if (startLine === null || endLine === null || totalLines === null) return null; + // An empty return is reported as 0/0; anything else must be a real, + // non-inverted range. A half-zero pair (0/5, 3/0) is incoherent. + const empty = startLine === 0 && endLine === 0; + if (!empty && (startLine < 1 || startLine > endLine)) return null; + return { path, contentHash, startLine, endLine, totalLines }; +} + +function asLineNumber(value: unknown): number | null { + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + return null; + } + return value; +} diff --git a/src/tools/os/fs-read.ts b/src/tools/os/fs-read.ts index bc2f2537..0fa2c986 100644 --- a/src/tools/os/fs-read.ts +++ b/src/tools/os/fs-read.ts @@ -1,6 +1,12 @@ -import { open, stat } from "node:fs/promises"; +import { open, realpath, stat } from "node:fs/promises"; import { compressToolResult } from "../../compressor/result-compressor.js"; import { resolveUserPath } from "./expand-home.js"; +import { + hashReadContent, + READ_COVERAGE_DETAIL_KEY, + splitReadLines, + type ReadCoverageDetail, +} from "./fs-read-coverage.js"; import type { ToolDefinition } from "../tool-registry.js"; const DEFAULT_MAX_BYTES = 64 * 1024; @@ -46,16 +52,38 @@ export const osFsReadTool: ToolDefinition = { if (!info.isFile()) { throw new Error(`os.fs.read: ${absolute} is not a regular file`); } + const canonical = await canonicalize(absolute); if (args.offset !== undefined || args.limit !== undefined) { - return readByLines(absolute, info.size, args); + return readByLines(absolute, canonical, info.size, args); } - return readByBytes(absolute, info.size, args); + return readByBytes(absolute, canonical, info.size, args); }, }; +/** + * Symlink-resolved path, used only as the read-coverage identity so that + * two reads of one file through different links are recognized as the + * same file. `details.path` keeps the unresolved absolute path the caller + * asked for — it is what the model and the UI have been shown all along, + * and rewriting it here would change every read's output. + * + * A failure (a race deleting the file between `stat` and here, a + * permission wall on a parent directory) degrades to the unresolved path: + * a slightly coarser identity is strictly better than failing a read that + * has already succeeded. + */ +async function canonicalize(absolute: string): Promise { + try { + return await realpath(absolute); + } catch { + return absolute; + } +} + async function readByBytes( absolute: string, + canonical: string, size: number, args: ReadArgs, ): Promise> { @@ -69,6 +97,13 @@ async function readByBytes( const truncated = size > args.maxBytes; const body = buffer.toString("utf8"); const text = args.lineNumbers ? prefixLineNumbers(body, 1) : body; + // Byte mode returns the whole (possibly byte-capped) prefix, so the + // returned range is lines 1..N of it. The top-level `startLine` / + // `totalLines` details stay absent here — byte-mode results have + // never carried them and consumers distinguish the two modes by + // exactly that — so the span is reported only inside the coverage + // detail, where it exists for the detector rather than for display. + const lineCount = splitReadLines(body).length; return compressToolResult( { tool: "os.fs.read", @@ -79,6 +114,11 @@ async function readByBytes( size, truncated, bytesRead: toRead, + [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, { + startLine: lineCount === 0 ? 0 : 1, + endLine: lineCount, + totalLines: lineCount, + }), }, }, { maxSummaryLength: args.maxBytes, maxTailLines: 500 }, @@ -90,24 +130,20 @@ async function readByBytes( async function readByLines( absolute: string, + canonical: string, size: number, args: ReadArgs, ): Promise> { const handle = await open(absolute, "r"); let allLines: string[]; + let buffer: Buffer; try { const toRead = Math.min(size, args.maxBytes); - const buffer = Buffer.alloc(toRead); + buffer = Buffer.alloc(toRead); if (toRead > 0) { await handle.read(buffer, 0, toRead, 0); } - const text = buffer.toString("utf8"); - allLines = text.split(/\r?\n/); - // A trailing newline produces an empty final element; drop it so the - // caller's line numbers align with typical editor line counts. - if (allLines.length > 0 && allLines[allLines.length - 1] === "") { - allLines.pop(); - } + allLines = splitReadLines(buffer.toString("utf8")); } finally { await handle.close(); } @@ -137,12 +173,34 @@ async function readByLines( startLine: total === 0 ? 0 : startIndex + 1, endLine: startIndex + sliced.length, returnedLines: sliced.length, + // A read that returned nothing (empty file, or an offset past the + // end) reports an empty 0/0 span rather than a range it did not + // return — the detector must never credit coverage for lines the + // model never saw. + [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, { + startLine: sliced.length === 0 ? 0 : startIndex + 1, + endLine: sliced.length === 0 ? 0 : startIndex + sliced.length, + totalLines: total, + }), }, }, { maxSummaryLength: args.maxBytes, maxTailLines: 500 }, ); } +/** Assemble the read-coverage detail for one successful read. */ +function coverage( + canonical: string, + bytesRead: Buffer, + span: Pick, +): ReadCoverageDetail { + return { + path: canonical, + contentHash: hashReadContent(bytesRead), + ...span, + }; +} + function resolveRange( total: number, offset: number | undefined, diff --git a/src/tracing/trace/trace-event.ts b/src/tracing/trace/trace-event.ts index ac69d542..68753be4 100644 --- a/src/tracing/trace/trace-event.ts +++ b/src/tracing/trace/trace-event.ts @@ -176,7 +176,26 @@ export interface TraceLoopDetected extends TraceEventBase { */ level?: "warn" | "critical" | "breaker"; /** Which sub-detector fired. Optional for back-compat. */ - detector?: "generic_repeat" | "no_progress" | "wandering" | "test_repeat"; + detector?: + | "generic_repeat" + | "no_progress" + | "wandering" + | "test_repeat" + | "read_repeat"; + /** + * `read_repeat` only (issue #114): the canonical file the reads landed + * on, the line range the triggering read returned, and the content + * fingerprint before and after it — equal fingerprints are what make + * the re-read redundant, so both are recorded and a trace reader can + * check the claim. Never carries file content. + */ + read?: { + path: string; + startLine: number; + endLine: number; + previousFingerprint: string; + fingerprint: string; + }; } /** diff --git a/src/tracing/trace/trace-recorder.ts b/src/tracing/trace/trace-recorder.ts index b903d574..4e6c0d71 100644 --- a/src/tracing/trace/trace-recorder.ts +++ b/src/tracing/trace/trace-recorder.ts @@ -383,6 +383,7 @@ export function createTraceRecorder( ...(event.detector !== undefined ? { detector: event.detector } : {}), + ...(event.read !== undefined ? { read: event.read } : {}), }); return; case "loop_failed": From 99a23b9847ed35e18cc1c00561aa0209bf47ce48 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:34:33 +0300 Subject: [PATCH 07/36] fix(tools): close the format enum and stop the read_document summary contradicting itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the issue #113 routing fix. Three gaps in the first commit, none of which made it wrong, all of which blunted it: - `format` was still an open `string` in both the stable-prefix argsSchema and the cloud JSON Schema, so the exact bad guess the issue is about — `format: "text"` — stayed fully representable, and the runtime rejection named neither the valid values nor os.fs.read. The repo convention (see the header of default-tool-args-schemas.ts, "enums match the runtime validators verbatim", and os.fs.archive.*) is to enumerate closed sets in both places. The set now lives once, as `DOCUMENT_FORMATS` in extractor-types.ts, with `DocumentFormat` derived from it; the validator, the rejection message and both prompt surfaces all read that declaration instead of copying it. - The new prompt strings said read_document is "NOT for source code or text files" while the same sentence advertised plain-text support — and the tool really does extract .txt/.md/.csv/.json/.yaml/.html as `plain`. For a change whose whole point is removing ambiguity, that is the same failure mode one level up. Both strings now say "NOT for source code" and stop there, which is true and equally pointed. - The `description` — the text the model reads in `### loaded-tools` after tool.view, i.e. at the moment it picks the tool — was the one of the three surfaces with no test. It has one now, on the same rationale as the stable-prefix pin. Two smaller inconsistencies the review turned up in the source-like set: - `.tsv` claimed to be a "source/text file" while its twin `.csv` extracted happily. `.tsv` now joins `.csv` on the `plain` arm; the error message no longer asserts a classification the switch next to it contradicts. The rejection wording moves to "source or config file", accurate for every remaining member of the set. - `env` only ever fires for the `name.env` shape — `extname("/x/.env")` is `""`, so bare dotfiles land on the extensionless arm and extract as `plain`. That is inherited from `extname` and not worth changing, but it is now written down next to the set. The document-format guard test only covered 11 of the switch's arms; it missed legacy `.doc` (whose extractor was not even in the injected bag), `.html`, `.xml`, `.log`, `.yml` and the extensionless case, so removing the `.doc` arm would not have failed it. All of them are in the loop now. --- src/prompt/build-prompt.test.ts | 11 +++- src/prompt/default-tool-args-schemas.test.ts | 33 +++++++++++ src/prompt/default-tool-args-schemas.ts | 7 ++- src/prompt/default-tool-descriptors-a.ts | 15 +++-- .../extractors/extractor-types.ts | 27 ++++++--- .../os/read-document/read-document.test.ts | 58 +++++++++++++++++-- src/tools/os/read-document/read-document.ts | 54 +++++++++-------- 7 files changed, 160 insertions(+), 45 deletions(-) diff --git a/src/prompt/build-prompt.test.ts b/src/prompt/build-prompt.test.ts index 44b32285..93f219cc 100644 --- a/src/prompt/build-prompt.test.ts +++ b/src/prompt/build-prompt.test.ts @@ -250,7 +250,16 @@ describe("buildPrompt", () => { "the default for source code and text files", ); expect(prompt.stablePrefix).toContain( - "NOT for source code or text files: use os.fs.read", + "NOT for source code: use os.fs.read", + ); + // The summary must not claim read_document rejects text files — it + // extracts .txt/.md/.csv as `plain`, and a summary that contradicts the + // tool re-creates the very ambiguity this change removes. + expect(prompt.stablePrefix).not.toContain("NOT for source code or text files"); + // The bad guess in issue #113 was `format: "text"`. The stable prefix + // carries the closed set so the guess is never reachable. + expect(prompt.stablePrefix).toContain( + "format?: 'pdf' | 'docx' | 'doc' | 'xlsx' | 'rtf' | 'odt' | 'pptx' | 'plain'", ); }); diff --git a/src/prompt/default-tool-args-schemas.test.ts b/src/prompt/default-tool-args-schemas.test.ts index 80bffb87..86526ef5 100644 --- a/src/prompt/default-tool-args-schemas.test.ts +++ b/src/prompt/default-tool-args-schemas.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; +import { DOCUMENT_FORMATS } from "../tools/os/read-document/extractors/extractor-types.js"; import { DEFAULT_TOOL_DESCRIPTORS } from "./tool-descriptors.js"; import { attachDefaultArgsJsonSchema, @@ -87,6 +88,38 @@ describe("default tool argsJsonSchema map", () => { }); }); + // Issue #113: `format` was an open string in both the JSON Schema and the + // stable-prefix argsSchema, so nothing stopped a model emitting the + // invented `format: "text"` that the issue is about. Both surfaces now + // carry the closed set, derived from the runtime declaration. + it("pins os.fs.read_document format as the closed DOCUMENT_FORMATS enum", () => { + const properties = ( + getDefaultArgsJsonSchema("os.fs.read_document") as { + properties: Record; + } + ).properties; + expect(properties.format).toEqual({ + type: "string", + enum: ["pdf", "docx", "doc", "xlsx", "rtf", "odt", "pptx", "plain"], + }); + // Verbatim against the runtime validator, not a second hand-written + // copy of it — that is the drift this module's header promises to avoid. + expect((properties.format as { enum: string[] }).enum).toEqual([ + ...DOCUMENT_FORMATS, + ]); + }); + + it("spells the same closed format set into the read_document argsSchema", () => { + const descriptor = DEFAULT_TOOL_DESCRIPTORS.find( + (d) => d.name === "os.fs.read_document", + ); + expect(descriptor).toBeDefined(); + expect(descriptor!.argsSchema).toContain( + "format?: 'pdf' | 'docx' | 'doc' | 'xlsx' | 'rtf' | 'odt' | 'pptx' | 'plain'", + ); + expect(descriptor!.argsSchema).not.toContain("format?: string"); + }); + it("does not leak vision.describe's maxItems onto other string[] schemas", () => { const properties = ( getDefaultArgsJsonSchema("os.shell.run") as { diff --git a/src/prompt/default-tool-args-schemas.ts b/src/prompt/default-tool-args-schemas.ts index dc1da011..617c7f6c 100644 --- a/src/prompt/default-tool-args-schemas.ts +++ b/src/prompt/default-tool-args-schemas.ts @@ -29,6 +29,7 @@ * guards the **shape**. */ +import { DOCUMENT_FORMATS } from "../tools/os/read-document/extractors/extractor-types.js"; import type { ToolDescriptor } from "./stable-prefix.js"; type Schema = Record; @@ -238,7 +239,11 @@ const DEFAULT_TOOL_ARGS_SCHEMAS: ReadonlyMap = new Map< obj( { path: stringSchema, - format: stringSchema, + // Closed set, per the "enums match the runtime validators verbatim" + // convention above: `detectFormat` accepts exactly DOCUMENT_FORMATS + // and rejects anything else. Leaving it an open string is what let + // models emit the invented `format: "text"` (issue #113). + format: { type: "string", enum: [...DOCUMENT_FORMATS] }, maxBytes: numberSchema, maxPages: numberSchema, pagesFrom: numberSchema, diff --git a/src/prompt/default-tool-descriptors-a.ts b/src/prompt/default-tool-descriptors-a.ts index 8346977c..e4093805 100644 --- a/src/prompt/default-tool-descriptors-a.ts +++ b/src/prompt/default-tool-descriptors-a.ts @@ -99,13 +99,20 @@ export const DEFAULT_TOOL_DESCRIPTORS_A: readonly ToolDescriptor[] = [ { // Models reach for read_document on `.py` / `.ts` source files, hit the // unsupported-extension error and burn a step guessing `format`. The - // summary therefore names the sibling tool explicitly: source and text - // go to os.fs.read, this one is for document extraction only. + // summary therefore names the sibling tool explicitly: source code goes + // to os.fs.read, this one is for document extraction. It deliberately + // does NOT say "not for text files" — this tool does read .txt/.md/.csv + // as `plain`, and a summary that contradicts the tool is the same + // ambiguity one level up. + // + // `format` is spelled out as a closed set for the same reason: the bad + // guess in issue #113 was `format: "text"`, and a model that guesses it + // preemptively never sees the runtime hint. name: "os.fs.read_document", summary: - "Extract plain text from documents — PDF, Office, ODF, RTF (markers in output). NOT for source code or text files: use os.fs.read. Read-only.", + "Extract plain text from documents — PDF, Office, ODF, RTF (markers in output). NOT for source code: use os.fs.read. Read-only.", argsSchema: - "{ path: string, format?: string, maxBytes?: number, maxPages?: number, pagesFrom?: number, pagesTo?: number, sheets?: (string | number)[], pageSeparators?: boolean, includeTables?: boolean }", + "{ path: string, format?: 'pdf' | 'docx' | 'doc' | 'xlsx' | 'rtf' | 'odt' | 'pptx' | 'plain', maxBytes?: number, maxPages?: number, pagesFrom?: number, pagesTo?: number, sheets?: (string | number)[], pageSeparators?: boolean, includeTables?: boolean }", }, { name: "os.fs.archive.list", diff --git a/src/tools/os/read-document/extractors/extractor-types.ts b/src/tools/os/read-document/extractors/extractor-types.ts index 25975a33..a0b700e8 100644 --- a/src/tools/os/read-document/extractors/extractor-types.ts +++ b/src/tools/os/read-document/extractors/extractor-types.ts @@ -6,15 +6,24 @@ * keeps extractors pure and easy to test. */ -export type DocumentFormat = - | "pdf" - | "docx" - | "doc" - | "xlsx" - | "rtf" - | "odt" - | "pptx" - | "plain"; +/** + * The closed set of formats the dispatcher accepts, as a value so that the + * runtime validator, the error message that lists the valid overrides and + * the prompt-side arg schemas can all be derived from one declaration + * instead of three hand-maintained copies that drift (issue #113). + */ +export const DOCUMENT_FORMATS = [ + "pdf", + "docx", + "doc", + "xlsx", + "rtf", + "odt", + "pptx", + "plain", +] as const; + +export type DocumentFormat = (typeof DOCUMENT_FORMATS)[number]; export interface ExtractorInput { data: Buffer; diff --git a/src/tools/os/read-document/read-document.test.ts b/src/tools/os/read-document/read-document.test.ts index 407d8824..b9f68db8 100644 --- a/src/tools/os/read-document/read-document.test.ts +++ b/src/tools/os/read-document/read-document.test.ts @@ -122,7 +122,7 @@ describe("os.fs.read_document dispatcher", () => { expect(error).toBeInstanceOf(Error); const message = (error as Error).message; - expect(message).toContain(`".${ext}" is a source/text file`); + expect(message).toContain(`".${ext}" is a source or config file`); expect(message).toContain("use os.fs.read instead"); expect(message).toContain('format: "plain"'); // The ambiguous bare-`format` phrasing is what produced the bad @@ -163,6 +163,23 @@ describe("os.fs.read_document dispatcher", () => { expect(result.summary).toContain("print(1)"); }); + it("advertises the os.fs.read hand-off in the tool description", async () => { + // The description is what lands in `### loaded-tools` after tool.view, + // i.e. the text the model reads at the moment it picks between the two + // readers — the same argument that earned the stable-prefix summary a + // pin test. Without this, a future edit can drop the routing hint (or + // re-introduce the `format: "text"` ambiguity) with a green suite. + const description = buildOsFsReadDocumentTool({}).description; + expect(description).toContain("NOT for source code"); + expect(description).toContain("os.fs.read"); + expect(description).toContain( + "pdf, docx, doc, xlsx, rtf, odt, pptx, plain", + ); + // It must not claim the tool refuses text files: it reads .txt/.md/.csv + // as `plain`, and the whole point of issue #113 is removing ambiguity. + expect(description).not.toMatch(/NOT for source code or other UTF-8 text/); + }); + it("keeps document extensions routed to their own extractors", async () => { // Guards the other half of issue #113: the new source-file branch must // not have moved any real document format into the reject path. @@ -171,6 +188,7 @@ describe("os.fs.read_document dispatcher", () => { extractors: { pdf: fakeExtractor({ format: "pdf", text: "p" }, () => seen.push("pdf")), docx: fakeExtractor({ format: "docx", text: "d" }, () => seen.push("docx")), + doc: fakeExtractor({ format: "doc", text: "l" }, () => seen.push("doc")), xlsx: fakeExtractor({ format: "xlsx", text: "x" }, () => seen.push("xlsx")), rtf: fakeExtractor({ format: "rtf", text: "r" }, () => seen.push("rtf")), odt: fakeExtractor({ format: "odt", text: "o" }, () => seen.push("odt")), @@ -178,18 +196,28 @@ describe("os.fs.read_document dispatcher", () => { plain: fakeExtractor({ format: "plain", text: "t" }, () => seen.push("plain")), }, }); + // Every arm of the switch is represented, including the ones the first + // version of this guard missed: legacy `.doc`, markup, `.log`, `.yml`, + // the delimited pair (`.csv`/`.tsv`) and the extensionless `case ""`. for (const name of [ "a.pdf", "a.docx", + "a.doc", "a.xlsx", "a.rtf", "a.odt", "a.pptx", "a.txt", "a.md", + "a.log", "a.csv", + "a.tsv", "a.json", + "a.html", + "a.xml", "a.yaml", + "a.yml", + "Makefile", ]) { const path = join(dir, name); await writeFile(path, Buffer.from("x")); @@ -198,15 +226,12 @@ describe("os.fs.read_document dispatcher", () => { expect(seen).toEqual([ "pdf", "docx", + "doc", "xlsx", "rtf", "odt", "pptx", - "plain", - "plain", - "plain", - "plain", - "plain", + ...Array(11).fill("plain"), ]); }); @@ -219,6 +244,27 @@ describe("os.fs.read_document dispatcher", () => { ).rejects.toThrow(/unknown format override/); }); + it('names the accepted formats and os.fs.read when rejecting format: "text"', async () => { + // A model can guess `format: "text"` before it ever sees detectFormat's + // hint (it reads "override with `format`" in the description and fills + // in a plausible value). This branch is that model's only feedback, so + // it has to carry both exits: the valid values, and the other tool. + const tool = buildOsFsReadDocumentTool({}); + const path = join(dir, "script.py"); + await writeFile(path, Buffer.from("print(1)\n")); + + const error = await tool + .run({ path, format: "text" }, makeCtx(dir)) + .then(() => undefined) + .catch((e: unknown) => e as Error); + + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain('unknown format override "text"'); + expect(message).toContain("pdf, docx, doc, xlsx, rtf, odt, pptx, plain"); + expect(message).toContain("os.fs.read"); + }); + it("forwards pagination args to the extractor", async () => { let captured: ExtractorInput | undefined; const tool = buildOsFsReadDocumentTool({ diff --git a/src/tools/os/read-document/read-document.ts b/src/tools/os/read-document/read-document.ts index 5c18aaa0..7b2c4773 100644 --- a/src/tools/os/read-document/read-document.ts +++ b/src/tools/os/read-document/read-document.ts @@ -4,6 +4,7 @@ import { extname } from "node:path"; import { compressToolResult } from "../../../compressor/result-compressor.js"; import { resolveUserPath } from "../expand-home.js"; import type { ToolDefinition } from "../../tool-registry.js"; +import { DOCUMENT_FORMATS } from "./extractors/extractor-types.js"; import type { DocumentFormat, Extractor, @@ -40,7 +41,7 @@ export function buildOsFsReadDocumentTool( return { name: "os.fs.read_document", description: - 'Extract plain text (with light structure markers) from PDF, DOCX, DOC (legacy), XLSX, RTF, ODT, PPTX, and plain-text files. NOT for source code or other UTF-8 text files — read those with os.fs.read. Auto-detects format by extension; override with `format` (the plain-text value is `format: "plain"`). Read-only, no approval required.', + 'Extract plain text (with light structure markers) from PDF, DOCX, DOC (legacy), XLSX, RTF, ODT, PPTX, and plain-text files. NOT for source code — read code with os.fs.read, which pages with offset/limit. Auto-detects format by extension; override with `format`, one of: pdf, docx, doc, xlsx, rtf, odt, pptx, plain. Read-only, no approval required.', readonly: true, async run(rawArgs, ctx) { const args = await parseArgs(rawArgs, ctx.workingDir); @@ -198,13 +199,22 @@ function parseSheetsArg( } /** - * Extensions that are almost certainly source code or other line-oriented - * text. They are deliberately NOT mapped to `plain`: `os.fs.read` is the - * right tool for them (offset/limit pagination, `lineNumbers`, no document - * extractor in the way). The set exists only so the rejection can say which - * tool to reach for next — without that, models retry `read_document` with - * an invented `format: "text"` and burn another step before discovering - * `os.fs.read` (issue #113). + * Extensions that are almost certainly source code or configuration — both + * line-oriented, both better served by `os.fs.read` (offset/limit + * pagination, `lineNumbers`, no document extractor in the way), so they are + * deliberately NOT mapped to `plain`. The set exists only so the rejection + * can say which tool to reach for next — without that, models retry + * `read_document` with an invented `format: "text"` and burn another step + * before discovering `os.fs.read` (issue #113). + * + * Deliberately absent: delimited data (`csv`, `tsv`) and markup (`html`, + * `xml`, `md`, `log`, `json`, `yaml`) — those route to `plain` in the switch + * below, and a set entry here would make the error message assert a + * classification the switch contradicts. + * + * `env` only fires for the `name.env` shape: `extname("/x/.env")` is `""`, + * so a bare dotfile lands on the extensionless `case ""` arm and extracts as + * `plain`. That asymmetry is inherited from `extname`, not introduced here. */ const SOURCE_LIKE_EXTENSIONS: ReadonlySet = new Set([ "bash", "c", "cc", "cfg", "cjs", "clj", "conf", "cpp", "cs", "css", "cxx", @@ -212,20 +222,24 @@ const SOURCE_LIKE_EXTENSIONS: ReadonlySet = new Set([ "hs", "ini", "ipynb", "java", "js", "jsonc", "jsx", "kt", "kts", "less", "lua", "m", "mjs", "mm", "php", "pl", "pm", "properties", "proto", "ps1", "py", "pyi", "r", "rb", "rs", "sass", "scala", "scss", "sh", "sql", "svelte", - "swift", "tf", "toml", "ts", "tsv", "tsx", "vue", "zsh", + "swift", "tf", "toml", "ts", "tsx", "vue", "zsh", ]); /** - * Resolve extension → canonical format. `.html/.xml/.json/.csv` currently - * fall through to `plain` — it's the safest default until we need format- - * specific pretty-printing for them. + * Resolve extension → canonical format. `.html/.xml/.json/.csv/.tsv` + * currently fall through to `plain` — it's the safest default until we need + * format-specific pretty-printing for them. */ function detectFormat(absolute: string, override: unknown): DocumentFormat { if (typeof override === "string" && override.length > 0) { const norm = override.toLowerCase(); if (isKnownFormat(norm)) return norm; + // Naming the accepted set here matters as much as it does in the + // extension branches below: a model that guesses `format: "text"` + // preemptively never sees the detectFormat hint, and a bare "unknown + // format override" leaves it with nowhere to go (issue #113). throw new Error( - `os.fs.read_document: unknown format override ${JSON.stringify(override)}`, + `os.fs.read_document: unknown format override ${JSON.stringify(override)} — expected one of ${DOCUMENT_FORMATS.join(", ")}; for source or text files use os.fs.read`, ); } const ext = extname(absolute).toLowerCase().replace(/^\./, ""); @@ -249,6 +263,7 @@ function detectFormat(absolute: string, override: unknown): DocumentFormat { case "md": case "log": case "csv": + case "tsv": case "json": case "html": case "xml": @@ -264,23 +279,14 @@ function detectFormat(absolute: string, override: unknown): DocumentFormat { // which is not a known format and costs another failed step. throw new Error( SOURCE_LIKE_EXTENSIONS.has(ext) - ? `os.fs.read_document: ".${ext}" is a source/text file — use os.fs.read instead; if you intended document extraction, retry with \`format: "plain"\`` + ? `os.fs.read_document: ".${ext}" is a source or config file — use os.fs.read instead; if you intended document extraction, retry with \`format: "plain"\`` : `os.fs.read_document: unsupported extension ".${ext}" — use os.fs.read for source or text files, or retry with an explicit format (e.g. \`format: "plain"\`)`, ); } } function isKnownFormat(value: string): value is DocumentFormat { - return ( - value === "pdf" || - value === "docx" || - value === "doc" || - value === "xlsx" || - value === "rtf" || - value === "odt" || - value === "pptx" || - value === "plain" - ); + return (DOCUMENT_FORMATS as readonly string[]).includes(value); } /** From 8946ad55525a66f378aa9a1175b2556bb414262e Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:41:21 +0300 Subject: [PATCH 08/36] feat(providers): probe streaming native-tool contract at setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live API key proves the account, not the route. `/v1/models` proves reachability, and the pre-save key check proves one non-streaming completion with no tools — neither exercises what an Atomic turn actually is: a streamed chat completion carrying a `tools` payload from which a native tool call must come back whole. Routes that pass both and then fail a turn are common (STREAM_EARLY_EOF, HTTP 400 only once `tools` is in the body, forced tool choice refused, incomplete tool-call deltas), and until now the operator's first message was the test. Adds a conformance probe next to the existing key check, built as a three-rung ladder so a healthy route costs exactly one request: 1. forced named tool, streaming, tools payload — a complete tool call here is the whole answer; 2. `tool_choice: auto` with the same payload, reached only when rung 1 was refused for a reason that is not the key, the quota or the model — a tool call now means it was the forcing the route refused; 3. the same streamed completion with no tools at all, reached only when both tool requests were refused — if this answers, the route works and it is specifically `tools` it rejects. Rung 3 is an experiment rather than a regex over error wording, which is not stable across providers; "refuses with tools, answers without them" is. A text answer under `auto` stays inconclusive and is never reported as "tools unsupported". The synthetic function is a diagnostic fixture: nothing registers it, nothing dispatches it, and its name sits outside the dotted namespace every built-in tool uses so it cannot shadow one. Details are bounded and credential-scrubbed — the exact key by match, key-shaped strings by pattern — and that redaction is now shared with the key check. Wired into the two setup paths (Providers wizard save, first-run cloud onboarding) as advisory only: it can never refuse a save, so a custom endpoint that blocks synthetic probes stays configurable, but a route it could not prove no longer reports as a working backend. `/llm check` runs it on demand against a provider that is already saved. It never runs on a turn path. --- .../verify/accumulate-probe-stream.test.ts | 120 ++++++ .../verify/accumulate-probe-stream.ts | 104 +++++ .../verify/classify-contract-probe.test.ts | 232 +++++++++++ .../verify/classify-contract-probe.ts | 134 ++++++ .../provider/verify/contract-probe-types.ts | 179 ++++++++ src/llm/provider/verify/index.ts | 28 ++ .../provider/verify/redact-provider-detail.ts | 42 ++ .../verify/run-contract-probe.test.ts | 313 ++++++++++++++ src/llm/provider/verify/run-contract-probe.ts | 389 ++++++++++++++++++ .../provider/verify/verify-provider-key.ts | 19 +- .../commands/slash-command-handler.test.ts | 15 + src/tui/commands/slash-command-handler.ts | 14 +- .../components/cloud-provider-onboarding.tsx | 19 +- src/tui/providers/contract-probe-target.ts | 110 +++++ src/tui/providers/describe-contract-probe.ts | 59 +++ .../providers/probe-wizard-contract.test.ts | 170 ++++++++ src/tui/providers/probe-wizard-contract.ts | 101 +++++ src/tui/providers/providers-actions.ts | 13 + .../providers/providers-orchestrator.test.ts | 67 +++ src/tui/providers/providers-orchestrator.ts | 126 +++++- src/tui/providers/providers-wizard-target.ts | 4 +- src/tui/submit-handler.ts | 7 + src/tui/tui-app.tsx | 7 + src/tui/tui-command.ts | 2 + 24 files changed, 2252 insertions(+), 22 deletions(-) create mode 100644 src/llm/provider/verify/accumulate-probe-stream.test.ts create mode 100644 src/llm/provider/verify/accumulate-probe-stream.ts create mode 100644 src/llm/provider/verify/classify-contract-probe.test.ts create mode 100644 src/llm/provider/verify/classify-contract-probe.ts create mode 100644 src/llm/provider/verify/contract-probe-types.ts create mode 100644 src/llm/provider/verify/redact-provider-detail.ts create mode 100644 src/llm/provider/verify/run-contract-probe.test.ts create mode 100644 src/llm/provider/verify/run-contract-probe.ts create mode 100644 src/tui/providers/contract-probe-target.ts create mode 100644 src/tui/providers/describe-contract-probe.ts create mode 100644 src/tui/providers/probe-wizard-contract.test.ts create mode 100644 src/tui/providers/probe-wizard-contract.ts diff --git a/src/llm/provider/verify/accumulate-probe-stream.test.ts b/src/llm/provider/verify/accumulate-probe-stream.test.ts new file mode 100644 index 00000000..f80b1720 --- /dev/null +++ b/src/llm/provider/verify/accumulate-probe-stream.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; + +import { accumulateProbeStream } from "./accumulate-probe-stream.js"; + +function sse(...events: unknown[]): string { + return events + .map((event) => + typeof event === "string" ? `data: ${event}\n\n` : `data: ${JSON.stringify(event)}\n\n`, + ) + .join(""); +} + +function toolCallChunk( + parts: { name?: string; arguments?: string; index?: number }, + finishReason: string | null = null, +): Record { + return { + model: "probe-model", + choices: [ + { + delta: { + tool_calls: [ + { + index: parts.index ?? 0, + id: "call_1", + type: "function", + function: { + ...(parts.name !== undefined ? { name: parts.name } : {}), + ...(parts.arguments !== undefined ? { arguments: parts.arguments } : {}), + }, + }, + ], + }, + ...(finishReason ? { finish_reason: finishReason } : {}), + }, + ], + }; +} + +describe("accumulateProbeStream", () => { + it("assembles a native tool call split across deltas", () => { + const observation = accumulateProbeStream( + sse( + toolCallChunk({ name: "atomic_contract_probe", arguments: '{"ok"' }), + toolCallChunk({ arguments: ":true}" }), + toolCallChunk({}, "tool_calls"), + "[DONE]", + ), + ); + expect(observation.toolCalls).toEqual([ + { index: 0, name: "atomic_contract_probe", arguments: '{"ok":true}' }, + ]); + expect(observation.sawToolCallDelta).toBe(true); + expect(observation.terminalObserved).toBe(true); + expect(observation.finishReason).toBe("tool_calls"); + }); + + it("keeps a tool-call delta that never carried a function name", () => { + // The production stream consumer drops this call, because a nameless + // call cannot be dispatched. The probe has to see it: "deltas came + // but assembled into nothing" is the diagnosis, and a probe that + // dropped it would report the far friendlier "no tool call". + const observation = accumulateProbeStream( + sse(toolCallChunk({ arguments: '{"ok":true}' }), toolCallChunk({}, "tool_calls")), + ); + expect(observation.sawToolCallDelta).toBe(true); + expect(observation.toolCalls[0]?.name).toBe(""); + }); + + it("does not mistake a repeated whole name for fragments", () => { + // Anthropic-compatible endpoints resend the full name every delta. + const observation = accumulateProbeStream( + sse( + toolCallChunk({ name: "atomic_contract_probe", arguments: "{" }), + toolCallChunk({ name: "atomic_contract_probe", arguments: "}" }), + toolCallChunk({}, "tool_calls"), + ), + ); + expect(observation.toolCalls[0]?.name).toBe("atomic_contract_probe"); + }); + + it("collects plain assistant text and its finish reason", () => { + const observation = accumulateProbeStream( + sse( + { choices: [{ delta: { content: "I can " } }] }, + { choices: [{ delta: { content: "help." }, finish_reason: "stop" }] }, + "[DONE]", + ), + ); + expect(observation.text).toBe("I can help."); + expect(observation.sawToolCallDelta).toBe(false); + expect(observation.terminalObserved).toBe(true); + }); + + it("reports no terminal signal when the body simply stops", () => { + const observation = accumulateProbeStream( + sse(toolCallChunk({ name: "atomic_contract_probe", arguments: '{"ok"' })), + ); + expect(observation.terminalObserved).toBe(false); + expect(observation.finishReason).toBeNull(); + }); + + it("reads a final event that has no trailing blank line", () => { + // Providers and proxies close the response right after the terminal + // event often enough that treating it as noise would turn healthy + // routes into false early-EOF reports. + const observation = accumulateProbeStream( + `${sse({ choices: [{ delta: { content: "hi" }, finish_reason: "stop" }] })}data: [DONE]`, + ); + expect(observation.terminalObserved).toBe(true); + }); + + it("survives a truncated JSON payload without inventing content", () => { + const observation = accumulateProbeStream( + `${sse({ choices: [{ delta: { content: "hi" } }] })}data: {"choices":[{"delta":{"too`, + ); + expect(observation.text).toBe("hi"); + expect(observation.terminalObserved).toBe(false); + }); +}); diff --git a/src/llm/provider/verify/accumulate-probe-stream.ts b/src/llm/provider/verify/accumulate-probe-stream.ts new file mode 100644 index 00000000..0b5d8121 --- /dev/null +++ b/src/llm/provider/verify/accumulate-probe-stream.ts @@ -0,0 +1,104 @@ +/** + * What actually came back over a probe's SSE stream. + * + * The probe reads the whole (small, bounded) body and then replays it + * through `parseOpenAiSseEvent` and `mergeToolName` — the very parser + * and name-merge rule a real turn uses. Reimplementing either here + * would let the probe and the turn disagree about what a provider sent, + * which is the one thing a conformance check must never do. + * + * It stops short of `createOpenAiStreamConsumer` on purpose. That + * consumer answers "what should the agent act on", and to do it it + * *drops* a tool call whose function name never arrived. A probe needs + * the opposite: knowing that tool-call deltas were streamed but never + * assembled into a callable tool is the whole diagnosis of a route with + * malformed deltas. + */ + +import { mergeToolName } from "../openai/openai-stream-consumer.js"; +import { parseOpenAiSseEvent } from "../openai/parse-sse-chunk.js"; +import { createReasoningExtractor } from "../openai/reasoning-extractor.js"; + +export interface ProbeToolCallObservation { + readonly index: number; + /** Empty when deltas for this index never carried a function name. */ + readonly name: string; + /** Concatenated argument fragments, exactly as they arrived. */ + readonly arguments: string; +} + +export interface ProbeStreamObservation { + /** Assistant text, for the auto-mode "answered in prose" case. */ + readonly text: string; + readonly toolCalls: readonly ProbeToolCallObservation[]; + /** A `tool_calls` delta was seen at all — even one naming nothing. */ + readonly sawToolCallDelta: boolean; + readonly finishReason: string | null; + /** + * The provider said it was finished: an explicit `finish_reason` on + * some chunk, or a `[DONE]` event. A body that simply stops carries + * neither, and that is precisely `STREAM_EARLY_EOF`. Same rule the + * stream consumer applies before it trusts a tool call. + */ + readonly terminalObserved: boolean; +} + +export function accumulateProbeStream(sse: string): ProbeStreamObservation { + // The probe never asks for reasoning, and no probe verdict depends on + // it, so the no-op extractor keeps the parser call honest without + // pulling provider reasoning formats into the check. + const reasoning = createReasoningExtractor("none"); + const calls = new Map(); + let text = ""; + let sawToolCallDelta = false; + let finishReason: string | null = null; + let terminalObserved = false; + + for (const rawEvent of splitSseEvents(sse)) { + const chunk = parseOpenAiSseEvent(rawEvent, reasoning, ""); + text += chunk.delta; + if (chunk.finishReason !== null) { + finishReason = chunk.finishReason; + terminalObserved = true; + } + if (chunk.done) terminalObserved = true; + if (chunk.toolArgsDelta === true) sawToolCallDelta = true; + for (const delta of chunk.toolCallDeltas) { + const current = calls.get(delta.index) ?? { name: "", arguments: "" }; + if (delta.function?.name) { + current.name = mergeToolName(current.name, delta.function.name); + } + if (delta.function?.arguments) { + current.arguments += delta.function.arguments; + } + calls.set(delta.index, current); + } + } + + return { + text, + toolCalls: [...calls.entries()] + .sort(([a], [b]) => a - b) + .map(([index, call]) => ({ + index, + name: call.name, + arguments: call.arguments, + })), + sawToolCallDelta, + finishReason, + terminalObserved, + }; +} + +/** + * SSE events are blank-line separated. A trailing chunk with no closing + * blank line still counts: providers and proxies routinely close the + * response straight after the terminal event, and treating that last + * event as noise would turn a clean finish into a false early EOF. + */ +function splitSseEvents(sse: string): string[] { + return sse + .split("\n\n") + .map((part) => part.trim()) + .filter((part) => part.length > 0); +} diff --git a/src/llm/provider/verify/classify-contract-probe.test.ts b/src/llm/provider/verify/classify-contract-probe.test.ts new file mode 100644 index 00000000..d2f1a9a6 --- /dev/null +++ b/src/llm/provider/verify/classify-contract-probe.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it } from "vitest"; + +import { accumulateProbeStream } from "./accumulate-probe-stream.js"; +import { + classifyContractProbeHttpFailure, + classifyProbeStream, + contractProbeFailureIsTerminal, +} from "./classify-contract-probe.js"; +import { + CONTRACT_PROBE_TOOL_NAME, + contractProbeFoundDefect, + contractProbeProvesToolSupport, + contractProbeToolDefinition, +} from "./contract-probe-types.js"; +import { DEFAULT_TOOL_DESCRIPTORS } from "../../../prompt/tool-descriptors.js"; + +function stream(body: string): ReturnType { + return accumulateProbeStream(body); +} + +const CALL_EVENT = + `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok":true}' }, + }, + ], + }, + }, + ], + })}\n\n`; +const FINISH_EVENT = `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "tool_calls" }] })}\n\ndata: [DONE]\n\n`; + +describe("classifyContractProbeHttpFailure", () => { + it("names an authentication refusal", () => { + expect( + classifyContractProbeHttpFailure(401, '{"error":"No auth credentials found"}'), + ).toBe("endpoint_auth_failed"); + }); + + it("buckets quota exhaustion and gateway throttling together", () => { + expect( + classifyContractProbeHttpFailure(429, '{"error":{"code":"insufficient_quota"}}'), + ).toBe("quota_or_routing_failed"); + expect(classifyContractProbeHttpFailure(402, "Insufficient credits")).toBe( + "quota_or_routing_failed", + ); + expect(classifyContractProbeHttpFailure(429, "slow down")).toBe( + "quota_or_routing_failed", + ); + }); + + it("names an unknown model", () => { + expect(classifyContractProbeHttpFailure(404, "no such model")).toBe( + "model_unavailable", + ); + expect( + classifyContractProbeHttpFailure(400, '{"error":"model does not exist"}'), + ).toBe("model_unavailable"); + }); + + it("leaves an unexplained 400 open rather than blaming tools", () => { + // This is the case the ladder exists for: a bare 400 with `tools` in + // the body proves nothing on its own, and guessing from wording is + // exactly what makes a diagnosis wrong. + expect(classifyContractProbeHttpFailure(400, "Bad Request")).toBe("provider_error"); + expect(contractProbeFailureIsTerminal("provider_error")).toBe(false); + }); + + it("treats key, quota and model refusals as terminal", () => { + expect(contractProbeFailureIsTerminal("endpoint_auth_failed")).toBe(true); + expect(contractProbeFailureIsTerminal("quota_or_routing_failed")).toBe(true); + expect(contractProbeFailureIsTerminal("model_unavailable")).toBe(true); + }); +}); + +describe("classifyProbeStream", () => { + it("accepts one complete native tool call", () => { + expect( + classifyProbeStream(stream(CALL_EVENT + FINISH_EVENT), "required_named"), + ).toBe("tools_supported"); + }); + + it("accepts a forced call that carried no arguments", () => { + // The synthetic schema requires no field, so empty arguments are a + // legal answer; calling them malformed would fail healthy routes. + const empty = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: "" }, + }, + ], + }, + }, + ], + })}\n\n`; + expect(classifyProbeStream(stream(empty + FINISH_EVENT), "required_named")).toBe( + "tools_supported", + ); + }); + + it("calls a text answer under auto inconclusive, never unsupported", () => { + const text = `data: ${JSON.stringify({ + choices: [{ delta: { content: "Sure!" }, finish_reason: "stop" }], + })}\n\ndata: [DONE]\n\n`; + expect(classifyProbeStream(stream(text), "auto")).toBe( + "inconclusive_no_tool_call", + ); + }); + + it("calls the same text answer under a forced choice a route defect", () => { + const text = `data: ${JSON.stringify({ + choices: [{ delta: { content: "Sure!" }, finish_reason: "stop" }], + })}\n\ndata: [DONE]\n\n`; + expect(classifyProbeStream(stream(text), "required_named")).toBe( + "forced_tool_choice_ignored", + ); + }); + + it("reports a stream that ended before announcing it was done", () => { + // A complete-looking call in a truncated body is not trustworthy: + // argument fragments may still have been in flight. + expect(classifyProbeStream(stream(CALL_EVENT), "required_named")).toBe( + "stream_early_eof", + ); + }); + + it("reports arguments that are not JSON", () => { + const truncated = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok":' }, + }, + ], + }, + }, + ], + })}\n\n`; + expect(classifyProbeStream(stream(truncated + FINISH_EVENT), "required_named")).toBe( + "malformed_tool_call", + ); + }); + + it("reports tool-call deltas that never named a function", () => { + const nameless = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [{ index: 0, type: "function", function: { arguments: "{}" } }], + }, + }, + ], + })}\n\n`; + expect(classifyProbeStream(stream(nameless + FINISH_EVENT), "required_named")).toBe( + "malformed_tool_call", + ); + }); + + it("reports a call naming a function that was never offered", () => { + const wrong = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "os.fs.read", arguments: "{}" }, + }, + ], + }, + }, + ], + })}\n\n`; + expect(classifyProbeStream(stream(wrong + FINISH_EVENT), "required_named")).toBe( + "malformed_tool_call", + ); + }); +}); + +describe("the synthetic probe tool", () => { + it("is not a tool this agent can dispatch", () => { + // The probe reads the call and throws it away; nothing dispatches + // it. This pins the other half of that promise: the name cannot + // collide with a real tool, now or after the catalog grows. + const registered = DEFAULT_TOOL_DESCRIPTORS.map((d) => d.name); + expect(registered).not.toContain(CONTRACT_PROBE_TOOL_NAME); + }); + + it("offers a schema with no required field", () => { + const fn = (contractProbeToolDefinition().function ?? {}) as { + name: string; + parameters: { required: string[] }; + }; + expect(fn.name).toBe(CONTRACT_PROBE_TOOL_NAME); + expect(fn.parameters.required).toEqual([]); + }); + + it("treats only a complete tool call as proven support", () => { + expect(contractProbeProvesToolSupport("tools_supported")).toBe(true); + for (const status of [ + "inconclusive_no_tool_call", + "stream_early_eof", + "malformed_tool_call", + "tools_payload_rejected", + "provider_error", + ] as const) { + expect(contractProbeProvesToolSupport(status)).toBe(false); + } + }); + + it("does not count an inconclusive auto answer as a defect", () => { + expect(contractProbeFoundDefect("inconclusive_no_tool_call")).toBe(false); + expect(contractProbeFoundDefect("stream_early_eof")).toBe(true); + }); +}); diff --git a/src/llm/provider/verify/classify-contract-probe.ts b/src/llm/provider/verify/classify-contract-probe.ts new file mode 100644 index 00000000..86060ee6 --- /dev/null +++ b/src/llm/provider/verify/classify-contract-probe.ts @@ -0,0 +1,134 @@ +/** + * Turning one probe request into a verdict about the *route*. + * + * Two classifiers live here, and the split matters: + * + * - `classifyContractProbeHttpFailure` reads a refusal. It answers + * only the questions a single status code and body can settle — + * credential, quota, model — and delegates that reading to + * `classifyVerifyResponse`, so the contract probe and the key check + * can never disagree about what a 402 or a Gemini 400 means. + * - `classifyProbeStream` reads a stream that was accepted, and + * settles whether a dispatchable native tool call actually arrived. + * + * Nothing here guesses "tools are unsupported" from error wording. + * Provider phrasing for that is not stable enough to hang a verdict on, + * and it does not need to be: the runner establishes it by experiment — + * refuse with tools, answer without them — which is both stronger + * evidence and the same evidence a human would gather by hand. + */ + +import { classifyVerifyResponse } from "./classify-verify-response.js"; +import { + CONTRACT_PROBE_TOOL_NAME, + type ProbeToolChoiceMode, + type ProviderContractStatus, +} from "./contract-probe-types.js"; +import type { ProbeStreamObservation } from "./accumulate-probe-stream.js"; + +/** + * Classify a non-2xx answer to a probe request. + * + * Only the terminal classes are named here (see + * `contractProbeFailureIsTerminal`). Anything else comes back as + * `provider_error`, which the runner reads as "not settled yet" and + * follows up on with the next rung of the ladder. + */ +export function classifyContractProbeHttpFailure( + httpStatus: number, + body: string, +): ProviderContractStatus { + const verdict = classifyVerifyResponse(httpStatus, body); + if (verdict.kind === "retry_next_model") return "model_unavailable"; + // The key check's "resend with the other max-tokens field" hint is + // meaningless here: the probe deliberately sends no token cap (see + // `run-contract-probe`), so a body naming those fields is the route + // complaining about something else it read in our request. + if (verdict.kind === "retry_token_field") return "provider_error"; + switch (verdict.status) { + case "invalid_key": + return "endpoint_auth_failed"; + case "no_balance": + case "rate_limited": + // One bucket on purpose: from a setup screen, "you are out of + // credit" and "this gateway is throttling you" lead to the same + // action — sort out the account, then probe again. + return "quota_or_routing_failed"; + case "model_unavailable": + return "model_unavailable"; + default: + return "provider_error"; + } +} + +/** + * `true` when the refusal already explains itself and no further + * request can teach us anything about tool support. Retrying a dead + * key without `tools` would only spend another request to be told the + * same thing. + */ +export function contractProbeFailureIsTerminal( + status: ProviderContractStatus, +): boolean { + return ( + status === "endpoint_auth_failed" || + status === "quota_or_routing_failed" || + status === "model_unavailable" + ); +} + +/** + * Read an accepted stream. + * + * Order is deliberate. A stream that never announced its own end is + * judged first and unconditionally: a tool call assembled out of a + * truncated body may be missing argument fragments that were still in + * flight, so trusting it is exactly the mistake + * `applyToolCallTerminationSafety` exists to prevent on the real path. + */ +export function classifyProbeStream( + observation: ProbeStreamObservation, + mode: ProbeToolChoiceMode, +): ProviderContractStatus { + if (!observation.terminalObserved) return "stream_early_eof"; + + if (observation.sawToolCallDelta) { + const call = + observation.toolCalls.find((c) => c.name === CONTRACT_PROBE_TOOL_NAME) ?? + observation.toolCalls[0]; + // Deltas arrived and still produced nothing callable. On the real + // path this is the failure that surfaces as `tool not registered in + // this agent`, several minutes into a turn. + if (!call || call.name.length === 0) return "malformed_tool_call"; + // Only one function was offered, so any other name is the route + // inventing one — the call could never be dispatched. + if (call.name !== CONTRACT_PROBE_TOOL_NAME) return "malformed_tool_call"; + if (!argumentsAreDispatchable(call.arguments)) return "malformed_tool_call"; + return "tools_supported"; + } + + // No tool call at all. What that means depends entirely on what we + // asked for, and conflating the two cases is the specific mistake + // this probe is built to avoid. + return mode === "required_named" + ? "forced_tool_choice_ignored" + : "inconclusive_no_tool_call"; +} + +/** + * Arguments Atomic could actually hand to a tool. Empty is fine — the + * probe's schema requires no field, and a model answering a forced call + * with nothing to say legitimately sends `""` or `{}`. Anything else + * has to parse as a JSON object; a truncated `{"ok":` is the shape a + * route with incomplete deltas produces. + */ +function argumentsAreDispatchable(raw: string): boolean { + const trimmed = raw.trim(); + if (trimmed.length === 0) return true; + try { + const parsed: unknown = JSON.parse(trimmed); + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed); + } catch { + return false; + } +} diff --git a/src/llm/provider/verify/contract-probe-types.ts b/src/llm/provider/verify/contract-probe-types.ts new file mode 100644 index 00000000..b021513a --- /dev/null +++ b/src/llm/provider/verify/contract-probe-types.ts @@ -0,0 +1,179 @@ +/** + * Shapes for the provider *contract* probe — the second question a + * setup flow has to ask. + * + * `verify-provider-key` settles whether the credential is live and + * funded. It cannot settle whether the route behind that credential can + * run an Atomic turn, because a turn is not a one-token completion: it + * is a **streamed** chat completion carrying a `tools` payload, from + * which a **native tool call** has to come back whole. Routes exist that + * pass the key check and then fail exactly one of those: the stream ends + * early, HTTP 400 appears only once `tools` is in the body, forced tool + * choice is refused, or the tool-call deltas arrive incomplete. + * + * Kept free of config and UI imports for the same reason the key check + * is: the wizard, onboarding and an explicit "test this provider" + * action all have to be able to run it. + */ + +/** + * The tool the probe offers. It is a *diagnostic fixture*, never an + * Atomic tool: nothing registers it, nothing dispatches it, and the + * model's call to it is read and thrown away. The name is deliberately + * outside the dotted namespace every built-in tool uses + * (`os.fs.read`, `browser.navigate`, …) so it cannot shadow a real one, + * and it stays inside the `^[A-Za-z0-9_-]{1,64}$` shape the strict + * providers validate function names against. + */ +export const CONTRACT_PROBE_TOOL_NAME = "atomic_contract_probe"; + +/** + * Which tool-choice mode a probe request ran in. `required_named` is the + * primary instrument — it is the only mode where "no tool call" is a + * real answer about the route rather than about the model's mood. + */ +export type ProbeToolChoiceMode = "required_named" | "auto"; + +export type ProviderContractStatus = + /** One complete native tool call came back over the stream. */ + | "tools_supported" + /** + * The route took the tools payload but chose to answer in prose under + * `tool_choice: auto`. That is a legal answer for a model, so it says + * nothing about whether the route can emit tool calls at all. + */ + | "inconclusive_no_tool_call" + /** + * The route accepted a forced named tool choice and then answered + * text anyway — it advertises the parameter without honoring it. + */ + | "forced_tool_choice_ignored" + /** + * The route refused the request while forcing a named tool, but ran + * the same tools payload under `auto` and produced a tool call. + */ + | "forced_tool_choice_rejected" + /** + * The route refused every request that carried `tools`, and answered + * the same streamed completion once `tools` was removed. + */ + | "tools_payload_rejected" + /** The configured model is unknown to this route. */ + | "model_unavailable" + /** The endpoint refused the credential; nothing else was learned. */ + | "endpoint_auth_failed" + /** Out of quota, or the gateway could not route to a backend. */ + | "quota_or_routing_failed" + /** The stream closed with neither a finish reason nor `[DONE]`. */ + | "stream_early_eof" + /** + * Tool-call deltas arrived but never assembled into a dispatchable + * call: no function name, a name nobody offered, or arguments that + * are not JSON. + */ + | "malformed_tool_call" + /** No HTTP response at all: DNS, refused connection, TLS, offline. */ + | "unreachable" + /** The probe's own deadline fired first. */ + | "timeout" + /** The operator (or the caller) aborted the probe. */ + | "cancelled" + /** The route failed in a way that says nothing about tool support. */ + | "provider_error"; + +export interface ProviderContractProbeTarget { + /** Service name for user-facing wording ("OpenRouter", "Groq"). */ + readonly label: string; + /** API root without the version prefix, already normalized. */ + readonly baseUrl: string; + /** Version prefix the service uses: `/v1`, Gemini's `/v1beta/openai`. */ + readonly apiPathPrefix: string; + /** Trimmed key, or `""` for a service that authenticates without one. */ + readonly apiKey: string; + /** + * The model the operator is about to run turns with. Not a cheap + * stand-in: route limitations are per-model, so probing anything else + * would answer a question nobody asked. + */ + readonly model: string; + readonly extraHeaders?: Record; +} + +export interface ProviderContractProbeResult { + readonly status: ProviderContractStatus; + readonly probedModel: string; + /** Status of the request the verdict came from, `null` if none did. */ + readonly httpStatus: number | null; + /** The mode the verdict came from; `null` when no request was made. */ + readonly toolChoiceMode: ProbeToolChoiceMode | null; + /** + * A bounded, credential-scrubbed excerpt of what the provider said — + * enough for a status line and the log, never the whole body. + */ + readonly detail: string; + readonly latencyMs: number; + /** How many HTTP requests the probe spent reaching this verdict. */ + readonly requests: number; +} + +/** + * The one verdict that means "this route can run a turn". Everything + * else is either a failure or an open question, and neither may be + * reported as proven compatibility. + */ +export function contractProbeProvesToolSupport( + status: ProviderContractStatus, +): boolean { + return status === "tools_supported"; +} + +/** + * `true` when the probe learned something about the *route* rather than + * about the model's whim. Used to decide whether a warning is worth + * showing: an inconclusive auto-mode answer is not a defect to report. + */ +export function contractProbeFoundDefect( + status: ProviderContractStatus, +): boolean { + return ( + status === "forced_tool_choice_ignored" || + status === "forced_tool_choice_rejected" || + status === "tools_payload_rejected" || + status === "model_unavailable" || + status === "endpoint_auth_failed" || + status === "quota_or_routing_failed" || + status === "stream_early_eof" || + status === "malformed_tool_call" + ); +} + +/** + * The synthetic function definition, in the exact shape + * `buildOpenAiChatBody` puts real tools in, so a route that validates + * tool schemas judges this one by the same rules it will judge Atomic's. + * + * No required properties: a model answering a forced call with empty + * arguments is honoring the contract, and demanding a field would turn + * that legal answer into a false "malformed" verdict. + */ +export function contractProbeToolDefinition(): Record { + return { + type: "function", + function: { + name: CONTRACT_PROBE_TOOL_NAME, + description: + "Diagnostic no-op used to check that this endpoint can emit a native tool call. Has no effect.", + parameters: { + type: "object", + properties: { + ok: { + type: "boolean", + description: "Always true.", + }, + }, + required: [], + additionalProperties: false, + }, + }, + }; +} diff --git a/src/llm/provider/verify/index.ts b/src/llm/provider/verify/index.ts index a100e465..87a43b75 100644 --- a/src/llm/provider/verify/index.ts +++ b/src/llm/provider/verify/index.ts @@ -1,12 +1,40 @@ +export { + accumulateProbeStream, + type ProbeStreamObservation, + type ProbeToolCallObservation, +} from "./accumulate-probe-stream.js"; +export { + classifyContractProbeHttpFailure, + classifyProbeStream, + contractProbeFailureIsTerminal, +} from "./classify-contract-probe.js"; export { classifyVerifyResponse, classifyVerifyTransportError, type VerifyResponseVerdict, } from "./classify-verify-response.js"; +export { + CONTRACT_PROBE_TOOL_NAME, + contractProbeFoundDefect, + contractProbeProvesToolSupport, + contractProbeToolDefinition, + type ProbeToolChoiceMode, + type ProviderContractProbeResult, + type ProviderContractProbeTarget, + type ProviderContractStatus, +} from "./contract-probe-types.js"; export { cheapestPaidOpenRouterModel, pickProbeModels, } from "./pick-probe-models.js"; +export { + PROVIDER_DETAIL_MAX_LEN, + redactProviderDetail, +} from "./redact-provider-detail.js"; +export { + PROVIDER_CONTRACT_PROBE_TIMEOUT_MS, + runProviderContractProbe, +} from "./run-contract-probe.js"; export { PROVIDER_VERIFY_TIMEOUT_MS, verifyProviderKey, diff --git a/src/llm/provider/verify/redact-provider-detail.ts b/src/llm/provider/verify/redact-provider-detail.ts new file mode 100644 index 00000000..641bf568 --- /dev/null +++ b/src/llm/provider/verify/redact-provider-detail.ts @@ -0,0 +1,42 @@ +/** + * What a provider said, made safe to put on a status line and in a log. + * + * Two rules, both learned the hard way from error bodies: + * + * - Providers echo the offending credential back. The key we sent is + * therefore removed by exact match, and anything else shaped like an + * API key is removed by pattern — a proxy in front of the service can + * quote a *different* key than the one under test, and the exact + * match would sail straight past it. + * - The body itself is never reproduced whole. A verdict needs a + * sentence of evidence, not a provider's entire JSON, which on some + * gateways carries request echoes and upstream headers. + */ + +/** Same cap the OpenAI HTTP layer uses when folding a body into an error. */ +export const PROVIDER_DETAIL_MAX_LEN = 300; + +/** + * Vendor key shapes common enough to be worth removing on sight: + * OpenAI-style `sk-…`, OpenRouter's `sk-or-…`, Google's `AIza…`, and a + * bearer token quoted out of an echoed header. Deliberately narrow — + * a pattern loose enough to catch every possible secret would redact + * model ids and error codes along with them. + */ +const KEY_SHAPED = [ + /\bsk-[A-Za-z0-9_-]{6,}/g, + /\bAIza[A-Za-z0-9_-]{10,}/g, + /\b[Bb]earer\s+[A-Za-z0-9._-]{8,}/g, +]; + +export function redactProviderDetail( + detail: string, + apiKey = "", + maxLen: number = PROVIDER_DETAIL_MAX_LEN, +): string { + // Short strings are not keys; splitting on one would shred ordinary + // words out of the message. + let out = apiKey.length >= 8 ? detail.split(apiKey).join("***") : detail; + for (const pattern of KEY_SHAPED) out = out.replace(pattern, "***"); + return out.slice(0, maxLen); +} diff --git a/src/llm/provider/verify/run-contract-probe.test.ts b/src/llm/provider/verify/run-contract-probe.test.ts new file mode 100644 index 00000000..0d39d62f --- /dev/null +++ b/src/llm/provider/verify/run-contract-probe.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + CONTRACT_PROBE_TOOL_NAME, + type ProviderContractProbeTarget, +} from "./contract-probe-types.js"; +import { runProviderContractProbe } from "./run-contract-probe.js"; + +const TARGET: ProviderContractProbeTarget = { + label: "OmniRoute", + baseUrl: "https://route.example", + apiPathPrefix: "/v1", + apiKey: "sk-secret-probe-key", + model: "vendor/some-model", +}; + +function sseEvent(payload: unknown): string { + return `data: ${JSON.stringify(payload)}\n\n`; +} + +const TOOL_CALL_STREAM = + sseEvent({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok"' }, + }, + ], + }, + }, + ], + }) + + sseEvent({ + choices: [ + { + delta: { + tool_calls: [{ index: 0, function: { arguments: ":true}" } }], + }, + }, + ], + }) + + sseEvent({ choices: [{ delta: {}, finish_reason: "tool_calls" }] }) + + "data: [DONE]\n\n"; + +const TEXT_STREAM = + sseEvent({ choices: [{ delta: { content: "Happy to help." } }] }) + + sseEvent({ choices: [{ delta: {}, finish_reason: "stop" }] }) + + "data: [DONE]\n\n"; + +/** Truncated: the arguments never close and nothing announces the end. */ +const EARLY_EOF_STREAM = sseEvent({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok"' }, + }, + ], + }, + }, + ], +}); + +const MALFORMED_STREAM = + sseEvent({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok":' }, + }, + ], + }, + }, + ], + }) + sseEvent({ choices: [{ delta: {}, finish_reason: "tool_calls" }] }); + +function streamResponse(body: string): Response { + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +function errorResponse(status: number, body: unknown): Response { + return new Response(typeof body === "string" ? body : JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +/** A fetch that answers the calls in order and records the bodies sent. */ +function scriptedFetch(responses: readonly (() => Response)[]): { + fetchImpl: typeof fetch; + bodies: () => Record[]; + calls: () => number; +} { + const sent: Record[] = []; + let index = 0; + const fetchImpl = vi.fn(async (_url: unknown, init?: RequestInit) => { + sent.push(JSON.parse(String(init?.body ?? "{}")) as Record); + const next = responses[index] ?? responses[responses.length - 1]; + index += 1; + return next!(); + }) as unknown as typeof fetch; + return { + fetchImpl, + bodies: () => sent, + calls: () => index, + }; +} + +describe("runProviderContractProbe", () => { + it("proves support from one forced, streamed native tool call", async () => { + const script = scriptedFetch([() => streamResponse(TOOL_CALL_STREAM)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("tools_supported"); + expect(result.toolChoiceMode).toBe("required_named"); + expect(result.probedModel).toBe("vendor/some-model"); + // A healthy route costs exactly one request: no probe ladder, and + // nothing that could run per turn. + expect(script.calls()).toBe(1); + expect(result.requests).toBe(1); + + const body = script.bodies()[0]!; + expect(body.stream).toBe(true); + expect(body.model).toBe("vendor/some-model"); + expect(body.tool_choice).toEqual({ + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME }, + }); + expect(JSON.stringify(body.tools)).toContain(CONTRACT_PROBE_TOOL_NAME); + }); + + it("reports a forced tool choice the route refuses but tools it accepts", async () => { + const script = scriptedFetch([ + () => errorResponse(400, { error: "tool_choice of type function is not supported" }), + () => streamResponse(TOOL_CALL_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("forced_tool_choice_rejected"); + expect(result.httpStatus).toBe(400); + expect(script.calls()).toBe(2); + expect(script.bodies()[1]!.tool_choice).toBe("auto"); + }); + + it("calls a plain text answer under auto inconclusive, not unsupported", async () => { + const script = scriptedFetch([ + () => errorResponse(400, { error: "unexpected parameter" }), + () => streamResponse(TEXT_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("inconclusive_no_tool_call"); + expect(result.toolChoiceMode).toBe("auto"); + }); + + it("proves the tools payload is the problem by answering without it", async () => { + // Endpoint success with no tools, refusal with them: the route works, + // and it is specifically `tools` it will not take. + const script = scriptedFetch([ + () => errorResponse(400, "Bad Request"), + () => errorResponse(400, "Bad Request"), + () => streamResponse(TEXT_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("tools_payload_rejected"); + expect(script.calls()).toBe(3); + // The control request is the same streamed completion minus tools. + const control = script.bodies()[2]!; + expect(control.stream).toBe(true); + expect(control.tools).toBeUndefined(); + expect(control.tool_choice).toBeUndefined(); + }); + + it("blames the route, not tools, when the no-tools control fails too", async () => { + const script = scriptedFetch([ + () => errorResponse(500, "upstream exploded"), + () => errorResponse(500, "upstream exploded"), + () => errorResponse(500, "upstream exploded"), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("provider_error"); + expect(result.httpStatus).toBe(500); + }); + + it("reports a stream that ended early", async () => { + const script = scriptedFetch([() => streamResponse(EARLY_EOF_STREAM)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("stream_early_eof"); + expect(script.calls()).toBe(1); + }); + + it("reports malformed tool-call deltas", async () => { + const script = scriptedFetch([() => streamResponse(MALFORMED_STREAM)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("malformed_tool_call"); + }); + + it("stops at a quota refusal instead of climbing the ladder", async () => { + const script = scriptedFetch([ + () => errorResponse(429, { error: { code: "insufficient_quota" } }), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("quota_or_routing_failed"); + // Nothing another request could add: the account, not the route. + expect(script.calls()).toBe(1); + }); + + it("stops at an authentication refusal", async () => { + const script = scriptedFetch([ + () => errorResponse(401, { error: "No auth credentials found" }), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("endpoint_auth_failed"); + expect(script.calls()).toBe(1); + }); + + it("stops at an unknown model", async () => { + const script = scriptedFetch([ + () => errorResponse(404, { error: "model not found" }), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("model_unavailable"); + expect(script.calls()).toBe(1); + }); + + it("reports an unreachable endpoint without claiming anything about tools", async () => { + const fetchImpl = vi.fn(async () => { + throw new TypeError("fetch failed"); + }) as unknown as typeof fetch; + const result = await runProviderContractProbe(TARGET, { fetchImpl }); + + expect(result.status).toBe("unreachable"); + expect(result.httpStatus).toBeNull(); + }); + + it("keeps credentials and whole response bodies out of the detail", async () => { + const leak = `${"x".repeat(400)} key=${TARGET.apiKey} other=sk-someoneelseskey123`; + const script = scriptedFetch([() => errorResponse(401, leak)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.detail).not.toContain(TARGET.apiKey); + expect(result.detail).not.toContain("sk-someoneelseskey123"); + expect(result.detail.length).toBeLessThanOrEqual(300); + }); + + it("refuses to probe with no model rather than inventing one", async () => { + const script = scriptedFetch([() => streamResponse(TOOL_CALL_STREAM)]); + const result = await runProviderContractProbe( + { ...TARGET, model: " " }, + { fetchImpl: script.fetchImpl }, + ); + + expect(result.status).toBe("model_unavailable"); + expect(script.calls()).toBe(0); + }); + + it("reports a cancelled probe as cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + const script = scriptedFetch([() => streamResponse(TOOL_CALL_STREAM)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + signal: controller.signal, + }); + + expect(result.status).toBe("cancelled"); + expect(script.calls()).toBe(0); + }); +}); diff --git a/src/llm/provider/verify/run-contract-probe.ts b/src/llm/provider/verify/run-contract-probe.ts new file mode 100644 index 00000000..f9983b9f --- /dev/null +++ b/src/llm/provider/verify/run-contract-probe.ts @@ -0,0 +1,389 @@ +/** + * Prove — or fail to prove — that a route can run an Atomic turn. + * + * A turn is a streamed chat completion that carries a `tools` payload + * and gets a native tool call back. `/v1/models` proves none of that, + * and neither does the one-token key check: both are non-streaming, both + * send no tools. Routes that pass them and then fail a real turn are + * common enough that the failure has a name in the field reports + * (`STREAM_EARLY_EOF`, "HTTP 400, but only with tools"). + * + * So this sends the real thing, once, with a synthetic function nobody + * has registered, and reads what comes back. + * + * ## The ladder + * + * Each rung only runs when the one above it left the question open, so + * a healthy route costs exactly one request: + * + * 1. **forced named tool**, streaming, tools payload. A complete tool + * call here is the whole answer: `tools_supported`. + * 2. **`tool_choice: auto`**, same tools payload — reached only when + * rung 1 was *refused* for a reason that is not the key, the quota + * or the model. A tool call now means the payload is fine and it + * was the forcing the route would not take. + * 3. **no tools at all**, same model, same streaming transport — + * reached only when both tool requests were refused. If this + * answers, the route works and it is specifically `tools` it + * rejects; if it fails too, the failure was never about tools. + * + * Rung 3 is what turns "HTTP 400" into a sentence an operator can act + * on, and it is deliberately an experiment rather than a regex over the + * error body: provider wording for "tools unsupported" is not stable, + * but "refuses with tools, answers without them" is unambiguous. + * + * ## What it never does + * + * The synthetic call is read and discarded. It is never looked up in, + * dispatched to, or registered with the tool registry — the probe does + * not import it and could not reach it. And nothing here runs per turn: + * the only callers are setup-time or an explicit operator request. + */ + +import { openAiFetch, type OpenAiHttpDeps } from "../openai/openai-http.js"; +import { + accumulateProbeStream, + type ProbeStreamObservation, +} from "./accumulate-probe-stream.js"; +import { + classifyContractProbeHttpFailure, + classifyProbeStream, + contractProbeFailureIsTerminal, +} from "./classify-contract-probe.js"; +import { isAbortError, classifyVerifyTransportError } from "./classify-verify-response.js"; +import { + CONTRACT_PROBE_TOOL_NAME, + contractProbeToolDefinition, + type ProbeToolChoiceMode, + type ProviderContractProbeResult, + type ProviderContractProbeTarget, + type ProviderContractStatus, +} from "./contract-probe-types.js"; +import { redactProviderDetail } from "./redact-provider-detail.js"; + +/** + * Whole-probe budget, not per request. Longer than the key check's 8s + * because this one waits for a model to actually generate, but still + * short enough that a wizard screen does not feel hung: a route that + * cannot produce a two-field tool call inside this is a finding in + * itself. + */ +export const PROVIDER_CONTRACT_PROBE_TIMEOUT_MS = 20_000; + +/** Ceiling on the SSE body we buffer. A probe answer is a few hundred bytes. */ +const MAX_STREAM_BYTES = 64 * 1024; + +/** The whole ladder: forced → auto → no-tools. Never more than that. */ +const MAX_PROBE_REQUESTS = 3; + +const PROBE_PROMPT = + `Call the ${CONTRACT_PROBE_TOOL_NAME} function with ok set to true. ` + + `Do not answer in words.`; + +export async function runProviderContractProbe( + target: ProviderContractProbeTarget, + opts: { + signal?: AbortSignal; + timeoutMs?: number; + fetchImpl?: typeof fetch; + } = {}, +): Promise { + const startedAt = Date.now(); + const budgetMs = opts.timeoutMs ?? PROVIDER_CONTRACT_PROBE_TIMEOUT_MS; + const deadline = startedAt + budgetMs; + const model = target.model.trim(); + const state = { requests: 0 }; + + const emit = ( + status: ProviderContractStatus, + httpStatus: number | null, + mode: ProbeToolChoiceMode | null, + detail: string, + ): ProviderContractProbeResult => ({ + status, + probedModel: model, + httpStatus, + toolChoiceMode: mode, + detail: redactProviderDetail(detail, target.apiKey), + latencyMs: Date.now() - startedAt, + requests: state.requests, + }); + + if (model.length === 0) { + return emit("model_unavailable", null, null, "no model configured to probe"); + } + + const run = async (body: Record): Promise => + runRung(target, body, { + deadline, + state, + ...(opts.signal ? { signal: opts.signal } : {}), + ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}), + }); + + // Rung 1 — the real contract, forced. + const forced = await run(probeBody(model, "required_named")); + if (forced.kind === "aborted") { + return emit(forced.status, null, "required_named", forced.detail); + } + if (forced.kind === "stream") { + return emit( + classifyProbeStream(forced.observation, "required_named"), + forced.httpStatus, + "required_named", + streamDetail(forced.observation), + ); + } + const forcedStatus = classifyContractProbeHttpFailure( + forced.httpStatus, + forced.body, + ); + if (contractProbeFailureIsTerminal(forcedStatus)) { + return emit(forcedStatus, forced.httpStatus, "required_named", forced.body); + } + + // Rung 2 — same payload, no forcing. + const auto = await run(probeBody(model, "auto")); + if (auto.kind === "aborted") { + return emit(auto.status, null, "auto", auto.detail); + } + if (auto.kind === "stream") { + const autoStatus = classifyProbeStream(auto.observation, "auto"); + // Tools work; it was the forced choice rung 1 asked for that this + // route would not take. Reporting rung 2's own verdict here would + // hide the actual limitation behind a cheerful "supported". + if (autoStatus === "tools_supported") { + return emit( + "forced_tool_choice_rejected", + forced.httpStatus, + "required_named", + forced.body, + ); + } + return emit(autoStatus, auto.httpStatus, "auto", streamDetail(auto.observation)); + } + const autoStatus = classifyContractProbeHttpFailure(auto.httpStatus, auto.body); + if (contractProbeFailureIsTerminal(autoStatus)) { + return emit(autoStatus, auto.httpStatus, "auto", auto.body); + } + + // Rung 3 — the control. Same model, same streaming transport, no tools. + const control = await run(probeBody(model, null)); + if (control.kind === "aborted") { + return emit(control.status, null, null, control.detail); + } + if (control.kind === "stream") { + // It answers without tools and refuses with them. That is the + // finding, stated from evidence rather than from error wording. + return emit("tools_payload_rejected", auto.httpStatus, "auto", auto.body); + } + const controlStatus = classifyContractProbeHttpFailure( + control.httpStatus, + control.body, + ); + // The control failed too, so nothing here was ever about tools. + return emit( + contractProbeFailureIsTerminal(controlStatus) ? controlStatus : "provider_error", + control.httpStatus, + null, + control.body, + ); +} + +type RungOutcome = + | { kind: "stream"; httpStatus: number; observation: ProbeStreamObservation } + | { kind: "http_error"; httpStatus: number; body: string } + | { + kind: "aborted"; + status: Extract< + ProviderContractStatus, + "timeout" | "cancelled" | "unreachable" | "provider_error" + >; + detail: string; + }; + +async function runRung( + target: ProviderContractProbeTarget, + body: Record, + ctx: { + deadline: number; + state: { requests: number }; + signal?: AbortSignal; + fetchImpl?: typeof fetch; + }, +): Promise { + if (ctx.signal?.aborted) { + return { kind: "aborted", status: "cancelled", detail: "probe cancelled" }; + } + if (ctx.state.requests >= MAX_PROBE_REQUESTS) { + return { + kind: "aborted", + status: "provider_error", + detail: "probe budget spent", + }; + } + const remainingMs = ctx.deadline - Date.now(); + if (remainingMs <= 0) { + return { kind: "aborted", status: "timeout", detail: "probe deadline reached" }; + } + ctx.state.requests += 1; + + const deps: OpenAiHttpDeps = { + baseUrl: target.baseUrl, + apiKey: target.apiKey, + extraHeaders: target.extraHeaders ?? {}, + requestTimeoutMs: remainingMs, + fetchImpl: ctx.fetchImpl ?? fetch, + label: target.label, + }; + + let res: Response; + try { + res = await openAiFetch( + deps, + `${target.apiPathPrefix}/chat/completions`, + body, + { ...(ctx.signal ? { signal: ctx.signal } : {}) }, + true, + "POST", + ); + } catch (err) { + if (ctx.signal?.aborted || isAbortError(err)) { + return { kind: "aborted", status: "cancelled", detail: "probe cancelled" }; + } + // Transport failures are read by the key check's classifier, so a + // refused connection or an expired deadline means the same thing in + // both checks. Only its `cancelled` verdict is unreachable here — + // that case is handled above. + const transport = classifyVerifyTransportError(err); + const status = + transport === "timeout" || transport === "unreachable" + ? transport + : "provider_error"; + return { + kind: "aborted", + status, + detail: err instanceof Error ? err.message : String(err), + }; + } + + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { kind: "http_error", httpStatus: res.status, body: text }; + } + + // `openAiFetch`'s own timeout covers the connect only — it clears the + // timer the moment headers arrive. A route that opens a stream and + // then stalls forever would hang here, so the body read carries the + // remaining budget itself. + const sse = await readStreamBounded(res, ctx.deadline - Date.now()); + if (sse.timedOut && sse.text.length === 0) { + return { + kind: "aborted", + status: "timeout", + detail: "no stream data before deadline", + }; + } + return { + kind: "stream", + httpStatus: res.status, + observation: accumulateProbeStream(sse.text), + }; +} + +/** + * The probe request, in the same shape `buildOpenAiChatBody` gives a + * real turn — including `parallel_tool_calls`, which some routes + * validate and which a turn always sends alongside tools. + * + * `mode === null` is the control: no tools, no tool choice, otherwise + * identical, so a difference in outcome can only be the tools payload. + * + * No `max_tokens`. The one-token key check has to cap spend and + * therefore has to guess between `max_tokens` and + * `max_completion_tokens` (newer OpenAI models reject the former). + * Here a cap would risk truncating the very tool call being measured, + * reporting malformed deltas that were in fact our own doing — and one + * forced call to a single-field function is cheap enough uncapped. + */ +function probeBody( + model: string, + mode: ProbeToolChoiceMode | null, +): Record { + const body: Record = { + model, + messages: [{ role: "user", content: PROBE_PROMPT }], + temperature: 0, + stream: true, + }; + if (mode === null) return body; + body.tools = [contractProbeToolDefinition()]; + body.parallel_tool_calls = true; + body.tool_choice = + mode === "required_named" + ? { type: "function", function: { name: CONTRACT_PROBE_TOOL_NAME } } + : "auto"; + return body; +} + +/** + * A one-line summary of what the stream contained. Deliberately not the + * body: the assistant text is the model's own words about a synthetic + * function and has no diagnostic value worth logging. + */ +function streamDetail(observation: ProbeStreamObservation): string { + const names = observation.toolCalls + .map((call) => (call.name.length > 0 ? call.name : "")) + .join(", "); + return [ + `finish_reason=${observation.finishReason ?? "none"}`, + `terminal=${observation.terminalObserved}`, + `tool_call_deltas=${observation.sawToolCallDelta}`, + `tool_calls=[${names}]`, + `text_chars=${observation.text.length}`, + ].join(" "); +} + +/** + * Buffer the SSE body under a byte ceiling and a deadline, cancelling + * the stream rather than leaving a socket open behind us. + */ +async function readStreamBounded( + res: Response, + budgetMs: number, +): Promise<{ text: string; timedOut: boolean }> { + if (!res.body) return { text: await res.text().catch(() => ""), timedOut: false }; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + let timedOut = false; + let timer: ReturnType | undefined; + try { + const deadline = + budgetMs > 0 + ? new Promise<"deadline">((resolve) => { + timer = setTimeout(() => resolve("deadline"), budgetMs); + }) + : Promise.resolve<"deadline">("deadline"); + for (;;) { + const next = await Promise.race([reader.read(), deadline]); + if (next === "deadline") { + timedOut = true; + break; + } + if (next.done) { + text += decoder.decode(); + break; + } + text += decoder.decode(next.value, { stream: true }); + if (text.length >= MAX_STREAM_BYTES) break; + } + } catch { + // A stream that breaks mid-body is exactly the early-EOF case: keep + // what arrived and let the classifier see that it never terminated. + } finally { + if (timer) clearTimeout(timer); + void reader.cancel().catch(() => {}); + } + return { text, timedOut }; +} diff --git a/src/llm/provider/verify/verify-provider-key.ts b/src/llm/provider/verify/verify-provider-key.ts index 8a9f8f10..5ab7e98c 100644 --- a/src/llm/provider/verify/verify-provider-key.ts +++ b/src/llm/provider/verify/verify-provider-key.ts @@ -19,6 +19,10 @@ import { classifyVerifyTransportError, isAbortError, } from "./classify-verify-response.js"; +import { + redactProviderDetail, + PROVIDER_DETAIL_MAX_LEN, +} from "./redact-provider-detail.js"; import type { ProviderVerifyResult, ProviderVerifyStatus, @@ -35,8 +39,6 @@ export const PROVIDER_VERIFY_TIMEOUT_MS = 8_000; /** model → other token field → next model. Never more than that. */ const MAX_VERIFY_REQUESTS = 3; -/** Provider error bodies are quoted back bounded, same cap as the HTTP layer. */ -const VERIFY_DETAIL_MAX_LEN = 300; export async function verifyProviderKey( target: ProviderVerifyTarget, @@ -168,7 +170,7 @@ function probeBody( async function readBounded(res: Response): Promise { const text = await res.text().catch(() => ""); - return text.slice(0, VERIFY_DETAIL_MAX_LEN); + return text.slice(0, PROVIDER_DETAIL_MAX_LEN); } function result( @@ -183,16 +185,7 @@ function result( status, probedModel, httpStatus, - detail: redactKey(detail, apiKey).slice(0, VERIFY_DETAIL_MAX_LEN), + detail: redactProviderDetail(detail, apiKey), latencyMs: Date.now() - startedAt, }; } - -/** - * Some providers echo the offending credential back in the error body, - * and this detail is headed for a status line and the log file. - */ -function redactKey(detail: string, apiKey: string): string { - if (apiKey.length < 8) return detail; - return detail.split(apiKey).join("***"); -} diff --git a/src/tui/commands/slash-command-handler.test.ts b/src/tui/commands/slash-command-handler.test.ts index 65e52e2d..83e02b8d 100644 --- a/src/tui/commands/slash-command-handler.test.ts +++ b/src/tui/commands/slash-command-handler.test.ts @@ -263,6 +263,21 @@ describe("dispatchSlashCommand", () => { ]); }); + it("asks for a contract probe of the active provider on /llm check", () => { + const result = dispatchSlashCommand("/llm check"); + expect(result.actions).toEqual([ + { type: "providers_contract_probe_requested", providerId: null }, + ]); + // The operator is told a request is about to be spent on their key. + expect(result.systemMessage).toContain("one request"); + }); + + it("names /llm check in the usage line so it is discoverable", () => { + const result = dispatchSlashCommand("/llm nonsense"); + expect(result.actions).toEqual([]); + expect(result.systemMessage).toContain("/llm check"); + }); + it("signals triggerLocalModelsStatus for /models status", () => { const result = dispatchSlashCommand("/models status"); expect(result.triggerLocalModelsStatus).toBe(true); diff --git a/src/tui/commands/slash-command-handler.ts b/src/tui/commands/slash-command-handler.ts index 2bb2d2c1..121ad895 100644 --- a/src/tui/commands/slash-command-handler.ts +++ b/src/tui/commands/slash-command-handler.ts @@ -697,8 +697,20 @@ function dispatchLlmSub(rawArgs: string): SlashDispatchResult { { type: "providers_refresh_requested" }, ]); } + // `/llm check` exercises the real turn contract against the active + // provider — streaming, tools payload, a forced native tool call — + // and reports what came back. A reachable `/v1/models` says nothing + // about any of that, so until now the first real message was the + // test. Explicit only: it spends requests against the operator's own + // account and must never run on a turn path. + if (/^check$/i.test(argPart)) { + return pureActions([{ type: "providers_contract_probe_requested", providerId: null }], { + systemMessage: + "checking the active provider's streaming tool-call contract — this sends one request", + }); + } return pureActions([], { - systemMessage: "usage: /llm | /llm provider | /llm fallback", + systemMessage: "usage: /llm | /llm provider | /llm check | /llm fallback", }); } diff --git a/src/tui/components/cloud-provider-onboarding.tsx b/src/tui/components/cloud-provider-onboarding.tsx index 7fcf3b90..7854ad05 100644 --- a/src/tui/components/cloud-provider-onboarding.tsx +++ b/src/tui/components/cloud-provider-onboarding.tsx @@ -14,6 +14,7 @@ import { createProvidersWizardState } from "../providers/providers-wizard-state. import type { ProvidersWizardState } from "../providers/providers-wizard-state.js"; import { saveProviderWizardToConfig } from "../providers/save-provider-wizard.js"; import { verifyWizardBeforeSave } from "../providers/verify-wizard-before-save.js"; +import { probeWizardContract } from "../providers/probe-wizard-contract.js"; import { theme } from "../theme/theme.js"; import { ProvidersWizard } from "./providers-wizard.js"; @@ -86,8 +87,24 @@ export function CloudProviderOnboarding(props: { setSubmitting(false); return; } + // The key is good; whether the route can run a turn is a + // separate question, and first-run is exactly where getting it + // wrong costs the most — the operator's first message is + // otherwise the test. Advisory only: it cannot stop the save, + // and it rides the same abort, so Esc still abandons the whole + // submit with nothing written. + const contract = await probeWizardContract(nextWizard, { + signal: abort.signal, + }); + if (!checkStillWanted(abort)) return; saveProviderWizardToConfig(nextWizard); - props.onFinished("saved_cloud", gate.warning ?? undefined); + const notes = [gate.warning, contract.warning].filter( + (note): note is string => Boolean(note), + ); + props.onFinished( + "saved_cloud", + notes.length > 0 ? notes.join(" ") : undefined, + ); } catch (err) { // An abandoned run does not get to report a failure either: it // would paint over the screen the operator was handed back, and diff --git a/src/tui/providers/contract-probe-target.ts b/src/tui/providers/contract-probe-target.ts new file mode 100644 index 00000000..cd926373 --- /dev/null +++ b/src/tui/providers/contract-probe-target.ts @@ -0,0 +1,110 @@ +/** + * What to send the contract probe, derived from a wizard run — or why + * there is nothing here to probe. + * + * Same endpoint resolution the key check uses (`endpointForKind`, + * `apiKeyForWizard`), with one deliberate difference: the model. + * + * `pickProbeModels` picks the *cheapest paid* model it can find, because + * the key check's question is "can this account pay for a token". The + * contract probe's question is "can the route I am about to use run a + * turn", and route limitations are per-model — a gateway can stream + * native tool calls for one model and refuse `tools` outright for + * another. Probing anything but the configured model would answer a + * question nobody asked. + * + * The skip cases are named rather than collapsed into `null` because an + * explicitly requested check has to report them: "this is a server on + * your own machine" and "this provider has no key yet" are different + * answers, and telling an operator the second when the first is true is + * how a diagnostic tool loses their trust. + */ + +import type { ProviderContractProbeTarget } from "../../llm/provider/verify/index.js"; +import { isLocalProviderUrl } from "./is-local-provider-url.js"; +import { + apiKeyForWizard, + chosenModelForWizard, + endpointForKind, + providerLabelForWizard, + wizardKeyIsOptional, +} from "./providers-wizard-target.js"; +import type { ProvidersWizardState } from "./providers-wizard-state.js"; + +export type ContractProbeSkipReason = + /** The wizard has not settled on a provider kind yet. */ + | "no_kind" + /** A subscription CLI: a subprocess, not an HTTP contract. */ + | "cli_backed" + /** A server on the operator's own machine. */ + | "local_endpoint" + /** No key resolved, and this service is not one that works without one. */ + | "no_api_key" + /** No model chosen, so there is nothing to probe *with*. */ + | "no_model"; + +export type ContractProbeTargetResolution = + | { readonly kind: "target"; readonly target: ProviderContractProbeTarget } + | { readonly kind: "skipped"; readonly reason: ContractProbeSkipReason }; + +export function contractProbeTargetForWizard( + wizard: ProvidersWizardState, +): ContractProbeTargetResolution { + const kind = wizard.kind; + if (!kind) return skip("no_kind"); + // A CLI-backed provider speaks its vendor's own protocol through a + // subprocess; there is no OpenAI-compatible endpoint here to hold to + // this contract, and inventing one would probe a URL it never uses. + if (kind === "claude-cli" || kind === "codex-cli") return skip("cli_backed"); + + const apiKey = apiKeyForWizard(wizard)?.trim() ?? ""; + // Keyless is legitimate for local servers and keyless-listing + // services; a missing key everywhere else is already refused by the + // key screen, and probing without one would only re-report that. + if (!apiKey && !wizardKeyIsOptional(wizard)) return skip("no_api_key"); + + const endpoint = endpointForKind(kind, wizard); + // A server on this machine is the operator's own: reachable, free to + // call, and a probe against it says more about their llama-server + // flags than about a provider. The key check skips it for the same + // reason. + if (isLocalProviderUrl(endpoint.baseUrl)) return skip("local_endpoint"); + + const model = chosenModelForWizard(wizard).trim(); + if (!model) return skip("no_model"); + + return { + kind: "target", + target: { + label: providerLabelForWizard(wizard), + baseUrl: endpoint.baseUrl, + apiPathPrefix: endpoint.apiPathPrefix, + apiKey, + model, + ...(endpoint.extraHeaders ? { extraHeaders: endpoint.extraHeaders } : {}), + }, + }; +} + +/** Why nothing was probed, in the operator's words. */ +export function describeContractProbeSkip( + reason: ContractProbeSkipReason, + label: string, +): string { + switch (reason) { + case "cli_backed": + return `${label} runs through a CLI, not an HTTP endpoint — there is no streaming tool contract to check.`; + case "local_endpoint": + return `${label} is a server on this machine — nothing to check against a provider.`; + case "no_api_key": + return `${label} has no API key yet, so the contract check has nothing to authenticate with.`; + case "no_model": + return `${label} has no chat model set, so there is nothing to run the contract check with.`; + default: + return `${label} is not configured far enough to run a contract check.`; + } +} + +function skip(reason: ContractProbeSkipReason): ContractProbeTargetResolution { + return { kind: "skipped", reason }; +} diff --git a/src/tui/providers/describe-contract-probe.ts b/src/tui/providers/describe-contract-probe.ts new file mode 100644 index 00000000..c1df4bfb --- /dev/null +++ b/src/tui/providers/describe-contract-probe.ts @@ -0,0 +1,59 @@ +/** + * One sentence per contract-probe verdict, in the voice the key check + * and the cloud error path already use: who answered, what happened, + * what to do about it. + * + * Three rules the wording has to keep: + * + * - Never say "incompatible" for something the probe did not establish. + * `inconclusive_no_tool_call` is the model declining to call a + * pointless function, which is legal behaviour, not a broken route. + * - Never say "compatible" for anything but a completed tool call. + * - Always name the next move. `HTTP 400` on a setup screen reads as a + * product failure; "answers fine until `tools` is in the request" + * reads as a route to change. + */ + +import type { ProviderContractProbeResult } from "../../llm/provider/verify/index.js"; + +export function describeContractProbeOutcome( + result: ProviderContractProbeResult, + label: string, +): string { + const who = `"${label}"`; + const on = ` on ${result.probedModel}`; + switch (result.status) { + case "tools_supported": + return `${who} streamed a native tool call${on} — this route can run a turn.`; + case "inconclusive_no_tool_call": + return `${who} answered in text instead of calling a tool${on}. Inconclusive: it would not take a forced tool choice, so whether it can emit tool calls is still unknown.`; + case "forced_tool_choice_ignored": + return `${who} accepted a forced tool choice${on} and answered in text anyway. Turns that must call a tool may loop or stall on this route.`; + case "forced_tool_choice_rejected": + return `${who} refuses a forced tool choice${on} but does emit tool calls without one${statusSuffix(result)}. Usable, with less control over when tools fire.`; + case "tools_payload_rejected": + return `${who} answers this model until "tools" is in the request, then refuses it${statusSuffix(result)}. Pick another model or route — Atomic sends tools on every turn.`; + case "model_unavailable": + return `${who} does not recognise ${result.probedModel}${statusSuffix(result)}. Pick a model this route actually serves.`; + case "endpoint_auth_failed": + return `${who} rejected the key when asked for a streamed tool call${statusSuffix(result)}. Tool support is still untested.`; + case "quota_or_routing_failed": + return `${who} could not run the check — out of quota, or no backend to route to${statusSuffix(result)}. Tool support is still untested.`; + case "stream_early_eof": + return `${who} closed the stream${on} before finishing the answer. Turns will end mid-tool-call on this route.`; + case "malformed_tool_call": + return `${who} streamed tool-call deltas${on} that never formed a callable tool. Atomic would fail the turn with a tool it cannot look up.`; + case "unreachable": + return `Could not reach ${who} for the contract check. Tool support is untested — check the connection or the base URL.`; + case "timeout": + return `${who} did not finish the contract check in time${on}. Tool support is untested.`; + case "cancelled": + return `Contract check cancelled. Tool support is untested.`; + default: + return `${who} failed the contract check${statusSuffix(result)}. Tool support is untested — this looks like the route, not your setup.`; + } +} + +function statusSuffix(result: ProviderContractProbeResult): string { + return result.httpStatus === null ? "" : ` (${result.httpStatus})`; +} diff --git a/src/tui/providers/probe-wizard-contract.test.ts b/src/tui/providers/probe-wizard-contract.test.ts new file mode 100644 index 00000000..dae85423 --- /dev/null +++ b/src/tui/providers/probe-wizard-contract.test.ts @@ -0,0 +1,170 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { CONTRACT_PROBE_TOOL_NAME } from "../../llm/provider/verify/index.js"; +import { probeWizardContract } from "./probe-wizard-contract.js"; +import { createProvidersWizardState } from "./providers-wizard-state.js"; +import type { + ProvidersWizardKind, + ProvidersWizardState, +} from "./providers-wizard-state.js"; + +const ENV_KEYS = [ + "OPENROUTER_API_KEY", + "AIMLAPI_API_KEY", + "GEMINI_API_KEY", + "OPENAI_COMPAT_API_KEY", + "OPENAI_API_KEY", + "LMSTUDIO_API_KEY", +] as const; + +function wizard( + kind: ProvidersWizardKind, + overrides: Partial = {}, +): ProvidersWizardState { + return { + ...createProvidersWizardState("add", { kind }), + phase: "api_key", + apiKeyBuffer: "sk-test-key", + ...overrides, + }; +} + +function sseEvent(payload: unknown): string { + return `data: ${JSON.stringify(payload)}\n\n`; +} + +const TOOL_CALL_STREAM = + sseEvent({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: CONTRACT_PROBE_TOOL_NAME, arguments: '{"ok":true}' }, + }, + ], + }, + }, + ], + }) + + sseEvent({ choices: [{ delta: {}, finish_reason: "tool_calls" }] }) + + "data: [DONE]\n\n"; + +beforeEach(() => { + for (const key of ENV_KEYS) delete process.env[key]; +}); +afterEach(() => { + vi.unstubAllGlobals(); + for (const key of ENV_KEYS) delete process.env[key]; +}); + +describe("probeWizardContract", () => { + it("proves a route that streams a native tool call", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(TOOL_CALL_STREAM, { status: 200 })), + ); + const outcome = await probeWizardContract(wizard("openrouter")); + expect(outcome.proven).toBe(true); + expect(outcome.warning).toBeNull(); + expect(outcome.summary).toContain("can run a turn"); + }); + + it("warns without blocking when the route refuses the tools payload", async () => { + // Refuses with tools twice, answers the no-tools control: the + // route works, and it is `tools` it will not take. + const bodies: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: unknown, init?: RequestInit) => { + bodies.push(String(init?.body ?? "")); + if (bodies.length <= 2) return new Response("Bad Request", { status: 400 }); + return new Response( + sseEvent({ choices: [{ delta: { content: "hi" }, finish_reason: "stop" }] }), + { status: 200 }, + ); + }), + ); + const outcome = await probeWizardContract(wizard("aimlapi")); + expect(outcome.proven).toBe(false); + expect(outcome.warning).toContain('"tools"'); + // Advisory only: nothing here can refuse a save. + expect(outcome.result?.status).toBe("tools_payload_rejected"); + }); + + it("does not call an inconclusive auto answer a failure of the route", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (_url: unknown, init?: RequestInit) => { + const body = String(init?.body ?? ""); + if (body.includes('"tool_choice":{')) { + return new Response("Bad Request", { status: 400 }); + } + return new Response( + sseEvent({ choices: [{ delta: { content: "Sure!" }, finish_reason: "stop" }] }), + { status: 200 }, + ); + }), + ); + const outcome = await probeWizardContract(wizard("openrouter")); + expect(outcome.result?.status).toBe("inconclusive_no_tool_call"); + expect(outcome.warning).toContain("Inconclusive"); + // "Unproven" is not "incompatible", and the wording must not drift. + expect(outcome.warning).not.toContain("cannot"); + }); + + it("never calls out for a server on this machine", async () => { + const fetchMock = vi.fn(async () => new Response(TOOL_CALL_STREAM, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const outcome = await probeWizardContract( + wizard("openai-compatible", { + apiKeyBuffer: "", + baseUrlLine: "http://127.0.0.1:8000", + }), + ); + expect(outcome.proven).toBe(false); + // A skip is not a warning: a local server is not a defect to report. + expect(outcome.warning).toBeNull(); + expect(outcome.skipped).toBe("local_endpoint"); + expect(outcome.summary).toContain("server on this machine"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("never calls out for a CLI-backed provider", async () => { + const fetchMock = vi.fn(async () => new Response(TOOL_CALL_STREAM, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const outcome = await probeWizardContract(wizard("claude-cli")); + expect(outcome.skipped).toBe("cli_backed"); + expect(outcome.warning).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("says a key is missing rather than pretending there is nothing to check", async () => { + const fetchMock = vi.fn(async () => new Response(TOOL_CALL_STREAM, { status: 200 })); + vi.stubGlobal("fetch", fetchMock); + const outcome = await probeWizardContract( + wizard("openrouter", { apiKeyBuffer: "" }), + ); + expect(outcome.skipped).toBe("no_api_key"); + expect(outcome.summary).toContain("no API key"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("probes the model the operator picked, not a cheap stand-in", async () => { + const bodies: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: unknown, init?: RequestInit) => { + bodies.push(String(init?.body ?? "")); + return new Response(TOOL_CALL_STREAM, { status: 200 }); + }), + ); + await probeWizardContract( + wizard("openrouter", { selectedChatModelId: "vendor/chosen-model" }), + ); + expect(bodies[0]).toContain("vendor/chosen-model"); + }); +}); diff --git a/src/tui/providers/probe-wizard-contract.ts b/src/tui/providers/probe-wizard-contract.ts new file mode 100644 index 00000000..529d5fae --- /dev/null +++ b/src/tui/providers/probe-wizard-contract.ts @@ -0,0 +1,101 @@ +/** + * The setup-time contract check, run once per provider save. + * + * Deliberately separate from `verifyWizardBeforeSave`, and deliberately + * unable to stop a save: + * + * - The key gate answers "is this credential usable", and a dead key is + * worth refusing because nothing downstream can work without one. + * - This answers "can this route run a turn", and the honest response + * to "no" is a warning, not a refusal. Some providers block synthetic + * probes outright; a custom endpoint the operator knows works must + * still be savable, and the operator is the one who decides whether + * to live with a route that fires tools only under `auto`. + * + * What it must not do is let a failed probe pass for a proven one — the + * caller keys "this install has a working cloud backend" off a clean + * result, so an unproven route reports as unproven. + */ + +import { + contractProbeProvesToolSupport, + runProviderContractProbe, + type ProviderContractProbeResult, +} from "../../llm/provider/verify/index.js"; +import { + contractProbeTargetForWizard, + describeContractProbeSkip, + type ContractProbeSkipReason, +} from "./contract-probe-target.js"; +import { describeContractProbeOutcome } from "./describe-contract-probe.js"; +import { providerLabelForWizard } from "./providers-wizard-target.js"; +import type { ProvidersWizardState } from "./providers-wizard-state.js"; + +export interface WizardContractProbeOutcome { + /** + * `true` only when a complete native tool call came back. Anything + * else — including a probe that never ran — leaves the route + * unproven, and callers must treat unproven as unproven. + */ + readonly proven: boolean; + /** + * What to show the operator, or `null` when there is nothing worth + * saying: the route passed, or there was nothing here to probe. + */ + readonly warning: string | null; + /** + * One sentence for the operator, whatever happened — a clean pass, a + * defect, or the reason no probe ran. An explicitly requested check + * has to be able to report all three; a save only wants `warning`. + */ + readonly summary: string; + /** Set when no request was made, saying which case this was. */ + readonly skipped: ContractProbeSkipReason | null; + /** The raw verdict, for callers that log or branch on it. */ + readonly result: ProviderContractProbeResult | null; +} + +/** + * Tighter than the probe module's own budget. This one runs with an + * operator watching a wizard screen: a route slow enough to blow + * through it has told us something already, and the save can proceed + * with the warning rather than holding the screen. + */ +export const WIZARD_CONTRACT_PROBE_TIMEOUT_MS = 12_000; + +export async function probeWizardContract( + wizard: ProvidersWizardState, + opts: { signal?: AbortSignal; timeoutMs?: number } = {}, +): Promise { + const resolved = contractProbeTargetForWizard(wizard); + if (resolved.kind === "skipped") { + return { + proven: false, + // Not a warning: none of the skip cases is a defect to act on, + // and a wizard that reported one would cry wolf on every local + // server it ever saved. + warning: null, + summary: describeContractProbeSkip( + resolved.reason, + providerLabelForWizard(wizard), + ), + skipped: resolved.reason, + result: null, + }; + } + const target = resolved.target; + + const result = await runProviderContractProbe(target, { + ...(opts.signal ? { signal: opts.signal } : {}), + timeoutMs: opts.timeoutMs ?? WIZARD_CONTRACT_PROBE_TIMEOUT_MS, + }); + const proven = contractProbeProvesToolSupport(result.status); + const summary = describeContractProbeOutcome(result, target.label); + return { + proven, + warning: proven ? null : summary, + summary, + skipped: null, + result, + }; +} diff --git a/src/tui/providers/providers-actions.ts b/src/tui/providers/providers-actions.ts index 9eb5cc21..97040b16 100644 --- a/src/tui/providers/providers-actions.ts +++ b/src/tui/providers/providers-actions.ts @@ -89,6 +89,19 @@ export type ProvidersAction = generation: number; error: string; } + | { + /** + * Run the side-effect-free provider contract probe against + * `providerId` (`null` = active text provider), on explicit + * operator request (`/llm check`). Handled by + * `ProvidersOrchestrator.runContractProbe`; `submit-handler` + * routes it through `onProvidersContractProbeRequested` for the + * same reason as the picker request above — a dispatched reducer + * action never reaches the event bus the orchestrator listens on. + */ + type: "providers_contract_probe_requested"; + providerId: string | null; + } | { type: "providers_wizard_updated"; wizard: ProvidersWizardState } | { type: "providers_wizard_closed" } | { type: "providers_wizard_submit_started" } diff --git a/src/tui/providers/providers-orchestrator.test.ts b/src/tui/providers/providers-orchestrator.test.ts index 1e47cf99..fb2de59a 100644 --- a/src/tui/providers/providers-orchestrator.test.ts +++ b/src/tui/providers/providers-orchestrator.test.ts @@ -363,6 +363,73 @@ describe("ProvidersOrchestrator.completeWizard", () => { expect(failure?.error).toContain("rejected this key"); }); + it("runs the contract probe on explicit request and reports the verdict", async () => { + currentConfig = { + llm: { + activeTextProvider: "openrouter", + activeEmbeddingProvider: "local-llama-embed", + toolTransport: "auto", + providers: [ + { + id: "openrouter", + kind: "openrouter", + apiKey: "sk-saved-key", + defaultChatModel: "vendor/configured-model", + }, + ], + }, + } as AtomicAgentConfig; + const bodies: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: unknown, init?: RequestInit) => { + bodies.push(String(init?.body ?? "")); + return new Response( + `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { + name: "atomic_contract_probe", + arguments: '{"ok":true}', + }, + }, + ], + }, + }, + ], + })}\n\ndata: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: "tool_calls" }], + })}\n\ndata: [DONE]\n\n`, + { status: 200 }, + ); + }), + ); + const { ProvidersOrchestrator } = await importFreshOrchestrator(); + const bus = fakeBus(); + const orchestrator = new ProvidersOrchestrator(fakeRuntime(), bus as never); + + await orchestrator.runContractProbe(null); + + const lines = bus.emit.mock.calls + .map((call) => call[0] as { type: string; line?: string }) + .filter((action) => action.type === "providers_status") + .map((action) => action.line ?? ""); + expect(lines.some((line) => line.includes("can run a turn"))).toBe(true); + // The configured model, streamed, with the tools payload — the real + // turn contract, not a cheap stand-in. + expect(bodies[0]).toContain("vendor/configured-model"); + expect(bodies[0]).toContain('"stream":true'); + expect(bodies[0]).toContain("atomic_contract_probe"); + // Nothing was written and nothing was activated: this is a read-only + // check an operator can run whenever they like. + expect(bodies).toHaveLength(1); + }); + it("hands the cancel back to the wizard while a check is in flight", async () => { currentConfig = configWithGemini(); let releaseFetch: () => void = () => {}; diff --git a/src/tui/providers/providers-orchestrator.ts b/src/tui/providers/providers-orchestrator.ts index 67e18892..aea038ca 100644 --- a/src/tui/providers/providers-orchestrator.ts +++ b/src/tui/providers/providers-orchestrator.ts @@ -31,7 +31,11 @@ import { isProvidersAction } from "./providers-actions.js"; import type { ProviderRow } from "./providers-panel-state.js"; import { saveProviderWizardToConfig } from "./save-provider-wizard.js"; import { verifyWizardBeforeSave } from "./verify-wizard-before-save.js"; -import { wizardKindForSubscriptionCli } from "./providers-wizard-state.js"; +import { probeWizardContract } from "./probe-wizard-contract.js"; +import { + createProvidersWizardState, + wizardKindForSubscriptionCli, +} from "./providers-wizard-state.js"; import type { ProvidersWizardKind, ProvidersWizardState, @@ -51,6 +55,13 @@ export class ProvidersOrchestrator { /** Aborts the pre-save key check when the operator presses Esc. */ private wizardVerifyAbort: AbortController | null = null; + /** + * Guards `/llm check` against a second run while one is in flight. + * The probe spends real requests against a route the operator may be + * paying per token for; a held-down key must not queue three of them. + */ + private contractProbeRunning = false; + constructor( private readonly runtime: AgentRuntime, private readonly bus: TuiEventBus & { emit(action: unknown): void }, @@ -365,13 +376,25 @@ export class ProvidersOrchestrator { // A cancel already put the wizard back in an editable state; a // late verdict from the abandoned check must not overwrite it. if (abort.signal.aborted) return; - // The check is over; from here Esc has nothing to cancel and must - // not interrupt the save that follows. - this.wizardVerifyAbort = null; if (!gate.proceed) { + this.wizardVerifyAbort = null; this.bus.emit({ type: "providers_wizard_failed", error: gate.error }); return; } + // A live key proves the account, not the route. Exercise the real + // contract once — streaming, tools payload, forced native tool + // call — before this provider is reported as working, so a route + // that only fails with `tools` in the body is named here instead + // of on the operator's first message. + // + // It cannot refuse the save (see `probe-wizard-contract`), but it + // runs while Esc can still abandon the whole submit, so nothing + // has reached disk yet if the operator gives up on a slow route. + const contract = await probeWizardContract(wizard, { signal: abort.signal }); + if (abort.signal.aborted) return; + // Both checks are over; from here Esc has nothing to cancel and + // must not interrupt the save that follows. + this.wizardVerifyAbort = null; const built = saveProviderWizardToConfig(wizard); const exists = this.runtime.providerRegistry .listIds() @@ -392,7 +415,12 @@ export class ProvidersOrchestrator { // an unreachable check is not a proven backend. A warned save that // turns out to work reports on a later verified save instead. // Only the provider id travels — never the key or the base URL. - if (gate.warning === null) { + // + // The contract probe joins the same gate: a route that could not + // stream a native tool call has not been shown to run a turn, and + // reporting it as a working backend would be the silent + // "fully compatible" this check exists to prevent. + if (gate.warning === null && contract.warning === null) { this.runtime.reportModelConfigured(built.entry.id, "cloud"); } if (gate.warning) { @@ -401,6 +429,10 @@ export class ProvidersOrchestrator { this.bus.emit({ type: "providers_status", line: gate.warning }); this.bus.emit({ type: "runtime_info", line: gate.warning }); } + if (contract.warning) { + this.bus.emit({ type: "providers_status", line: contract.warning }); + this.bus.emit({ type: "runtime_info", line: contract.warning }); + } this.bus.emit({ type: "runtime_info", line: `Active text provider: ${built.entry.id} (${built.entry.defaultChatModel ?? "default model"}). Chat uses cloud native tools now.`, @@ -432,6 +464,90 @@ export class ProvidersOrchestrator { } } + /** + * Run the contract probe against a provider that is already saved + * (`null` = the active text provider), on explicit request — the + * `/llm check` command. + * + * This is the second of exactly two ways the probe ever runs: here, + * and once per wizard save. It is never on a turn path. A route can + * degrade after setup (a gateway drops a backend, an account runs + * dry), and until now the only way to find out was to send a real + * message and watch it fail. + * + * The provider is described to the probe through a `configure` wizard + * state — the same object the panel builds when the operator presses + * `c` on that row — so the endpoint, key and model resolution is + * literally the wizard's, not a second implementation that could + * drift from it. + */ + async runContractProbe(providerId: string | null): Promise { + if (this.contractProbeRunning) { + this.bus.emit({ + type: "providers_status", + line: "A provider check is already running.", + }); + return; + } + const config = getConfig(); + const resolved = resolveLlmConfig(config); + const id = providerId ?? resolved.activeTextProvider; + const provider = id ? resolved.providers.find((p) => p.id === id) : undefined; + if (!id || !provider) { + this.bus.emit({ + type: "providers_status", + line: "No provider to check — configure one first.", + }); + return; + } + const fileEntry = config.llm?.providers.find((e) => e.id === id); + const kind = configureWizardKindForRow({ + kind: provider.kind, + ...(fileEntry?.subscriptionCli + ? { subscriptionCli: { cli: fileEntry.subscriptionCli.cli } } + : {}), + }); + const chatModel = fileEntry?.defaultChatModel ?? fileEntry?.model ?? null; + const wizard = kind + ? { + ...createProvidersWizardState("configure", { + providerId: id, + kind, + ...(fileEntry?.baseUrl ? { baseUrl: fileEntry.baseUrl } : {}), + ...(chatModel ? { chatModel } : {}), + }), + // The factory prefills `chatModelLine` only for CLI-backed + // kinds — a cloud reconfigure re-picks its model on the model + // screen. Nothing is being re-picked here, so the saved model + // is pinned directly; without it the probe would silently + // test the kind's default instead of the model this provider + // actually runs, and report a verdict about the wrong route. + ...(chatModel ? { selectedChatModelId: chatModel } : {}), + } + : null; + + this.contractProbeRunning = true; + this.bus.emit({ type: "providers_busy", busy: true }); + try { + // Every path here says what actually happened, including the ones + // where no request went out. Reporting a skip as a pass would be + // the silent "fully compatible" this whole check exists to stop. + const line = wizard + ? (await probeWizardContract(wizard)).summary + : `"${id}" has no provider kind that can be contract-checked.`; + this.bus.emit({ type: "providers_status", line }); + this.bus.emit({ type: "runtime_info", line }); + } catch (err) { + this.bus.emit({ + type: "providers_status", + line: err instanceof Error ? err.message : String(err), + }); + } finally { + this.contractProbeRunning = false; + this.bus.emit({ type: "providers_busy", busy: false }); + } + } + async removeProviderById(id: string): Promise { this.bus.emit({ type: "providers_busy", busy: true }); try { diff --git a/src/tui/providers/providers-wizard-target.ts b/src/tui/providers/providers-wizard-target.ts index f1b906e4..30c6f048 100644 --- a/src/tui/providers/providers-wizard-target.ts +++ b/src/tui/providers/providers-wizard-target.ts @@ -212,7 +212,7 @@ export function providerLabelForWizard(wizard: ProvidersWizardState): string { } /** The model this wizard run is about to save, before any defaulting. */ -function chosenModelForWizard(wizard: ProvidersWizardState): string { +export function chosenModelForWizard(wizard: ProvidersWizardState): string { const typed = wizard.chatModelLine.trim(); if (wizard.selectedChatModelId) return wizard.selectedChatModelId; if (typed.length > 0) return typed; @@ -222,7 +222,7 @@ function chosenModelForWizard(wizard: ProvidersWizardState): string { return OPENAI_COMPAT_DEFAULT_CHAT_MODEL; } -function endpointForKind( +export function endpointForKind( kind: ProvidersWizardKind, wizard: ProvidersWizardState, ): { baseUrl: string; apiPathPrefix: string; extraHeaders?: Record } { diff --git a/src/tui/submit-handler.ts b/src/tui/submit-handler.ts index e73eca52..64dcbbd7 100644 --- a/src/tui/submit-handler.ts +++ b/src/tui/submit-handler.ts @@ -201,6 +201,13 @@ export function runSlashCommand( callbacks.onProvidersChatModelPickerRequested?.(action.providerId); continue; } + if (action.type === "providers_contract_probe_requested") { + // Same wiring rule again for `/llm check`: the probe lives on + // `ProvidersOrchestrator.runContractProbe`, which only the + // callback layer can reach. + callbacks.onProvidersContractProbeRequested?.(action.providerId); + continue; + } if (action.type === "providers_inline_models_ensure_requested") { // Same wiring rule for the inline Cloud-pane model list (`/model`): // the catalog ensure must reach diff --git a/src/tui/tui-app.tsx b/src/tui/tui-app.tsx index b38cbbb4..739d090c 100644 --- a/src/tui/tui-app.tsx +++ b/src/tui/tui-app.tsx @@ -436,6 +436,13 @@ export interface TuiAppCallbacks { * above: only the callback layer reaches the orchestrator's bus. */ onProvidersInlineModelsEnsureRequested?(providerId: string | null): void; + /** + * `/llm check`: run the provider contract probe against `providerId` + * (`null` = active text provider). Callback for the same reason as + * the two above. Explicit request only — the probe spends real + * requests and never runs on a turn path. + */ + onProvidersContractProbeRequested?(providerId: string | null): void; /** Providers tab / LLM panel: switch the active embedding provider. */ onProvidersSetActiveEmbedding?(id: string): void; /** Providers tab / LLM panel: select an exact embedding model. */ diff --git a/src/tui/tui-command.ts b/src/tui/tui-command.ts index 7d6b5e08..3b0f91bf 100644 --- a/src/tui/tui-command.ts +++ b/src/tui/tui-command.ts @@ -540,6 +540,8 @@ export async function tuiCommand(args: string[]): Promise { void orchestrator.providers.openChatModelPicker(providerId), onProvidersInlineModelsEnsureRequested: (providerId) => void orchestrator.providers.ensureInlineModels(providerId), + onProvidersContractProbeRequested: (providerId) => + void orchestrator.providers.runContractProbe(providerId), onProvidersSetActiveEmbedding: (id) => void orchestrator.providers.setActiveEmbedding(id), onProvidersSelectEmbeddingModel: (providerId, modelId) => From 9ebdc4fb95ed661ee7e360fb13c88f8589861a25 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:51:01 +0300 Subject: [PATCH 09/36] fix(openai): make the stream retry budget actually shared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit claimed, in three places, that one streaming completion gets `OPENAI_MAX_ATTEMPTS` requests "shared with the open, not a fresh budget per layer". It did not. Every reopen called `openAiStartStream`, which starts a fresh `runOpenAiWithRetry` with its own three attempts, so only the outer counter was shared. A provider that answers two 500s before each successful open and then drops the body before its first chunk issued nine requests for one turn — exactly the "3 x 3 = 9" outcome the `OpenAiHttpError` guard was justified by preventing. Nine billable prompt submissions, and on a reasoning model that thinks for a minute before dropping, minutes of frozen UI. `OpenAiAttemptBudget` is a mutable counter created once per `completeStream` and passed into every `openAiStartStream`, so opens and reopens add instead of multiplying. `runOpenAiWithRetry` spends it and indexes the backoff by requests already made, so the delay keeps growing across the open/reopen seam. Callers that pass no budget (every non-streaming path) get a private full one, unchanged. Four further corrections, each now pinned by a test that fails without the line it describes: - Esc during the inter-attempt backoff walked into the next open, because `sleep()` resolves on abort rather than throwing. The turn then failed as `transport` — `classifyFailure` files every `OpenAiHttpError` that way before it looks for an abort — and `shouldAdvance` read that as an immediate provider-down signal, switching links and restarting the completion the user had just stopped. Cancellation is now checked before the guards and after the backoff, and throws `signal.reason`, which classifies as `cancelled`. - The guard-3 doc justified `OpenAiHttpError` with budget squaring and "a bad API key would be tried nine times". A 401 is stopped by the shape guard, not that one. The guard's real job is narrower and is now described and tested as what it is: a non-retryable open failure whose message happens to look transport-shaped (a 400 whose body mentions a socket) must not have the HTTP client's verdict overridden here. - `discardResponseBody` was a no-op on every reachable path. The reopen is only reached when the body reader raised, so the stream is already errored, `cancel()` rejects with the stored error into an empty catch, and undici has already destroyed the socket. Deleted, with the reason written down where the call used to be. - The reopen loop was an unbounded `for (;;)` whose only exit was a helper's return value. Bounded by `OPENAI_MAX_ATTEMPTS`, with an explicit throw if the bound is ever what stops it — an empty completion would otherwise look to the user like a model that said nothing. Tool-call arguments never set `committed`, because the consumer yields on a `function.arguments` delta only once the arguments contain reply text. That widens the retry window past "any byte the provider sent" and was undocumented; it is safe (nothing reached the user, tools are dispatched only after the completion returns) and is now stated and pinned rather than left to be rediscovered. AGENTS.md also now records the two costs the layer does carry: no observability event, and up to a 3x wall-clock multiplier on a provider that hangs before dropping. --- AGENTS.md | 10 +- src/llm/provider/openai/openai-http.ts | 79 ++++- src/llm/provider/openai/openai-provider.ts | 186 ++++++++---- .../openai/openai-stream-retry.test.ts | 270 +++++++++++++++++- 4 files changed, 475 insertions(+), 70 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4c97fdd8..f1a312c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1791,7 +1791,15 @@ Three narrow retry layers sit between the agent loop and the model server. All a 1. **Parser retry (step-executor).** If the first `parseToolCall` on a completion throws, the executor calls the unary `llmComplete` exactly once more with the same prompt/slot and re-parses. A `parse_retry` event is emitted for observability. If the second attempt also fails, the original error (with a raw-output preview) is thrown. The streaming path always falls back to unary for the retry so partial SSE deltas are not double-emitted. 2. **Transport retry (LlamaServerClient).** `complete()` and the initial pre-body fetch of `completeStream()` are wrapped in a bounded retry governed by `llama.completionRetries` (default 3) and `llama.completionRetryBackoffMs` (default 150ms, exponential with ±20% jitter). Retries fire **only** for network errors (`LlamaServerError.status === null`) and HTTP 5xx. Grammar/validation 4xx and abort signals short-circuit immediately. Once the SSE body starts streaming, no further retries happen — the conversation state on the server is considered indeterminate. -3. **Pre-first-chunk stream retry (`OpenAiProvider.completeStream`).** Closes the one window nobody owned: a cloud provider answers 2xx, thinks for a long time (reasoning models, cold routes), then drops the socket **before emitting a single delta**. `openAiStartStream`'s budget is already spent by then, and undici surfaces the death as a bare `Error: terminated` from the body reader, so the whole turn used to fail instantly with `Turn failed [transport]: terminated`. `completeStream` now reopens the stream while `committed` is still false — i.e. while not one chunk has been yielded to its caller, which is exactly when a replay is unobservable. The guards, in order: **committed** (any yielded chunk, reasoning or even a bare `role` preamble, ends retrying forever), **cancellation** (`request.signal.aborted` is checked before the error is inspected — an Esc is not a network failure), **`OpenAiHttpError`** (came from the *open*, already spent `OPENAI_MAX_ATTEMPTS`; retrying here would square the budget to 9 requests), **shape** (`isNetworkError`, so a consumer bug is not replayed three times), and the shared `OPENAI_MAX_ATTEMPTS` budget with `openAiRetryBackoff`'s pacing. The dead response body is cancelled before reopening so a retry does not leak a socket. Resuming a stream that has *already* emitted output is deliberately out of scope — it would either duplicate text or need prefill continuation, and non-deterministic sampling rules out a prefix dedupe. Pinned by [src/llm/provider/openai/openai-stream-retry.test.ts](src/llm/provider/openai/openai-stream-retry.test.ts). +3. **Pre-first-chunk stream retry (`OpenAiProvider.completeStream`).** Closes the one window nobody owned: a cloud provider answers 2xx, thinks for a long time (reasoning models, cold routes), then drops the socket **before emitting a single delta**. `openAiStartStream` has already returned by then, and undici surfaces the death as a bare `Error: terminated` from the body reader, so the whole turn used to fail instantly with `Turn failed [transport]: terminated`. `completeStream` reopens the stream while `committed` is still false — i.e. while not one chunk has been yielded to its caller, which is exactly when a replay is unobservable. + + Cancellation is handled before the guards run and *throws* rather than returning a verdict: `signal.reason` is thrown so the failure classifies as `cancelled`, because both shapes that otherwise reach the classifier here (`Error: terminated`, and the `OpenAiHttpError` an abort produces inside `runOpenAiWithRetry`) are filed as `transport`, which makes `shouldAdvance` switch providers and restart the very turn the user stopped. The signal is re-checked after the backoff, since `sleep()` resolves on abort instead of rejecting. + + The guards, in order: **committed** (any yielded chunk, reasoning or even a bare `role` preamble, ends retrying forever — but *not* a completion that streamed only tool-call arguments, which yields nothing and is therefore replayed; safe, because nothing reached the user and tools are dispatched only after the completion returns), **`OpenAiHttpError`** (came from the *open*, where this client's own retry policy has already judged it; reopening would replace that policy with `isNetworkError`'s looser one), **shape** (`isNetworkError`, so a consumer bug is not replayed three times), and **budget**. + + The budget is genuinely shared, not shared by assertion: `OpenAiAttemptBudget` is a mutable counter created once per `completeStream` and passed into every `openAiStartStream`, so opens and reopens add up to `OPENAI_MAX_ATTEMPTS` instead of nesting into `3 x 3 = 9` requests for one turn. Pacing is `openAiRetryBackoff`, indexed by requests already spent so the delay keeps growing across the open/reopen seam. Nothing is done to the dead response body: the only way to reach the reopen with a response in hand is a body-read failure, so the stream is already errored, `cancel()` on it rejects, and undici has already destroyed the socket. + + Known costs, accepted: this layer emits **no observability event** (unlike layer 1's `parse_retry`) and the body-read phase is untimed (`openAiFetch`'s timeout controller is cleared once headers arrive), so a provider that thinks for a minute and then drops now freezes the UI for up to three of those minutes before the same message appears. Bounded by the shared budget, but a wall-clock cap and a `stream_reopen` event are the obvious follow-ups. Resuming a stream that has *already* emitted output is deliberately out of scope — it would either duplicate text or need prefill continuation, and non-deterministic sampling rules out a prefix dedupe. Pinned by [src/llm/provider/openai/openai-stream-retry.test.ts](src/llm/provider/openai/openai-stream-retry.test.ts). ### Failure taxonomy diff --git a/src/llm/provider/openai/openai-http.ts b/src/llm/provider/openai/openai-http.ts index 7b5a1170..165dd0d8 100644 --- a/src/llm/provider/openai/openai-http.ts +++ b/src/llm/provider/openai/openai-http.ts @@ -113,10 +113,37 @@ const OPENAI_ERROR_DETAIL_MAX_LEN = 300; * * Exported because it is the budget for *one* streaming completion, not * just for one HTTP call: `OpenAiProvider.completeStream` reopens a - * stream that died before its first chunk, and that reopen has to come - * out of this same budget rather than multiply it. + * stream that died before its first chunk, and that reopen comes out of + * this same budget — see `OpenAiAttemptBudget`, which is what actually + * makes the sharing true rather than merely intended. */ export const OPENAI_MAX_ATTEMPTS = 3; + +/** + * The remaining attempts of ONE logical completion, carried across the + * calls that make it up. + * + * Without this, "shared budget" is a sentence in a comment and nothing + * else: every `openAiStartStream` call opens a fresh `runOpenAiWithRetry` + * loop with a fresh count, so a `completeStream` that reopens a dead + * stream three times, each reopen paying for two 500s first, issues + * 3 x 3 = 9 requests for one turn. That is nine billable prompt + * submissions and, on a reasoning model that thinks for a minute before + * dropping, minutes of frozen UI. + * + * The budget is a mutable counter rather than a number passed down and + * returned because the spending happens inside the retry loop and has to + * stay visible to the caller even when that loop throws. + */ +export interface OpenAiAttemptBudget { + /** Attempts still available. Decremented once per HTTP request made. */ + remaining: number; +} + +/** A budget for one logical completion: the full `OPENAI_MAX_ATTEMPTS`. */ +export function createOpenAiAttemptBudget(): OpenAiAttemptBudget { + return { remaining: OPENAI_MAX_ATTEMPTS }; +} const OPENAI_BACKOFF_BASE_MS = 150; /** * Ceiling on how long a provider's `retry-after` can stall one attempt. @@ -180,15 +207,18 @@ export async function openAiPostJson( * and failures downstream are not retryable at this layer — the caller * owns that window. `OpenAiProvider.completeStream` extends the same * "nothing emitted yet, so a replay is free" argument a little further - * by reopening when the body dies before its first chunk; a failure that - * escapes *this* function has already spent `OPENAI_MAX_ATTEMPTS` and - * must not be retried again there. + * by reopening when the body dies before its first chunk. + * + * Pass `budget` to make those reopens share one completion's attempts + * with the opens: without it each call starts a fresh count and the two + * layers multiply instead of adding. */ export async function openAiStartStream( deps: OpenAiHttpDeps, path: string, body: Record, request: { signal?: AbortSignal }, + budget?: OpenAiAttemptBudget, ): Promise }> { return runOpenAiWithRetry(deps, path, request.signal, async () => { const res = await openAiFetch(deps, path, body, request, true, "POST"); @@ -196,7 +226,7 @@ export async function openAiStartStream( throw await httpErrorFromResponse(deps, path, res); } return res as Response & { body: NonNullable }; - }); + }, budget); } /** @@ -205,8 +235,9 @@ export async function openAiStartStream( * sleep. Exported so the one retry that lives *outside* this file * (`OpenAiProvider.completeStream` reopening a stream that died before * its first chunk) reuses this client's pacing instead of inventing a - * second set of magic numbers. `attemptNumber` is the 1-based attempt - * that just failed. + * second set of magic numbers. `attemptNumber` is the 1-based count of + * requests this completion has already made, so the delay keeps growing + * across the open/reopen seam instead of restarting at the base. */ export async function openAiRetryBackoff( attemptNumber: number, @@ -343,14 +374,35 @@ function isRetryableOpenAiError(err: unknown): boolean { return err.status >= 500 || err.status === 429 || err.status === 408; } +/** + * Run `attempt` under the bounded retry policy, spending `budget`. + * + * A caller that does not pass a budget gets a private full one, which is + * the historical behaviour and stays right for every one-shot call. The + * streaming path passes the completion's budget so its opens and its + * reopens draw from one pot. + */ async function runOpenAiWithRetry( deps: OpenAiHttpDeps, path: string, signal: AbortSignal | undefined, attempt: () => Promise, + budget: OpenAiAttemptBudget = createOpenAiAttemptBudget(), ): Promise { + if (budget.remaining <= 0) { + // Only reachable if a caller keeps using an exhausted budget. Fail + // typed rather than falling through to a bare `Error("undefined")`. + throw new OpenAiHttpError( + `openai provider retry budget exhausted: ${deps.baseUrl}${path}`, + null, + `${deps.baseUrl}${path}`, + false, + null, + deps.label, + ); + } let lastError: unknown; - for (let i = 1; i <= OPENAI_MAX_ATTEMPTS; i += 1) { + while (budget.remaining > 0) { if (signal?.aborted) { throw new OpenAiHttpError( "completion aborted by caller", @@ -358,6 +410,11 @@ async function runOpenAiWithRetry( `${deps.baseUrl}${path}`, ); } + budget.remaining -= 1; + // 1-based index of the request about to be made *within this + // completion*, so the backoff keeps growing across a reopen instead + // of resetting to the base delay every time the stream is retried. + const attemptNumber = OPENAI_MAX_ATTEMPTS - budget.remaining; try { return await attempt(); } catch (err) { @@ -365,8 +422,8 @@ async function runOpenAiWithRetry( // A caller-triggered abort is never retryable, whatever shape it // surfaced as. if (signal?.aborted) throw err; - if (!isRetryableOpenAiError(err) || i >= OPENAI_MAX_ATTEMPTS) throw err; - await sleep(resolveWaitMs(err, i), signal); + if (!isRetryableOpenAiError(err) || budget.remaining <= 0) throw err; + await sleep(resolveWaitMs(err, attemptNumber), signal); } } // Unreachable: the loop either returns or throws. diff --git a/src/llm/provider/openai/openai-provider.ts b/src/llm/provider/openai/openai-provider.ts index f1341124..7fbf3e7c 100644 --- a/src/llm/provider/openai/openai-provider.ts +++ b/src/llm/provider/openai/openai-provider.ts @@ -20,12 +20,14 @@ import { createOpenAiStreamConsumer } from "./openai-stream-consumer.js"; import { buildOpenAiChatBody } from "./openai-build-body.js"; import { buildOpenAiHeaders, + createOpenAiAttemptBudget, openAiGetJson, openAiPostJson, openAiRetryBackoff, openAiStartStream, OpenAiHttpError, OPENAI_MAX_ATTEMPTS, + type OpenAiAttemptBudget, type OpenAiHttpDeps, } from "./openai-http.js"; import { isNetworkError } from "../../reliability/network-error.js"; @@ -138,6 +140,13 @@ export class OpenAiProvider implements LlmProvider { // from the top and duplicate text the user already read, and because // sampling is non-deterministic no prefix dedupe can repair that. let committed = false; + // One completion, one budget. Passed into every open so a reopen + // draws from the same pot the opens do — the alternative is two + // nested loops of `OPENAI_MAX_ATTEMPTS` and 9 requests per turn. + const budget = createOpenAiAttemptBudget(); + // Distinguishes "the loop finished a stream" from "the loop ran out + // of iterations", which is what makes the bound below safe to add. + let streamEnded = false; // The window this loop exists for: a provider answers 2xx, thinks for // a long time (reasoning models, cold routes), then drops the socket @@ -146,15 +155,25 @@ export class OpenAiProvider implements LlmProvider { // death as a bare `Error: terminated` from the body reader — which // used to fail the whole turn ("Turn failed [transport]: terminated") // even though not one byte of output existed. - attempts: for (let attempt = 1; ; attempt += 1) { - let res: (Response & { body: NonNullable }) | undefined; + // + // The `attempt <= OPENAI_MAX_ATTEMPTS` bound is a structural + // backstop, not the working exit: every iteration spends at least one + // unit of `budget` and `canReopenStream` stops at zero, so the + // condition should never be what ends this loop. It is written anyway + // because this loop issues network requests, and a loop whose only + // termination is a helper's return value is one edit away from + // hammering a provider forever. + attempts: for (let attempt = 1; attempt <= OPENAI_MAX_ATTEMPTS; attempt += 1) { try { // Opening the stream (connect + status check) happens inside the // client's bounded retry, strictly before the first chunk exists. - res = await openAiStartStream(this.http, path, body, request); + const res = await openAiStartStream(this.http, path, body, request, budget); // A reopen starts from an empty transcript: whatever the dead // attempt accumulated was never yielded and must not be mixed - // into the fresh one. + // into the fresh one. With the built-in stream consumer nothing + // can accumulate without also committing, so this is dead weight + // for it; it is not dead for an injected `streamConsumer`, which + // may report content on a `done` chunk this loop does not yield. accumulated = ""; accumulatedReasoning = ""; streamFinal = undefined; @@ -163,25 +182,66 @@ export class OpenAiProvider implements LlmProvider { const next = await stream.next(); if (next.done) { streamFinal = next.value; + streamEnded = true; break attempts; } const chunk = next.value; if (chunk.delta) accumulated += chunk.delta; if (chunk.reasoningDelta) accumulatedReasoning += chunk.reasoningDelta; if (!chunk.done) { + // Set before the yield, deliberately: a caller that throws + // into this generator (`generator.throw()`, which is how a + // consumer reports its own failure into a stream it is + // draining) resumes us *inside* the catch below, with the + // yield never having returned. Set after the yield, that + // error would find `committed === false` and replay a + // completion the caller has already shown part of. committed = true; yield chunk; } } } catch (err) { - if (!canReopenStream(err, request.signal, committed, attempt)) throw err; - // Hand the dead socket back before opening a new one, or the - // retry leaks a connection out of undici's pool for the rest of - // the process. - await discardResponseBody(res); - await openAiRetryBackoff(attempt, request.signal); + // Cancellation first, and it throws rather than returning a + // verdict, because the *shape* of the error decides what the user + // gets. `classifyFailure` files every `OpenAiHttpError` as + // `transport` before it ever looks for an abort, and a bare + // `Error: terminated` from a body that died while the abort was + // in flight is `transport` too — either one makes `shouldAdvance` + // report an immediate provider-down signal, so the fallback chain + // switches links and starts the very completion the user just + // stopped. `signal.reason` is abort-shaped by construction. + if (request.signal?.aborted) throw cancellationError(request.signal, err); + if (!canReopenStream(err, committed, budget)) throw err; + // No `res.body.cancel()` here, on purpose. The only way to reach + // this line with a response in hand is `isNetworkError(err)` on + // an error raised by the body reader — i.e. the stream is already + // errored, `cancel()` on an errored stream rejects with the + // stored error, and undici has already destroyed the socket. A + // cancel call would be a swallowed no-op dressed up as hygiene. + await openAiRetryBackoff(OPENAI_MAX_ATTEMPTS - budget.remaining, request.signal); + // `sleep()` resolves on abort instead of rejecting, so without + // this the loop walks out of the backoff straight into the next + // open. Today that open throws before it fetches + // (`runOpenAiWithRetry` checks the signal first) and the check + // above catches it on the way through, which makes this line + // redundant *given* that behaviour — no test can tell the two + // apart, and the mutation sweep confirms it: dropping either + // check alone keeps the suite green, dropping both fails two + // tests. It stays because the invariant belongs to this loop: + // once the caller has cancelled, this loop issues nothing more, + // whatever the HTTP client decides to do about aborted signals. + if (request.signal?.aborted) throw cancellationError(request.signal, err); } } + if (!streamEnded) { + // The backstop fired: `canReopenStream` let the loop run past its + // budget. Returning the empty completion assembled below would look + // to the user like a model that said nothing, so say what actually + // happened instead. + throw new Error( + "openai stream retry loop ended without a completion — canReopenStream and OPENAI_MAX_ATTEMPTS disagree", + ); + } const final = completionFromStreamFinal( streamFinal, this.defaultChatModel, @@ -292,64 +352,86 @@ function completionFromStreamFinal( }; } +/** + * The error a turn the user cancelled should fail with. + * + * `classifyFailure` reads cancellation off the error's *shape*, and the + * shapes that reach the retry loop when someone presses Esc are not + * reliably abort-shaped: a body that dies while the abort is in flight + * arrives as `Error: terminated` (→ `transport`), and an abort noticed + * inside `runOpenAiWithRetry` arrives as an `OpenAiHttpError` whose + * `transport` branch is checked *before* the abort branch. Either one + * makes `shouldAdvance` report an immediate provider-down signal, so the + * fallback chain switches links and starts the very completion the user + * just stopped. + * + * `signal.reason` is the abort's own error — a `DOMException` named + * `AbortError` when the aborter supplied nothing — so it classifies as + * `cancelled`. The original error is the fallback for signal doubles + * that never populate `reason`. + */ +function cancellationError(signal: AbortSignal, fallback: unknown): unknown { + const reason: unknown = signal.reason; + return reason ?? fallback; +} + /** * May a failure raised between "2xx headers received" and "first chunk * handed to our caller" be recovered by reopening the stream? * - * The order of these guards is the contract, not a stylistic choice: + * Cancellation is deliberately NOT one of these guards: the caller checks + * the signal before asking, because an aborted turn needs a specific + * error *thrown*, not a boolean returned (see `cancellationError`). + * + * The order of the guards that are here is the contract, not a stylistic + * choice: * * 1. **Committed.** Once a chunk has been yielded, nothing below matters. - * This is deliberately the strictest reading of "output": a chunk that - * carries only the provider's opening `role` delta commits the stream - * just as a text delta does. We cannot know what a downstream consumer - * did with it, and being wrong here means duplicating a user's reply. - * Reasoning deltas are output for the same reason — the TUI renders - * them live. - * 2. **Cancellation.** A user pressing Esc is not a network failure, and - * an abort reaches us in several disguises (`AbortError`, a raw - * `Error: aborted`, or a custom `fetchImpl`'s own shape). The signal - * is the only reliable oracle, so it is consulted before the error is - * inspected at all — the ordering `network-error.ts` documents. - * 3. **`OpenAiHttpError`.** The failure came from *opening* the stream, - * which already ran inside `runOpenAiWithRetry` and already spent the - * whole `OPENAI_MAX_ATTEMPTS` budget on 429s/5xx/connect errors. - * Retrying it here would silently square the budget (3 × 3) and delay - * a real, actionable message — a bad API key would be tried nine - * times. Only untyped body-read deaths get past this guard. - * 4. **Shape.** Anything that is not a recognisable transport death — a + * This is a strict reading of "output": a chunk carrying only the + * provider's opening `role` delta commits the stream just as a text + * delta does. We cannot know what a downstream consumer did with it, + * and being wrong here means duplicating a user's reply. Reasoning + * deltas are output for the same reason — the TUI renders them live. + * + * What this does NOT cover is a completion that streamed nothing but + * tool-call arguments: `createOpenAiStreamConsumer` yields on a + * `function.arguments` delta only once the accumulated arguments + * contain reply text, so a `{"path":"a.txt"` in flight leaves + * `committed` false and such a stream IS reopened. That is safe — no + * text reached the user, and tools are dispatched only after the + * completion returns — and it is what the reporter's case wants, since + * a tool call whose arguments never finished streaming is not a usable + * turn. It is spelled out because "any chunk we yielded" and "any byte + * the provider sent" are not the same line. + * 2. **`OpenAiHttpError`.** The failure came from *opening* the stream, + * inside `runOpenAiWithRetry`, which has already applied this client's + * retry policy to it. Reopening here would quietly replace that policy + * with a looser one, because `isNetworkError` says yes to failures + * `isRetryableOpenAiError` deliberately says no to: our own request + * timeout, and any non-retryable status whose body preview happens to + * mention a socket. The shared budget makes this cheap rather than + * catastrophic now — a *retryable* open failure has already drained + * the budget, so guard 4 would stop it anyway — but a non-retryable + * one still has budget left, and re-deciding a call the HTTP client + * already made is how a deterministic failure turns into three + * requests and a delayed, actionable message. + * 3. **Shape.** Anything that is not a recognisable transport death — a * bug in a stream consumer, a parse error — is a real error. Replaying * it would just hide it behind three identical failures. - * 5. **Budget.** One streaming completion gets `OPENAI_MAX_ATTEMPTS` - * total, shared with the open, not a fresh budget per layer. + * 4. **Budget.** One streaming completion gets `OPENAI_MAX_ATTEMPTS` + * requests in total, shared with the opens: the same counter is passed + * into `openAiStartStream`, so opens and reopens add up instead of + * multiplying. */ function canReopenStream( err: unknown, - signal: AbortSignal | undefined, committed: boolean, - attempt: number, + budget: OpenAiAttemptBudget, ): boolean { if (committed) return false; - if (signal?.aborted) return false; if (err instanceof OpenAiHttpError) return false; if (!isNetworkError(err)) return false; - return attempt < OPENAI_MAX_ATTEMPTS; -} - -/** - * Release a response whose body died mid-read, so the reopen does not - * leak the socket. By the time we get here the stream consumer's - * `finally` has released its reader lock, but the body itself still owns - * the connection until it is cancelled. A body that refuses to cancel - * (already errored, still locked) is not worth failing the turn over — - * we are on our way to a fresh request either way. - */ -async function discardResponseBody(res: Response | undefined): Promise { - if (!res?.body) return; - try { - await res.body.cancel(); - } catch { - // Nothing left to release. - } + return budget.remaining > 0; } function applyToolCallTerminationSafety( diff --git a/src/llm/provider/openai/openai-stream-retry.test.ts b/src/llm/provider/openai/openai-stream-retry.test.ts index 42be7a95..7707e165 100644 --- a/src/llm/provider/openai/openai-stream-retry.test.ts +++ b/src/llm/provider/openai/openai-stream-retry.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it, vi } from "vitest"; -import type { CompletionResult, StreamChunk } from "../completion-types.js"; +import type { + CompletionResult, + StreamChunk, + StreamFinalResult, +} from "../completion-types.js"; +import type { StreamConsumer } from "../adapters/stream-consumer.js"; +import { classifyFailure } from "../../reliability/classify-failure.js"; +import { shouldAdvance } from "../../fallback/should-advance.js"; import { OpenAiHttpError } from "./openai-http.js"; import { OpenAiProvider } from "./openai-provider.js"; @@ -71,13 +78,46 @@ function streamingResponse(parts: readonly BodyPart[]): Response { }); } -function provider(fetchImpl: unknown): OpenAiProvider { +function provider( + fetchImpl: unknown, + streamConsumer?: StreamConsumer, +): OpenAiProvider { return new OpenAiProvider({ id: "test", baseUrl: "https://example.invalid", apiKey: "", defaultChatModel: "qwen-test", fetchImpl: fetchImpl as typeof fetch, + ...(streamConsumer ? { streamConsumer } : {}), + }); +} + +/** One SSE event carrying a streamed `function.arguments` fragment. */ +function toolArgsFrame( + fragment: string, + extras: { name?: string; id?: string } = {}, +): string { + return frame({ + model: "qwen-test", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + ...(extras.id ? { id: extras.id } : {}), + type: "function", + function: { + ...(extras.name ? { name: extras.name } : {}), + arguments: fragment, + }, + }, + ], + }, + finish_reason: null, + }, + ], }); } @@ -225,8 +265,12 @@ describe("OpenAiProvider stream transport retry (pre-first-chunk)", () => { openGate(); const { error } = await pending; - expect((error as Error).message).toBe("terminated"); expect(fetchImpl).toHaveBeenCalledTimes(1); + // The shape matters as much as the absent retry: a turn the user + // stopped has to classify as `cancelled`, or the fallback chain reads + // it as a dead provider and restarts the completion on another link. + expect(classifyFailure(error)).toBe("cancelled"); + expect(shouldAdvance(error).advance).toBe(false); }); it("does not retry a failure that is not a transport death", async () => { @@ -246,9 +290,9 @@ describe("OpenAiProvider stream transport retry (pre-first-chunk)", () => { }); it("does not re-retry an open failure that already spent the HTTP budget", async () => { - // 500s are retried by `runOpenAiWithRetry` itself. If this layer - // retried the resulting `OpenAiHttpError` too, the budget would be - // squared (3 × 3 = 9 requests) instead of shared. + // 500s are retried by `runOpenAiWithRetry` itself, and those retries + // come out of the completion's budget — so by the time the error + // reaches this layer there is nothing left to spend on a reopen. const fetchImpl = vi.fn( async () => new Response("upstream exploded", { status: 500 }), ); @@ -267,6 +311,41 @@ describe("OpenAiProvider stream transport retry (pre-first-chunk)", () => { streamingResponse([new Error("terminated")]), ); + const startedAt = Date.now(); + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect((error as Error).message).toBe("terminated"); + expect(fetchImpl).toHaveBeenCalledTimes(3); + // Two reopens means two `openAiRetryBackoff` waits (~150ms and + // ~300ms before jitter). Asserted as a floor because a reopen loop + // with no pacing is a provider-hammering loop, and nothing else here + // would notice if the backoff were dropped. + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(200); + }); +}); + +/** + * The budget is the whole reason this layer is safe to have: two bounded + * retry loops that do not share a counter are one multiplying loop. Each + * of these pins a scenario that cost 9 HTTP requests for a single turn + * while the sharing was only asserted in a comment. + */ +describe("OpenAiProvider stream transport retry (shared attempt budget)", () => { + it("spends one budget across opens and reopens, not one per layer", async () => { + // The worst shape: every open pays two 500s before it succeeds, and + // every body then dies before its first chunk. With a per-layer + // budget that is 3 opens x 3 attempts = 9 requests for one turn — + // nine billable prompt submissions. + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + return calls % 3 === 0 + ? streamingResponse([new Error("terminated")]) + : new Response("upstream exploded", { status: 500 }); + }); + const { error } = await drainToError( provider(fetchImpl).completeStream({ prompt: "hi" }), ); @@ -274,4 +353,183 @@ describe("OpenAiProvider stream transport retry (pre-first-chunk)", () => { expect((error as Error).message).toBe("terminated"); expect(fetchImpl).toHaveBeenCalledTimes(3); }); + + it("costs an unreachable provider three requests, not nine", async () => { + // `ECONNREFUSED` is the one open failure that is BOTH typed as an + // `OpenAiHttpError` and recognised by `isNetworkError`, so it is + // where the two layers most want to retry each other's work. + const fetchImpl = vi.fn(async () => { + throw Object.assign(new TypeError("fetch failed"), { + code: "ECONNREFUSED", + }); + }); + + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect(error).toBeInstanceOf(OpenAiHttpError); + expect((error as OpenAiHttpError).code).toBe("ECONNREFUSED"); + expect(fetchImpl).toHaveBeenCalledTimes(3); + }); + + it("does not reopen a non-retryable open failure that merely looks like a socket death", async () => { + // The case the `OpenAiHttpError` guard actually owns. A 400 is not + // retryable, so the HTTP client throws it with budget to spare — and + // its message carries the provider's body, which here mentions a + // socket, so `isNetworkError` says yes. Without that guard this layer + // would override the client's "deterministic, do not retry" verdict + // and spend the rest of the completion's budget on it. + const fetchImpl = vi.fn( + async () => new Response("socket hang up upstream", { status: 400 }), + ); + + const { error } = await drainToError( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect(error).toBeInstanceOf(OpenAiHttpError); + expect((error as OpenAiHttpError).status).toBe(400); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); + +describe("OpenAiProvider stream transport retry (cancellation and commit ordering)", () => { + it("fails as cancelled when Esc lands during the inter-attempt backoff", async () => { + // The window this layer added: the first body is already dead, the + // reopen has not happened yet, and the user — who has been staring at + // a frozen screen for a minute — presses Esc. `sleep()` resolves on + // abort rather than throwing, so without an explicit re-check the + // loop walks straight into another request, and the failure it ends + // up producing classifies as `transport`, which makes the fallback + // chain switch providers and restart the turn the user just stopped. + const controller = new AbortController(); + let firstFetchSeen: () => void = () => {}; + const firstFetch = new Promise((resolve) => { + firstFetchSeen = resolve; + }); + const fetchImpl = vi.fn(async () => { + firstFetchSeen(); + return streamingResponse([new Error("terminated")]); + }); + + const pending = drainToError( + provider(fetchImpl).completeStream({ + prompt: "hi", + signal: controller.signal, + }), + ); + await firstFetch; + // Comfortably inside the ~150ms first backoff, and after the body has + // errored (it errors on its very first `pull`). + await new Promise((resolve) => setTimeout(resolve, 20)); + controller.abort(); + const { error } = await pending; + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(classifyFailure(error)).toBe("cancelled"); + expect(shouldAdvance(error).advance).toBe(false); + }); + + it("treats output as delivered the moment it is yielded, not after", async () => { + // `generator.throw()` is how a consumer reports its own failure into + // a stream it is draining, and it resumes this generator *inside* the + // catch, with the yield never having returned. If `committed` were + // set after the yield instead of before, that error would find + // `committed === false` and replay a completion the caller has + // already shown part of. + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + streamingResponse([contentFrame("part one"), STOP_FRAMES]), + ) + .mockResolvedValueOnce( + streamingResponse([contentFrame("part one again"), STOP_FRAMES]), + ); + + const stream = provider(fetchImpl).completeStream({ prompt: "hi" }); + const first = await stream.next(); + + expect(first.done).toBe(false); + await expect(stream.throw(new Error("terminated"))).rejects.toThrow( + "terminated", + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("reopens a stream that only ever streamed tool-call arguments", async () => { + // Not a hole in the committed rule but a documented edge of it: the + // consumer yields on a `function.arguments` delta only once the + // accumulated arguments contain reply text, so a tool call whose + // arguments were still in flight commits nothing and is replayed. + // Safe — nothing reached the user and tools are dispatched only after + // the completion returns — but it is a wider window than "any byte + // the provider sent", so it is pinned rather than left to be + // rediscovered as a surprise. + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + streamingResponse([ + toolArgsFrame("", { id: "c1", name: "read_file" }), + toolArgsFrame('{"path":"a.txt"'), + new Error("terminated"), + ]), + ) + .mockResolvedValueOnce( + streamingResponse([ + toolArgsFrame('{"path":"b.txt"}', { id: "c2", name: "read_file" }), + STOP_FRAMES, + ]), + ); + + const { chunks, result } = await drain( + provider(fetchImpl).completeStream({ prompt: "hi" }), + ); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(chunks.map((c) => c.delta).join("")).toBe(""); + // Only the second attempt's tool call survives — the half-streamed + // arguments of the dead one must not stack onto it. + expect(result.toolCalls).toEqual([ + { + id: "c2", + type: "function", + function: { name: "read_file", arguments: '{"path":"b.txt"}' }, + }, + ]); + }); + + it("starts a reopened stream from an empty transcript", async () => { + // The built-in consumer cannot accumulate without also committing, so + // the per-reopen reset only bites for an injected `streamConsumer` — + // a public constructor option — that reports content on a chunk this + // loop does not yield. Without the reset the replay stacks on top of + // the dead attempt's text. + let consumeCalls = 0; + const consumer: StreamConsumer = { + async *consume(): AsyncGenerator { + consumeCalls += 1; + if (consumeCalls === 1) { + yield { delta: "ghost", reasoningDelta: "", done: true }; + throw new Error("terminated"); + } + yield { delta: "real", reasoningDelta: "", done: true }; + return { + content: "", + reasoningContent: "", + finishReason: "stop", + modelId: "qwen-test", + terminalObserved: true, + }; + }, + }; + const fetchImpl = vi.fn(async () => streamingResponse([STOP_FRAMES])); + + const { result } = await drain( + provider(fetchImpl, consumer).completeStream({ prompt: "hi" }), + ); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(result.content).toBe("real"); + }); }); From 4ae2d0182a3272d2c16571cf5e209eaeba987885 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:06:47 +0300 Subject: [PATCH 10/36] fix(loop-detector): stop read_repeat misreading two honest reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the read-coverage detector turned up two cases where it either fired on a read that was not redundant, or fired correctly but gave advice the model could not act on. Rendering is part of the file's version. Coverage was keyed on (canonical path, content hash) alone, so re-reading an already-covered range with `lineNumbers: true` — the ordinary last step before a precise edit — counted as no progress. It is not: the numbered text carries line numbers the plain read never showed, and the notice's "re-reading a covered range returns the same text" was simply untrue there. Two plain reads followed by two numbered ones reached the warn floor. `os.fs.read` now publishes `readCoverage.numbered`, and the tracker treats a rendering switch exactly like a content change: fresh coverage, no warning. The cost is a missed detection when a model alternates renderings forever, which is the direction this detector is allowed to fail in. An empty return is not a covered-range re-read. On a file larger than `maxBytes` only the byte-capped prefix is reachable, so `offset: 3000` on a 1905-line window returns nothing and counts as no progress — correctly, it is pure waste — but the notice then told the model to "read a range you have not covered", which is the request that had just come back empty, and claimed a covered range had been re-read, which had not happened. `readCoverage.truncated` now records whether content sits behind the byte cap (the byte cap specifically, not the top-level `truncated` detail, which is also true when a line window merely ended early and another offset can still reach the rest), and the notice picks one of three remediations: name the byte cap and tell the model to raise `maxBytes`; or name the reachable window for an offset past the end of a fully readable file; or keep the original covered-range advice when the read really did return lines the turn already had. Both flags parse leniently (`=== true`, default false) so a replayed trace or an older session degrades to "plain, un-capped read" instead of switching the detector off. Also pins two pieces of the range algebra that had no test: `mergeRange` merging a span adjacent to the interval AFTER it (losing that branch leaves a one-line seam and breaks the module's own non-adjacent invariant), and `newlyCoveredCount` clamping at zero for overlapping input — callers test its result both with `> 0` and with `=== 0`, so an unclamped negative would answer "no" to both and silently reset the no-progress streak. --- AGENTS.md | 6 +- src/agent/batch-executor.ts | 8 + src/agent/loop-detector.ts | 109 ++++++++++--- src/agent/read-coverage.test.ts | 217 +++++++++++++++++++++++++- src/agent/read-coverage.ts | 31 +++- src/tools/os/fs-read-coverage.test.ts | 79 +++++++++- src/tools/os/fs-read-coverage.ts | 42 ++++- src/tools/os/fs-read.ts | 19 ++- 8 files changed, 475 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 780d7884..f11e0277 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -124,7 +124,9 @@ The runtime guards against "stuck" turns where the model re-emits the same tool **Wandering detector (distinct-spread).** `getRepeatCount` / `getNoProgressStreak` only catch the *same* signature repeating. A model probing endless **distinct** URLs / queries / pages on one tool (e.g. guessing 8 different `os.web.fetch` URLs, or firing 21 search POSTs that differ only in volatile result fields) is a different failure mode. For wandering-prone tools (`isWanderingProneTool`: `os.web.fetch`, `os.http.request`, `browser.*`), `check()` computes `effectiveSpread` — the count of distinct completed `argsHash`es for that tool in the window, plus one when the prospective call introduces a new signature. Crossing `loopWanderingThreshold` ⇒ a `wandering` warn whose notice is an **actionable redirect** (`formatWanderingRedirect`: "stop probing URLs, run a web search or reply best-effort") rather than the repeat advisory. Crossing `loopWanderingEscalation` ⇒ `isWanderingEscalated()` returns true and the gate raises a `breaker` signal — the unique call **is** vetoed and the turn ends gracefully. Bulk reads over distinct files (`os.fs.read`) are deliberately **not** wandering-prone — scanning many files is legitimate work. -**Read-coverage detector (semantic progress on `os.fs.read`).** Argument hashing cannot see that `offset: 40, limit: 30` and `offset: 90, limit: 30` returned text the model already has, and reads are (correctly) not wandering-prone, so an overlapping re-read of one unchanged file used to be invisible. `os.fs.read` therefore publishes a `details.readCoverage` block — canonical (symlink-resolved) path, a hash of the bytes it actually read, and the line range it actually returned (see [src/tools/os/fs-read-coverage.ts](src/tools/os/fs-read-coverage.ts)) — and after each call the gate folds that range into a per-file coverage set (`checkReadRepeat` / `recordRead`, [src/agent/read-coverage.ts](src/agent/read-coverage.ts)). A read whose returned range is already fully covered at the same content hash makes no progress; newly covered lines do; a changed hash discards the file's coverage, so a same-size in-place edit with an untouched mtime resets it. Crossing `READ_REPEAT_WARNING_THRESHOLD` (2 consecutive no-progress reads) ⇒ a `read_repeat` **warn** (`formatReadRepeatNotice`, line numbers only — never file content). Warn-only by construction: the read has already executed, so there is nothing to veto. Failed reads record nothing, a truncated read banks only the prefix it returned, and a scan over many distinct files never signals. +**Read-coverage detector (semantic progress on `os.fs.read`).** Argument hashing cannot see that `offset: 40, limit: 30` and `offset: 90, limit: 30` returned text the model already has, and reads are (correctly) not wandering-prone, so an overlapping re-read of one unchanged file used to be invisible. `os.fs.read` therefore publishes a `details.readCoverage` block — canonical (symlink-resolved) path, a hash of the bytes it actually read, and the line range it actually returned (see [src/tools/os/fs-read-coverage.ts](src/tools/os/fs-read-coverage.ts)) — and after each call the gate folds that range into a per-file coverage set (`checkReadRepeat` / `recordRead`, [src/agent/read-coverage.ts](src/agent/read-coverage.ts)). A read whose returned range is already fully covered at the same *version* makes no progress; newly covered lines do. Version is `(contentHash, numbered)`: a changed hash discards the file's coverage, so a same-size in-place edit with an untouched mtime resets it, and so does a switch to or from `lineNumbers: true` — the same lines with `LINE_NUMBER|` prefixes are text the model did not have, and calling that a repeat would be a false positive on the ordinary read-then-re-read-numbered-to-edit workflow. Crossing `READ_REPEAT_WARNING_THRESHOLD` (2 consecutive no-progress reads) ⇒ a `read_repeat` **warn** (`formatReadRepeatNotice`, line numbers only — never file content). Warn-only by construction: the read has already executed, so there is nothing to veto. Failed reads record nothing, a truncated read banks only the prefix it returned, and a scan over many distinct files never signals. + +The notice picks its remediation from three cases, because a repeat that returned *nothing* is not the same failure as one that returned covered lines. An empty return means the requested range does not exist — either past the end of the file, or (when the coverage detail's `truncated` flag is set) behind this read's `maxBytes` budget, which no `offset` can reach. Telling that model to "read a range you have not covered" would point it straight back at the request that just came back empty, so it is told the reachable window instead, and the byte cap is named where it is the cause. **Volatile-stripping in `hashToolOutcome`.** Before hashing a generic (non-shell) result's `details`, `stripVolatile` recursively drops `VOLATILE_RESULT_KEYS` (`timestamp`, `ts`, `date`, `time`, `timeTotal`, `timeTotalSeconds`, `durationMs`, `sizeDownload`, `requestId`/`request_id`, `id`, `traceId`/`trace_id`, `sentAt`, `createdAt`, `deliveredAt`). Without this, per-call timings/sizes (e.g. `timeTotalSeconds`, `sizeDownload` on `os.http.request`) make every result hash unique, so a repeated dead/identical endpoint never registers as a no-progress streak. Mirrors OpenClaw's `stripVolatileSendIds`. @@ -132,7 +134,7 @@ The runtime guards against "stuck" turns where the model re-emits the same tool **Trace.** `loop_detected` events carry `level` (`warn` | `critical` | `breaker`) and `detector` (`generic_repeat` | `no_progress` | `wandering` | `test_repeat` | `read_repeat`) — see [src/tracing/trace/trace-event.ts](src/tracing/trace/trace-event.ts). A `read_repeat` event also carries `read` (resolved path, returned range, and the fingerprint on either side of the read), which is enough to audit why it fired without recording a line of the file. -Pinned by [src/agent/loop-detector.test.ts](src/agent/loop-detector.test.ts), [src/agent/read-coverage.test.ts](src/agent/read-coverage.test.ts) (containment / partial overlap / pagination / symlink identity / same-size replacement / truncation / mutation / multi-file scan), [src/agent/batch-executor.test.ts](src/agent/batch-executor.test.ts) (veto single call / siblings survive / terminal never vetoed / breaker escalation), and [src/agent/agent-loop.test.ts](src/agent/agent-loop.test.ts) ("ends the turn with a graceful reply (not loop_failed) when the breaker trips"). +Pinned by [src/agent/loop-detector.test.ts](src/agent/loop-detector.test.ts), [src/agent/read-coverage.test.ts](src/agent/read-coverage.test.ts) (containment / partial overlap / pagination / symlink identity / same-size replacement / rendering switch / byte-cap wording / warn-bucket keying / truncation / multi-file scan), [src/cli/trace-formatter.test.ts](src/cli/trace-formatter.test.ts) (`loop_detected` rendering, including traces that predate `detector`/`read`), [src/agent/batch-executor.test.ts](src/agent/batch-executor.test.ts) (veto single call / siblings survive / terminal never vetoed / breaker escalation), and [src/agent/agent-loop.test.ts](src/agent/agent-loop.test.ts) ("ends the turn with a graceful reply (not loop_failed) when the breaker trips", "warns on the second no-progress re-read of one unchanged file" — the latter drives the real `os.fs.read` through the production path and pins the detector's own threshold, its notice and the event payload). ### Out of scope (deferred) diff --git a/src/agent/batch-executor.ts b/src/agent/batch-executor.ts index ff12ee82..d7d245c7 100644 --- a/src/agent/batch-executor.ts +++ b/src/agent/batch-executor.ts @@ -62,6 +62,13 @@ export interface BatchLoopSignal { endLine: number; /** Lines visible in the read window. */ totalLines: number; + /** + * Whether the file has content past `totalLines` that the read's + * byte budget hid. The notice needs it to tell "you asked for a line + * past the end of the file" apart from "you asked for a line the + * byte cap hid", which have opposite fixes. + */ + truncated: boolean; /** Compact list of lines already read this turn, e.g. `"1-40, 88-120"`. */ covered: string; /** Content fingerprint this read saw. */ @@ -677,6 +684,7 @@ function observeReadCoverage( startLine: observation.span?.start ?? 0, endLine: observation.span?.end ?? 0, totalLines: observation.totalLines, + truncated: observation.truncated, covered: repeat.covered, fingerprint: observation.contentHash, // `checkReadRepeat` only reports a repeat when it has seen this diff --git a/src/agent/loop-detector.ts b/src/agent/loop-detector.ts index 5db8390f..1c5ca0cb 100644 --- a/src/agent/loop-detector.ts +++ b/src/agent/loop-detector.ts @@ -216,14 +216,19 @@ export class ToolLoopTracker { private readonly pendingTestKeys = new Map(); /** * Read-coverage detector state (issue #114): canonical file path → the - * content fingerprint that path was last read at, the merged set of - * lines read at THAT fingerprint, and how many reads in a row have - * returned nothing outside it. Insertion order doubles as a + * content fingerprint and rendering that path was last read at, the + * merged set of lines read at THAT version, and how many reads in a + * row have returned nothing outside it. Insertion order doubles as a * least-recently-read order for eviction (see `MAX_TRACKED_READ_FILES`). */ private readonly readCoverage = new Map< string, - { contentHash: string; covered: LineRange[]; noProgress: number } + { + contentHash: string; + numbered: boolean; + covered: LineRange[]; + noProgress: number; + } >(); constructor(options: ToolLoopTrackerOptions = {}) { @@ -428,17 +433,23 @@ export class ToolLoopTracker { * blocking it would cost the model information without saving anything. * * No progress means: the file's content is byte-identical to what it - * was when this turn last read it, and every line this read returned - * was already returned earlier in the turn. A read that returned no - * lines at all (an offset past the end) also counts — it cannot have - * shown anything new — but only once the file has been seen at this - * version, so the first such read is never flagged. + * was when this turn last read it, it was rendered the same way, and + * every line this read returned was already returned earlier in the + * turn. A read that returned no lines at all (an offset past the end) + * also counts — it cannot have shown anything new — but only once the + * file has been seen at this version, so the first such read is never + * flagged. + * + * The rendering half of "version" is what keeps a plain read followed + * by a `lineNumbers: true` re-read of the same lines — the normal + * preparation for a precise edit — off this detector: that re-read + * does return text the model did not have. */ checkReadRepeat(observation: ReadObservation): ReadRepeatCheck { const prev = this.readCoverage.get(observation.path); if (prev === undefined) return { repeat: false, count: 0, covered: "" }; const previousFingerprint = prev.contentHash; - if (prev.contentHash !== observation.contentHash) { + if (!sameReadVersion(prev, observation)) { return { repeat: false, count: 0, covered: "", previousFingerprint }; } const fresh = @@ -460,16 +471,16 @@ export class ToolLoopTracker { * Fold a completed read into its file's coverage. Call AFTER * `checkReadRepeat`. * - * A different content fingerprint discards the previous coverage - * outright: the lines the turn read before belong to a version of the - * file that no longer exists, so counting them again would mark a - * genuinely new read as no progress. That reset is also what makes an - * edit-then-re-read cycle free of false warnings. + * A different content fingerprint — or a different rendering — discards + * the previous coverage outright: the lines the turn read before belong + * to a version of the file that no longer exists, or were rendered + * without the line numbers this read added, so counting them again + * would mark a genuinely new read as no progress. That reset is also + * what makes an edit-then-re-read cycle free of false warnings. */ recordRead(observation: ReadObservation): void { const prev = this.readCoverage.get(observation.path); - const sameVersion = - prev !== undefined && prev.contentHash === observation.contentHash; + const sameVersion = prev !== undefined && sameReadVersion(prev, observation); const covered = sameVersion ? prev.covered : []; const fresh = observation.span === null ? 0 : newlyCoveredCount(covered, observation.span); @@ -478,6 +489,7 @@ export class ToolLoopTracker { this.readCoverage.delete(observation.path); this.readCoverage.set(observation.path, { contentHash: observation.contentHash, + numbered: observation.numbered, covered: observation.span === null ? covered : mergeRange(covered, observation.span), noProgress: sameVersion && fresh === 0 ? prev.noProgress + 1 : 0, @@ -878,6 +890,22 @@ export function formatTestRepeatNotice(verdict: { return lines.join("\n"); } +/** + * Is this read looking at the same version of the file, rendered the + * same way, as the coverage already banked for it? Both halves have to + * hold: different bytes are different text, and so are the same bytes + * with `LINE_NUMBER|` prefixes the previous read did not have. + */ +function sameReadVersion( + entry: { contentHash: string; numbered: boolean }, + observation: ReadObservation, +): boolean { + return ( + entry.contentHash === observation.contentHash && + entry.numbered === observation.numbered + ); +} + /** * Notice injected when the same unchanged file was read again without * reaching a new line (issue #114, warn-only). @@ -887,6 +915,15 @@ export function formatTestRepeatNotice(verdict: { * is the model not realising its shifted `offset`/`limit` landed inside * text it already has. Line numbers and the path only: no file content * appears here, in the event, or in the log line. + * + * The remediation sentence is chosen from three cases, because the same + * advice is not true of all of them. A read that returned nothing did + * NOT re-read a covered range — it asked for a range that does not + * exist, either past the end of the file or (when `truncated`) behind + * the read's byte budget — and telling that model to "read a range you + * have not covered" points it straight back at the request that just + * failed. Naming the reachable window, and the byte cap when there is + * one, is the only advice that can actually unstick it. */ export function formatReadRepeatNotice(verdict: { count: number; @@ -895,22 +932,44 @@ export function formatReadRepeatNotice(verdict: { endLine: number; totalLines: number; covered: string; + truncated?: boolean; }): string { const label = sanitizeReadPath(verdict.path); - const returned = - verdict.startLine === 0 - ? "returned no lines at all" - : `returned lines ${verdict.startLine}-${verdict.endLine}`; - const lines = [ - `You read ${label} ${verdict.count} times in a row without reaching a line you had not already read this turn. The last read ${returned}, and the file's content has not changed since the previous read.`, - ]; + const empty = verdict.startLine === 0; + const reach = + verdict.totalLines > 0 + ? `lines 1-${verdict.totalLines}` + : "no lines at all"; + const lines: string[] = []; + if (empty) { + lines.push( + `You read ${label} ${verdict.count} times in a row without reaching a line you had not already read this turn. The last read returned no lines at all: the range you asked for is outside the part of the file this read can reach, which is ${reach}.`, + ); + } else { + lines.push( + `You read ${label} ${verdict.count} times in a row without reaching a line you had not already read this turn. The last read returned lines ${verdict.startLine}-${verdict.endLine}, and the file's content has not changed since the previous read.`, + ); + } if (verdict.covered.length > 0) { lines.push( `Already read this turn: lines ${verdict.covered}${verdict.totalLines > 0 ? ` (of ${verdict.totalLines} readable lines)` : ""}.`, ); } + if (empty && verdict.truncated === true) { + lines.push( + `The file is larger than this read's \`maxBytes\` budget, so everything past line ${verdict.totalLines} is invisible to it no matter which \`offset\` you pass. Raise \`maxBytes\` to reach further into the file, or work with the part you can already see.`, + ); + } else if (empty) { + lines.push( + `Asking for an \`offset\` past the end returns nothing. Stay inside ${reach}, open a different file, or act on what you already have.`, + ); + } else { + lines.push( + "Re-reading a covered range returns the same text. Read a range you have not covered, open a different file, or act on what you already have.", + ); + } lines.push( - "Re-reading a covered range returns the same text. Read a range you have not covered, open a different file, or act on what you already have. If the repeat was intentional, continue — this is a warning, nothing was blocked.", + "If the repeat was intentional, continue — this is a warning, nothing was blocked.", ); return lines.join("\n"); } diff --git a/src/agent/read-coverage.test.ts b/src/agent/read-coverage.test.ts index 02ff3df0..3962811c 100644 --- a/src/agent/read-coverage.test.ts +++ b/src/agent/read-coverage.test.ts @@ -44,6 +44,39 @@ describe("read coverage range algebra", () => { expect(newlyCoveredCount(covered, { start: 38, end: 45 })).toBe(0); }); + it("merges a span that closes the gap between two intervals", () => { + // The mirror image of the case above: the new span is adjacent to + // the interval AFTER it, not before it. Both adjacency tests use a + // ±1 slack, and losing either one leaves a one-line seam that + // `describeCoverage` then renders as two ranges ("1-19, 20-30") + // while the module documents its coverage as non-adjacent. + const covered: LineRange[] = [ + { start: 1, end: 10 }, + { start: 20, end: 30 }, + ]; + expect(mergeRange(covered, { start: 11, end: 19 })).toEqual([ + { start: 1, end: 30 }, + ]); + // Adjacency on one side only still merges only that side. + expect(mergeRange(covered, { start: 12, end: 19 })).toEqual([ + { start: 1, end: 10 }, + { start: 12, end: 30 }, + ]); + }); + + it("never reports a negative count for overlapping input", () => { + // `covered` is documented as disjoint, but `newlyCoveredCount` is + // exported and a caller can break that. Overlapping ranges subtract + // their shared lines once each, so the raw arithmetic goes negative — + // and a negative answers "no" to both `> 0` (progress) and `=== 0` + // (extend the streak), quietly disabling the detector. + const overlapping: LineRange[] = [ + { start: 1, end: 10 }, + { start: 5, end: 15 }, + ]; + expect(newlyCoveredCount(overlapping, { start: 5, end: 10 })).toBe(0); + }); + it("keeps disjoint ranges separate and sorted", () => { let covered = mergeRange([], { start: 50, end: 60 }); covered = mergeRange(covered, { start: 1, end: 10 }); @@ -82,6 +115,8 @@ describe("classifyReadResult", () => { startLine: 3, endLine: 9, totalLines: 40, + numbered: true, + truncated: true, }; it("extracts the observation from a successful read", () => { @@ -96,6 +131,10 @@ describe("classifyReadResult", () => { contentHash: "hash1", span: { start: 3, end: 9 }, totalLines: 40, + // The rendering and byte-cap flags are part of the observation: + // one decides coverage identity, the other the notice wording. + numbered: true, + truncated: true, }); }); @@ -139,8 +178,9 @@ function observe( contentHash: string, span: LineRange | null, totalLines = 200, + numbered = false, ): Parameters[0] { - return { path, contentHash, span, totalLines }; + return { path, contentHash, span, totalLines, numbered, truncated: false }; } describe("ToolLoopTracker read-coverage detector", () => { @@ -231,6 +271,34 @@ describe("ToolLoopTracker read-coverage detector", () => { ).toBe(false); }); + it("resets coverage when the rendering changes", () => { + // Re-reading covered lines WITH line numbers returns text the model + // did not have — the numbers themselves, which is the usual last step + // before a precise edit. Flagging that would be a false positive, and + // the notice's "re-reading a covered range returns the same text" + // would be an untrue claim. + const tracker = new ToolLoopTracker(); + tracker.recordRead(observe("/a.ts", "v1", { start: 1, end: 100 })); + const numbered = observe("/a.ts", "v1", { start: 10, end: 20 }, 200, true); + const check = tracker.checkReadRepeat(numbered); + expect(check.repeat).toBe(false); + // The content genuinely did not change; only the rendering did. + expect(check.previousFingerprint).toBe("v1"); + tracker.recordRead(numbered); + // …and the numbered coverage starts fresh, so 1-100 is worth reading + // again in the new rendering, while a numbered re-read of 10-20 is not. + expect( + tracker.checkReadRepeat( + observe("/a.ts", "v1", { start: 1, end: 100 }, 200, true), + ).repeat, + ).toBe(false); + expect( + tracker.checkReadRepeat( + observe("/a.ts", "v1", { start: 12, end: 18 }, 200, true), + ).repeat, + ).toBe(true); + }); + it("keeps files independent, so a multi-file scan never trips", () => { const tracker = new ToolLoopTracker(); for (let i = 0; i < 50; i += 1) { @@ -299,6 +367,39 @@ describe("formatReadRepeatNotice", () => { it("words an empty return honestly", () => { const notice = formatReadRepeatNotice({ ...base, startLine: 0, endLine: 0 }); expect(notice).toContain("returned no lines at all"); + // An empty return did NOT re-read a covered range, so the notice must + // not say it did — that advice points back at the request that just + // came back empty. + expect(notice).not.toContain("Re-reading a covered range"); + expect(notice).toContain("outside the part of the file this read can reach"); + expect(notice).toContain("Stay inside lines 1-900"); + }); + + it("blames the byte cap, not the offset, when the cap is what hid the lines", () => { + // The model asked for a line behind `maxBytes`. Telling it to "read a + // range you have not covered" is advice it cannot act on: no offset + // reaches past the cap. Naming the cap is the only way out. + const notice = formatReadRepeatNotice({ + ...base, + startLine: 0, + endLine: 0, + truncated: true, + }); + expect(notice).toContain("`maxBytes`"); + expect(notice).toContain("Raise `maxBytes`"); + expect(notice).toContain("past line 900"); + expect(notice).not.toContain("Re-reading a covered range"); + expect(notice).not.toContain("Stay inside"); + }); + + it("keeps the covered-range advice for a genuinely redundant re-read", () => { + // The other side of the same fork: a read that DID return lines the + // turn already had gets the original advice, and never the byte-cap + // wording — the cap is irrelevant when the range came back. + const notice = formatReadRepeatNotice({ ...base, truncated: true }); + expect(notice).toContain("Re-reading a covered range returns the same text"); + expect(notice).not.toContain("Raise `maxBytes`"); + expect(notice).toContain("content has not changed since the previous read"); }); }); @@ -460,6 +561,120 @@ describe("read-coverage detection end to end", () => { ).toEqual([]); }); + it("does not flag a numbered re-read of a range read plainly", async () => { + // The workflow this protects: read the file to understand it, then + // re-read the interesting range with `lineNumbers: true` to anchor an + // edit. The second read returns text the first one did not — the line + // numbers — so it is progress, and two more of them must still not + // reach the warn floor on the strength of the plain read alone. + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + expect( + await runRead(dir, tracker, { + path: "src.ts", + offset: 20, + limit: 30, + lineNumbers: true, + }), + ).toEqual([]); + // A shifted numbered read that stays inside the numbered coverage is + // still caught — the reset is per rendering, not an amnesty. + const signals = await runRead(dir, tracker, { + path: "src.ts", + offset: 25, + limit: 10, + lineNumbers: true, + }); + expect(signals).toHaveLength(1); + expect(signals[0]?.detector).toBe("read_repeat"); + }); + + it("blames the byte cap when the requested lines are behind it", async () => { + // 200 lines of ~9 bytes; a 300-byte cap makes everything past line ~30 + // unreachable. The model asking for line 150 twice is not re-reading a + // covered range — it is asking for something no `offset` can deliver — + // so the signal has to carry that fact through to the notice. + const cap = { path: "src.ts", maxBytes: 300 }; + expect(await runRead(dir, tracker, { ...cap, offset: 1, limit: 5 })).toEqual( + [], + ); + expect( + await runRead(dir, tracker, { ...cap, offset: 150, limit: 10 }), + ).toHaveLength(1); + const signals = await runRead(dir, tracker, { + ...cap, + offset: 170, + limit: 10, + }); + expect(signals).toHaveLength(1); + const read = signals[0]!.read!; + expect(read.startLine).toBe(0); + expect(read.endLine).toBe(0); + expect(read.truncated).toBe(true); + expect(read.totalLines).toBeLessThan(200); + const notice = formatReadRepeatNotice({ count: signals[0]!.count, ...read }); + expect(notice).toContain("Raise `maxBytes`"); + expect(notice).not.toContain("Re-reading a covered range"); + }); + + it("blames the offset, not the cap, past the end of a fully readable file", async () => { + // Same empty return, opposite cause and opposite fix: the whole file + // fits in the byte budget, so the model simply asked past its end. + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + expect( + await runRead(dir, tracker, { path: "src.ts", offset: 500, limit: 10 }), + ).toHaveLength(1); + const signals = await runRead(dir, tracker, { + path: "src.ts", + offset: 900, + limit: 10, + }); + const read = signals[0]!.read!; + expect(read.truncated).toBe(false); + const notice = formatReadRepeatNotice({ count: signals[0]!.count, ...read }); + expect(notice).toContain("Stay inside lines 1-200"); + expect(notice).not.toContain("Raise `maxBytes`"); + }); + + it("keys the warn bucket by file version so a post-edit nudge survives", async () => { + // `shouldEmitWarning` de-duplicates per `warningKey`. Keyed by path + // alone, the bucket set while warning about the OLD content would + // swallow the first warning about the new content after an edit. + const file = join(dir, "src.ts"); + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + const before = await runRead(dir, tracker, { + path: "src.ts", + offset: 10, + limit: 5, + }); + expect(before[0]?.warningKey).toContain(before[0]!.read!.fingerprint); + + await writeFile(file, "changed\nbody\nhere\n", "utf8"); + expect(await runRead(dir, tracker, { path: "src.ts" })).toEqual([]); + const after = await runRead(dir, tracker, { + path: "src.ts", + offset: 2, + limit: 2, + }); + expect(after[0]?.detector).toBe("read_repeat"); + // Different content ⇒ different key ⇒ a fresh, unspent warn bucket. + expect(after[0]?.warningKey).not.toBe(before[0]?.warningKey); + const tracker2 = new ToolLoopTracker(); + expect( + tracker2.shouldEmitWarning( + before[0]!.warningKey, + READ_REPEAT_WARNING_THRESHOLD, + READ_REPEAT_WARNING_THRESHOLD, + ), + ).toBe(true); + expect( + tracker2.shouldEmitWarning( + after[0]!.warningKey, + READ_REPEAT_WARNING_THRESHOLD, + READ_REPEAT_WARNING_THRESHOLD, + ), + ).toBe(true); + }); + it("records nothing for a failed read", async () => { const outcome = await executeBatch( toBatchInputs([ diff --git a/src/agent/read-coverage.ts b/src/agent/read-coverage.ts index 5d759f19..92a979ba 100644 --- a/src/agent/read-coverage.ts +++ b/src/agent/read-coverage.ts @@ -39,6 +39,20 @@ export interface ReadObservation { span: LineRange | null; /** Lines visible in the read window (see `ReadCoverageDetail`). */ totalLines: number; + /** + * Whether this read rendered `LINE_NUMBER|` prefixes. Part of the + * coverage IDENTITY, not of the span: the same lines rendered + * differently are different text, so a rendering switch starts a fresh + * coverage set exactly the way a content change does. + */ + numbered: boolean; + /** + * Whether the file has content past `totalLines` that the read's byte + * budget hid. Only used to word the notice honestly — a repeat of an + * unreachable range is still a repeat, but the fix for it is a bigger + * `maxBytes`, not a different offset. + */ + truncated: boolean; } /** @@ -68,17 +82,28 @@ export function classifyReadResult( ? null : { start: detail.startLine, end: detail.endLine }, totalLines: detail.totalLines, + numbered: detail.numbered, + truncated: detail.truncated, }; } /** * How many lines of `span` are not already in `covered`. * - * `covered` must be sorted, disjoint and non-adjacent (the shape - * `mergeRange` maintains). Zero means the read was fully contained in - * what the turn had already seen — the no-progress case the issue is + * `covered` is expected to be sorted, disjoint and non-adjacent (the + * shape `mergeRange` maintains). Zero means the read was fully contained + * in what the turn had already seen — the no-progress case the issue is * about, which a plain "same start line?" check misses whenever the model * shifts the offset. + * + * The result is clamped at zero so it is always a count of lines, never + * a negative number. `covered` reaching here overlapping violates that + * contract but is possible — this function is exported — and overlapping + * ranges subtract their shared lines once per range, so the raw + * arithmetic can go below zero. That matters because callers test the + * result BOTH ways: `> 0` means progress, and `=== 0` is what extends + * the no-progress streak. An unclamped `-6` would answer "no" to both + * and silently reset the streak. */ export function newlyCoveredCount( covered: readonly LineRange[], diff --git a/src/tools/os/fs-read-coverage.test.ts b/src/tools/os/fs-read-coverage.test.ts index 4acf1674..9e015515 100644 --- a/src/tools/os/fs-read-coverage.test.ts +++ b/src/tools/os/fs-read-coverage.test.ts @@ -157,6 +157,63 @@ describe("os.fs.read coverage detail", () => { expect(coverage.totalLines).toBe(2); }); + it("reports the rendering mode the read actually used", async () => { + // The mode is part of the coverage identity: the same lines with + // `LINE_NUMBER|` prefixes are not the same text, so the detector has + // to be able to tell the two renderings apart. + await writeFile(join(dir, "a.txt"), "one\ntwo\nthree\n", "utf8"); + expect((await readCoverageOf(dir, { path: "a.txt" })).numbered).toBe(false); + expect( + (await readCoverageOf(dir, { path: "a.txt", lineNumbers: true })).numbered, + ).toBe(true); + expect( + ( + await readCoverageOf(dir, { + path: "a.txt", + offset: 1, + limit: 2, + lineNumbers: true, + }) + ).numbered, + ).toBe(true); + }); + + it("flags a byte-capped read, in both modes, and only when the cap bites", async () => { + const body = Array.from({ length: 400 }, (_, i) => `line ${i + 1}`).join("\n"); + await writeFile(join(dir, "big.txt"), `${body}\n`, "utf8"); + expect((await readCoverageOf(dir, { path: "big.txt" })).truncated).toBe(false); + expect( + (await readCoverageOf(dir, { path: "big.txt", maxBytes: 200 })).truncated, + ).toBe(true); + expect( + ( + await readCoverageOf(dir, { + path: "big.txt", + maxBytes: 200, + offset: 1, + limit: 5, + }) + ).truncated, + ).toBe(true); + }); + + it("does not call a short line window byte-capped", async () => { + // The top-level `truncated` detail is also true when the requested + // window merely ended before the end of the file — a range another + // `offset` can still reach. The coverage flag answers the narrower + // question (is there content NO offset of this call can reach?), so + // it must stay false here or the notice would blame the byte cap for + // a range the model can simply ask for. + const body = Array.from({ length: 40 }, (_, i) => `line ${i + 1}`).join("\n"); + await writeFile(join(dir, "mid.txt"), `${body}\n`, "utf8"); + const result = await osFsReadTool.run( + { path: "mid.txt", offset: 1, limit: 5 }, + makeCtx(dir), + ); + expect(result.details.truncated).toBe(true); + expect(parseReadCoverage(result.details)?.truncated).toBe(false); + }); + it("reports an empty span for an empty file", async () => { await writeFile(join(dir, "empty.txt"), "", "utf8"); const coverage = await readCoverageOf(dir, { path: "empty.txt" }); @@ -176,7 +233,27 @@ describe("parseReadCoverage", () => { }; it("accepts a well-formed detail", () => { - expect(parseReadCoverage({ readCoverage: valid })).toEqual(valid); + const full = { ...valid, numbered: true, truncated: true }; + expect(parseReadCoverage({ readCoverage: full })).toEqual(full); + }); + + it("defaults the rendering and byte-cap flags for a detail that predates them", () => { + // A replayed trace or an older session carries neither flag. The + // detector must keep working against it (degrading to "plain, + // un-capped read"), not reject the whole detail and switch itself off. + expect(parseReadCoverage({ readCoverage: valid })).toEqual({ + ...valid, + numbered: false, + truncated: false, + }); + }); + + it("reads a non-boolean rendering or cap flag as false", () => { + expect( + parseReadCoverage({ + readCoverage: { ...valid, numbered: "yes", truncated: 1 }, + }), + ).toMatchObject({ numbered: false, truncated: false }); }); it("returns null when the detail is absent (older or replayed results)", () => { diff --git a/src/tools/os/fs-read-coverage.ts b/src/tools/os/fs-read-coverage.ts index 2178401d..530fd0f4 100644 --- a/src/tools/os/fs-read-coverage.ts +++ b/src/tools/os/fs-read-coverage.ts @@ -52,6 +52,33 @@ export interface ReadCoverageDetail { * largest `endLine` any range of this call could have returned. */ totalLines: number; + /** + * Whether the returned text carried `LINE_NUMBER|` prefixes. + * + * Coverage is per rendering as well as per version, because the same + * lines rendered differently are not the same text: a model that read + * a file plainly and then re-reads a range with `lineNumbers: true` — + * the ordinary preparation for a precise edit — genuinely learns + * something it did not have. Treating that as a repeat would be a + * false positive, and the notice's "re-reading a covered range returns + * the same text" would be untrue. + * + * Absent on results produced before this field existed (a replayed + * trace, an older session); `parseReadCoverage` reads a missing value + * as `false`, which matches the tool's own default. + */ + numbered: boolean; + /** + * Whether the file is larger than this read's byte budget, i.e. there + * is content past `totalLines` that NO range of this call could reach. + * + * The detector still flags repeated reads of an unreachable range — + * they return nothing and are pure waste — but the remediation it + * offers has to be different: "read a range you have not covered" is + * useless advice when the range the model wants is behind the byte + * cap. See `formatReadRepeatNotice`. + */ + truncated: boolean; } /** Digest of the bytes a read looked at. Short — this is an identity, not a checksum. */ @@ -98,7 +125,20 @@ export function parseReadCoverage( // non-inverted range. A half-zero pair (0/5, 3/0) is incoherent. const empty = startLine === 0 && endLine === 0; if (!empty && (startLine < 1 || startLine > endLine)) return null; - return { path, contentHash, startLine, endLine, totalLines }; + // `numbered` and `truncated` are booleans with a meaningful default: + // an older result that predates them is a plain, un-capped read as far + // as anything downstream is concerned. Reading them leniently (rather + // than rejecting the whole detail) keeps the detector working against + // replayed traces instead of silently switching itself off. + return { + path, + contentHash, + startLine, + endLine, + totalLines, + numbered: record.numbered === true, + truncated: record.truncated === true, + }; } function asLineNumber(value: unknown): number | null { diff --git a/src/tools/os/fs-read.ts b/src/tools/os/fs-read.ts index 0fa2c986..b2e312e3 100644 --- a/src/tools/os/fs-read.ts +++ b/src/tools/os/fs-read.ts @@ -114,10 +114,11 @@ async function readByBytes( size, truncated, bytesRead: toRead, - [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, { + [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, args, { startLine: lineCount === 0 ? 0 : 1, endLine: lineCount, totalLines: lineCount, + truncated, }), }, }, @@ -177,10 +178,17 @@ async function readByLines( // end) reports an empty 0/0 span rather than a range it did not // return — the detector must never credit coverage for lines the // model never saw. - [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, { + [READ_COVERAGE_DETAIL_KEY]: coverage(canonical, buffer, args, { startLine: sliced.length === 0 ? 0 : startIndex + 1, endLine: sliced.length === 0 ? 0 : startIndex + sliced.length, totalLines: total, + // Deliberately NOT `truncated` from the details above: that one + // also fires when the requested window merely ended before the + // end of the readable prefix, which another offset can still + // reach. The detector asks a narrower question — is there + // content no range of this call could return? — and only the + // byte cap makes that true. + truncated: fileBytesTruncated, }), }, }, @@ -192,11 +200,16 @@ async function readByLines( function coverage( canonical: string, bytesRead: Buffer, - span: Pick, + args: ReadArgs, + span: Pick< + ReadCoverageDetail, + "startLine" | "endLine" | "totalLines" | "truncated" + >, ): ReadCoverageDetail { return { path: canonical, contentHash: hashReadContent(bytesRead), + numbered: args.lineNumbers, ...span, }; } From dd35ddc4417132b0949b1a20258ca0ee02b427b9 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:07:00 +0300 Subject: [PATCH 11/36] test(loop-detector): pin the read_repeat event, trace and notice wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector's core was tested; everything downstream of the batch signal was not. The new tests asserted `signal.read` at the batch-executor layer and stopped there, so deleting the event payload, the recorder passthrough or the trace-formatter interpolation left the suite green — and acceptance criterion 8 (events identify the detector, path, range and fingerprint transition) rested on code nothing executed. - `src/agent/agent-loop.test.ts`: one turn through the production path — real `os.fs.read`, real batch gate, real agent-loop wiring. Three reads with three different argument hashes, only the coverage detector able to see that the last two returned nothing new. It pins the detector's own floor of 2 (the generic threshold of 3 would never fire inside the turn), the read-specific notice reaching the next prompt (the generic one talks about repeated arguments, which was never true here), the `read` payload on the emitted event, and that no line of the file appears in it. - `src/tracing/trace/trace-recorder.test.ts`: the `read` payload survives into the recorded trace event, and is absent for detectors that have none. - `src/cli/trace-formatter.test.ts`: new file. `formatTraceChronology` had no test at all, so the `loop_detected` line — the only place a human sees why the detector fired — was unpinned, including the ` detector=` field this branch adds to EVERY loop_detected line and the back-compat path for older NDJSON traces that carry neither `detector` nor `read`. Verified by mutation: with these tests in place, deleting the threshold branch, the notice call, the event payload, the recorder passthrough or either half of the formatter interpolation each fails the suite. --- src/agent/agent-loop.test.ts | 82 +++++++++++++++++- src/cli/trace-formatter.test.ts | 101 +++++++++++++++++++++++ src/tracing/trace/trace-recorder.test.ts | 52 ++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 src/cli/trace-formatter.test.ts diff --git a/src/agent/agent-loop.test.ts b/src/agent/agent-loop.test.ts index 798f6d5a..06dfb459 100644 --- a/src/agent/agent-loop.test.ts +++ b/src/agent/agent-loop.test.ts @@ -1,9 +1,11 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { AgentLoop } from "./agent-loop.js"; +import type { AgentLoopEvent } from "./agent-loop.js"; import { buildDefaultToolRegistry } from "../tools/index.js"; +import { osFsReadTool } from "../tools/os/fs-read.js"; import { SlotManager } from "../llm/slot-manager.js"; import { createEmptySessionState } from "../session/session-state.js"; import type { @@ -612,6 +614,84 @@ describe("AgentLoop end-to-end with mock LLM", () => { expect(events[0]?.count).toBeGreaterThanOrEqual(3); }); + it("warns on the second no-progress re-read of one unchanged file", async () => { + // The read-coverage detector (issue #114) end to end, through the + // production path: the real `os.fs.read`, the real batch gate, the + // agent-loop's own threshold branch and notice formatting. Every one + // of the three reads below hashes to a different argument signature, + // so nothing but the coverage detector can see that the second and + // third returned only lines the first already showed. + const registry = buildDefaultToolRegistry(); + registry.register(osFsReadTool); + const body = Array.from({ length: 200 }, (_, i) => `line ${i + 1}`).join("\n"); + writeFileSync(join(workingDir, "src.ts"), `${body}\n`, "utf8"); + const script = [ + { tool: "os.fs.read", args: { path: "src.ts" } }, + { tool: "os.fs.read", args: { path: "src.ts", offset: 40, limit: 30 } }, + { tool: "os.fs.read", args: { path: "src.ts", offset: 90, limit: 30 } }, + { tool: "finish", args: { summary: "done" } }, + ]; + const prompts: string[] = []; + const detected: Extract[] = []; + const loop = new AgentLoop({ + registry, + slotManager: new SlotManager(2), + grammar: 'root ::= "ok"', + llmComplete: async ({ prompt }) => { + const step = prompts.length; + prompts.push(prompt); + return makeCompletion( + JSON.stringify(script[Math.min(step, script.length - 1)]), + ); + }, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + onEvent: (event) => { + if (event.type === "loop_detected") detected.push(event); + if (process.env.DBG && event.type === "loop_failed") console.log("ERRMSG", (event as any).error?.message); + }, + }); + const session = createEmptySessionState({ id: "s-read-loop", workingDir }); + await loop.runTurn(session, { + userMessage: "look at src.ts", + maxSteps: 6, + signal: new AbortController().signal, + }); + + // Exactly one warning, and it lands on the SECOND no-progress read — + // the detector's own floor of 2, not the generic warning threshold of + // 3, which would never have been reached inside this turn. + expect(detected).toHaveLength(1); + const event = detected[0]!; + expect(event.detector).toBe("read_repeat"); + expect(event.level).toBe("warn"); + expect(event.count).toBe(2); + expect(event.tool).toBe("os.fs.read"); + // The payload acceptance criterion 8 asks for: which file, which + // range came back, and the fingerprints on either side. Equal + // fingerprints are the evidence that the content did not move. + expect(event.read?.path).toContain("src.ts"); + expect(event.read?.startLine).toBe(90); + expect(event.read?.endLine).toBe(119); + expect(event.read?.fingerprint).toBeTruthy(); + expect(event.read?.previousFingerprint).toBe(event.read?.fingerprint); + // Line numbers and a path only — no line of the file in the event. + expect(JSON.stringify(event)).not.toContain("line 90"); + + // The read-specific notice — not the generic repeat one — reaches the + // next prompt. The generic notice talks about repeated ARGUMENTS, + // which is precisely the thing that was never true here. + const after = prompts.slice(3).join("\n"); + expect(after).toContain("without reaching a line you had not already read"); + expect(after).toContain("Already read this turn: lines 1-200"); + expect(after).not.toMatch(/same arguments \d+ times/); + // No notice before the detector fired. + expect(prompts.slice(0, 3).join("\n")).not.toContain( + "without reaching a line you had not already read", + ); + }); + it("ends the turn with a graceful reply (not loop_failed) when the breaker trips", async () => { const registry = buildDefaultToolRegistry(); let runCount = 0; diff --git a/src/cli/trace-formatter.test.ts b/src/cli/trace-formatter.test.ts new file mode 100644 index 00000000..52ea941a --- /dev/null +++ b/src/cli/trace-formatter.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect } from "vitest"; +import { formatTraceChronology } from "./trace-formatter.js"; +import type { TraceEvent, TraceLoopDetected } from "../tracing/index.js"; + +/** + * `trace show` rendering. The `loop_detected` line is the interesting one + * here: it is the only place a human ever sees why the read-coverage + * detector (issue #114) fired, and its payload is optional on the event, + * so every branch of the interpolation needs pinning — including the + * back-compat one, since old NDJSON traces carry neither `detector` nor + * `read`. + */ + +function loopDetected( + extra: Partial = {}, +): TraceLoopDetected { + return { + type: "loop_detected", + seq: 7, + sessionId: "s-1", + ts: Date.parse("2026-09-01T10:00:00.000Z"), + stepIndex: 4, + tool: "os.fs.read", + count: 2, + ...extra, + }; +} + +function render(events: readonly TraceEvent[]): string { + return formatTraceChronology(events); +} + +describe("formatTraceChronology loop_detected", () => { + it("renders the bare line for a trace that predates the detector fields", () => { + const line = render([loopDetected({ tool: "noop", count: 3 })]); + expect(line).toContain("#7 loop_detected"); + expect(line).toContain("step=4 tool=noop count=3"); + // Nothing invented for fields the event does not carry. + expect(line).not.toContain("detector="); + expect(line).not.toContain("path="); + expect(line).not.toContain("undefined"); + }); + + it("names the sub-detector when the event carries one", () => { + // Every detector benefits: "count=3" alone never said whether the + // generic repeat counter, the wandering spread or a test re-run was + // what tripped. + const line = render([loopDetected({ detector: "wandering", tool: "noop" })]); + expect(line).toContain("detector=wandering"); + expect(line).not.toContain("path="); + }); + + it("renders the file, the range and the fingerprint transition for a read repeat", () => { + const line = render([ + loopDetected({ + detector: "read_repeat", + level: "warn", + read: { + path: "/repo/src/agent/loop-detector.ts", + startLine: 90, + endLine: 119, + previousFingerprint: "ab12", + fingerprint: "ab12", + }, + }), + ]); + expect(line).toContain("detector=read_repeat"); + expect(line).toContain("path=/repo/src/agent/loop-detector.ts"); + expect(line).toContain("lines=90-119"); + // Equal fingerprints on either side are the evidence that the content + // stood still, which is what made the re-read redundant; a reader has + // to be able to check that claim rather than take it on faith. + expect(line).toContain("fingerprint=ab12→ab12"); + }); + + it("renders an empty return as the 0-0 range it was", () => { + const line = render([ + loopDetected({ + detector: "read_repeat", + read: { + path: "/repo/big.ts", + startLine: 0, + endLine: 0, + previousFingerprint: "cd34", + fingerprint: "cd34", + }, + }), + ]); + expect(line).toContain("lines=0-0"); + }); + + it("keeps the --step filter working on loop_detected lines", () => { + const events: TraceEvent[] = [ + loopDetected({ stepIndex: 1, detector: "read_repeat" }), + loopDetected({ seq: 8, stepIndex: 4, detector: "generic_repeat" }), + ]; + const only = formatTraceChronology(events, { step: 4 }); + expect(only.split("\n")).toHaveLength(1); + expect(only).toContain("detector=generic_repeat"); + }); +}); diff --git a/src/tracing/trace/trace-recorder.test.ts b/src/tracing/trace/trace-recorder.test.ts index 1991c453..23249f55 100644 --- a/src/tracing/trace/trace-recorder.test.ts +++ b/src/tracing/trace/trace-recorder.test.ts @@ -270,6 +270,58 @@ describe("createTraceRecorder", () => { }); }); + it("carries the read-repeat detector payload into the recorded event", () => { + // A `read_repeat` trace line is unreadable without the file, the + // range and the fingerprint pair: those three are the whole evidence + // that the re-read was redundant. They have to survive the recorder, + // not just the agent-loop event. + const { events, emit } = collector(); + const rec = createTraceRecorder({ sessionId: "s-7b", emit, now }); + rec.onAgentEvent({ type: "turn_started", turnIndex: 0 }); + rec.onAgentEvent({ + type: "loop_detected", + tool: "os.fs.read", + count: 2, + stepIndex: 4, + level: "warn", + detector: "read_repeat", + read: { + path: "/repo/src/a.ts", + startLine: 90, + endLine: 119, + previousFingerprint: "ff01", + fingerprint: "ff01", + }, + }); + expect(events.find((e) => e.type === "loop_detected")).toMatchObject({ + type: "loop_detected", + detector: "read_repeat", + read: { + path: "/repo/src/a.ts", + startLine: 90, + endLine: 119, + previousFingerprint: "ff01", + fingerprint: "ff01", + }, + }); + }); + + it("omits the read payload for detectors that have none", () => { + const { events, emit } = collector(); + const rec = createTraceRecorder({ sessionId: "s-7c", emit, now }); + rec.onAgentEvent({ type: "turn_started", turnIndex: 0 }); + rec.onAgentEvent({ + type: "loop_detected", + tool: "noop", + count: 3, + stepIndex: 1, + detector: "generic_repeat", + }); + const event = events.find((e) => e.type === "loop_detected"); + expect(event).toBeDefined(); + expect("read" in event!).toBe(false); + }); + it("ignores noisy events (reasoning_delta, assistant_delta, llm_completed)", () => { const { events, emit } = collector(); const rec = createTraceRecorder({ sessionId: "s-8", emit, now }); From b070c39310fc8b4754eb8372d7e6ed18936cadfd Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:24:48 +0300 Subject: [PATCH 12/36] fix(providers): stop the probe reporting our own limits as route defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the contract probe found several verdicts that describe something Atomic did, not something the route did. Each one reaches an operator as a sentence about their provider, so each is a bug. * **Our deadline was reported as `stream_early_eof`.** The timeout flag was discarded the moment any byte arrived, so a queued OpenRouter request (it sends `: OPENROUTER PROCESSING` while it waits) or a slow first token was classified as "closed the stream before finishing the answer. Turns will end mid-tool-call on this route" — and blocked from `reportModelConfigured`. The flag now survives: only a stream that announced its own end before the timer fired is classified at all, anything else is `timeout`, which is what it always was. * **An abort mid-stream was reported the same way.** Once headers had arrived the reader's rejection was swallowed and the partial body classified, so cancelling produced an invented route defect. The read now races the caller's signal, which also means Esc gets the screen back at once instead of at the deadline. * **Forced-tool-choice verdicts warned about a request Atomic never makes.** `step-executor` sends `tool_choice: "auto"` on every turn, deliberately, with production-observed reasons written down beside it. A route that refuses a forced choice and streams a complete call under `auto` therefore runs every real turn — it is now `proven`, not a warning that suppresses "this install has a working backend". A route that *accepts* forcing and ignores it no longer stops the ladder either: rung 2 asks the way a turn asks, and only if that also produces nothing is `forced_tool_choice_ignored` reported. * **The probe sent no `max_tokens`, which every turn sends.** `buildOpenAiChatBody` sets it unconditionally and has no `max_completion_tokens` fallback, so a model that rejects the field fails every turn — exactly the class of failure this probe exists to catch, and one it passed. The cap is now in the body, generous enough (1024) that it cannot truncate a one-field tool call, a `retry_token_field` refusal is reported as the new `token_cap_rejected` instead of being retried in a shape Atomic never uses, and `finish_reason: "length"` is read as inconclusive so our own cap can never be reported as a route defect either. Smaller corrections from the same review: * `parallel_tool_calls` is no longer hardcoded: the wizard resolves what a turn would send (`agent.maxParallelToolCalls > 1`). * Keyless-listing presets (Nous, Novita, Ollama Cloud, SambaNova, Sarvam) are skipped rather than probed with an empty credential and told their absent key "was rejected" — that flag is about listing models, not about completions. * `contractProbeFoundDefect` is gone. It had no caller and its doc described a warning policy the shipped code contradicts. * One timeout constant instead of two: the 20s default no caller passed described a budget that never existed. * The unreachable `MAX_PROBE_REQUESTS` guard is gone; the ladder is three fixed rungs and the deadline is the real bound. * `/llm check` is now in the `/llm` menu description, and prints the redacted provider detail when the route did not pass — until now that string was computed, scrubbed and never shown to anyone. --- .../verify/classify-contract-probe.test.ts | 16 +- .../verify/classify-contract-probe.ts | 42 +++- .../provider/verify/contract-probe-types.ts | 67 ++--- src/llm/provider/verify/index.ts | 1 - .../verify/run-contract-probe.test.ts | 180 +++++++++++++- src/llm/provider/verify/run-contract-probe.ts | 230 +++++++++++++----- src/tui/menu/menu-registry.test.ts | 8 +- src/tui/menu/menu-registry.ts | 2 +- src/tui/providers/contract-probe-target.ts | 28 ++- src/tui/providers/describe-contract-probe.ts | 10 +- .../providers/probe-wizard-contract.test.ts | 45 ++++ src/tui/providers/probe-wizard-contract.ts | 30 +-- src/tui/providers/providers-orchestrator.ts | 20 +- 13 files changed, 539 insertions(+), 140 deletions(-) diff --git a/src/llm/provider/verify/classify-contract-probe.test.ts b/src/llm/provider/verify/classify-contract-probe.test.ts index d2f1a9a6..8a0e0129 100644 --- a/src/llm/provider/verify/classify-contract-probe.test.ts +++ b/src/llm/provider/verify/classify-contract-probe.test.ts @@ -8,7 +8,6 @@ import { } from "./classify-contract-probe.js"; import { CONTRACT_PROBE_TOOL_NAME, - contractProbeFoundDefect, contractProbeProvesToolSupport, contractProbeToolDefinition, } from "./contract-probe-types.js"; @@ -216,17 +215,26 @@ describe("the synthetic probe tool", () => { expect(contractProbeProvesToolSupport("tools_supported")).toBe(true); for (const status of [ "inconclusive_no_tool_call", + "forced_tool_choice_ignored", "stream_early_eof", "malformed_tool_call", "tools_payload_rejected", + "token_cap_rejected", "provider_error", ] as const) { expect(contractProbeProvesToolSupport(status)).toBe(false); } }); - it("does not count an inconclusive auto answer as a defect", () => { - expect(contractProbeFoundDefect("inconclusive_no_tool_call")).toBe(false); - expect(contractProbeFoundDefect("stream_early_eof")).toBe(true); + it("counts a route that only refuses the forcing as proven", () => { + // Atomic never sends a forced tool choice (`step-executor` sends + // `auto` on every turn), so a route that refuses one and streams a + // complete call without it runs every real turn. Marking it + // unproven would warn operators about a bug they cannot hit and + // withhold "this install has a working backend" from an install + // that has one. + expect(contractProbeProvesToolSupport("forced_tool_choice_rejected")).toBe( + true, + ); }); }); diff --git a/src/llm/provider/verify/classify-contract-probe.ts b/src/llm/provider/verify/classify-contract-probe.ts index 86060ee6..4b524039 100644 --- a/src/llm/provider/verify/classify-contract-probe.ts +++ b/src/llm/provider/verify/classify-contract-probe.ts @@ -40,11 +40,13 @@ export function classifyContractProbeHttpFailure( ): ProviderContractStatus { const verdict = classifyVerifyResponse(httpStatus, body); if (verdict.kind === "retry_next_model") return "model_unavailable"; - // The key check's "resend with the other max-tokens field" hint is - // meaningless here: the probe deliberately sends no token cap (see - // `run-contract-probe`), so a body naming those fields is the route - // complaining about something else it read in our request. - if (verdict.kind === "retry_token_field") return "provider_error"; + // The key check answers this hint by resending with the other field. + // The probe must not: it sends the same `max_tokens` a turn sends + // (see `run-contract-probe`), and `buildOpenAiChatBody` has no + // `max_completion_tokens` fallback to switch to. Retrying would prove + // a route works in a shape Atomic never uses and report a pass for a + // route whose every turn 400s. So the refusal is the verdict. + if (verdict.kind === "retry_token_field") return "token_cap_rejected"; switch (verdict.status) { case "invalid_key": return "endpoint_auth_failed"; @@ -73,7 +75,10 @@ export function contractProbeFailureIsTerminal( return ( status === "endpoint_auth_failed" || status === "quota_or_routing_failed" || - status === "model_unavailable" + status === "model_unavailable" || + // Every rung carries the same token cap, so the next one would be + // refused for the same reason — and it is a real finding already. + status === "token_cap_rejected" ); } @@ -92,6 +97,14 @@ export function classifyProbeStream( ): ProviderContractStatus { if (!observation.terminalObserved) return "stream_early_eof"; + // The probe's own `max_tokens` ran out. A verbose or thinking model + // can spend it honestly before it gets to a tool call, or in the + // middle of one, so everything below this line would be blaming the + // route for a limit we set. The one thing still worth reading is a + // call that arrived *complete* despite the cut — that is proof, and + // it is checked below by the same rules as any other. + const truncatedByOurCap = observation.finishReason === "length"; + if (observation.sawToolCallDelta) { const call = observation.toolCalls.find((c) => c.name === CONTRACT_PROBE_TOOL_NAME) ?? @@ -99,14 +112,19 @@ export function classifyProbeStream( // Deltas arrived and still produced nothing callable. On the real // path this is the failure that surfaces as `tool not registered in // this agent`, several minutes into a turn. - if (!call || call.name.length === 0) return "malformed_tool_call"; - // Only one function was offered, so any other name is the route - // inventing one — the call could never be dispatched. - if (call.name !== CONTRACT_PROBE_TOOL_NAME) return "malformed_tool_call"; - if (!argumentsAreDispatchable(call.arguments)) return "malformed_tool_call"; - return "tools_supported"; + const dispatchable = + call !== undefined && + call.name.length > 0 && + // Only one function was offered, so any other name is the route + // inventing one — the call could never be dispatched. + call.name === CONTRACT_PROBE_TOOL_NAME && + argumentsAreDispatchable(call.arguments); + if (dispatchable) return "tools_supported"; + return truncatedByOurCap ? "inconclusive_no_tool_call" : "malformed_tool_call"; } + if (truncatedByOurCap) return "inconclusive_no_tool_call"; + // No tool call at all. What that means depends entirely on what we // asked for, and conflating the two cases is the specific mistake // this probe is built to avoid. diff --git a/src/llm/provider/verify/contract-probe-types.ts b/src/llm/provider/verify/contract-probe-types.ts index b021513a..3d55b8fa 100644 --- a/src/llm/provider/verify/contract-probe-types.ts +++ b/src/llm/provider/verify/contract-probe-types.ts @@ -38,14 +38,18 @@ export type ProviderContractStatus = /** One complete native tool call came back over the stream. */ | "tools_supported" /** - * The route took the tools payload but chose to answer in prose under - * `tool_choice: auto`. That is a legal answer for a model, so it says - * nothing about whether the route can emit tool calls at all. + * The route took the tools payload but produced no callable tool for + * a reason that says nothing about the route: it answered in prose + * under `tool_choice: auto` (legal for a model), or the probe's own + * `max_tokens` cut the answer short. */ | "inconclusive_no_tool_call" /** * The route accepted a forced named tool choice and then answered - * text anyway — it advertises the parameter without honoring it. + * text anyway — it advertises the parameter without honoring it — and + * called nothing under `auto` either. The second half matters: `auto` + * is the only mode a turn uses, so a route that ignores forcing but + * calls tools without it is reported as working, not as this. */ | "forced_tool_choice_ignored" /** @@ -58,6 +62,13 @@ export type ProviderContractStatus = * the same streamed completion once `tools` was removed. */ | "tools_payload_rejected" + /** + * The route refused the `max_tokens` cap every Atomic turn carries + * (newer OpenAI models want `max_completion_tokens` instead). Nothing + * was learned about tools, and nothing needed to be: the turn path + * has no second field to try, so it would fail the same way. + */ + | "token_cap_rejected" /** The configured model is unknown to this route. */ | "model_unavailable" /** The endpoint refused the credential; nothing else was learned. */ @@ -97,6 +108,14 @@ export interface ProviderContractProbeTarget { */ readonly model: string; readonly extraHeaders?: Record; + /** + * What a turn would put in `parallel_tool_calls` for this provider — + * the executor's cap and the provider's declared capability, resolved + * by the caller because neither is visible from here. Defaults to + * `true`, which is what `buildOpenAiChatBody` sends for a provider + * that declares nothing. + */ + readonly parallelToolCalls?: boolean; } export interface ProviderContractProbeResult { @@ -117,34 +136,26 @@ export interface ProviderContractProbeResult { } /** - * The one verdict that means "this route can run a turn". Everything - * else is either a failure or an open question, and neither may be - * reported as proven compatibility. + * The verdicts that mean "this route can run a turn". Everything else + * is either a failure or an open question, and neither may be reported + * as proven compatibility. + * + * Two of them qualify, because a turn is a narrower thing than the + * probe's primary instrument. `step-executor` sends + * `tool_choice: "auto"` on every request and never a forced or named + * choice — deliberately, with production-observed reasons written down + * beside it (Alibaba's Qwen-thinking gate answers `400 InvalidParameter` + * to a forced choice at all). A route that refuses the forcing and then + * streams a complete native tool call under `auto` therefore runs every + * real Atomic turn correctly, and calling it unproven would warn + * operators who are not hitting any bug. The forced rung stays first + * because it is the only mode in which "no tool call" is a statement + * about the route rather than about the model's mood. */ export function contractProbeProvesToolSupport( status: ProviderContractStatus, ): boolean { - return status === "tools_supported"; -} - -/** - * `true` when the probe learned something about the *route* rather than - * about the model's whim. Used to decide whether a warning is worth - * showing: an inconclusive auto-mode answer is not a defect to report. - */ -export function contractProbeFoundDefect( - status: ProviderContractStatus, -): boolean { - return ( - status === "forced_tool_choice_ignored" || - status === "forced_tool_choice_rejected" || - status === "tools_payload_rejected" || - status === "model_unavailable" || - status === "endpoint_auth_failed" || - status === "quota_or_routing_failed" || - status === "stream_early_eof" || - status === "malformed_tool_call" - ); + return status === "tools_supported" || status === "forced_tool_choice_rejected"; } /** diff --git a/src/llm/provider/verify/index.ts b/src/llm/provider/verify/index.ts index 87a43b75..560e7a45 100644 --- a/src/llm/provider/verify/index.ts +++ b/src/llm/provider/verify/index.ts @@ -15,7 +15,6 @@ export { } from "./classify-verify-response.js"; export { CONTRACT_PROBE_TOOL_NAME, - contractProbeFoundDefect, contractProbeProvesToolSupport, contractProbeToolDefinition, type ProbeToolChoiceMode, diff --git a/src/llm/provider/verify/run-contract-probe.test.ts b/src/llm/provider/verify/run-contract-probe.test.ts index 0d39d62f..1b7879e6 100644 --- a/src/llm/provider/verify/run-contract-probe.test.ts +++ b/src/llm/provider/verify/run-contract-probe.test.ts @@ -86,6 +86,38 @@ const MALFORMED_STREAM = ], }) + sseEvent({ choices: [{ delta: {}, finish_reason: "tool_calls" }] }); +/** + * A response whose body opens, says something, and then never ends — + * a queued OpenRouter request (`: OPENROUTER PROCESSING`) or a model + * still thinking about its first token. `stalled` resolves once the + * first chunk has been handed over, so a test can act mid-stream. + */ +function stallingStreamResponse(preamble: string): { + response: () => Response; + firstChunkRead: Promise; +} { + let seen = () => {}; + const firstChunkRead = new Promise((resolve) => { + seen = resolve; + }); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(preamble)); + // Never closed and never enqueued again: the socket is open and + // the route is thinking. + setTimeout(seen, 0); + }, + }); + return { + response: () => + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + firstChunkRead, + }; +} + function streamResponse(body: string): Response { return new Response(body, { status: 200, @@ -276,7 +308,11 @@ describe("runProviderContractProbe", () => { }); it("keeps credentials and whole response bodies out of the detail", async () => { - const leak = `${"x".repeat(400)} key=${TARGET.apiKey} other=sk-someoneelseskey123`; + // Both keys sit inside the first 300 characters, so the length cap + // cannot be what hides them: only redaction can. (The rules + // themselves are pinned in `redact-provider-detail.test.ts`.) + const leak = + `key=${TARGET.apiKey} other=sk-someoneelseskey123 ` + "x".repeat(400); const script = scriptedFetch([() => errorResponse(401, leak)]); const result = await runProviderContractProbe(TARGET, { fetchImpl: script.fetchImpl, @@ -284,6 +320,7 @@ describe("runProviderContractProbe", () => { expect(result.detail).not.toContain(TARGET.apiKey); expect(result.detail).not.toContain("sk-someoneelseskey123"); + expect(result.detail).toContain("key=***"); expect(result.detail.length).toBeLessThanOrEqual(300); }); @@ -310,4 +347,145 @@ describe("runProviderContractProbe", () => { expect(result.status).toBe("cancelled"); expect(script.calls()).toBe(0); }); + it("sends the token cap a real turn sends", async () => { + const script = scriptedFetch([() => streamResponse(TOOL_CALL_STREAM)]); + await runProviderContractProbe(TARGET, { fetchImpl: script.fetchImpl }); + + // `buildOpenAiChatBody` puts `max_tokens` on every turn and has no + // second field to fall back on, so a probe that left it out could + // pass on a route where every real message 400s. + expect(script.bodies()[0]!.max_tokens).toBeTypeOf("number"); + expect(script.bodies()[0]!.parallel_tool_calls).toBe(true); + }); + + it("sends the parallel_tool_calls the caller says a turn would send", async () => { + const script = scriptedFetch([() => streamResponse(TOOL_CALL_STREAM)]); + await runProviderContractProbe( + { ...TARGET, parallelToolCalls: false }, + { fetchImpl: script.fetchImpl }, + ); + + expect(script.bodies()[0]!.parallel_tool_calls).toBe(false); + }); + + it("reports a rejected token cap instead of retrying with the other field", async () => { + const script = scriptedFetch([ + () => + errorResponse(400, { + error: { + message: + "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.", + }, + }), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + // The turn path has no `max_completion_tokens` fallback, so this is + // the verdict, not a request to resend: every real turn would be + // refused the same way. + expect(result.status).toBe("token_cap_rejected"); + expect(script.calls()).toBe(1); + }); + + it("settles a route that ignores forcing by asking the way a turn asks", async () => { + const script = scriptedFetch([ + () => streamResponse(TEXT_STREAM), + () => streamResponse(TOOL_CALL_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + // Rung 1 was accepted and ignored, which says nothing about the + // mode Atomic runs in. Rung 2 asks in that mode and gets a complete + // tool call: the route works, and warning about the forced-choice + // quirk would be warning about a request Atomic never makes. + expect(result.status).toBe("tools_supported"); + expect(result.toolChoiceMode).toBe("auto"); + expect(script.calls()).toBe(2); + expect(script.bodies()[1]!.tool_choice).toBe("auto"); + }); + + it("reports an ignored forcing when auto declines as well", async () => { + const script = scriptedFetch([ + () => streamResponse(TEXT_STREAM), + () => streamResponse(TEXT_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + expect(result.status).toBe("forced_tool_choice_ignored"); + expect(script.calls()).toBe(2); + }); + + it("does not blame tools when rung 1 streamed and rung 2 failed", async () => { + const script = scriptedFetch([ + () => streamResponse(TEXT_STREAM), + () => errorResponse(500, "upstream exploded"), + () => streamResponse(TEXT_STREAM), + ]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + // The first rung carried `tools` and streamed, so the no-tools + // control could not attribute anything to them: spending a third + // request could only produce a wrong sentence. + expect(result.status).toBe("provider_error"); + expect(script.calls()).toBe(2); + }); + + it("calls a cap-truncated answer inconclusive, not a route defect", async () => { + const truncated = + sseEvent({ choices: [{ delta: { content: "Let me think about" } }] }) + + sseEvent({ choices: [{ delta: {}, finish_reason: "length" }] }) + + "data: [DONE]\n\n"; + const script = scriptedFetch([() => streamResponse(truncated)]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + }); + + // Our own `max_tokens` ended that answer. Reporting it as "ignored + // a forced tool choice" would blame the route for our limit. + expect(result.status).toBe("inconclusive_no_tool_call"); + }); + + it("calls its own deadline a timeout, even once bytes have arrived", async () => { + // OpenRouter's real queue keepalive, then silence. Under the old + // rule ("timed out with zero bytes") this comment alone turned a + // slow route into `stream_early_eof` — "turns will end + // mid-tool-call" — a defect invented by our own budget. + const stalling = stallingStreamResponse(": OPENROUTER PROCESSING\n\n"); + const script = scriptedFetch([stalling.response]); + const result = await runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + timeoutMs: 250, + }); + + expect(result.status).toBe("timeout"); + expect(script.calls()).toBe(1); + }); + + it("reports an abort that lands mid-stream as cancelled", async () => { + const stalling = stallingStreamResponse( + sseEvent({ choices: [{ delta: { content: "thinking" } }] }), + ); + const script = scriptedFetch([stalling.response]); + const controller = new AbortController(); + const running = runProviderContractProbe(TARGET, { + fetchImpl: script.fetchImpl, + signal: controller.signal, + timeoutMs: 5_000, + }); + await stalling.firstChunkRead; + controller.abort(); + + // The stream is open and half-read: without the signal in the read + // race this is a partial body, which classifies as + // `stream_early_eof` — a route defect we caused by giving up. + await expect(running).resolves.toMatchObject({ status: "cancelled" }); + }); }); diff --git a/src/llm/provider/verify/run-contract-probe.ts b/src/llm/provider/verify/run-contract-probe.ts index f9983b9f..8420962c 100644 --- a/src/llm/provider/verify/run-contract-probe.ts +++ b/src/llm/provider/verify/run-contract-probe.ts @@ -18,14 +18,21 @@ * * 1. **forced named tool**, streaming, tools payload. A complete tool * call here is the whole answer: `tools_supported`. - * 2. **`tool_choice: auto`**, same tools payload — reached only when - * rung 1 was *refused* for a reason that is not the key, the quota - * or the model. A tool call now means the payload is fine and it - * was the forcing the route would not take. + * 2. **`tool_choice: auto`**, same tools payload — reached when rung 1 + * was *refused* for a reason that is not the key, the quota or the + * model, or when it was *accepted and ignored*. This is the mode a + * real turn runs in (`step-executor` sends `auto` on every request, + * deliberately — several providers reject a forced choice outright), + * so a tool call here is proof about the shape Atomic actually uses: + * after a refusal it means only the forcing was unacceptable, and + * after an ignored forcing it means the route emits tool calls + * regardless. * 3. **no tools at all**, same model, same streaming transport — - * reached only when both tool requests were refused. If this + * reached only when both tool *requests* were refused. If this * answers, the route works and it is specifically `tools` it - * rejects; if it fails too, the failure was never about tools. + * rejects; if it fails too, the failure was never about tools. It is + * skipped when rung 1 streamed, because a stream already proved the + * route takes `tools` and the control could only mislead. * * Rung 3 is what turns "HTTP 400" into a sentence an operator can act * on, and it is deliberately an experiment rather than a regex over the @@ -64,17 +71,36 @@ import { redactProviderDetail } from "./redact-provider-detail.js"; /** * Whole-probe budget, not per request. Longer than the key check's 8s * because this one waits for a model to actually generate, but still - * short enough that a wizard screen does not feel hung: a route that - * cannot produce a two-field tool call inside this is a finding in - * itself. + * short enough that a wizard screen does not feel hung — every caller + * runs it with an operator watching, which is why this is the only + * budget in the module: a second, laxer default nobody passes would + * only describe a timeout that never happens. + * + * Blowing it is *our* verdict, not the route's: see `runRung`, which + * reports `timeout` rather than inventing a stream defect out of a slow + * route. */ -export const PROVIDER_CONTRACT_PROBE_TIMEOUT_MS = 20_000; +export const PROVIDER_CONTRACT_PROBE_TIMEOUT_MS = 12_000; /** Ceiling on the SSE body we buffer. A probe answer is a few hundred bytes. */ const MAX_STREAM_BYTES = 64 * 1024; -/** The whole ladder: forced → auto → no-tools. Never more than that. */ -const MAX_PROBE_REQUESTS = 3; +/** + * The cap a real turn always carries. `buildOpenAiChatBody` sets + * `max_tokens` on every request unconditionally and has no + * `max_completion_tokens` fallback, so a route that refuses the field + * refuses every Atomic turn — exactly the class of failure this probe + * exists to catch, and one it would miss by leaving the cap out. + * + * The value only has to be far above what one call to a single-field + * function costs; what a route validates is the field, not the number. + * Generous on purpose: a thinking model can spend hundreds of tokens + * before it calls anything, and a cap that truncated the answer would + * report our own doing as a route defect. `classifyProbeStream` guards + * the remainder of that risk by reading `finish_reason: "length"` as + * inconclusive. + */ +const PROBE_MAX_TOKENS = 1024; const PROBE_PROMPT = `Call the ${CONTRACT_PROBE_TOOL_NAME} function with ok set to true. ` + @@ -122,28 +148,47 @@ export async function runProviderContractProbe( }); // Rung 1 — the real contract, forced. - const forced = await run(probeBody(model, "required_named")); + const forced = await run(probeBody(target, model, "required_named")); if (forced.kind === "aborted") { return emit(forced.status, null, "required_named", forced.detail); } + // What rung 1 left open, in the two shapes rung 2 has to tell apart. + // Exactly one of these is set by the time rung 2 runs; anything else + // rung 1 saw was already the verdict and returned above. + let forcedRefusal: { httpStatus: number; body: string } | null = null; + let forcedIgnored: { httpStatus: number; detail: string } | null = null; if (forced.kind === "stream") { - return emit( - classifyProbeStream(forced.observation, "required_named"), + const forcedStatus = classifyProbeStream(forced.observation, "required_named"); + if (forcedStatus !== "forced_tool_choice_ignored") { + return emit( + forcedStatus, + forced.httpStatus, + "required_named", + streamDetail(forced.observation), + ); + } + // The route took `tool_choice` and then answered prose anyway. That + // is a real observation about the route, but it is not yet an + // answer about Atomic: turns never force a tool (`step-executor` + // sends `auto`), so what still has to be settled is whether this + // route emits tool calls in the mode it will actually be run in. + forcedIgnored = { + httpStatus: forced.httpStatus, + detail: streamDetail(forced.observation), + }; + } else { + const forcedStatus = classifyContractProbeHttpFailure( forced.httpStatus, - "required_named", - streamDetail(forced.observation), + forced.body, ); - } - const forcedStatus = classifyContractProbeHttpFailure( - forced.httpStatus, - forced.body, - ); - if (contractProbeFailureIsTerminal(forcedStatus)) { - return emit(forcedStatus, forced.httpStatus, "required_named", forced.body); + if (contractProbeFailureIsTerminal(forcedStatus)) { + return emit(forcedStatus, forced.httpStatus, "required_named", forced.body); + } + forcedRefusal = { httpStatus: forced.httpStatus, body: forced.body }; } - // Rung 2 — same payload, no forcing. - const auto = await run(probeBody(model, "auto")); + // Rung 2 — same payload, no forcing. This is the request a turn makes. + const auto = await run(probeBody(target, model, "auto")); if (auto.kind === "aborted") { return emit(auto.status, null, "auto", auto.detail); } @@ -151,13 +196,27 @@ export async function runProviderContractProbe( const autoStatus = classifyProbeStream(auto.observation, "auto"); // Tools work; it was the forced choice rung 1 asked for that this // route would not take. Reporting rung 2's own verdict here would - // hide the actual limitation behind a cheerful "supported". - if (autoStatus === "tools_supported") { + // hide the limitation, so it keeps its own status — one that says + // "usable", because `auto` is all a turn ever sends. + if (autoStatus === "tools_supported" && forcedRefusal) { return emit( "forced_tool_choice_rejected", - forced.httpStatus, + forcedRefusal.httpStatus, + "required_named", + forcedRefusal.body, + ); + } + // Forcing ignored *and* nothing called under `auto`: two requests + // and not one tool call. Still not proof that the route cannot emit + // them — a model may simply keep declining a pointless function — + // but "it ignores a forced choice" is the sharper of the two + // observations, so that is the one reported. + if (forcedIgnored && autoStatus === "inconclusive_no_tool_call") { + return emit( + "forced_tool_choice_ignored", + forcedIgnored.httpStatus, "required_named", - forced.body, + forcedIgnored.detail, ); } return emit(autoStatus, auto.httpStatus, "auto", streamDetail(auto.observation)); @@ -166,9 +225,15 @@ export async function runProviderContractProbe( if (contractProbeFailureIsTerminal(autoStatus)) { return emit(autoStatus, auto.httpStatus, "auto", auto.body); } + if (forcedIgnored) { + // Rung 1 streamed, so this route demonstrably accepts `tools`; the + // no-tools control could not attribute rung 2's refusal to them and + // would only spend a request to reach a wrong sentence. + return emit(autoStatus, auto.httpStatus, "auto", auto.body); + } // Rung 3 — the control. Same model, same streaming transport, no tools. - const control = await run(probeBody(model, null)); + const control = await run(probeBody(target, model, null)); if (control.kind === "aborted") { return emit(control.status, null, null, control.detail); } @@ -215,13 +280,6 @@ async function runRung( if (ctx.signal?.aborted) { return { kind: "aborted", status: "cancelled", detail: "probe cancelled" }; } - if (ctx.state.requests >= MAX_PROBE_REQUESTS) { - return { - kind: "aborted", - status: "provider_error", - detail: "probe budget spent", - }; - } const remainingMs = ctx.deadline - Date.now(); if (remainingMs <= 0) { return { kind: "aborted", status: "timeout", detail: "probe deadline reached" }; @@ -275,38 +333,51 @@ async function runRung( // `openAiFetch`'s own timeout covers the connect only — it clears the // timer the moment headers arrive. A route that opens a stream and // then stalls forever would hang here, so the body read carries the - // remaining budget itself. - const sse = await readStreamBounded(res, ctx.deadline - Date.now()); - if (sse.timedOut && sse.text.length === 0) { + // remaining budget itself, and the operator's abort as well. + const sse = await readStreamBounded( + res, + ctx.deadline - Date.now(), + ctx.signal, + ); + if (sse.aborted || ctx.signal?.aborted) { + return { kind: "aborted", status: "cancelled", detail: "probe cancelled" }; + } + const observation = accumulateProbeStream(sse.text); + // Our own deadline is not a route defect. A single byte is enough to + // put text in the buffer — OpenRouter sends `: OPENROUTER PROCESSING` + // while a request is queued, and a reasoning model can take seconds + // over its first token — so keying this off "no bytes arrived" would + // report every slow route as `stream_early_eof` ("turns will end + // mid-tool-call"), a defect we invented by giving up first. Only a + // stream that announced its own end before the timer fired is a + // complete observation worth classifying. + if (sse.timedOut && !observation.terminalObserved) { return { kind: "aborted", status: "timeout", - detail: "no stream data before deadline", + detail: `no complete stream before deadline (${sse.text.length} bytes read)`, }; } - return { - kind: "stream", - httpStatus: res.status, - observation: accumulateProbeStream(sse.text), - }; + return { kind: "stream", httpStatus: res.status, observation }; } /** * The probe request, in the same shape `buildOpenAiChatBody` gives a - * real turn — including `parallel_tool_calls`, which some routes - * validate and which a turn always sends alongside tools. + * real turn — the streamed transport, the tools payload, + * `parallel_tool_calls`, and the `max_tokens` cap a turn always carries. + * Sending anything less would let a route pass the probe and then fail + * the first message on a field the probe never showed it. * * `mode === null` is the control: no tools, no tool choice, otherwise * identical, so a difference in outcome can only be the tools payload. * - * No `max_tokens`. The one-token key check has to cap spend and - * therefore has to guess between `max_tokens` and - * `max_completion_tokens` (newer OpenAI models reject the former). - * Here a cap would risk truncating the very tool call being measured, - * reporting malformed deltas that were in fact our own doing — and one - * forced call to a single-field function is cheap enough uncapped. + * The one field a turn sends that this cannot is `parallel_tool_calls`' + * *value* — a turn computes it from the provider's declared capability + * and the executor's cap. The target carries it when the caller knows + * it; `true` is what a wizard-saved cloud provider gets by default. */ function probeBody( + target: ProviderContractProbeTarget, model: string, mode: ProbeToolChoiceMode | null, ): Record { @@ -314,11 +385,12 @@ function probeBody( model, messages: [{ role: "user", content: PROBE_PROMPT }], temperature: 0, + max_tokens: PROBE_MAX_TOKENS, stream: true, }; if (mode === null) return body; body.tools = [contractProbeToolDefinition()]; - body.parallel_tool_calls = true; + body.parallel_tool_calls = target.parallelToolCalls ?? true; body.tool_choice = mode === "required_named" ? { type: "function", function: { name: CONTRACT_PROBE_TOOL_NAME } } @@ -345,19 +417,30 @@ function streamDetail(observation: ProbeStreamObservation): string { } /** - * Buffer the SSE body under a byte ceiling and a deadline, cancelling - * the stream rather than leaving a socket open behind us. + * Buffer the SSE body under a byte ceiling, a deadline and the caller's + * abort, cancelling the stream rather than leaving a socket open behind + * us. + * + * All three outcomes are reported separately because they mean + * different things about the route: a body that simply stopped is the + * early EOF the probe is hunting for, while a deadline or an abort is + * something *we* did and must never be dressed up as one. */ async function readStreamBounded( res: Response, budgetMs: number, -): Promise<{ text: string; timedOut: boolean }> { - if (!res.body) return { text: await res.text().catch(() => ""), timedOut: false }; + signal?: AbortSignal, +): Promise<{ text: string; timedOut: boolean; aborted: boolean }> { + if (!res.body) { + return { text: await res.text().catch(() => ""), timedOut: false, aborted: false }; + } const reader = res.body.getReader(); const decoder = new TextDecoder(); let text = ""; let timedOut = false; + let aborted = false; let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; try { const deadline = budgetMs > 0 @@ -365,8 +448,25 @@ async function readStreamBounded( timer = setTimeout(() => resolve("deadline"), budgetMs); }) : Promise.resolve<"deadline">("deadline"); + // A stalled stream does not reject on abort by itself on every + // transport, so the signal is raced rather than waited for: an + // operator pressing Esc must get the screen back now, not at the + // deadline. + const cancelled = new Promise<"aborted">((resolve) => { + if (!signal) return; + if (signal.aborted) { + resolve("aborted"); + return; + } + onAbort = () => resolve("aborted"); + signal.addEventListener("abort", onAbort, { once: true }); + }); for (;;) { - const next = await Promise.race([reader.read(), deadline]); + const next = await Promise.race([reader.read(), deadline, cancelled]); + if (next === "aborted") { + aborted = true; + break; + } if (next === "deadline") { timedOut = true; break; @@ -378,12 +478,16 @@ async function readStreamBounded( text += decoder.decode(next.value, { stream: true }); if (text.length >= MAX_STREAM_BYTES) break; } - } catch { + } catch (err) { // A stream that breaks mid-body is exactly the early-EOF case: keep // what arrived and let the classifier see that it never terminated. + // Unless it broke because the caller aborted the request, which is + // not a fact about the route at all. + if (signal?.aborted || isAbortError(err)) aborted = true; } finally { if (timer) clearTimeout(timer); + if (onAbort) signal?.removeEventListener("abort", onAbort); void reader.cancel().catch(() => {}); } - return { text, timedOut }; + return { text, timedOut, aborted }; } diff --git a/src/tui/menu/menu-registry.test.ts b/src/tui/menu/menu-registry.test.ts index 748e90d5..a05da31e 100644 --- a/src/tui/menu/menu-registry.test.ts +++ b/src/tui/menu/menu-registry.test.ts @@ -145,10 +145,12 @@ const V0_2_2_SLASH_COMMANDS = [ }, { name: "llm", - // Updated when the Fallback pane got its deep link: the palette must - // advertise all four panes, not the three that predate it. + // Updated when the Fallback pane got its deep link, and again for + // `/llm check`: every subcommand the handler answers has to be + // reachable from here, or it exists only for whoever types an + // invalid one and reads the usage line. description: - "open LLM Local/Cloud/External/Fallback panel · `/llm provider ` switch text provider · `/llm fallback` edit the fallover chain", + "open LLM Local/Cloud/External/Fallback panel · `/llm provider ` switch text provider · `/llm check` test the active route's streaming tool contract · `/llm fallback` edit the fallover chain", }, { name: "mcp", diff --git a/src/tui/menu/menu-registry.ts b/src/tui/menu/menu-registry.ts index 96040018..b63242ba 100644 --- a/src/tui/menu/menu-registry.ts +++ b/src/tui/menu/menu-registry.ts @@ -290,7 +290,7 @@ export const MENU: readonly MenuNode[] = [ slash: { name: "llm", description: - "open LLM Local/Cloud/External/Fallback panel · `/llm provider ` switch text provider · `/llm fallback` edit the fallover chain", + "open LLM Local/Cloud/External/Fallback panel · `/llm provider ` switch text provider · `/llm check` test the active route's streaming tool contract · `/llm fallback` edit the fallover chain", rank: 24, }, section: "manage", diff --git a/src/tui/providers/contract-probe-target.ts b/src/tui/providers/contract-probe-target.ts index cd926373..efbe9da6 100644 --- a/src/tui/providers/contract-probe-target.ts +++ b/src/tui/providers/contract-probe-target.ts @@ -20,6 +20,7 @@ * how a diagnostic tool loses their trust. */ +import { getConfig } from "../../config/index.js"; import type { ProviderContractProbeTarget } from "../../llm/provider/verify/index.js"; import { isLocalProviderUrl } from "./is-local-provider-url.js"; import { @@ -27,7 +28,6 @@ import { chosenModelForWizard, endpointForKind, providerLabelForWizard, - wizardKeyIsOptional, } from "./providers-wizard-target.js"; import type { ProvidersWizardState } from "./providers-wizard-state.js"; @@ -57,19 +57,24 @@ export function contractProbeTargetForWizard( // this contract, and inventing one would probe a URL it never uses. if (kind === "claude-cli" || kind === "codex-cli") return skip("cli_backed"); - const apiKey = apiKeyForWizard(wizard)?.trim() ?? ""; - // Keyless is legitimate for local servers and keyless-listing - // services; a missing key everywhere else is already refused by the - // key screen, and probing without one would only re-report that. - if (!apiKey && !wizardKeyIsOptional(wizard)) return skip("no_api_key"); - const endpoint = endpointForKind(kind, wizard); // A server on this machine is the operator's own: reachable, free to // call, and a probe against it says more about their llama-server // flags than about a provider. The key check skips it for the same - // reason. + // reason. Checked before the key, because a local server having none + // is the *reason* it has none. if (isLocalProviderUrl(endpoint.baseUrl)) return skip("local_endpoint"); + const apiKey = apiKeyForWizard(wizard)?.trim() ?? ""; + // No key, no probe — including for the presets `wizardKeyIsOptional` + // lets through. That flag means "this service lists its models + // without a key", which is true of Nous, Novita, Ollama Cloud, + // SambaNova and Sarvam and says nothing about completions: they all + // answer a keyless one with a 401. Probing anyway would spend a + // request to tell the operator their key "was rejected" when no key + // was ever sent. + if (!apiKey) return skip("no_api_key"); + const model = chosenModelForWizard(wizard).trim(); if (!model) return skip("no_model"); @@ -81,6 +86,13 @@ export function contractProbeTargetForWizard( apiPathPrefix: endpoint.apiPathPrefix, apiKey, model, + // What `buildOpenAiChatBody` would put in `parallel_tool_calls` + // for a provider saved from this wizard. The wizard has no screen + // for the per-provider `supportsTools` flag, so the executor's cap + // is the only half of the turn's expression that can differ here + // (`step-executor`: `maxParallelToolCalls > 1 && + // supportsParallelTools`). + parallelToolCalls: getConfig().agent.maxParallelToolCalls > 1, ...(endpoint.extraHeaders ? { extraHeaders: endpoint.extraHeaders } : {}), }, }; diff --git a/src/tui/providers/describe-contract-probe.ts b/src/tui/providers/describe-contract-probe.ts index c1df4bfb..b9642615 100644 --- a/src/tui/providers/describe-contract-probe.ts +++ b/src/tui/providers/describe-contract-probe.ts @@ -9,6 +9,10 @@ * `inconclusive_no_tool_call` is the model declining to call a * pointless function, which is legal behaviour, not a broken route. * - Never say "compatible" for anything but a completed tool call. + * - Never describe a limitation Atomic cannot hit. A turn always sends + * `tool_choice: "auto"` (see `step-executor`), so a route that + * refuses a *forced* choice and streams a call without one is + * working, and its sentence has to read that way. * - Always name the next move. `HTTP 400` on a setup screen reads as a * product failure; "answers fine until `tools` is in the request" * reads as a route to change. @@ -28,9 +32,11 @@ export function describeContractProbeOutcome( case "inconclusive_no_tool_call": return `${who} answered in text instead of calling a tool${on}. Inconclusive: it would not take a forced tool choice, so whether it can emit tool calls is still unknown.`; case "forced_tool_choice_ignored": - return `${who} accepted a forced tool choice${on} and answered in text anyway. Turns that must call a tool may loop or stall on this route.`; + return `${who} took a forced tool choice${on} and answered in text anyway, and called nothing under "auto" either. Inconclusive, but no request has yet produced a tool call on this route.`; case "forced_tool_choice_rejected": - return `${who} refuses a forced tool choice${on} but does emit tool calls without one${statusSuffix(result)}. Usable, with less control over when tools fire.`; + return `${who} refuses a forced tool choice${on} but streams native tool calls without one${statusSuffix(result)} — which is all Atomic ever asks for, so this route can run a turn.`; + case "token_cap_rejected": + return `${who} rejected the "max_tokens" cap Atomic puts on every request${statusSuffix(result)}. This model wants "max_completion_tokens"; real turns would fail the same way, so pick another model or route.`; case "tools_payload_rejected": return `${who} answers this model until "tools" is in the request, then refuses it${statusSuffix(result)}. Pick another model or route — Atomic sends tools on every turn.`; case "model_unavailable": diff --git a/src/tui/providers/probe-wizard-contract.test.ts b/src/tui/providers/probe-wizard-contract.test.ts index dae85423..c03cb533 100644 --- a/src/tui/providers/probe-wizard-contract.test.ts +++ b/src/tui/providers/probe-wizard-contract.test.ts @@ -167,4 +167,49 @@ describe("probeWizardContract", () => { ); expect(bodies[0]).toContain("vendor/chosen-model"); }); + it("treats a route that only refuses the forcing as one that can run a turn", async () => { + // Refuses `tool_choice: {type:"function"}`, streams a complete call + // under `auto` — which is the only mode `step-executor` ever sends. + // Every real turn on this route works, so warning about it would be + // warning about a bug the operator is not having. + vi.stubGlobal( + "fetch", + vi.fn(async (_url: unknown, init?: RequestInit) => { + const body = String(init?.body ?? ""); + if (body.includes('"tool_choice":{')) { + return new Response( + JSON.stringify({ error: "tool_choice does not support being set to object" }), + { status: 400 }, + ); + } + return new Response(TOOL_CALL_STREAM, { status: 200 }); + }), + ); + const outcome = await probeWizardContract(wizard("openrouter")); + + expect(outcome.result?.status).toBe("forced_tool_choice_rejected"); + expect(outcome.proven).toBe(true); + expect(outcome.warning).toBeNull(); + expect(outcome.summary).toContain("can run a turn"); + }); + + it("does not tell a keyless-listing service its absent key was rejected", async () => { + // `wizardKeyIsOptional` is true for these presets because they list + // models without a key — not because a completion works without + // one. Probing anyway earns a 401 and the sentence "rejected the + // key" about a key nobody sent. + const fetchMock = vi.fn(async () => new Response("Unauthorized", { status: 401 })); + vi.stubGlobal("fetch", fetchMock); + const outcome = await probeWizardContract( + wizard("openai-compatible", { + apiKeyBuffer: "", + presetId: "nous", + baseUrlLine: "https://inference-api.nousresearch.com", + }), + ); + + expect(outcome.skipped).toBe("no_api_key"); + expect(outcome.summary).toContain("no API key"); + expect(fetchMock).not.toHaveBeenCalled(); + }); }); diff --git a/src/tui/providers/probe-wizard-contract.ts b/src/tui/providers/probe-wizard-contract.ts index 529d5fae..d02ad014 100644 --- a/src/tui/providers/probe-wizard-contract.ts +++ b/src/tui/providers/probe-wizard-contract.ts @@ -8,13 +8,17 @@ * worth refusing because nothing downstream can work without one. * - This answers "can this route run a turn", and the honest response * to "no" is a warning, not a refusal. Some providers block synthetic - * probes outright; a custom endpoint the operator knows works must - * still be savable, and the operator is the one who decides whether - * to live with a route that fires tools only under `auto`. + * probes outright, and a custom endpoint the operator knows works + * must still be savable. * * What it must not do is let a failed probe pass for a proven one — the * caller keys "this install has a working cloud backend" off a clean - * result, so an unproven route reports as unproven. + * result, so an unproven route reports as unproven. Nor may it warn + * about something a turn cannot hit: `contractProbeProvesToolSupport` + * counts a route that refuses a *forced* tool choice and streams a call + * under `auto` as proven, because `auto` is the only mode Atomic ever + * sends, and a warning there would be about a bug the operator is not + * having. */ import { @@ -40,7 +44,10 @@ export interface WizardContractProbeOutcome { readonly proven: boolean; /** * What to show the operator, or `null` when there is nothing worth - * saying: the route passed, or there was nothing here to probe. + * saying: the route ran a turn's worth of work, or there was nothing + * here to probe. `null` is also what the caller's + * "report a working backend" gate keys off, so a probe that ran and + * did not prove the route always fills this in. */ readonly warning: string | null; /** @@ -55,14 +62,6 @@ export interface WizardContractProbeOutcome { readonly result: ProviderContractProbeResult | null; } -/** - * Tighter than the probe module's own budget. This one runs with an - * operator watching a wizard screen: a route slow enough to blow - * through it has told us something already, and the save can proceed - * with the warning rather than holding the screen. - */ -export const WIZARD_CONTRACT_PROBE_TIMEOUT_MS = 12_000; - export async function probeWizardContract( wizard: ProvidersWizardState, opts: { signal?: AbortSignal; timeoutMs?: number } = {}, @@ -85,9 +84,12 @@ export async function probeWizardContract( } const target = resolved.target; + // No budget of its own: `PROVIDER_CONTRACT_PROBE_TIMEOUT_MS` is + // already sized for an operator watching a wizard screen, and this is + // the only entry point the probe has. const result = await runProviderContractProbe(target, { ...(opts.signal ? { signal: opts.signal } : {}), - timeoutMs: opts.timeoutMs ?? WIZARD_CONTRACT_PROBE_TIMEOUT_MS, + ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), }); const proven = contractProbeProvesToolSupport(result.status); const summary = describeContractProbeOutcome(result, target.label); diff --git a/src/tui/providers/providers-orchestrator.ts b/src/tui/providers/providers-orchestrator.ts index aea038ca..4d660bd0 100644 --- a/src/tui/providers/providers-orchestrator.ts +++ b/src/tui/providers/providers-orchestrator.ts @@ -532,11 +532,25 @@ export class ProvidersOrchestrator { // Every path here says what actually happened, including the ones // where no request went out. Reporting a skip as a pass would be // the silent "fully compatible" this whole check exists to stop. - const line = wizard - ? (await probeWizardContract(wizard)).summary - : `"${id}" has no provider kind that can be contract-checked.`; + const outcome = wizard ? await probeWizardContract(wizard) : null; + const line = + outcome?.summary ?? + `"${id}" has no provider kind that can be contract-checked.`; this.bus.emit({ type: "providers_status", line }); this.bus.emit({ type: "runtime_info", line }); + // What the route actually said, on the one surface where it is + // worth the noise: this command was typed to diagnose something, + // and a verdict sentence alone leaves the operator guessing which + // 400 they are looking at. Bounded and credential-scrubbed at the + // source (`redactProviderDetail`), and only when there is a + // problem — a passing route's stream summary tells nobody + // anything. + if (outcome?.result && !outcome.proven && outcome.result.detail.length > 0) { + this.bus.emit({ + type: "runtime_info", + line: `Route said: ${outcome.result.detail}`, + }); + } } catch (err) { this.bus.emit({ type: "providers_status", From 6322a0d2d7301d97519e49c1b3739a2b256d57e4 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:25:01 +0300 Subject: [PATCH 13/36] test(providers): cover the probe's wiring, its gate and its redaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three claims the feature is sold on had no test that could fail. Both save paths could be deleted wholesale and every test still passed: the wizard's `completeWizard` tests all stop at a refused key, so nothing reached the probe call, the gate, or the note the operator gets. `completeWizard` now runs end to end with only the disk writes stubbed — real key check, real probe, real gate — in two shapes: a route that streams a native tool call is saved *and* reported as a working backend, and a route whose stream ends early is saved, warned about, and not reported. First-run onboarding gets the same pair. Deleting either call site, or weakening the gate to `gate.warning === null`, now fails four tests. The redaction test was vacuous: 400 filler characters sat in front of the credentials, so the 300-character cap satisfied both assertions on its own and the test passed with `redactProviderDetail` reduced to a plain slice. The fixture now puts both keys inside the cap, and `redact-provider-detail.test.ts` pins each rule on strings short enough that truncation cannot do the work — including the pattern pass, the one capability this adds over the key check's exact-key removal, which nothing in the repo detected the loss of. Verified by mutation, each of these now fails at least one test: redaction disabled; the `KEY_SHAPED` loop deleted; the probe removed from either save path; the gate weakened; the old "timed out with zero bytes" rule restored; the abort dropped from the read race; `max_tokens` removed from the probe body; `forced_tool_choice_rejected` dropped from the proven set. --- .../verify/redact-provider-detail.test.ts | 78 +++++++++ .../cloud-provider-onboarding.test.tsx | 116 +++++++++++++ .../providers/providers-orchestrator.test.ts | 155 ++++++++++++++++++ 3 files changed, 349 insertions(+) create mode 100644 src/llm/provider/verify/redact-provider-detail.test.ts diff --git a/src/llm/provider/verify/redact-provider-detail.test.ts b/src/llm/provider/verify/redact-provider-detail.test.ts new file mode 100644 index 00000000..2c4a8433 --- /dev/null +++ b/src/llm/provider/verify/redact-provider-detail.test.ts @@ -0,0 +1,78 @@ +/** + * Redaction is the only thing standing between a provider's error body + * and a status line, so each rule is exercised on a string short enough + * that the length cap cannot do the work for it. A test whose fixture + * is longer than `PROVIDER_DETAIL_MAX_LEN` passes with redaction + * removed entirely, which is worse than no test at all. + */ + +import { describe, expect, it } from "vitest"; + +import { + PROVIDER_DETAIL_MAX_LEN, + redactProviderDetail, +} from "./redact-provider-detail.js"; + +const KEY = "sk-ours-1234567890"; + +describe("redactProviderDetail", () => { + it("removes the key we sent, wherever the provider echoed it", () => { + const detail = redactProviderDetail( + `invalid key ${KEY} for org (header: Bearer ${KEY})`, + KEY, + ); + + expect(detail).not.toContain(KEY); + expect(detail.length).toBeLessThan(PROVIDER_DETAIL_MAX_LEN); + }); + + it("removes a key-shaped string that is not the one under test", () => { + // The case the exact-match rule cannot reach: a gateway quoting the + // upstream credential it uses on our behalf. Short on purpose — + // truncation must not be what hides this. + const detail = redactProviderDetail( + "upstream rejected sk-or-v1-abcdef0123456789 (routed)", + KEY, + ); + + expect(detail).not.toContain("sk-or-v1-abcdef0123456789"); + expect(detail).toContain("upstream rejected"); + expect(detail).toContain("(routed)"); + }); + + it("removes Google keys and quoted bearer tokens", () => { + const detail = redactProviderDetail( + 'API key AIzaSyD-0123456789abcdef invalid; sent "Bearer ghp_0123456789abcd"', + "", + ); + + expect(detail).not.toContain("AIzaSyD-0123456789abcdef"); + expect(detail).not.toContain("ghp_0123456789abcd"); + }); + + it("leaves an ordinary provider message readable", () => { + // The other half of the contract: a pattern loose enough to redact + // model ids and error codes would make every verdict unreadable. + const message = + "400 InvalidParameter: tool_choice does not support being set to object"; + expect(redactProviderDetail(message, KEY)).toBe(message); + expect(redactProviderDetail("model deepseek-v4-flash not found", KEY)).toBe( + "model deepseek-v4-flash not found", + ); + }); + + it("never lets a whole body through, redacted or not", () => { + const body = `{"error":{"message":"${"detail ".repeat(200)}"}}`; + expect(redactProviderDetail(body, KEY)).toHaveLength( + PROVIDER_DETAIL_MAX_LEN, + ); + }); + + it("ignores a key too short to be one, rather than shredding words", () => { + // `split(apiKey).join("***")` on a 3-character "key" would cut the + // message to pieces; the length floor is what stops it. + expect(redactProviderDetail("the model was not found", "the")).toBe( + "the model was not found", + ); + }); +}); diff --git a/src/tui/components/cloud-provider-onboarding.test.tsx b/src/tui/components/cloud-provider-onboarding.test.tsx index f90697f1..40064972 100644 --- a/src/tui/components/cloud-provider-onboarding.test.tsx +++ b/src/tui/components/cloud-provider-onboarding.test.tsx @@ -61,6 +61,9 @@ vi.mock("../providers/verify-wizard-before-save.js", async (importOriginal) => { }); const currentConfig = { + // The contract probe reads `agent.maxParallelToolCalls` to send the + // `parallel_tool_calls` a real turn would send. + agent: { maxParallelToolCalls: 8 }, llm: { activeTextProvider: "local-llama", activeEmbeddingProvider: "local-llama-embed", @@ -125,6 +128,72 @@ function stubGatedProbe(status = 429): ProbeGate { }; } +/** + * A fetch that lets the key check pass and answers the contract probe — + * the streamed request carrying `tools` — with `sse`. + */ +function stubProbeFetch(sse: string): { probeBodies: () => string[] } { + const probeBodies: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: unknown, init?: RequestInit) => { + if (!String(url).includes("/chat/completions")) { + return new Response(JSON.stringify({ data: [] }), { status: 200 }); + } + const body = String(init?.body ?? "{}"); + if (!body.includes('"stream":true')) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "ok" } }] }), + { status: 200 }, + ); + } + probeBodies.push(body); + return new Response(sse, { status: 200 }); + }), + ); + return { probeBodies: () => probeBodies }; +} + +/** A complete native tool call: the one verdict that proves the route. */ +const PROBE_TOOL_CALL_SSE = + `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { + name: "atomic_contract_probe", + arguments: '{"ok":true}', + }, + }, + ], + }, + }, + ], + })}\n\ndata: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: "tool_calls" }], + })}\n\ndata: [DONE]\n\n`; + +/** Truncated mid-argument, with nothing announcing the end. */ +const PROBE_EARLY_EOF_SSE = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "atomic_contract_probe", arguments: '{"ok"' }, + }, + ], + }, + }, + ], +})}\n\n`; + async function flush(times = 6): Promise { for (let i = 0; i < times; i += 1) { await new Promise((resolve) => setImmediate(resolve)); @@ -321,3 +390,50 @@ describe("CloudProviderOnboarding cancellation", () => { unmount(); }); }); + +describe("CloudProviderOnboarding contract probe", () => { + beforeEach(() => { + saveMock.mockClear(); + gateOverrides.length = 0; + // The key itself is not what these tests are about; they are about + // what happens between a good key and the save. + gateOverrides.push(async () => ({ proceed: true, warning: null })); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("exercises the streaming tool contract before finishing first-run", async () => { + const probe = stubProbeFetch(PROBE_TOOL_CALL_SSE); + const { stdin, onFinished, unmount } = await mountAtSubmitPoint(); + + stdin.write("\r"); + await flush(12); + + // First-run is where a route that cannot run a turn costs the most: + // the operator's very first message would otherwise be the test. + expect(probe.probeBodies()).toHaveLength(1); + expect(probe.probeBodies()[0]).toContain("atomic_contract_probe"); + expect(saveMock).toHaveBeenCalledTimes(1); + expect(onFinished).toHaveBeenCalledWith("saved_cloud", undefined); + unmount(); + }); + + it("carries the probe's verdict into the finish note without blocking the save", async () => { + stubProbeFetch(PROBE_EARLY_EOF_SSE); + const { stdin, onFinished, unmount } = await mountAtSubmitPoint(); + + stdin.write("\r"); + await flush(12); + + // Advisory, not a gate: the provider is saved either way, and the + // operator is told what the route did instead of finding out on + // their first message. + expect(saveMock).toHaveBeenCalledTimes(1); + expect(onFinished).toHaveBeenCalledTimes(1); + expect(onFinished.mock.calls[0]?.[0]).toBe("saved_cloud"); + expect(String(onFinished.mock.calls[0]?.[1])).toContain("closed the stream"); + unmount(); + }); +}); diff --git a/src/tui/providers/providers-orchestrator.test.ts b/src/tui/providers/providers-orchestrator.test.ts index fb2de59a..ff0b08e5 100644 --- a/src/tui/providers/providers-orchestrator.test.ts +++ b/src/tui/providers/providers-orchestrator.test.ts @@ -15,8 +15,17 @@ vi.mock("../../config/index.js", async (importOriginal) => { let currentConfig: AtomicAgentConfig; +/** + * Enough of the real config for the paths under test: the contract + * probe reads `agent.maxParallelToolCalls` to send the + * `parallel_tool_calls` a turn would send, so a fixture without it + * would fail for a reason no user has. + */ +const AGENT_CONFIG = { maxParallelToolCalls: 8 } as AtomicAgentConfig["agent"]; + function configWithGemini(): AtomicAgentConfig { return { + agent: AGENT_CONFIG, llm: { activeTextProvider: "gemini", activeEmbeddingProvider: "local-llama-embed", @@ -294,6 +303,7 @@ describe("ProvidersOrchestrator.completeWizard", () => { afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); + vi.doUnmock("../persist-llm-provider.js"); }); function wizardFor(kind: "openrouter" | "aimlapi"): ProvidersWizardState { @@ -363,8 +373,153 @@ describe("ProvidersOrchestrator.completeWizard", () => { expect(failure?.error).toContain("rejected this key"); }); + /** + * The save path with only the disk writes stubbed: the real key + * check, the real contract probe and the real gate between them. + * Everything above this point in the file stops at a refused key, so + * without it nothing ever reaches the code that decides whether this + * install may be reported as having a working backend. + */ + async function importOrchestratorWithStubbedDisk() { + vi.doMock("../persist-llm-provider.js", async (importOriginal) => { + const original = + await importOriginal(); + return { + ...original, + upsertLlmProvider: vi.fn(), + writeProviderApiKeyToDotenv: vi.fn(), + setActiveTextProviderInConfig: vi.fn(), + }; + }); + return importFreshOrchestrator(); + } + + /** + * Answers the pre-save key check with a live key, and hands the probe + * request — the streamed one, carrying `tools` — to the caller. + */ + function stubSaveFetch(probeAnswer: () => Response) { + const bodies: Record[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: unknown, init?: RequestInit) => { + if (!String(url).includes("/chat/completions")) { + return new Response(JSON.stringify({ data: [] }), { status: 200 }); + } + const body = JSON.parse(String(init?.body ?? "{}")) as Record< + string, + unknown + >; + bodies.push(body); + if (body.stream !== true) { + return new Response( + JSON.stringify({ choices: [{ message: { content: "ok" } }] }), + { status: 200 }, + ); + } + return probeAnswer(); + }), + ); + return { bodies: () => bodies }; + } + + const PROBE_TOOL_CALL_SSE = + `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { + name: "atomic_contract_probe", + arguments: '{"ok":true}', + }, + }, + ], + }, + }, + ], + })}\n\ndata: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: "tool_calls" }], + })}\n\ndata: [DONE]\n\n`; + + /** Truncated mid-argument, with nothing announcing the end. */ + const PROBE_EARLY_EOF_SSE = `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "atomic_contract_probe", arguments: '{"ok"' }, + }, + ], + }, + }, + ], + })}\n\n`; + + function wizardWithModel(): ProvidersWizardState { + return { ...wizardFor("openrouter"), selectedChatModelId: "vendor/picked-model" }; + } + + it("probes the route on save and reports a backend only once it is proven", async () => { + currentConfig = configWithGemini(); + const fetches = stubSaveFetch( + () => new Response(PROBE_TOOL_CALL_SSE, { status: 200 }), + ); + const { ProvidersOrchestrator } = await importOrchestratorWithStubbedDisk(); + const bus = fakeBus(); + const runtime = fakeRuntime(); + const orchestrator = new ProvidersOrchestrator(runtime, bus as never); + + await orchestrator.completeWizard(wizardWithModel()); + + // The key check, then the turn contract itself: streamed, with the + // tools payload, on the model the operator picked. + const probeBody = fetches.bodies()[1]; + expect(probeBody?.stream).toBe(true); + expect(probeBody?.model).toBe("vendor/picked-model"); + expect(JSON.stringify(probeBody?.tools)).toContain("atomic_contract_probe"); + + const types = bus.emit.mock.calls.map((call) => (call[0] as { type: string }).type); + expect(types).toContain("providers_wizard_succeeded"); + expect(runtime.reportModelConfigured).toHaveBeenCalledWith( + "openrouter", + "cloud", + ); + }); + + it("saves but reports no working backend when the probe fails", async () => { + currentConfig = configWithGemini(); + stubSaveFetch(() => new Response(PROBE_EARLY_EOF_SSE, { status: 200 })); + const { ProvidersOrchestrator } = await importOrchestratorWithStubbedDisk(); + const bus = fakeBus(); + const runtime = fakeRuntime(); + const orchestrator = new ProvidersOrchestrator(runtime, bus as never); + + await orchestrator.completeWizard(wizardWithModel()); + + // The key is live, so the save stands — the probe is advisory and + // may never refuse one. + const types = bus.emit.mock.calls.map((call) => (call[0] as { type: string }).type); + expect(types).toContain("providers_wizard_succeeded"); + // But the route was never shown to run a turn, so "this install has + // a working cloud backend" must not be claimed on its behalf. + expect(runtime.reportModelConfigured).not.toHaveBeenCalled(); + const lines = bus.emit.mock.calls + .map((call) => call[0] as { type: string; line?: string }) + .filter((action) => action.type === "providers_status") + .map((action) => action.line ?? ""); + expect(lines.some((line) => line.includes("closed the stream"))).toBe(true); + }); + it("runs the contract probe on explicit request and reports the verdict", async () => { currentConfig = { + agent: AGENT_CONFIG, llm: { activeTextProvider: "openrouter", activeEmbeddingProvider: "local-llama-embed", From 4fcad177c7ce35dd8dcaac44a9b2b793787d7baf Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:13:44 +0300 Subject: [PATCH 14/36] tui: stop button in the composer while a turn is running Esc, Ctrl+C and /abort all stop the agent, but they are keyboard lore - nothing on screen says a running turn can be stopped at all. Put a clickable stop chip inside the input field, next to Send, rendered only while status is running; a press takes exactly the path Esc does (onAbort + abort_requested), so there is one abort path however it was asked for. The chip paints on the palette's error ground with a measured ink (readableOn), the same trick the composer buffer uses, so it stays legible across all eleven themes. --- .../composer-stop-button.mouse.test.tsx | 160 ++++++++++++++++++ src/tui/components/composer-stop-button.tsx | 72 ++++++++ src/tui/components/prompt-shell.tsx | 25 +++ src/tui/tui-app.tsx | 13 ++ 4 files changed, 270 insertions(+) create mode 100644 src/tui/components/composer-stop-button.mouse.test.tsx create mode 100644 src/tui/components/composer-stop-button.tsx diff --git a/src/tui/components/composer-stop-button.mouse.test.tsx b/src/tui/components/composer-stop-button.mouse.test.tsx new file mode 100644 index 00000000..aaa305a6 --- /dev/null +++ b/src/tui/components/composer-stop-button.mouse.test.tsx @@ -0,0 +1,160 @@ +import { render } from "ink-testing-library"; +import { describe, expect, it } from "vitest"; + +import { ClipboardProvider } from "../clipboard/clipboard-context.js"; +import { makeMouseSource } from "../mouse/mouse-source.js"; +import type { TuiMouseEvent } from "../mouse/mouse-event.js"; +import { makeTuiEventBus, TuiApp, type TuiAppCallbacks } from "../tui-app.js"; +import type { TuiSessionInfo } from "../tui-state.js"; + +/** + * The stop chip versus the running turn. + * + * Esc, Ctrl+C and `/abort` all stop the agent, but they are keyboard + * lore; the chip is the one *visible* control. These cases pin down the + * whole loop from the outside — through `TuiApp`, real Ink layout, real + * hit-testing: absent while idle, present while a turn is in flight, + * a click on it lands on `onAbort` (the same callback Esc reaches), + * and it leaves the field when the run ends. + */ + +const SESSION: TuiSessionInfo = { + sessionId: "s1", + workingDir: "/tmp/stop-button-mouse", + llamaUrl: "http://127.0.0.1:8080", + browserChannel: "chrome", + browserHeadless: false, + approvalLevel: 5, + maxSteps: 10, + skillCount: 0, +}; + +/** The chip's on-screen label — glyph included, so a hint strip's plain + * "stop" wording can never satisfy the assertions below. */ +const STOP_LABEL = "■ stop"; + +const strip = (value: string): string => value.replace(/\[[0-9;]*m/g, ""); + +const delay = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitUntil( + condition: () => boolean, + what: string, + timeoutMs = 10_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (condition()) return; + await delay(25); + } + throw new Error(`timed out waiting for ${what}`); +} + +function click(x: number, y: number): TuiMouseEvent { + return { + kind: "press", + button: "left", + wheel: null, + x, + y, + shift: false, + alt: false, + ctrl: false, + }; +} + +/** Screen cell of the LAST line containing `needle`. */ +function locateLast( + frame: string, + needle: string, +): { x: number; y: number } { + const lines = frame.split("\n"); + for (let y = lines.length - 1; y >= 0; y -= 1) { + const x = (lines[y] ?? "").indexOf(needle); + if (x !== -1) return { x, y }; + } + throw new Error(`"${needle}" is not on screen:\n${frame}`); +} + +function mountApp() { + const bus = makeTuiEventBus(); + const mouse = makeMouseSource(); + const submitted: string[] = []; + let aborted = 0; + const clipboard = { + copy: async () => true, + }; + const callbacks: TuiAppCallbacks = { + onApprovalDecision: () => {}, + onAbort: () => { + aborted += 1; + }, + onQuit: () => {}, + onMessageSubmitted: (message) => { + submitted.push(message); + }, + }; + const app = render( + + + , + ); + return { + ...app, + mouse, + submitted, + aborted: () => aborted, + finishRun: () => + bus.emit({ + type: "agent_event", + event: { type: "loop_completed", reason: "reply" }, + }), + frame: () => strip(app.lastFrame() ?? ""), + }; +} + +describe("composer stop button", () => { + it("appears while a turn runs, aborts on click, leaves when the run ends", async () => { + const app = mountApp(); + await waitUntil(() => app.frame().includes("send"), "composer on screen"); + // Idle: no run to stop, so no chip to press. + expect(app.frame()).not.toContain(STOP_LABEL); + + // Submitting a message starts a turn; the chip must come with it. + app.stdin.write("do the thing"); + await waitUntil(() => app.frame().includes("do the thing"), "typed text"); + app.stdin.write("\r"); + await waitUntil(() => app.submitted.length === 1, "message submitted"); + await waitUntil( + () => app.frame().includes(STOP_LABEL), + "stop chip on screen while running", + ); + + // A click on the chip is exactly Esc: `onAbort`, once per press. + // Re-click until it lands — targets register a frame after they + // first paint. + const spot = locateLast(app.frame(), STOP_LABEL); + await waitUntil(() => { + app.mouse.emit(click(spot.x + 1, spot.y)); + return app.aborted() > 0; + }, "stop click to land"); + const landed = app.aborted(); + // The click stopped at the chip: it must not have doubled as a + // submit / steer of the (empty) buffer. + expect(app.submitted.length).toBe(1); + + // One settled press must not have queued extra aborts behind the + // first that landed. + await delay(100); + expect(app.aborted()).toBe(landed); + + // Run over: the chip has nothing left to act on and leaves the field. + app.finishRun(); + await waitUntil( + () => !app.frame().includes(STOP_LABEL), + "stop chip gone after the run ended", + ); + app.unmount(); + }); +}); diff --git a/src/tui/components/composer-stop-button.tsx b/src/tui/components/composer-stop-button.tsx new file mode 100644 index 00000000..dfc262ef --- /dev/null +++ b/src/tui/components/composer-stop-button.tsx @@ -0,0 +1,72 @@ +import { Text } from "ink"; +import type { ReactElement } from "react"; +import { MouseTarget, useMouseCommands } from "../mouse/mouse-context.js"; +import { isPrimaryPress } from "../mouse/mouse-event.js"; +import { readableOn } from "../theme/readable-foreground.js"; +import { theme } from "../theme/theme.js"; + +/** The label carries its own padding so the chip's ground reads as a button. */ +const STOP_LABEL = " ■ stop "; + +export interface ComposerStopButtonProps { + onPress: () => void; + /** + * Mouse layer for the click target. Same story as the send chip: the + * composer overlay floats over the chat log, so its button registers + * above the base layer — otherwise a covered chat control could win + * the click. + */ + layer?: number; +} + +/** + * The composer's stop chip, drawn inside the input field while a turn + * is in flight. + * + * Esc, Ctrl+C and `/abort` all stop the run already, but every one of + * them is invisible: an operator watching a turn go wrong has no + * on-screen control that says the run *can* be stopped, let alone where. + * The hint strip advertises `[esc] abort`, yet the strip is one row of + * muted text under everything else — a mouse user staring at a runaway + * task deserves a button next to the field they are typing into. + * + * Unlike Send this chip has no disabled state: it only renders while + * `status === "running"`, and a stop button that renders but refuses to + * press would be worse than none. The caller owns that condition, the + * same way it owns wiring the press to the one abort path Esc uses. + * + * The ground is the palette's `error` — stop is the composer's one + * destructive verb and it should not dress like Send. `error` is a page + * token, not one of the guaranteed chip pairs, so the ink is *measured* + * against it (`readableOn`) instead of assumed; that is what keeps the + * label legible across all eleven palettes without a per-theme table. + */ +export function ComposerStopButton({ + onPress, + layer, +}: ComposerStopButtonProps): ReactElement { + const background = theme.colors.error; + const chip = ( + + {STOP_LABEL} + + ); + const mouse = useMouseCommands(); + // No provider (component tests, the wizard's separate Ink tree): + // render the label and stop. Registering a target that swallows the + // click without acting would be worse than no target. + if (!mouse) return chip; + return ( + { + if (!isPrimaryPress(hit.event)) return false; + onPress(); + return true; + }} + > + {chip} + + ); +} diff --git a/src/tui/components/prompt-shell.tsx b/src/tui/components/prompt-shell.tsx index 8ecc150d..8a8fec1a 100644 --- a/src/tui/components/prompt-shell.tsx +++ b/src/tui/components/prompt-shell.tsx @@ -5,6 +5,7 @@ import { useRotatingPlaceholder } from "../hooks/use-rotating-placeholder.js"; import { readableOn } from "../theme/readable-foreground.js"; import { theme } from "../theme/theme.js"; import { ComposerSendButton } from "./composer-send-button.js"; +import { ComposerStopButton } from "./composer-stop-button.js"; import { MultiLineEditor, type MultiLineEditorProps } from "./multi-line-editor.js"; import { PromptMetaBar } from "./prompt-meta-bar.js"; @@ -82,6 +83,16 @@ export interface PromptShellProps /** Optional context readout, rendered at the action bar's right end. */ contextSlot?: ReactElement | null; modeSlot?: ReactElement | null; + /** + * A turn is in flight. Puts the stop chip into the field, next to + * Send — the one moment the composer has a destructive verb to offer. + */ + running?: boolean; + /** + * Stop the running turn. The chat surface passes the same path Esc + * takes; the chip renders only when both `running` and this are set. + */ + onStop?: () => void; } export function PromptShell(props: PromptShellProps): ReactElement { @@ -97,6 +108,8 @@ export function PromptShell(props: PromptShellProps): ReactElement { rightSlot, contextSlot, modeSlot, + running, + onStop, focus, disabled, value, @@ -196,6 +209,18 @@ export function PromptShell(props: PromptShellProps): ReactElement { bare /> + {running && onStop ? ( + + {/* + Stop sits between the buffer and Send, on exactly the + turns it can act on. Inside the field like Send, and for + the same reason: it is a verb for the run the operator + is watching, and the bar below already spends its slots + on status readouts. + */} + + + ) : null} { + callbacks.onAbort(); + dispatch({ type: "abort_requested" }); + }, [callbacks]); + const onEditorChange = useCallback( (next: string) => { // An editor that is unmounting keeps its `useInput` subscription @@ -2001,6 +2012,8 @@ export function TuiApp({ rightSlot={promptRightSlot} contextSlot={promptContextSlot} modeSlot={promptModeSlot} + running={state.status === "running"} + onStop={onStopRun} focus={editorFocus} disabled={!canTypeMessage(state)} claimKey={composerClaimKey} From b657a65ed30dd01d244c62eec3b4c2dae5db6d79 Mon Sep 17 00:00:00 2001 From: Valerii Date: Wed, 2 Sep 2026 17:32:01 +0300 Subject: [PATCH 15/36] tui: persistent update banner in the status bar's top-right corner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The startup update modal is skippable, and skipping it left no trace on screen — an operator who pressed n once stayed on the old version with nothing reminding them a newer one exists. The offer now also raises a banner pinned to the right edge of the status bar: 'new version vX.Y.Z available [ Update ]'. It renders inverse-video, so it is the opposite of whatever ground the palette and terminal use — distinguishable by construction, and static, so it never pulls the eye away from the work. Clicking Update runs the same path as the modal's y (including the refusal while a turn is in flight); the modal itself is untouched and remains the keyboard route. The banner survives update_dismissed (the modal is the question, the banner is the memory), yields while an update runs or has finished, and returns on a failed install so there is still a way to retry. On narrow rows it degrades — full sentence, bare version, button alone, nothing — instead of wrapping the one-row bar. Testing ground: 'atomic-agent tui --fake-update 9.9.9' pretends that version is released — the real check and the real installer are both bypassed, so the modal, the banner and its degradations can be eyeballed on a dev build. --- src/tui/agent-event-reducer.test.ts | 31 ++++++ src/tui/agent-event-reducer.ts | 1 + src/tui/components/status-bar-update.test.tsx | 69 ++++++++++++ src/tui/components/status-bar.tsx | 61 +++++++++- src/tui/components/update-banner.test.tsx | 44 ++++++++ src/tui/components/update-banner.tsx | 104 ++++++++++++++++++ src/tui/tui-app.tsx | 1 + src/tui/tui-args.test.ts | 27 +++++ src/tui/tui-args.ts | 21 ++++ src/tui/tui-command.ts | 26 ++++- src/tui/tui-state.ts | 10 ++ 11 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 src/tui/components/status-bar-update.test.tsx create mode 100644 src/tui/components/update-banner.test.tsx create mode 100644 src/tui/components/update-banner.tsx diff --git a/src/tui/agent-event-reducer.test.ts b/src/tui/agent-event-reducer.test.ts index a78e62f0..340c6324 100644 --- a/src/tui/agent-event-reducer.test.ts +++ b/src/tui/agent-event-reducer.test.ts @@ -834,3 +834,34 @@ describe("turn_gate_blocked", () => { expect(blocked.feed.at(-1)?.line).not.toContain("\n"); }); }); + +describe("update banner state", () => { + const offer: TuiAction = { + type: "update_available", + current: "0.5.4", + latest: "9.9.9", + }; + + it("update_available raises both the modal and the banner", () => { + const next = reduceTuiState(createInitialTuiState(fakeSession()), offer); + expect(next.updatePrompt).toEqual({ current: "0.5.4", latest: "9.9.9" }); + expect(next.updateBanner).toEqual({ current: "0.5.4", latest: "9.9.9" }); + }); + + it("update_dismissed clears only the modal — the banner is the memory", () => { + const next = apply(createInitialTuiState(fakeSession()), [ + offer, + { type: "update_dismissed" }, + ]); + expect(next.updatePrompt).toBeNull(); + expect(next.updateBanner).toEqual({ current: "0.5.4", latest: "9.9.9" }); + }); + + it("a repeat offer while an update runs still changes nothing", () => { + const running = apply(createInitialTuiState(fakeSession()), [ + offer, + { type: "update_started" }, + ]); + expect(reduceTuiState(running, offer)).toBe(running); + }); +}); diff --git a/src/tui/agent-event-reducer.ts b/src/tui/agent-event-reducer.ts index 2f8baaf6..bbaf06ac 100644 --- a/src/tui/agent-event-reducer.ts +++ b/src/tui/agent-event-reducer.ts @@ -310,6 +310,7 @@ export function reduceTuiState(state: TuiState, action: TuiAction): TuiState { return { ...state, updatePrompt: { current: action.current, latest: action.latest }, + updateBanner: { current: action.current, latest: action.latest }, }; case "update_dismissed": return { ...state, updatePrompt: null }; diff --git a/src/tui/components/status-bar-update.test.tsx b/src/tui/components/status-bar-update.test.tsx new file mode 100644 index 00000000..fb4fb9d5 --- /dev/null +++ b/src/tui/components/status-bar-update.test.tsx @@ -0,0 +1,69 @@ +import { render } from "ink-testing-library"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { reduceTuiState } from "../agent-event-reducer.js"; +import { apply, fakeSession } from "../test-fixtures.js"; +import { createInitialTuiState, type TuiState } from "../tui-state.js"; +import { StatusBar } from "./status-bar.js"; + +const strip = (s: string): string => s.replace(/\u001b\[[0-9;]*m/g, ""); + +function offered(): TuiState { + return reduceTuiState(createInitialTuiState(fakeSession()), { + type: "update_available", + current: "0.5.4", + latest: "9.9.9", + }); +} + +describe("StatusBar update banner", () => { + it("shows the offer at the end of the bar", () => { + const frame = strip(render().lastFrame() ?? ""); + expect(frame).toContain("v9.9.9"); + expect(frame).toContain("Update"); + }); + + it("stays on screen after the startup modal is dismissed", () => { + const state = apply(offered(), [{ type: "update_dismissed" }]); + expect(state.updatePrompt).toBeNull(); + const frame = strip(render().lastFrame() ?? ""); + expect(frame).toContain("Update"); + }); + + it("yields while the installer runs, and returns on failure", () => { + const running = apply(offered(), [{ type: "update_started" }]); + expect( + strip(render().lastFrame() ?? ""), + ).not.toContain("Update"); + + const failed = apply(running, [ + { type: "update_finished", ok: false, error: "boom" }, + ]); + expect( + strip(render().lastFrame() ?? ""), + ).toContain("Update"); + }); + + it("says nothing when no newer version exists", () => { + const state = createInitialTuiState(fakeSession()); + const frame = strip(render().lastFrame() ?? ""); + expect(frame).not.toContain("Update"); + }); + + it("pins the banner to the right edge when given the row width", () => { + const view = render(); + const line = strip(view.lastFrame() ?? "").split("\n")[0] ?? ""; + // The button's trailing pad cell sits on the last column; everything + // before the banner is left-flowing content and a stretched spacer. + expect(line.trimEnd().endsWith("Update")).toBe(true); + expect(line.trimEnd().length).toBeGreaterThan(60); + }); + + it("keeps the bar one row tall with the banner up", () => { + const view = render(); + const rows = strip(view.lastFrame() ?? "") + .split("\n") + .filter((line) => line.trim().length > 0); + expect(rows).toHaveLength(1); + }); +}); diff --git a/src/tui/components/status-bar.tsx b/src/tui/components/status-bar.tsx index d7bcf427..4782332a 100644 --- a/src/tui/components/status-bar.tsx +++ b/src/tui/components/status-bar.tsx @@ -12,9 +12,16 @@ import type { TuiState } from "../tui-state.js"; import { getAppVersion } from "../../version.js"; import { Chip, tracked } from "./chip.js"; import { sessionTitleLine } from "./session-title.js"; +import { planUpdateBanner, UpdateBanner } from "./update-banner.js"; interface StatusBarProps { state: TuiState; + /** + * Row width in cells. When set, the bar claims the full row and pins + * the update banner to its right edge; without it (unit tests, odd + * hosts) the bar stays content-sized and the banner trails the text. + */ + width?: number; /** * Draw the `atomic-agent vX.Y.Z` lockup. False when the rail is on * screen: the rail already carries the brand and the version, and two @@ -53,14 +60,35 @@ interface StatusBarProps { */ export function StatusBar({ state, + width, brand = true, railRestore = false, }: StatusBarProps): ReactElement { const section = getCurrentSection(state); const title = currentSessionTitle(state); const { columns } = useTerminalSize(); + // The banner outlives the modal (`updateBanner` survives + // `update_dismissed`) and yields only to an update actually running + // or finished — `failed` keeps it up, because the banner is then the + // one remaining way to retry. + const banner = + state.updateStatus === "idle" || state.updateStatus === "failed" + ? state.updateBanner + : null; + // `chipBudget` reserves cells for a session tag whether or not one is + // drawn — safe slack for the download chip, but it starves the banner + // out of a fresh 70-column session where the corner is visibly empty. + // Reclaim the reservation when no tag renders. + const bannerBudget = Math.max( + 0, + rawBudget(columns, brand, title) + + (state.session.sessionId ? 0 : SESSION_TAG), + ); + const bannerPlan = banner + ? planUpdateBanner(banner.latest, bannerBudget) + : null; return ( - + {railRestore ? : null} {brand ? ( <> @@ -76,7 +104,13 @@ export function StatusBar({ {state.localModelsPanel.pull ? ( ) : null} {title ? ( @@ -88,6 +122,15 @@ export function StatusBar({ ) : null} + {banner && bannerPlan ? ( + <> + {/* flexGrow pushes the banner into the top-right corner when + the bar knows its row width; content-sized bars (no + `width`) collapse the spacer to two plain cells. */} + + + + ) : null} ); } @@ -105,14 +148,24 @@ export function StatusBar({ * the header into a paragraph and push the whole app down the screen. */ function chipBudget(columns: number, brand: boolean, title: string | null): number { + return Math.max(0, rawBudget(columns, brand, title)); +} + +/** + * The same leftover before clamping. The banner's session-tag reclaim + * must be added to THIS number — adding it after the clamp turned a + * 42-column deficit into 18 phantom cells and wrapped the bar. + */ +function rawBudget(columns: number, brand: boolean, title: string | null): number { const BRAND = 22; const BREADCRUMB = 14; - const SESSION_TAG = 18; const used = (brand ? BRAND : 0) + BREADCRUMB + SESSION_TAG + (title ? title.length + 4 : 0); - return Math.max(0, columns - used - 2); + return columns - used - 2; } +const SESSION_TAG = 18; + function currentSessionTitle(state: TuiState): string | null { const id = state.session.sessionId; if (!id) return null; diff --git a/src/tui/components/update-banner.test.tsx b/src/tui/components/update-banner.test.tsx new file mode 100644 index 00000000..607a3aa3 --- /dev/null +++ b/src/tui/components/update-banner.test.tsx @@ -0,0 +1,44 @@ +import { render } from "ink-testing-library"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { planUpdateBanner, UpdateBanner } from "./update-banner.js"; + +const strip = (s: string): string => s.replace(/\u001b\[[0-9;]*m/g, ""); + +describe("UpdateBanner", () => { + it("says the whole sentence when the row has room", () => { + const view = render(); + const frame = strip(view.lastFrame() ?? ""); + expect(frame).toContain("new version v9.9.9 available"); + expect(frame).toContain("Update"); + }); + + it("sheds the sentence, then the version, as the row fills up", () => { + const medium = strip( + render().lastFrame() ?? "", + ); + expect(medium).toContain("v9.9.9"); + expect(medium).not.toContain("new version"); + expect(medium).toContain("Update"); + + const tight = strip( + render().lastFrame() ?? "", + ); + expect(tight).toContain("Update"); + expect(tight).not.toContain("9.9.9"); + }); + + it("disappears rather than wrapping the one-row bar", () => { + const view = render(); + expect(strip(view.lastFrame() ?? "").trim()).toBe(""); + }); + + it("never plans a form wider than its budget", () => { + for (const latest of ["1.0.0", "10.20.30", "0.5.5-rc.1"]) { + for (let budget = 0; budget <= 60; budget += 1) { + const plan = planUpdateBanner(latest, budget); + if (plan) expect(plan.width).toBeLessThanOrEqual(budget); + } + } + }); +}); diff --git a/src/tui/components/update-banner.tsx b/src/tui/components/update-banner.tsx new file mode 100644 index 00000000..41284221 --- /dev/null +++ b/src/tui/components/update-banner.tsx @@ -0,0 +1,104 @@ +import { Text } from "ink"; +import type { ReactElement } from "react"; +import { MouseTarget, useMouseCommands } from "../mouse/mouse-context.js"; +import { isPrimaryPress } from "../mouse/mouse-event.js"; +import { theme } from "../theme/theme.js"; + +/** + * The persistent "a newer release exists" strip at the right end of the + * status bar. + * + * The startup {@link UpdateModal} already offers the update once; this + * banner is what remains after the operator skips it. It has to survive + * the whole session without stealing attention from the work — so it + * sits in the one corner the eye only visits deliberately, and it never + * blinks, animates, or claims a key. What it *does* claim is contrast: + * the strip renders inverse-video, swapping ink and ground, which is + * distinguishable on every palette by construction — whatever the + * terminal's background is, the banner is its opposite. No hand-picked + * colour can promise that across twelve palettes and user terminals. + * + * `Update` is the click target and runs the same path as the modal's + * `y` (`onUpdateConfirmed` → `runUpdate`), including its refusal while + * a turn is in flight. Without mouse support the banner is inert + * signage, like every other chip — the modal remains the keyboard route. + */ +export interface UpdateBannerProps { + latest: string; + /** + * Columns the banner may use. Ink wraps rather than clips, so an + * over-wide banner would fold the one-row status bar into a + * paragraph; the banner degrades instead — full sentence, then bare + * version, then the button alone, then nothing. + */ + budget: number; +} + +/** The click target. Fixed label, so its width is a constant. */ +const BUTTON = " Update "; + +/** Cell between the label and the button. */ +const GAP = 1; + +export interface UpdateBannerPlan { + /** Inverse-video label before the button; `null` for button-only. */ + label: string | null; + /** Total cells the banner occupies, button included. */ + width: number; +} + +/** + * Which form fits the budget. Exported so the status bar can subtract + * the banner's real width from the download chip's budget instead of + * guessing — the two share the same row. + */ +export function planUpdateBanner( + latest: string, + budget: number, +): UpdateBannerPlan | null { + const full = ` new version v${latest} available `; + const short = ` v${latest} `; + for (const label of [full, short]) { + const width = label.length + GAP + BUTTON.length; + if (width <= budget) return { label, width }; + } + if (BUTTON.length <= budget) return { label: null, width: BUTTON.length }; + return null; +} + +export function UpdateBanner({ + latest, + budget, +}: UpdateBannerProps): ReactElement | null { + const mouse = useMouseCommands(); + const plan = planUpdateBanner(latest, budget); + if (!plan) return null; + // Inverse accent: the palette's accent as ground, the terminal's own + // background as ink. Louder than the inverse label beside it, so the + // actionable cell reads as the button and the sentence as its caption. + const button = ( + + {BUTTON} + + ); + // Siblings, not one parent: `MouseTarget` wraps its child in a + // Box to own a measurable region, and Ink refuses a Box inside Text. + return ( + <> + {plan.label ? {`${plan.label} `} : null} + {mouse ? ( + { + if (!isPrimaryPress(hit.event)) return false; + mouse.callbacks.onUpdateConfirmed?.(); + return true; + }} + > + {button} + + ) : ( + button + )} + + ); +} diff --git a/src/tui/tui-app.tsx b/src/tui/tui-app.tsx index b38cbbb4..2d275bc8 100644 --- a/src/tui/tui-app.tsx +++ b/src/tui/tui-app.tsx @@ -1735,6 +1735,7 @@ export function TuiApp({ diff --git a/src/tui/tui-args.test.ts b/src/tui/tui-args.test.ts index 3abbed21..54953653 100644 --- a/src/tui/tui-args.test.ts +++ b/src/tui/tui-args.test.ts @@ -45,3 +45,30 @@ describe("parseTuiArgs mouse flags", () => { expect(TUI_HELP).toContain("--mouse"); }); }); + +describe("parseTuiArgs --fake-update", () => { + it("stays off by default", () => { + expect(parseTuiArgs([])).toMatchObject({ fakeUpdateVersion: null }); + }); + + it("captures the pretended version", () => { + expect(parseTuiArgs(["--fake-update", "9.9.9"])).toMatchObject({ + fakeUpdateVersion: "9.9.9", + }); + }); + + it("tolerates a v-prefixed version, since releases are tagged that way", () => { + expect(parseTuiArgs(["--fake-update", "v9.9.9"])).toMatchObject({ + fakeUpdateVersion: "9.9.9", + }); + }); + + it("refuses a missing or flag-shaped value", () => { + expect(parseTuiArgs(["--fake-update"])).toHaveProperty("error"); + expect(parseTuiArgs(["--fake-update", "--no-mouse"])).toHaveProperty("error"); + }); + + it("advertises the flag in --help", () => { + expect(TUI_HELP).toContain("--fake-update"); + }); +}); diff --git a/src/tui/tui-args.ts b/src/tui/tui-args.ts index fbd862ca..20f01f27 100644 --- a/src/tui/tui-args.ts +++ b/src/tui/tui-args.ts @@ -17,6 +17,14 @@ export interface TuiArgs { * text selection for this run. */ mouse: boolean | null; + /** + * Dev testing ground for the update surfaces: pretend this version is + * available on GitHub Releases. Skips the real check (and the real + * installer on accept), so the modal, the status-bar banner and their + * degradations can be eyeballed without publishing a release or + * running a stale binary. `null` in normal operation. + */ + fakeUpdateVersion: string | null; } export type TuiArgsResult = TuiArgs | { error: string } | { help: true }; @@ -36,6 +44,7 @@ export const TUI_HELP = " --skip-llama-setup Skip the first-run local-model setup gate", " --mouse Force terminal mouse support on for this run", " --no-mouse Disable mouse support; restores drag-to-select", + " --fake-update Dev: pretend version is released (no real install)", "", "Needs an interactive terminal; in scripts use `atomic-agent run`.", ].join("\n") + "\n"; @@ -58,6 +67,7 @@ export function parseTuiArgs(args: string[]): TuiArgsResult { let noApproval = false; let skipLlamaSetup = false; let mouse: boolean | null = null; + let fakeUpdateVersion: string | null = null; for (let i = 0; i < args.length; i += 1) { const flag = args[i]; switch (flag) { @@ -90,6 +100,16 @@ export function parseTuiArgs(args: string[]): TuiArgsResult { case "--no-mouse": mouse = false; break; + case "--fake-update": { + const value = args[++i]; + // A bare version, not a flag that happened to follow. Catching + // `--fake-update --no-mouse` here beats a banner advertising + // "v--no-mouse" ten minutes into a test session. + if (!value || value.startsWith("-")) + return { error: "--fake-update requires a version (e.g. --fake-update 9.9.9)" }; + fakeUpdateVersion = value.replace(/^v/, ""); + break; + } default: return { error: `unknown flag: ${flag}` }; } @@ -100,6 +120,7 @@ export function parseTuiArgs(args: string[]): TuiArgsResult { noApproval, skipLlamaSetup, mouse, + fakeUpdateVersion, }; } diff --git a/src/tui/tui-command.ts b/src/tui/tui-command.ts index 7d6b5e08..963a0ac8 100644 --- a/src/tui/tui-command.ts +++ b/src/tui/tui-command.ts @@ -12,6 +12,7 @@ import { import { checkLlamaServer } from "../llm/llama-server-health.js"; import { describeLlamaHealthFailure } from "../llm/describe-llama-health-failure.js"; import { createAgentRuntime, type AgentRuntime } from "../runtime/bootstrap.js"; +import { getAppVersion } from "../version.js"; import type { LogRecord, LogSink } from "../tracing/structured-logger.js"; import type { MetricSample, MetricSink } from "../tracing/metrics-collector.js"; import { isKnownLocalModelId } from "../local-llm/index.js"; @@ -660,7 +661,17 @@ export async function tuiCommand(args: string[]): Promise { onAnalyticsSetEnabledRequested: (enabled) => orchestrator.privacy.setAnalyticsEnabled(enabled), onPrivacyRefreshRequested: () => orchestrator.privacy.refresh(), - onUpdateConfirmed: () => orchestrator.runUpdate(), + onUpdateConfirmed: () => + parsed.fakeUpdateVersion + ? // The testing ground must never reach install.sh: the + // point of `--fake-update` is to look at the surfaces, and + // "accept" on a dev build would install the real latest + // release over whatever is being worked on. + bus.emit({ + type: "system_message", + text: `--fake-update: accepted (v${parsed.fakeUpdateVersion}); install skipped in fake mode`, + }) + : orchestrator.runUpdate(), onUpdateRestart: () => { restartRequested = true; }, @@ -751,7 +762,18 @@ export async function tuiCommand(args: string[]): Promise { // Fire-and-forget startup version check. Surfaces an in-app update // offer when a newer release is published; silently no-ops when // disabled, offline, rate-limited, or running a dev build. - void orchestrator.checkForUpdate(); + // `--fake-update` bypasses the check (a dev build fails + // `canSelfUpdate` anyway) and emits the offer directly, so the modal + // and the status-bar banner can be exercised on demand. + if (parsed.fakeUpdateVersion) { + bus.emit({ + type: "update_available", + current: getAppVersion(), + latest: parsed.fakeUpdateVersion, + }); + } else { + void orchestrator.checkForUpdate(); + } try { await ink.waitUntilExit(); diff --git a/src/tui/tui-state.ts b/src/tui/tui-state.ts index 4c6d3a25..e5bb3ba7 100644 --- a/src/tui/tui-state.ts +++ b/src/tui/tui-state.ts @@ -527,6 +527,15 @@ export interface TuiState { * offer was dismissed / accepted). Drives the {@link UpdateModal}. */ updatePrompt: { current: string; latest: string } | null; + /** + * Persistent "a newer release exists" fact behind the status-bar + * banner. Set alongside {@link updatePrompt} and — unlike the prompt — + * NOT cleared by `update_dismissed`: skipping the modal means "not + * now", and the banner is what keeps the offer reachable afterwards. + * The bar hides it while an update is running or finished + * (`updateStatus`), so no reducer case ever needs to null it. + */ + updateBanner: { current: string; latest: string } | null; /** * Lifecycle of an accepted self-update. `running` while `install.sh` * executes; `done` / `failed` after it settles. Purely informational — @@ -790,6 +799,7 @@ export function createInitialTuiState( themePickerOriginal: "", aborting: false, updatePrompt: null, + updateBanner: null, updateStatus: "idle", ringBufferSize, tasksPanel: createInitialTasksPanelState(), From 57a485d701d7f7131e499ea876d1137dab723653 Mon Sep 17 00:00:00 2001 From: plombeer31 Date: Wed, 2 Sep 2026 18:38:21 +0300 Subject: [PATCH 16/36] llm: report the whole prompt in timing.promptTokens, cached prefix included MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llama-server's prompt_n / tokens_evaluated count only the tokens evaluated on this request; the prefix reused from the KV cache sits in tokens_cached and was dropped. Every consumer treats promptTokens as "how big was the prompt" (their fallback is prompt.tokens.total, and the TUI overwrites the context readout with it after each completion), so on a warm cache the occupied-context figure collapsed to the newly-evaluated slice — and then leapt ~4x back to the estimator's full figure the moment the context panel's task selector reprojected it. Report evaluated + cached from the llama client so the measured readout and the selector projection agree. Cloud adapters already include cached tokens in their prompt counts and are untouched. --- src/llm/llama-server-client.test.ts | 28 +++++++++++++++++++++++++++- src/llm/llama-server-client.ts | 15 +++++++++++++-- src/tui/agent-event-reducer.ts | 8 +++++--- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/llm/llama-server-client.test.ts b/src/llm/llama-server-client.test.ts index 0bfcc8d8..ff0e46e9 100644 --- a/src/llm/llama-server-client.test.ts +++ b/src/llm/llama-server-client.test.ts @@ -50,7 +50,10 @@ describe("LlamaServerClient.complete", () => { expect(result.content).toBe('{"tool":"finish","args":{}}'); expect(result.reasoningContent).toBe(""); - expect(result.timing.promptTokens).toBe(40); + // 40 evaluated this request + 30 reused from the KV cache: the + // prompt the model saw was 70 tokens, and that is what occupancy + // consumers (the TUI context chip among them) need reported. + expect(result.timing.promptTokens).toBe(70); expect(result.cacheHitTokens).toBe(30); expect(result.slotId).toBe(2); expect(result.modelId).toBe("qwen-test"); @@ -65,6 +68,29 @@ describe("LlamaServerClient.complete", () => { expect(snapshot.body.repeat_last_n).toBe(256); }); + it("reports the bare evaluated count when nothing was cached", async () => { + const client = new LlamaServerClient({ + baseUrl: "http://127.0.0.1:9999", + fetchImpl: createMockFetch(async () => + new Response( + JSON.stringify({ + content: "ok", + stop: true, + truncated: false, + timings: { prompt_ms: 10, predicted_ms: 20, prompt_n: 40, predicted_n: 8 }, + slot_id: 0, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ), + }); + + const result = await client.complete({ prompt: "hello", maxTokens: 16 }); + + expect(result.timing.promptTokens).toBe(40); + expect(result.cacheHitTokens).toBe(0); + }); + it("forwards explicit repeatPenalty / repeatLastN overrides", async () => { let captured: Record | null = null; const client = new LlamaServerClient({ diff --git a/src/llm/llama-server-client.ts b/src/llm/llama-server-client.ts index 3abe229a..f2d8c2e4 100644 --- a/src/llm/llama-server-client.ts +++ b/src/llm/llama-server-client.ts @@ -551,6 +551,17 @@ function normaliseCompletionResponse( payload: Record, ): CompletionResult { const timings = (payload.timings ?? {}) as Record; + // `prompt_n` / `tokens_evaluated` count only the tokens llama-server + // actually evaluated this request — the prefix reused from the KV + // cache (`tokens_cached`) is excluded. Every consumer of + // `timing.promptTokens` treats it as "how big was the prompt" (their + // fallback is `prompt.tokens.total`, and the TUI shows it as occupied + // context), so report the whole prompt: evaluated + cached. On a warm + // cache the raw `prompt_n` is a small fraction of the prompt and the + // context readout collapsed to it, then leapt back to the estimator's + // full figure the moment anything reprojected it. + const evaluatedTokens = toNumber(timings.prompt_n ?? payload.tokens_evaluated); + const cachedTokens = toNumber(payload.tokens_cached); return { content: typeof payload.content === "string" ? payload.content : "", reasoningContent: @@ -562,10 +573,10 @@ function normaliseCompletionResponse( timing: { promptMs: toNumber(timings.prompt_ms), predictedMs: toNumber(timings.predicted_ms), - promptTokens: toNumber(timings.prompt_n ?? payload.tokens_evaluated), + promptTokens: evaluatedTokens + cachedTokens, predictedTokens: toNumber(timings.predicted_n ?? payload.tokens_predicted), }, - cacheHitTokens: toNumber(payload.tokens_cached), + cacheHitTokens: cachedTokens, slotId: toNumber(payload.slot_id ?? payload.id_slot, -1), modelId: typeof payload.model === "string" ? payload.model : null, diff --git a/src/tui/agent-event-reducer.ts b/src/tui/agent-event-reducer.ts index 2f8baaf6..57d2f01f 100644 --- a/src/tui/agent-event-reducer.ts +++ b/src/tui/agent-event-reducer.ts @@ -716,9 +716,11 @@ function reduceStepEvent( case "llm_completed": { // `prompt_built` carried an estimate (`estimateTokens` over-counts // by design); the provider just reported what its own tokenizer - // actually counted — llama.cpp from `tokens_evaluated`, an - // OpenAI-compatible cloud from `usage.prompt_tokens`. Prefer it, - // and leave the estimate standing when nothing was reported. + // actually counted — llama.cpp from `prompt_n + tokens_cached` + // (the whole prompt, not just the slice evaluated past the KV + // cache), an OpenAI-compatible cloud from `usage.prompt_tokens`. + // Prefer it, and leave the estimate standing when nothing was + // reported. const counted = event.completion.timing?.promptTokens ?? 0; if (counted <= 0) return state; return { From 3a87513c645dcd2a9e3323f7922ee0c1b3e1d7ed Mon Sep 17 00:00:00 2001 From: plombeer31 Date: Wed, 2 Sep 2026 18:42:01 +0300 Subject: [PATCH 17/36] tui: the composer context chip recalculates live Two gaps between the context panel and the chip under it: - Working the panel's task selector reprojected every figure inside the panel and left the composer chip on the last built prompt, so the one readout that survives closing the panel never showed the number just chosen. The chip now renders the same projection the panel does (selectComposerContextUsage): stepping the dial moves the minibar gauge, token pair and task count on the same render, and the draft keeps driving the chip until a prompt is actually built against it. - Switching the active chat model (or provider) left the chip gauging against the window the *previous* model's prompt was built with, because resolveWindow prefers the prompt-derived window over every live source. A providers_refresh that changes the active text route now drops that stale window, so the chip re-gauges from the health poller / catalogue for the newly chosen model immediately instead of one prompt build later. --- src/tui/providers/providers-reducer.test.ts | 86 ++++++++++++++++++++ src/tui/providers/providers-reducer.ts | Bin 9806 -> 11317 bytes src/tui/select-context-usage.test.ts | 66 ++++++++++++++- src/tui/select-context-usage.ts | 25 ++++++ src/tui/tui-app.tsx | 14 +++- 5 files changed, 187 insertions(+), 4 deletions(-) diff --git a/src/tui/providers/providers-reducer.test.ts b/src/tui/providers/providers-reducer.test.ts index 1f68c2dc..92dcd584 100644 --- a/src/tui/providers/providers-reducer.test.ts +++ b/src/tui/providers/providers-reducer.test.ts @@ -35,3 +35,89 @@ describe("reduceProvidersPanel", () => { expect(down.providersPanel.cursor).toBe(1); }); }); + +describe("a refresh that switches the active text route", () => { + const row = (overrides: Record) => ({ + id: "a", + kind: "openrouter", + isActiveText: true, + isActiveEmbedding: false, + hasApiKey: true, + chatModel: "openai/gpt-4o-mini", + embeddingModel: null, + ...overrides, + }); + + /** A state that has measured a prompt against a 32k window. */ + function measuredState() { + const base = createInitialTuiState({ + session: { id: "s1", workingDir: "/tmp" }, + }); + const withRows = reduceProvidersPanel(base, { + type: "providers_refresh", + rows: [row({})], + })!; + return { + ...withRows, + contextUsage: { + ...withRows.contextUsage, + tokens: 14_000, + contextWindow: 32_768, + }, + }; + } + + /** + * The window the last prompt was built against belongs to the model + * that built it. `resolveWindow` prefers it over every live source, so + * left standing it has the composer chip gauging the freshly chosen + * model against the old model's window until the next prompt build. + */ + it("drops the prompt-derived window when the chat model changes", () => { + const next = reduceProvidersPanel(measuredState(), { + type: "providers_refresh", + rows: [row({ chatModel: "anthropic/claude-sonnet-5" })], + })!; + expect(next.contextUsage.contextWindow).toBeNull(); + // Only the window is stale — the measured prompt size still stands. + expect(next.contextUsage.tokens).toBe(14_000); + }); + + it("drops it when a different provider takes over chat", () => { + const next = reduceProvidersPanel(measuredState(), { + type: "providers_refresh", + rows: [ + row({ isActiveText: false }), + row({ id: "b", chatModel: "openai/gpt-4o-mini" }), + ], + })!; + expect(next.contextUsage.contextWindow).toBeNull(); + }); + + it("keeps it across an ordinary refresh of the same route", () => { + const next = reduceProvidersPanel(measuredState(), { + type: "providers_refresh", + rows: [row({ hasApiKey: false })], + })!; + expect(next.contextUsage.contextWindow).toBe(32_768); + }); + + it("does not treat the first population of the rows as a switch", () => { + const base = { + ...createInitialTuiState({ session: { id: "s1", workingDir: "/tmp" } }), + }; + const seeded = { + ...base, + contextUsage: { + ...base.contextUsage, + tokens: 14_000, + contextWindow: 32_768, + }, + }; + const next = reduceProvidersPanel(seeded, { + type: "providers_refresh", + rows: [row({})], + })!; + expect(next.contextUsage.contextWindow).toBe(32_768); + }); +}); diff --git a/src/tui/providers/providers-reducer.ts b/src/tui/providers/providers-reducer.ts index 2cd94926bb19c2126e6a407979321289c521e717..00062c7e8540f3fe7b7b916c2932fa5afb1c158a 100644 GIT binary patch delta 1530 zcmZux!EW0|5N*<%S_C{YY;}yArV#TBuQ7@Syzlr2s6PdIng`X`2jx%a2 zy{M=j(D2SLISA0AQgK4c=G5v~4Rx0;XOq!00LRu*t$YkXYqjliAbJkYG*LTCm9k~R zaYaFY|MokofN0CqpMl68lPs0dxr#i12*xF=(0I<;Dt6l zy=VNUi%*5!F^Gj322r`Q=4w}_m#@Y%3ejs@(ywG2W9Ti^TsRxzMy9z46S6|v{8*sI zG(VG9nGO$=kmwZ0r}#`4bbQcvAERA7YrIXMa)@Vnym;`ougG-1`egec%Kj)q()l?Z z9=?V~d-ic6JAGY)G0wa#2fGY!nD!gO_zg@BR-+tqSzI z(FQoB(n9hz8;7k>bm_a=^HRGe(6H6V9TyQx*#)VMTJVG}(q@UzqEFo{UM8 zpgLk-QJVy8A<`a5nk@~JPbl==7ns+qLT~Ez2Tp;@FVr)1FVfdjh_$G+; zM2EulR%t6|7-%)f#+G@HOn{To4f_{c2evtWXY|Osx|X*cSiqN1lwoV>lx8cG*&8KR zaAH|)WHgem;2OVxjgKo<263sKG3+O_P*E9IHawP|yM#P{n@AH}EUlL8yDKSH9X32R_Q(o{2tgadsCVjT~>XnlVWn-Bx=^{mi_Tnj5 zH%6tL4Y!G;gLEM4;3^u%+9LE(N-_ODWj&|)_K?{V1{i8mCoopIy%~xoYPI7 zhdAGSGp2+2{6FWCFJ}|z_;h`J`?Sll49Ue+FpRK87nbKK@&n=FAEuUsJrAnn+5emq z0Ww6~QrT|ljc_+`IR8qp`k2hpeXyxp|9S7rulv%~-c0DZ8cWss;O>L { }); }); +describe("the composer chip follows the task-count draft", () => { + /** A session of three tasks, measured under a cap of 20. */ + function measuredUsage(): ContextUsageState { + return usage({ + tokens: 8_200, + contextWindow: 128_000, + conversationTokens: 900, + conversationPairs: 3, + conversationPairsCap: 20, + droppedPairs: 0, + pairCosts: [300, 280, 320], + sections: [ + { label: "prompt scaffold", tokens: 6_100 }, + { label: "conversation", tokens: 900 }, + ], + }); + } + + it("shows the measurement while no draft is in force", () => { + const state = stateWith(measuredUsage()); + expect(selectComposerContextUsage(state)).toEqual( + selectContextUsage(state), + ); + }); + + /** + * The point of the selector: stepping the dial in the panel moves the + * chip on the same render, not one prompt build later. + */ + it("reprojects at the draft the moment one exists", () => { + const state = stateWith(measuredUsage(), { contextPanelPairsDraft: 22 }); + const view = selectComposerContextUsage(state); + // Everything outside the transcript plus every task that exists — + // dialing past the session's real size adds nothing. + expect(view?.tokens).toBe(6_100 + 900); + expect(view?.pairs).toBe(3); + expect(view?.pairsCap).toBe(22); + expect(view?.droppedPairs).toBe(0); + }); + + it("prices a draft below the measured count", () => { + const state = stateWith(measuredUsage(), { contextPanelPairsDraft: 2 }); + const view = selectComposerContextUsage(state); + // The two newest tasks survive; the oldest is priced out. + expect(view?.conversationTokens).toBe(280 + 320); + expect(view?.pairs).toBe(2); + expect(view?.droppedPairs).toBe(1); + }); + + /** + * `prompt_built` retires the draft when reality catches up with it; + * until that dispatch lands a draft equal to the cap must already + * read as the measurement, or the chip would swap a real tokenizer + * count for an estimate on a no-op. + */ + it("keeps the measurement when the draft equals the cap", () => { + const state = stateWith(measuredUsage(), { contextPanelPairsDraft: 20 }); + expect(selectComposerContextUsage(state)?.tokens).toBe(8_200); + }); +}); + describe("which limit holds the transcript down", () => { it("names config when the configured cap is what binds", () => { expect(selectContextUsage(stateWith(usage()))?.capSource).toBe("config"); diff --git a/src/tui/select-context-usage.ts b/src/tui/select-context-usage.ts index ef5a8e5d..206ecbf9 100644 --- a/src/tui/select-context-usage.ts +++ b/src/tui/select-context-usage.ts @@ -250,3 +250,28 @@ export function selectContextUsage(state: TuiState): ContextUsageView | null { sections, }; } + +/** + * What the composer's chip renders: the measured view, reprojected at + * the operator's draft task count whenever one is in force. + * + * The detail panel has always projected the draft; the chip kept + * showing the last built prompt, so working the selector moved the + * panel's numbers while the bar under it sat still — and the one + * readout that survives closing the panel never said what was just + * chosen. Sharing the panel's own condition (`draft === pairsCap` + * means reality already caught up — see `prompt_built`, which retires + * the draft on exactly that match) keeps the two surfaces telling one + * story, and the draft outliving the panel is deliberate: the chip + * carries the chosen figure until a prompt is actually built against + * it. + */ +export function selectComposerContextUsage( + state: TuiState, +): ContextUsageView | null { + const measured = selectContextUsage(state); + if (measured === null) return null; + const draft = state.contextPanelPairsDraft; + if (draft === null || draft === measured.pairsCap) return measured; + return usageAtPairs(measured, draft); +} diff --git a/src/tui/tui-app.tsx b/src/tui/tui-app.tsx index b38cbbb4..60429b94 100644 --- a/src/tui/tui-app.tsx +++ b/src/tui/tui-app.tsx @@ -15,7 +15,10 @@ import { CodingModePopup } from "./components/coding-mode-popup.js"; import { OnboardingScreen } from "./components/onboarding-screen.js"; import { TerminalTooSmall } from "./components/terminal-too-small.js"; import { ContextPanel } from "./components/context-panel.js"; -import { selectContextUsage } from "./select-context-usage.js"; +import { + selectComposerContextUsage, + selectContextUsage, +} from "./select-context-usage.js"; import { Box, Text, useApp, useInput, type DOMElement, type Key } from "ink"; import type { HuggingFaceRepoChoices } from "../local-llm/index.js"; import { @@ -1637,8 +1640,13 @@ export function TuiApp({ dispatch({ type: "context_pairs_selected", pairs: next }); }, []); - const promptContextSlot = contextUsage ? ( - + // The chip follows the operator's draft task count the instant the + // selector moves; the panel keeps the measured view and projects the + // draft itself, so the two stay in step. See + // `selectComposerContextUsage`. + const composerContextUsage = selectComposerContextUsage(state); + const promptContextSlot = composerContextUsage ? ( + ) : null; // Always drawn, including in `default`. A control that appears only // once you are in an unusual mode is a control nobody discovers, and From 0f6b8c56d83fdc9092c3db68d6e68f7979e8440c Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:43:08 +0300 Subject: [PATCH 18/36] session: persist the context gauge and restore it on session switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composer's context chip was computed only from the live turn's prompt_built event, so it vanished on relaunch and — worse — switching threads kept the previous thread's numbers on screen until the next turn ran. - Move the ContextUsageState shape + contextUsageFromPrompt projection from the TUI into src/session/context-usage.ts (TUI paths re-export) so the runtime can use them without a TUI dependency. - executeTurn now stamps the turn's window occupancy (prompt_built estimate, refined by the provider's real promptTokens) onto SessionState before the post-turn save — every origin funnels through it. - session_switched carries the stored snapshot; the reducer restores the chip from it and resets to empty when the target session has none, fixing the stale-gauge-on-switch bug. --- src/runtime/bootstrap.test.ts | 35 ++++++ src/runtime/bootstrap.ts | 47 +++++++- src/session/context-usage.ts | 154 +++++++++++++++++++++++++++ src/session/index.ts | 9 ++ src/session/session-state.ts | 11 ++ src/session/session-store.test.ts | 22 ++++ src/tui/chat-orchestrator.ts | 7 ++ src/tui/context-usage-from-prompt.ts | 87 ++------------- src/tui/reduce-ui-actions.test.ts | 39 +++++++ src/tui/reduce-ui-actions.ts | 5 + src/tui/tui-action.ts | 9 ++ src/tui/tui-state.ts | 72 ++----------- 12 files changed, 354 insertions(+), 143 deletions(-) create mode 100644 src/session/context-usage.ts diff --git a/src/runtime/bootstrap.test.ts b/src/runtime/bootstrap.test.ts index aaa7121a..26556235 100644 --- a/src/runtime/bootstrap.test.ts +++ b/src/runtime/bootstrap.test.ts @@ -755,6 +755,41 @@ describe("createAgentRuntime", () => { } }); + it("persists the turn's context usage onto the stored session", async () => { + const runtime = await createAgentRuntime({ + workingDir, + approvalLevel: 5, + overrides: { + browserBackend: backend, + skipLlamaHealthCheck: true, + llamaComplete: async () => ({ + content: JSON.stringify({ + tool: "reply", + args: { text: "hi back" }, + }), + timing: { promptTokens: 777, predictedTokens: 3 }, + slotId: 0, + cacheReused: false, + }), + }, + }); + try { + const session = runtime.createSession(); + const result = await runtime.runTurn(session, "hello", { maxSteps: 5 }); + // `prompt_built` seeded the snapshot; `llm_completed`'s tokenizer + // count (777) replaced the estimate before the stamp. + expect(result.session.contextUsage).toBeDefined(); + expect(result.session.contextUsage?.tokens).toBe(777); + expect(result.session.contextUsage?.sections.length).toBeGreaterThan(0); + // The stored row carries the same snapshot, so a later process — + // the TUI reopening this session — can restore the gauge. + const reloaded = runtime.sessionStore.load(session.id)!; + expect(reloaded.contextUsage).toEqual(result.session.contextUsage); + } finally { + await runtime.shutdown(); + } + }); + it("refreshSkills rebuilds the catalog and notifies listeners", async () => { let notified: Array<{ name: string }> = []; const runtime = await createAgentRuntime({ diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index 143bcc05..5a36bf68 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -157,6 +157,8 @@ import type { AgentLoopEvent, RunTurnResult } from "../agent/agent-loop.js"; import { SessionStore, createEmptySessionState, + contextUsageFromPrompt, + type ContextUsageState, type SessionState, } from "../session/index.js"; @@ -827,6 +829,15 @@ export async function createAgentRuntime( * pointer. */ const turnContext = new AsyncLocalStorage<{ sessionId: string }>(); + /** + * The running turn's window occupancy, per session. Written by + * `emitAgentLoopEvent` (`prompt_built`, refined by `llm_completed`), + * consumed once by `executeTurn` when it stamps the finished session, + * and always cleared in its `finally` so an aborted turn cannot leak + * an entry — or bleed one turn's gauge into a session that never + * built a prompt of its own. + */ + const lastTurnContextUsage = new Map(); const steeringInbox = new SteeringInbox(); const turnController = new TurnController({ onHookError: (err, ctxInfo) => { @@ -852,6 +863,28 @@ export async function createAgentRuntime( const recorder = touchRecorder(ctx.sessionId); recorder?.onAgentEvent(event); turnController.emit(ctx.sessionId, event); + // Track the turn's window occupancy so `executeTurn` can stamp it + // onto the session before the post-turn save. Mirrors the TUI's own + // reduction: the `prompt_built` estimate, refined by the provider's + // real tokenizer count when the completion reports one. + if (event.type === "llm_event") { + const step = event.event; + if (step.type === "prompt_built") { + lastTurnContextUsage.set( + ctx.sessionId, + contextUsageFromPrompt(step.prompt), + ); + } else if (step.type === "llm_completed") { + const counted = step.completion.timing?.promptTokens ?? 0; + const usage = lastTurnContextUsage.get(ctx.sessionId); + if (counted > 0 && usage) { + lastTurnContextUsage.set(ctx.sessionId, { + ...usage, + tokens: counted, + }); + } + } + } } if (event.type === "loop_failed") { captureError(errorReporter, event.error, { @@ -2310,9 +2343,19 @@ export async function createAgentRuntime( maxSteps: runOptions.maxSteps ?? config.agent.maxSteps, signal: runOptions.signal ?? new AbortController().signal, }); - sessionStore.save(result.session); - return result; + // Stamp the turn's window occupancy so the stored session can + // restore the TUI's context gauge when it is reopened. A turn + // that built no prompt (failed before step 1) leaves whatever + // snapshot the previous turn persisted. + const usage = lastTurnContextUsage.get(session.id); + const finished = + usage === undefined + ? result.session + : { ...result.session, contextUsage: usage }; + sessionStore.save(finished); + return usage === undefined ? result : { ...result, session: finished }; } finally { + lastTurnContextUsage.delete(session.id); activeTraceSessions.delete(session.id); // A delete that arrived mid-turn was deferred to keep the pin honest; // complete it now that nothing is writing through the recorder. diff --git a/src/session/context-usage.ts b/src/session/context-usage.ts new file mode 100644 index 00000000..15c21eb7 --- /dev/null +++ b/src/session/context-usage.ts @@ -0,0 +1,154 @@ +import type { BuiltPrompt } from "../prompt/build-prompt-types.js"; + +/** + * What the last built prompt actually put in the model's context window. + * + * Lives in the session module (not the TUI) because the figure is a + * property of the *session*, not of the screen that happens to render + * it: `executeTurn` stamps the latest snapshot onto `SessionState` so a + * session reopened tomorrow — or switched to from another thread — + * shows its window fill immediately instead of a blank chip until the + * next turn rebuilds a prompt. + * + * Every field is a snapshot of the most recent `prompt_built`, refined by + * the completion's own token count when the provider reports one. + */ +export interface ContextUsageState { + /** + * Tokens in the last prompt. An estimate at `prompt_built` time + * (`estimateTokens` over-counts by design), replaced by the real + * tokenizer count once the step completes and the provider reports + * `promptTokens`. + */ + tokens: number | null; + /** + * Physical window the prompt was built against, when the runtime knows + * it. `null` on cloud providers, where the model profile carries no + * window — the chip resolves those from the model catalogue instead. + */ + contextWindow: number | null; + /** Turns `packConversation` dropped to make the transcript fit. */ + droppedTurns: number; + /** Tokens the `### conversation` section actually rendered to. */ + conversationTokens: number; + /** + * Ceiling that section is packed to — `conversationCapEffective`. The + * one number that says when older turns start being dropped, and the + * only budget figure that is defined even when nobody knows the + * physical window (the clamp falls back to the configured cap). + */ + conversationCap: number | null; + /** + * The cap as configured (`agent.conversationMaxTokens`), before the + * window clamp. Equal to `conversationCap` when config is what binds; + * larger when the window is. That comparison is the only way to tell + * an operator which knob actually moves their limit. + */ + conversationCapConfigured: number | null; + /** + * The configured cap is `0` — auto. `conversationCapConfigured` is + * then a fallback rather than a ceiling, so the comparison above says + * nothing and the panel must not name `agent.conversationMaxTokens` + * as what is holding the transcript down. Nothing is: the window is. + */ + conversationCapAuto: boolean; + /** Macro-turns the prompt carried. */ + conversationPairs: number; + /** Macro-turns dropped whole. */ + droppedPairs: number; + /** `agent.conversationMaxPairs` in force. */ + conversationPairsCap: number; + /** Which limit trimmed history, when either did. */ + conversationBoundBy: "pairs" | "tokens" | null; + /** + * Token cost of each macro-turn, oldest first — enough to price a + * different pair count without building another prompt, so moving the + * dial redraws the gauge while the operator is looking at it. + */ + pairCosts: readonly number[]; + /** Per-section breakdown, for the detail view. Empty before the first prompt. */ + sections: readonly ContextUsageSection[]; +} + +export interface ContextUsageSection { + label: string; + tokens: number; +} + +/** A window nothing has been built against yet. */ +export const EMPTY_CONTEXT_USAGE: ContextUsageState = { + tokens: null, + contextWindow: null, + droppedTurns: 0, + conversationTokens: 0, + conversationCap: null, + conversationCapConfigured: null, + conversationCapAuto: false, + conversationPairs: 0, + droppedPairs: 0, + conversationPairsCap: 0, + conversationBoundBy: null, + pairCosts: [], + sections: [], +}; + +/** + * The transcript's row label. Exported because the context panel has to + * find that one row to recalculate it when the task count changes, and + * matching on a literal string in two files is a bug waiting for someone + * to reword one of them. + */ +export const CONVERSATION_SECTION_LABEL = "conversation"; + +/** + * Order the sections are shown in: the fixed cost first, then the + * transcript, then everything the memory fabric contributed, then the + * small stuff. Not the order `BuiltPrompt.tokens` declares them in — + * that one follows the prompt's own assembly, which is not how anyone + * reads a bill. + */ +const SECTIONS: readonly { + key: keyof BuiltPrompt["tokens"]; + label: string; +}[] = [ + { key: "stablePrefix", label: "prompt scaffold" }, + { key: "conversation", label: CONVERSATION_SECTION_LABEL }, + { key: "recalled", label: "recalled memory" }, + { key: "memoryIndex", label: "memory index" }, + { key: "worldSnapshot", label: "world snapshot" }, + { key: "loadedTools", label: "loaded tools" }, + { key: "loadedSkills", label: "loaded skills" }, + { key: "sessionFacts", label: "session facts" }, + { key: "profile", label: "profile" }, + { key: "taskPolicy", label: "task policy" }, +]; + +/** + * Project a built prompt into the readout the composer shows. + * + * Sections that cost nothing are dropped rather than listed as zeros: a + * session with no skills loaded should not have to read the word + * "skills" to find that out. + */ +export function contextUsageFromPrompt(prompt: BuiltPrompt): ContextUsageState { + const sections: ContextUsageSection[] = []; + for (const { key, label } of SECTIONS) { + const tokens = prompt.tokens[key]; + if (tokens > 0) sections.push({ label, tokens }); + } + return { + tokens: prompt.tokens.total, + contextWindow: prompt.contextWindow, + droppedTurns: prompt.droppedTurns, + conversationTokens: prompt.tokens.conversation, + conversationCap: prompt.conversationCapEffective, + conversationCapConfigured: prompt.limits.conversation, + conversationCapAuto: prompt.conversationCapAuto, + conversationPairs: prompt.conversationPairs, + droppedPairs: prompt.droppedPairs, + conversationPairsCap: prompt.conversationPairsCap, + conversationBoundBy: prompt.conversationBoundBy, + pairCosts: prompt.pairCosts, + sections, + }; +} diff --git a/src/session/index.ts b/src/session/index.ts index ed288af9..1435f690 100644 --- a/src/session/index.ts +++ b/src/session/index.ts @@ -37,3 +37,12 @@ export type { ConversationTurn, PackedConversation, } from "./conversation-turn.js"; +export { + CONVERSATION_SECTION_LABEL, + EMPTY_CONTEXT_USAGE, + contextUsageFromPrompt, +} from "./context-usage.js"; +export type { + ContextUsageState, + ContextUsageSection, +} from "./context-usage.js"; diff --git a/src/session/session-state.ts b/src/session/session-state.ts index 83fb35c4..c2263998 100644 --- a/src/session/session-state.ts +++ b/src/session/session-state.ts @@ -8,6 +8,7 @@ import { appendTurn, type ConversationTurn, } from "./conversation-turn.js"; +import type { ContextUsageState } from "./context-usage.js"; export type SessionStatus = | "pending" @@ -124,6 +125,16 @@ export interface SessionState { createdAt: number; updatedAt: number; lastError: string | null; + /** + * Snapshot of the last turn's window occupancy — what the TUI's + * context chip draws. Stamped by `executeTurn` right before the + * post-turn save (every origin funnels through it), so reopening or + * switching into a session restores the gauge immediately instead of + * showing nothing until the next prompt is built. Deliberately NOT + * ephemeral: the whole point is to survive the process. Absent on + * sessions that predate the field or have never run a turn. + */ + contextUsage?: ContextUsageState; /** * Free-form session metadata. Reserved keys (set by the runtime, not * the agent — agents may read but must not overwrite them): diff --git a/src/session/session-store.test.ts b/src/session/session-store.test.ts index 61d5ad17..a7ed5add 100644 --- a/src/session/session-store.test.ts +++ b/src/session/session-store.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { SessionStore } from "./session-store.js"; import { createEmptySessionState } from "./session-state.js"; +import { EMPTY_CONTEXT_USAGE } from "./context-usage.js"; describe("SessionStore", () => { let tmp: string; @@ -59,6 +60,27 @@ describe("SessionStore", () => { expect(loaded?.loadedTools[0]?.name).toBe("os.git.show"); }); + it("round-trips the persisted context-usage snapshot", () => { + const initial = createEmptySessionState({ + id: "s-ctx", + workingDir: "/w", + }); + const snapshot = { + ...EMPTY_CONTEXT_USAGE, + tokens: 12_345, + contextWindow: 131_072, + conversationTokens: 9_000, + conversationPairs: 4, + sections: [{ label: "conversation", tokens: 9_000 }], + }; + store.save({ ...initial, contextUsage: snapshot }); + const loaded = store.load("s-ctx"); + expect(loaded?.contextUsage).toEqual(snapshot); + // A session written before the field existed simply has none. + store.save(createEmptySessionState({ id: "s-old", workingDir: "/w" })); + expect(store.load("s-old")?.contextUsage).toBeUndefined(); + }); + it("updates an existing session in place", () => { const state = createEmptySessionState({ id: "s2", diff --git a/src/tui/chat-orchestrator.ts b/src/tui/chat-orchestrator.ts index 12c03a1c..4e861e6d 100644 --- a/src/tui/chat-orchestrator.ts +++ b/src/tui/chat-orchestrator.ts @@ -544,6 +544,10 @@ export class ChatOrchestrator { workingDir: loaded.workingDir, messages: turnsToMessages(loaded.turns), running, + // Restore the context gauge this session persisted with its last + // turn (absent on threads that never ran one — the reducer then + // resets the chip rather than keeping the old thread's figure). + ...(loaded.contextUsage ? { contextUsage: loaded.contextUsage } : {}), }); // The stored snapshot above misses everything the still-running // turn has said (a turn saves only when it finishes — for a thread @@ -1061,6 +1065,9 @@ export class ChatOrchestrator { sessionId: turnSessionId, workingDir: this.session.workingDir, messages: turnsToMessages(this.session.turns), + ...(this.session.contextUsage + ? { contextUsage: this.session.contextUsage } + : {}), }); } const next = this.queue.shift(); diff --git a/src/tui/context-usage-from-prompt.ts b/src/tui/context-usage-from-prompt.ts index 04a0f540..2a1125bd 100644 --- a/src/tui/context-usage-from-prompt.ts +++ b/src/tui/context-usage-from-prompt.ts @@ -1,80 +1,11 @@ -import type { BuiltPrompt } from "../prompt/build-prompt-types.js"; -import type { ContextUsageSection, ContextUsageState } from "./tui-state.js"; - -/** A window nothing has been built against yet. */ -export const EMPTY_CONTEXT_USAGE: ContextUsageState = { - tokens: null, - contextWindow: null, - droppedTurns: 0, - conversationTokens: 0, - conversationCap: null, - conversationCapConfigured: null, - conversationCapAuto: false, - conversationPairs: 0, - droppedPairs: 0, - conversationPairsCap: 0, - conversationBoundBy: null, - pairCosts: [], - sections: [], -}; - /** - * Order the sections are shown in: the fixed cost first, then the - * transcript, then everything the memory fabric contributed, then the - * small stuff. Not the order `BuiltPrompt.tokens` declares them in — - * that one follows the prompt's own assembly, which is not how anyone - * reads a bill. + * The projection and its constants moved to `src/session/context-usage.ts` + * so the runtime can stamp the same snapshot onto `SessionState` without + * reaching into the TUI. This re-export keeps the TUI-side import paths + * (reducers, panels, tests) stable. */ -/** - * The transcript's row label. Exported because the context panel has to - * find that one row to recalculate it when the task count changes, and - * matching on a literal string in two files is a bug waiting for someone - * to reword one of them. - */ -export const CONVERSATION_SECTION_LABEL = "conversation"; - -const SECTIONS: readonly { - key: keyof BuiltPrompt["tokens"]; - label: string; -}[] = [ - { key: "stablePrefix", label: "prompt scaffold" }, - { key: "conversation", label: CONVERSATION_SECTION_LABEL }, - { key: "recalled", label: "recalled memory" }, - { key: "memoryIndex", label: "memory index" }, - { key: "worldSnapshot", label: "world snapshot" }, - { key: "loadedTools", label: "loaded tools" }, - { key: "loadedSkills", label: "loaded skills" }, - { key: "sessionFacts", label: "session facts" }, - { key: "profile", label: "profile" }, - { key: "taskPolicy", label: "task policy" }, -]; - -/** - * Project a built prompt into the readout the composer shows. - * - * Sections that cost nothing are dropped rather than listed as zeros: a - * session with no skills loaded should not have to read the word - * "skills" to find that out. - */ -export function contextUsageFromPrompt(prompt: BuiltPrompt): ContextUsageState { - const sections: ContextUsageSection[] = []; - for (const { key, label } of SECTIONS) { - const tokens = prompt.tokens[key]; - if (tokens > 0) sections.push({ label, tokens }); - } - return { - tokens: prompt.tokens.total, - contextWindow: prompt.contextWindow, - droppedTurns: prompt.droppedTurns, - conversationTokens: prompt.tokens.conversation, - conversationCap: prompt.conversationCapEffective, - conversationCapConfigured: prompt.limits.conversation, - conversationCapAuto: prompt.conversationCapAuto, - conversationPairs: prompt.conversationPairs, - droppedPairs: prompt.droppedPairs, - conversationPairsCap: prompt.conversationPairsCap, - conversationBoundBy: prompt.conversationBoundBy, - pairCosts: prompt.pairCosts, - sections, - }; -} +export { + CONVERSATION_SECTION_LABEL, + EMPTY_CONTEXT_USAGE, + contextUsageFromPrompt, +} from "../session/context-usage.js"; diff --git a/src/tui/reduce-ui-actions.test.ts b/src/tui/reduce-ui-actions.test.ts index 67251504..cda3a968 100644 --- a/src/tui/reduce-ui-actions.test.ts +++ b/src/tui/reduce-ui-actions.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { reduceTuiState } from "./agent-event-reducer.js"; +import { EMPTY_CONTEXT_USAGE } from "./context-usage-from-prompt.js"; import { reduceUiAction } from "./reduce-ui-actions.js"; import { THEME_NAMES } from "./theme/theme.js"; import { createInitialTuiState, type TuiSessionInfo } from "./tui-state.js"; @@ -177,6 +178,44 @@ describe("reduceUiAction message_queued", () => { }); expect(next?.queuedMessages).toEqual([]); }); + + it("restores the target session's persisted context gauge on switch", () => { + const state = { + ...createInitialTuiState(SESSION), + contextUsage: { ...EMPTY_CONTEXT_USAGE, tokens: 999 }, + }; + const snapshot = { + ...EMPTY_CONTEXT_USAGE, + tokens: 4321, + conversationTokens: 2100, + conversationPairs: 2, + sections: [{ label: "conversation", tokens: 2100 }], + }; + const next = reduceUiAction(state, { + type: "session_switched", + sessionId: "s2", + workingDir: "/tmp", + messages: [], + contextUsage: snapshot, + }); + expect(next?.contextUsage).toEqual(snapshot); + }); + + it("resets the gauge when the target session carries no snapshot", () => { + // Carrying the old thread's figure over would claim the fresh + // session is exactly as full as the one just left. + const state = { + ...createInitialTuiState(SESSION), + contextUsage: { ...EMPTY_CONTEXT_USAGE, tokens: 999 }, + }; + const next = reduceUiAction(state, { + type: "session_switched", + sessionId: "s2", + workingDir: "/tmp", + messages: [], + }); + expect(next?.contextUsage).toEqual(EMPTY_CONTEXT_USAGE); + }); }); describe("reduceUiAction while_busy_mode_changed", () => { diff --git a/src/tui/reduce-ui-actions.ts b/src/tui/reduce-ui-actions.ts index 0b87ff26..3eaec0a1 100644 --- a/src/tui/reduce-ui-actions.ts +++ b/src/tui/reduce-ui-actions.ts @@ -393,6 +393,11 @@ export function reduceUiAction( // this surface did not watch the turn start, so "elapsed since // re-attach" is the honest figure it can show. status: action.running ? "running" : "idle", + // The gauge belongs to the thread on screen: restore the target + // session's persisted snapshot, or reset when it has none — + // carrying the old thread's figure over would claim this one is + // exactly as full as the one just left. + contextUsage: action.contextUsage ?? EMPTY_CONTEXT_USAGE, messages: [...action.messages], reasoning: [], feed: [], diff --git a/src/tui/tui-action.ts b/src/tui/tui-action.ts index 9016d614..c5161bec 100644 --- a/src/tui/tui-action.ts +++ b/src/tui/tui-action.ts @@ -20,6 +20,7 @@ import type { UninstallAction } from "./uninstall/uninstall-actions.js"; import type { FallbackPanelAction } from "./llm-panel/fallback/fallback-panel-actions.js"; import type { WhileBusySubmitMode } from "../config/index.js"; import type { ChatMessage, SessionPickerEntry, TuiTab, TuiUiMode } from "./tui-state.js"; +import type { ContextUsageState } from "../session/context-usage.js"; /** * Every action the reducer knows how to fold into `TuiState`. All side @@ -228,6 +229,14 @@ export type TuiAction = * pretending the session is idle. Absent means idle. */ running?: boolean; + /** + * The target session's persisted window-occupancy snapshot, when + * it has one. The reducer restores the context chip from it; + * absent resets the gauge (a fresh session, or one that predates + * the persisted field) instead of leaving the previous thread's + * numbers on screen. + */ + contextUsage?: ContextUsageState; } /** Header/runtime: user saved a new llama-server base URL (e.g. via /llama). */ | { type: "llama_url_changed"; url: string } diff --git a/src/tui/tui-state.ts b/src/tui/tui-state.ts index 4c6d3a25..662b5aae 100644 --- a/src/tui/tui-state.ts +++ b/src/tui/tui-state.ts @@ -4,6 +4,7 @@ import { } from "../approval/approval-level.js"; import type { CodingMode } from "./coding-mode.js"; import { EMPTY_CONTEXT_USAGE } from "./context-usage-from-prompt.js"; +import type { ContextUsageState } from "../session/context-usage.js"; import type { ComposerSwitchState } from "./composer-switch/composer-switch-state.js"; import type { ContextMenuState } from "./context-menu/context-menu-state.js"; import type { ApprovalRequest } from "../approval/approval-gate.js"; @@ -230,70 +231,15 @@ export interface RollingMetrics { * readout that answers "how full is the window right now". The window * does not empty when you press Enter. * - * Every field is a snapshot of the most recent `prompt_built`, refined by - * the completion's own token count when the provider reports one. + * The shape itself now lives in `src/session/context-usage.ts` — the + * runtime persists the same snapshot on `SessionState` so a reopened + * session can restore the gauge — and is re-exported here so TUI-side + * importers keep their path. */ -export interface ContextUsageState { - /** - * Tokens in the last prompt. An estimate at `prompt_built` time - * (`estimateTokens` over-counts by design), replaced by the real - * tokenizer count once the step completes and the provider reports - * `promptTokens`. - */ - tokens: number | null; - /** - * Physical window the prompt was built against, when the runtime knows - * it. `null` on cloud providers, where the model profile carries no - * window — the chip resolves those from the model catalogue instead. - */ - contextWindow: number | null; - /** Turns `packConversation` dropped to make the transcript fit. */ - droppedTurns: number; - /** Tokens the `### conversation` section actually rendered to. */ - conversationTokens: number; - /** - * Ceiling that section is packed to — `conversationCapEffective`. The - * one number that says when older turns start being dropped, and the - * only budget figure that is defined even when nobody knows the - * physical window (the clamp falls back to the configured cap). - */ - conversationCap: number | null; - /** - * The cap as configured (`agent.conversationMaxTokens`), before the - * window clamp. Equal to `conversationCap` when config is what binds; - * larger when the window is. That comparison is the only way to tell - * an operator which knob actually moves their limit. - */ - conversationCapConfigured: number | null; - /** - * The configured cap is `0` — auto. `conversationCapConfigured` is - * then a fallback rather than a ceiling, so the comparison above says - * nothing and the panel must not name `agent.conversationMaxTokens` - * as what is holding the transcript down. Nothing is: the window is. - */ - conversationCapAuto: boolean; - /** Macro-turns the prompt carried. */ - conversationPairs: number; - /** Macro-turns dropped whole. */ - droppedPairs: number; - /** `agent.conversationMaxPairs` in force. */ - conversationPairsCap: number; - /** Which limit trimmed history, when either did. */ - conversationBoundBy: "pairs" | "tokens" | null; - /** - * Token cost of each macro-turn, oldest first — enough to price a - * different pair count without building another prompt, so moving the - * dial redraws the gauge while the operator is looking at it. - */ - pairCosts: readonly number[]; - /** Per-section breakdown, for the detail view. Empty before the first prompt. */ - sections: readonly ContextUsageSection[]; -} - -export interface ContextUsageSection { - label: string; - tokens: number; -} +export type { + ContextUsageState, + ContextUsageSection, +} from "../session/context-usage.js"; export interface TuiSessionInfo { sessionId: string | null; From 38789fb69f22b0123b862ea494e0b0a9cc1b6592 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:49:39 +0300 Subject: [PATCH 19/36] session: pin the provider/model per session and restore it on switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The active text provider + chat model are one global config setting, so every session silently followed whatever the operator last picked — switch from an OpenRouter/glm thread into a local-llama thread and the OpenRouter model kept serving it. - executeTurn stamps metadata.llm = { providerId, chatModel } (resolved from the live config at turn start, not the fallback chain's substitute) onto the session with its post-turn save. - The TUI also stamps the open session the moment the operator picks a model, so a choice made between turns survives switching away. - switchSession re-applies the target session's stamp when it differs from the active model, through the LLM panel's own bus actions (providers_select_chat_model / providers_set_active_text) so config persistence + provider reload + panel refresh stay in the one place that owns them. A stamped provider that was removed changes nothing and says so; sessions without a stamp keep the current model. - planModelRestore/readSessionLlmStamp are pure and unit-tested; malformed metadata degrades to "no stamp", never a crash. --- src/runtime/bootstrap.test.ts | 41 +++++++++++ src/runtime/bootstrap.ts | 27 ++++++- src/session/index.ts | 5 ++ src/session/session-llm.test.ts | 44 +++++++++++ src/session/session-llm.ts | 53 ++++++++++++++ src/session/session-state.ts | 5 ++ src/tui/chat-orchestrator.ts | 79 ++++++++++++++++++++ src/tui/session-model-restore.test.ts | 101 ++++++++++++++++++++++++++ src/tui/session-model-restore.ts | 77 ++++++++++++++++++++ 9 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 src/session/session-llm.test.ts create mode 100644 src/session/session-llm.ts create mode 100644 src/tui/session-model-restore.test.ts create mode 100644 src/tui/session-model-restore.ts diff --git a/src/runtime/bootstrap.test.ts b/src/runtime/bootstrap.test.ts index aaa7121a..28023ca2 100644 --- a/src/runtime/bootstrap.test.ts +++ b/src/runtime/bootstrap.test.ts @@ -13,11 +13,17 @@ import { randomBytes } from "node:crypto"; import { createAgentRuntime, managedLocalLlmHealthFailureHint } from "./bootstrap.js"; import { + getConfig, getUserConfigPath, resetConfigCache, USER_CONFIG_DEFAULTS, writeUserConfigFileSync, } from "../config/index.js"; +import { resolveLlmConfig } from "../llm/provider/registry/index.js"; +import { + readSessionLlmStamp, + SESSION_LLM_METADATA_KEY, +} from "../session/session-llm.js"; import { buildSearchCacheKey, createPersistentSearchCache, @@ -755,6 +761,41 @@ describe("createAgentRuntime", () => { } }); + it("stamps the session with the provider/model the turn ran on", async () => { + const runtime = await createAgentRuntime({ + workingDir, + approvalLevel: 5, + overrides: { + browserBackend: backend, + skipLlamaHealthCheck: true, + llamaComplete: async () => ({ + content: JSON.stringify({ + tool: "reply", + args: { text: "hi back" }, + }), + timing: { promptTokens: 5, predictedTokens: 3 }, + slotId: 0, + cacheReused: false, + }), + }, + }); + try { + const session = runtime.createSession(); + const result = await runtime.runTurn(session, "hello", { maxSteps: 5 }); + const expected = { + providerId: resolveLlmConfig(getConfig()).activeTextProvider, + chatModel: null, + }; + // The returned state and the stored row agree, so switching back + // into this session later can restore its provider/model. + expect(result.session.metadata[SESSION_LLM_METADATA_KEY]).toEqual(expected); + const reloaded = runtime.sessionStore.load(session.id)!; + expect(readSessionLlmStamp(reloaded.metadata)).toEqual(expected); + } finally { + await runtime.shutdown(); + } + }); + it("refreshSkills rebuilds the catalog and notifies listeners", async () => { let notified: Array<{ name: string }> = []; const runtime = await createAgentRuntime({ diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index 143bcc05..055057b8 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -157,6 +157,8 @@ import type { AgentLoopEvent, RunTurnResult } from "../agent/agent-loop.js"; import { SessionStore, createEmptySessionState, + SESSION_LLM_METADATA_KEY, + type SessionLlmStamp, type SessionState, } from "../session/index.js"; @@ -2303,6 +2305,18 @@ export async function createAgentRuntime( // remaining event of the turn and any tool call whose `pendingCalls` // entry went with it is logged with empty args. activeTraceSessions.add(session.id); + // Resolved before the turn runs, from the live config: the model the + // operator chose for this turn is what the session should remember, + // not whatever the config says by the time the turn finishes — and + // deliberately not the fallback chain's emergency substitute either. + const llmResolved = resolveLlmConfig(getConfig()); + const llmEntry = llmResolved.providers.find( + (p) => p.id === llmResolved.activeTextProvider, + ); + const llmStamp: SessionLlmStamp = { + providerId: llmResolved.activeTextProvider, + chatModel: llmEntry?.defaultChatModel ?? llmEntry?.model ?? null, + }; return turnContext.run({ sessionId: session.id }, async () => { try { const result = await loop.runTurn(session, { @@ -2310,8 +2324,17 @@ export async function createAgentRuntime( maxSteps: runOptions.maxSteps ?? config.agent.maxSteps, signal: runOptions.signal ?? new AbortController().signal, }); - sessionStore.save(result.session); - return result; + // Stamp what this turn ran on so switching back into the session + // later can restore its provider/model (session-llm.ts). + const finished: SessionState = { + ...result.session, + metadata: { + ...result.session.metadata, + [SESSION_LLM_METADATA_KEY]: llmStamp, + }, + }; + sessionStore.save(finished); + return { ...result, session: finished }; } finally { activeTraceSessions.delete(session.id); // A delete that arrived mid-turn was deferred to keep the pin honest; diff --git a/src/session/index.ts b/src/session/index.ts index ed288af9..dd3d4844 100644 --- a/src/session/index.ts +++ b/src/session/index.ts @@ -37,3 +37,8 @@ export type { ConversationTurn, PackedConversation, } from "./conversation-turn.js"; +export { + SESSION_LLM_METADATA_KEY, + readSessionLlmStamp, +} from "./session-llm.js"; +export type { SessionLlmStamp } from "./session-llm.js"; diff --git a/src/session/session-llm.test.ts b/src/session/session-llm.test.ts new file mode 100644 index 00000000..dc3fb44f --- /dev/null +++ b/src/session/session-llm.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { readSessionLlmStamp, SESSION_LLM_METADATA_KEY } from "./session-llm.js"; + +describe("readSessionLlmStamp", () => { + it("reads a well-formed stamp", () => { + expect( + readSessionLlmStamp({ + [SESSION_LLM_METADATA_KEY]: { + providerId: "openrouter", + chatModel: "z-ai/glm-5.2", + }, + }), + ).toEqual({ providerId: "openrouter", chatModel: "z-ai/glm-5.2" }); + }); + + it("normalises a missing or empty model to null", () => { + expect( + readSessionLlmStamp({ + [SESSION_LLM_METADATA_KEY]: { providerId: "local-llama" }, + }), + ).toEqual({ providerId: "local-llama", chatModel: null }); + expect( + readSessionLlmStamp({ + [SESSION_LLM_METADATA_KEY]: { providerId: "local-llama", chatModel: "" }, + }), + ).toEqual({ providerId: "local-llama", chatModel: null }); + }); + + it("degrades malformed values to no stamp instead of crashing", () => { + // Metadata is a free-form JSON bag: old sessions, other writers and + // hand-edited stores all feed into it. + expect(readSessionLlmStamp(undefined)).toBeNull(); + expect(readSessionLlmStamp({})).toBeNull(); + expect(readSessionLlmStamp({ [SESSION_LLM_METADATA_KEY]: null })).toBeNull(); + expect(readSessionLlmStamp({ [SESSION_LLM_METADATA_KEY]: "gpt" })).toBeNull(); + expect(readSessionLlmStamp({ [SESSION_LLM_METADATA_KEY]: [] })).toBeNull(); + expect( + readSessionLlmStamp({ [SESSION_LLM_METADATA_KEY]: { providerId: "" } }), + ).toBeNull(); + expect( + readSessionLlmStamp({ [SESSION_LLM_METADATA_KEY]: { providerId: 7 } }), + ).toBeNull(); + }); +}); diff --git a/src/session/session-llm.ts b/src/session/session-llm.ts new file mode 100644 index 00000000..54128983 --- /dev/null +++ b/src/session/session-llm.ts @@ -0,0 +1,53 @@ +/** + * Which provider/model a session runs on, remembered per session. + * + * The active text provider and its default chat model are one global + * config setting, so historically every session silently followed + * whatever the operator last picked — switch from an OpenRouter thread + * into a local-llama thread and the OpenRouter model kept serving it. + * The stamp records the session's own choice in `metadata` (under + * {@link SESSION_LLM_METADATA_KEY}) so switching back into a thread can + * re-apply the provider/model it actually ran on. + * + * Written by two hands: `executeTurn` stamps the configured active + * provider/model at the start of every turn (all origins funnel through + * it), and the TUI stamps immediately when the operator picks a model + * while a session is open — so a choice made between turns is not lost + * by switching away before the next message. + */ + +/** Reserved `SessionState.metadata` key the stamp lives under. */ +export const SESSION_LLM_METADATA_KEY = "llm"; + +export interface SessionLlmStamp { + /** Config id of the text provider the session runs on. */ + providerId: string; + /** + * Chat model id on that provider, or `null` when the provider entry + * names none (a bare llama-server serves whatever it loaded). + */ + chatModel: string | null; +} + +/** + * Read the stamp back out of session metadata. Defensive on purpose: + * metadata is a free-form JSON bag that old sessions, other writers and + * hand-edited stores all feed into, so a malformed value degrades to + * "no stamp" rather than a crash or a garbage provider switch. + */ +export function readSessionLlmStamp( + metadata: Record | undefined, +): SessionLlmStamp | null { + const raw = metadata?.[SESSION_LLM_METADATA_KEY]; + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) { + return null; + } + const providerId = (raw as { providerId?: unknown }).providerId; + if (typeof providerId !== "string" || providerId.length === 0) return null; + const chatModel = (raw as { chatModel?: unknown }).chatModel; + return { + providerId, + chatModel: + typeof chatModel === "string" && chatModel.length > 0 ? chatModel : null, + }; +} diff --git a/src/session/session-state.ts b/src/session/session-state.ts index 83fb35c4..9ff7168f 100644 --- a/src/session/session-state.ts +++ b/src/session/session-state.ts @@ -140,6 +140,11 @@ export interface SessionState { * webhook. * - `ephemeralTask: true` + `scheduledBy: ` — stamped on * fresh sessions created by `tasks.schedule` with `newSession=true`. + * - `llm: { providerId, chatModel }` — the text provider/model this + * session runs on. Stamped by `executeTurn` at the top of every + * turn and by the TUI when the operator picks a model mid-session; + * read back on session switch to restore the session's own model. + * See `session-llm.ts`. */ metadata: Record; /** diff --git a/src/tui/chat-orchestrator.ts b/src/tui/chat-orchestrator.ts index 12c03a1c..f91cc627 100644 --- a/src/tui/chat-orchestrator.ts +++ b/src/tui/chat-orchestrator.ts @@ -8,6 +8,17 @@ import { isFailedSessionStatus, type SessionState, } from "../session/session-state.js"; +import { getConfig } from "../config/index.js"; +import { resolveLlmConfig } from "../llm/provider/registry/index.js"; +import { + readSessionLlmStamp, + SESSION_LLM_METADATA_KEY, + type SessionLlmStamp, +} from "../session/session-llm.js"; +import { + describeModelRestore, + planModelRestore, +} from "./session-model-restore.js"; import { checkForAppUpdate, runAppUpdate, canSelfUpdate } from "../update/index.js"; import { clearTtyScreen } from "./clear-tty-screen.js"; import { @@ -234,6 +245,27 @@ export class ChatOrchestrator { } this.turnEvents.record(action.sessionId, action.event); }); + // A model picked while a thread is open belongs to that thread: + // stamp it immediately so switching away before the next turn runs + // does not lose the choice. `providers_select_chat_model` carries + // both ids; `providers_set_active_text` names only the provider, + // whose current default model the config still knows (setActiveText + // does not change it, so reading here is not racing the write). + bus.subscribe((action) => { + if (action.type === "providers_select_chat_model") { + this.stampSessionModel({ + providerId: action.providerId, + chatModel: action.modelId, + }); + } else if (action.type === "providers_set_active_text") { + const resolved = resolveLlmConfig(getConfig()); + const entry = resolved.providers.find((p) => p.id === action.id); + this.stampSessionModel({ + providerId: action.id, + chatModel: entry?.defaultChatModel ?? entry?.model ?? null, + }); + } + }); this.chatPull.attach(bus); } @@ -557,6 +589,9 @@ export class ChatOrchestrator { running ? " — a turn is still running here" : "" }`, }); + // Each thread keeps the model it ran on: entering one whose stamp + // differs from the active provider/model re-applies it. + this.restoreSessionModel(loaded); // After `session_switched`, so they land in the new transcript // rather than the one that was just replaced. for (const notice of notices) this.notify(notice); @@ -572,6 +607,50 @@ export class ChatOrchestrator { } } + /** + * Re-apply the model the target session last ran on, when it differs + * from the active one (`planModelRestore` decides). Goes through the + * LLM panel's own bus actions so config persistence, provider reload + * and panel refresh all happen in the one place that already owns + * them. A stamped provider that has since been removed changes + * nothing and says so. + */ + private restoreSessionModel(loaded: SessionState): void { + const plan = planModelRestore( + readSessionLlmStamp(loaded.metadata), + resolveLlmConfig(getConfig()), + ); + const line = describeModelRestore(plan); + if (line) this.bus.emit({ type: "runtime_info", line }); + if (plan.kind === "select") { + this.bus.emit({ + type: "providers_select_chat_model", + providerId: plan.providerId, + modelId: plan.modelId, + }); + } else if (plan.kind === "activate") { + this.bus.emit({ + type: "providers_set_active_text", + id: plan.providerId, + }); + } + } + + /** + * Write the provider/model stamp onto the open session and persist + * it. No-op without a live session — the choice then simply stays the + * global default the next session inherits. + */ + private stampSessionModel(stamp: SessionLlmStamp): void { + const session = this.session; + if (!session) return; + this.session = { + ...session, + metadata: { ...session.metadata, [SESSION_LLM_METADATA_KEY]: stamp }, + }; + this.runtime.sessionStore.save(this.session); + } + /** * Re-offer the re-attached turn's buffered events to the reducer. * They are tagged with the now-visible session, so they apply; live diff --git a/src/tui/session-model-restore.test.ts b/src/tui/session-model-restore.test.ts new file mode 100644 index 00000000..74402c51 --- /dev/null +++ b/src/tui/session-model-restore.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import type { ResolvedLlmConfig } from "../llm/provider/registry/index.js"; +import { + describeModelRestore, + planModelRestore, +} from "./session-model-restore.js"; + +function resolved(overrides: Partial = {}): ResolvedLlmConfig { + return { + activeTextProvider: "openrouter", + activeEmbeddingProvider: "local-llama-embed", + toolTransport: "auto", + providers: [ + { + id: "openrouter", + kind: "openrouter", + defaultChatModel: "z-ai/glm-5.2", + }, + { id: "local-llama", kind: "llama-server" }, + { id: "aimlapi", kind: "aimlapi", model: "legacy-model" }, + ], + ...overrides, + }; +} + +describe("planModelRestore", () => { + it("does nothing without a stamp", () => { + expect(planModelRestore(null, resolved())).toEqual({ kind: "none" }); + }); + + it("does nothing when the stamp is already the active provider/model", () => { + expect( + planModelRestore( + { providerId: "openrouter", chatModel: "z-ai/glm-5.2" }, + resolved(), + ), + ).toEqual({ kind: "none" }); + }); + + it("selects the stamped model when the session ran on a different one", () => { + expect( + planModelRestore( + { providerId: "openrouter", chatModel: "another/model" }, + resolved(), + ), + ).toEqual({ + kind: "select", + providerId: "openrouter", + modelId: "another/model", + }); + }); + + it("selects across providers, falling back to the legacy `model` field", () => { + expect( + planModelRestore( + { providerId: "aimlapi", chatModel: "legacy-model" }, + resolved(), + ), + ).toEqual({ + kind: "select", + providerId: "aimlapi", + modelId: "legacy-model", + }); + }); + + it("activates a model-less provider instead of selecting", () => { + expect( + planModelRestore({ providerId: "local-llama", chatModel: null }, resolved()), + ).toEqual({ kind: "activate", providerId: "local-llama" }); + // …and stays put when that provider is already active. + expect( + planModelRestore( + { providerId: "local-llama", chatModel: null }, + resolved({ activeTextProvider: "local-llama" }), + ), + ).toEqual({ kind: "none" }); + }); + + it("reports a provider deleted since the session ran, changing nothing", () => { + const plan = planModelRestore( + { providerId: "gone", chatModel: "x/y" }, + resolved(), + ); + expect(plan).toEqual({ kind: "missing", providerId: "gone", chatModel: "x/y" }); + expect(describeModelRestore(plan)).toContain("no longer configured"); + }); + + it("describes only plans that act or warn", () => { + expect(describeModelRestore({ kind: "none" })).toBeNull(); + expect( + describeModelRestore({ + kind: "select", + providerId: "openrouter", + modelId: "another/model", + }), + ).toContain("openrouter/another/model"); + expect( + describeModelRestore({ kind: "activate", providerId: "local-llama" }), + ).toContain("local-llama"); + }); +}); diff --git a/src/tui/session-model-restore.ts b/src/tui/session-model-restore.ts new file mode 100644 index 00000000..d5e3a9d2 --- /dev/null +++ b/src/tui/session-model-restore.ts @@ -0,0 +1,77 @@ +import type { ResolvedLlmConfig } from "../llm/provider/registry/index.js"; +import type { SessionLlmStamp } from "../session/session-llm.js"; + +/** + * What switching into a session should do about the active model. + * + * Pure decision, separated from the orchestrator so the interesting + * cases (no stamp, provider deleted since, already active, model-less + * provider) are unit-testable without a bus or a config file. The + * orchestrator translates the plan into the same actions the LLM panel + * emits — `providers_select_chat_model` / `providers_set_active_text` — + * so restoring goes through the one code path that already knows how to + * persist the config and reload the provider. + */ +export type ModelRestorePlan = + /** Nothing to do: no stamp, or the stamp is already the active model. */ + | { kind: "none" } + /** The stamped provider is gone from the config; say so, change nothing. */ + | { kind: "missing"; providerId: string; chatModel: string | null } + /** Re-apply provider + chat model (the LLM panel's select-model path). */ + | { kind: "select"; providerId: string; modelId: string } + /** + * Re-apply the provider alone — the stamp names no model (e.g. a bare + * llama-server entry), so only the active-provider switch applies. + */ + | { kind: "activate"; providerId: string }; + +export function planModelRestore( + stamp: SessionLlmStamp | null, + resolved: ResolvedLlmConfig, +): ModelRestorePlan { + if (!stamp) return { kind: "none" }; + const entry = resolved.providers.find((p) => p.id === stamp.providerId); + if (!entry) { + return { + kind: "missing", + providerId: stamp.providerId, + chatModel: stamp.chatModel, + }; + } + const activeEntry = resolved.providers.find( + (p) => p.id === resolved.activeTextProvider, + ); + const activeModel = + activeEntry?.defaultChatModel ?? activeEntry?.model ?? null; + const sameProvider = stamp.providerId === resolved.activeTextProvider; + if (stamp.chatModel === null) { + // A model-less stamp asks only for the provider. When it is already + // active, whatever model it currently serves is as close to "what + // the session ran on" as the stamp can say. + return sameProvider + ? { kind: "none" } + : { kind: "activate", providerId: stamp.providerId }; + } + if (sameProvider && stamp.chatModel === activeModel) return { kind: "none" }; + return { + kind: "select", + providerId: stamp.providerId, + modelId: stamp.chatModel, + }; +} + +/** One line for the transcript describing what a plan is about to do. */ +export function describeModelRestore(plan: ModelRestorePlan): string | null { + switch (plan.kind) { + case "none": + return null; + case "missing": + return `this session last ran on "${plan.providerId}${ + plan.chatModel ? `/${plan.chatModel}` : "" + }", which is no longer configured — keeping the current model`; + case "select": + return `restoring this session's model: ${plan.providerId}/${plan.modelId}`; + case "activate": + return `restoring this session's provider: ${plan.providerId}`; + } +} From 2f0a9b41b39c9e1c19e9ec690a16afc6ac6e2bb6 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:54:19 +0300 Subject: [PATCH 20/36] tui: a stopped turn says who stopped it, and offers a retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aborting via the stop chip (or Esc / Ctrl+C / /abort) used to land a warn-styled 'Turn failed [cancelled]: This operation was aborted' in the chat — an error wall for something the operator did on purpose. Worse, the abort races the LLM stream, so it sometimes surfaced as 'Turn failed [model]: model returned empty content', dressing the user's own stop as a provider failure. The reducer now treats loop_failed as stopped-by-user when the category is 'cancelled' OR an abort_requested is on the books (state.aborting — every abort entry point sets it, finishRun clears it). It leaves a calm system notice — 'Agent stopped by user.' — that carries the aborted turn's prompt as retryText, and chat-log hangs the existing [try again] beside [copy] on exactly that notice, resending the user's prompt: a mistaken stop is one click to undo. --- src/tui/agent-event-reducer.test.ts | 92 ++++++++++++++++++++ src/tui/agent-event-reducer.ts | 37 ++++++++ src/tui/components/chat-log.test.tsx | 27 ++++++ src/tui/components/chat-log.tsx | 13 ++- src/tui/components/chat-try-again-button.tsx | 5 +- src/tui/tui-state.ts | 8 ++ 6 files changed, 180 insertions(+), 2 deletions(-) diff --git a/src/tui/agent-event-reducer.test.ts b/src/tui/agent-event-reducer.test.ts index a78e62f0..dd740d73 100644 --- a/src/tui/agent-event-reducer.test.ts +++ b/src/tui/agent-event-reducer.test.ts @@ -353,6 +353,98 @@ describe("reduceTuiState", () => { expect(errMsg?.text).toBe("Turn failed [tool]: boom"); }); + it("renders a calm stopped-by-user notice with a retry prompt on a cancelled loop_failed", () => { + const initial = createInitialTuiState(fakeSession()); + const next = apply(initial, [ + { type: "agent_event", event: { type: "user_message", text: "count the stars" } }, + { type: "message_submitted" }, + { + type: "agent_event", + event: { + type: "loop_failed", + error: new Error("This operation was aborted"), + category: "cancelled", + }, + }, + ]); + expect(next.status).toBe("idle"); + expect(next.lastRunStatus).toBe("stopped by user"); + expect(next.runHistory[0]?.outcome).toBe("cancelled"); + // No warn-styled "Turn failed" wall: the operator did this on + // purpose and the notice says so, carrying the aborted turn's + // prompt for the [try again] affordance. + const warn = next.messages.find( + (m) => m.role === "system" && m.variant === "warn", + ); + expect(warn).toBeUndefined(); + const notice = next.messages.find((m) => m.role === "system"); + expect(notice?.text).toBe("Agent stopped by user."); + expect(notice?.retryText).toBe("count the stars"); + }); + + it("treats any loop_failed during a requested abort as stopped-by-user", () => { + // The abort races the LLM stream: a killed response can surface as + // `[model] model returned empty content` before the AbortError + // does. With `abort_requested` on the books, that is still the + // operator's stop, not a provider failure. + const initial = createInitialTuiState(fakeSession()); + const next = apply(initial, [ + { type: "agent_event", event: { type: "user_message", text: "count the stars" } }, + { type: "message_submitted" }, + { type: "abort_requested" }, + { + type: "agent_event", + event: { + type: "loop_failed", + error: new Error("model returned empty content"), + category: "model", + }, + }, + ]); + expect(next.lastRunStatus).toBe("stopped by user"); + expect(next.aborting).toBe(false); + const notice = next.messages.find((m) => m.role === "system"); + expect(notice?.text).toBe("Agent stopped by user."); + expect(notice?.retryText).toBe("count the stars"); + }); + + it("keeps the warn styling for a loop_failed with no abort on the books", () => { + const initial = createInitialTuiState(fakeSession()); + const next = apply(initial, [ + { type: "message_submitted" }, + { + type: "agent_event", + event: { + type: "loop_failed", + error: new Error("model returned empty content"), + category: "model", + }, + }, + ]); + const warn = next.messages.find( + (m) => m.role === "system" && m.variant === "warn", + ); + expect(warn?.text).toBe("Turn failed [model]: model returned empty content"); + }); + + it("leaves retryText off the stopped notice when no user message exists to re-run", () => { + const initial = createInitialTuiState(fakeSession()); + const next = apply(initial, [ + { type: "message_submitted" }, + { + type: "agent_event", + event: { + type: "loop_failed", + error: new Error("This operation was aborted"), + category: "cancelled", + }, + }, + ]); + const notice = next.messages.find((m) => m.role === "system"); + expect(notice?.text).toBe("Agent stopped by user."); + expect(notice?.retryText).toBeUndefined(); + }); + it("appends the llama hint on transport failure for a custom-id llama-server route", () => { const initial = createInitialTuiState(fakeSession()); const next = apply(initial, [ diff --git a/src/tui/agent-event-reducer.ts b/src/tui/agent-event-reducer.ts index 2f8baaf6..db2abf89 100644 --- a/src/tui/agent-event-reducer.ts +++ b/src/tui/agent-event-reducer.ts @@ -17,6 +17,7 @@ import { finishRun, finishRunWithoutHistory, finishTurn, + lastUserMessage, pushRing, startNewRun, upsertReasoning, @@ -458,6 +459,42 @@ function reduceAgentEvent(state: TuiState, event: AgentLoopEvent): TuiState { ); } case "loop_failed": { + // A user-initiated abort is not a failure and must not dress like + // one: the operator pressed stop (the chip, Esc, Ctrl+C or + // `/abort`) and already knows the turn is dead. Instead of the + // warn-styled `Turn failed [cancelled]: This operation was + // aborted` wall, leave a calm system notice that says who stopped + // it — and carry the aborted turn's user message as `retryText`, + // so a mistaken click is one `[try again]` away from undone. + // + // `state.aborting` is checked alongside the category because the + // abort races the LLM stream: killing a response mid-flight can + // surface as `[model] model returned empty content` (or another + // category) before the AbortError ever propagates, and an + // operator who just pressed stop would read that as a provider + // failure they caused. Every abort entry point dispatches + // `abort_requested` first, and `finishRun` clears the flag, so + // the window is exactly the abort the operator asked for. + if (event.category === "cancelled" || state.aborting) { + const lastRunStatus = "stopped by user"; + const prompt = lastUserMessage(state); + return finishRun( + appendChatMessage( + appendFeed(state, { + kind: "loop_failed", + stepIndex: null, + line: `» ${lastRunStatus}`, + color: "yellow", + }), + { + role: "system", + text: "Agent stopped by user.", + retryText: prompt.length > 0 ? prompt : undefined, + }, + ), + { outcome: "cancelled", reason: lastRunStatus, lastRunStatus }, + ); + } const lastRunStatus = `failed [${event.category}]: ${event.error.message}`; const chatError = formatAgentErrorForChat( event.category, diff --git a/src/tui/components/chat-log.test.tsx b/src/tui/components/chat-log.test.tsx index f9ecc30d..e65dda54 100644 --- a/src/tui/components/chat-log.test.tsx +++ b/src/tui/components/chat-log.test.tsx @@ -228,4 +228,31 @@ describe("ChatLog", () => { expect(text).toContain("Hi"); expect(text).toMatch(/reasoning/); }); + + it("hangs a [try again] under the stopped-by-user notice, and only there", () => { + const state: TuiState = { + ...createInitialTuiState(BASE_SESSION), + messages: [ + { + id: "m1", + role: "system", + text: "Agent stopped by user.", + retryText: "count the stars", + timestamp: 1, + }, + { + id: "m2", + role: "system", + text: "queue cleared", + timestamp: 2, + }, + ], + }; + const { lastFrame } = render(); + const text = strip(lastFrame() ?? ""); + expect(text).toContain("Agent stopped by user."); + // Exactly one button: the notice with `retryText` earns it, the + // plain runtime notice under it does not. + expect(text.match(/\[try again\]/g)).toHaveLength(1); + }); }); diff --git a/src/tui/components/chat-log.tsx b/src/tui/components/chat-log.tsx index d93ee188..04e84751 100644 --- a/src/tui/components/chat-log.tsx +++ b/src/tui/components/chat-log.tsx @@ -270,7 +270,18 @@ function FinalisedMessage({ text={message.text} warn={message.variant === "warn"} /> - + + + {/* + Only the abort notice sets `retryText`, and the button resends + THAT — the stopped turn's user prompt — not the notice's own + text. Same shared footer row as every other role, so + `estimateMessageHeight` stays role-blind. + */} + {message.retryText !== undefined ? ( + + ) : null} + ); } diff --git a/src/tui/components/chat-try-again-button.tsx b/src/tui/components/chat-try-again-button.tsx index 20855b96..84a96dda 100644 --- a/src/tui/components/chat-try-again-button.tsx +++ b/src/tui/components/chat-try-again-button.tsx @@ -77,7 +77,10 @@ export function resubmitChatMessage( * prose; sending it back would open a turn whose prompt is the previous * answer, which is not "try again" in any sense an operator means. A * system message is TUI runtime output — queue listings, turn-failed - * lines — and re-sending one as a prompt is worse than nonsense. Asking + * lines — and re-sending one as a prompt is worse than nonsense. The + * one system notice that carries the button — "Agent stopped by user", + * via `ChatMessage.retryText` — is no exception: what it resends is the + * aborted turn's *user* prompt, never its own text. Asking * the model to have another go at the *same* question is a different * feature (it has to drop the last turn, not append one) and it is not * this button. diff --git a/src/tui/tui-state.ts b/src/tui/tui-state.ts index 4c6d3a25..f3ca8f30 100644 --- a/src/tui/tui-state.ts +++ b/src/tui/tui-state.ts @@ -135,6 +135,14 @@ export interface ChatMessage { text: string; /** `warn` — failure / runtime error styling in {@link SystemBubble}. */ variant?: ChatMessageVariant; + /** + * A user prompt this notice offers to re-run. Set on the system + * notice a user-initiated abort leaves in the chat: the stop was one + * click, so undoing a mistaken one should be too. `chat-log.tsx` + * renders a `[try again]` beside `[copy]` that resubmits THIS text — + * the aborted turn's user message — never the notice's own text. + */ + retryText?: string; /** Number of tool steps the assistant ran inside this turn. */ toolSteps?: number; /** Tool cards (call + result) attached to this assistant turn. */ From cc175ae51ca195b3f6e695bbcda244d9dc51c70a Mon Sep 17 00:00:00 2001 From: Valerii Date: Wed, 2 Sep 2026 19:07:42 +0300 Subject: [PATCH 21/36] tui: update banner clickable through the modal floor; lifecycle strip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two field reports from the first demo round: The obvious first click a fresh launch invites is the banner's Update button — while the startup offer modal is still on screen. The modal raises the mouse floor to the modal rung, so that click was silently swallowed. The button now registers on the modal rung itself: a click there is exactly the modal's y, so answering the offer from the corner is the same decision, not a bypass. And once an update is accepted, the corner went quiet at the very moment it had something worth saying. The strip now narrates the lifecycle instead of hiding: 'updating to vX — do not close' while the installer runs (degrading to 'updating — do not close', then 'updating…'), 'updated to vX — restart to apply' once it lands, and back to the offer — the retry path — after a failure. --fake-update's accept now walks the same event sequence the real installer emits (update_started, feed lines, update_finished ok) on a watchable timeline instead of a one-line notice, so the whole arc — modal, click-through, do-not-close strip, restart prompt — is on show; the restart re-execs the same dev command, so nothing is ever installed. --- src/tui/components/status-bar-update.test.tsx | 24 +++- src/tui/components/status-bar.tsx | 32 +++-- src/tui/components/update-banner.test.tsx | 59 +++++++-- src/tui/components/update-banner.tsx | 121 ++++++++++++------ src/tui/tui-command.ts | 37 +++++- 5 files changed, 207 insertions(+), 66 deletions(-) diff --git a/src/tui/components/status-bar-update.test.tsx b/src/tui/components/status-bar-update.test.tsx index fb4fb9d5..0d3ae456 100644 --- a/src/tui/components/status-bar-update.test.tsx +++ b/src/tui/components/status-bar-update.test.tsx @@ -30,13 +30,27 @@ describe("StatusBar update banner", () => { expect(frame).toContain("Update"); }); - it("yields while the installer runs, and returns on failure", () => { + it("narrates the install instead of offering it while the installer runs", () => { const running = apply(offered(), [{ type: "update_started" }]); - expect( - strip(render().lastFrame() ?? ""), - ).not.toContain("Update"); + const frame = strip(render().lastFrame() ?? ""); + expect(frame).toContain("do not close"); + // The button is gone: a second accept mid-install has no meaning. + expect(frame).not.toContain("Update"); + }); + + it("says a restart applies it once the installer lands", () => { + const done = apply(offered(), [ + { type: "update_started" }, + { type: "update_finished", ok: true, version: "9.9.9" }, + ]); + const frame = strip(render().lastFrame() ?? ""); + expect(frame).toContain("restart to apply"); + expect(frame).not.toContain("Update"); + }); - const failed = apply(running, [ + it("returns to the offer — the retry path — after a failed install", () => { + const failed = apply(offered(), [ + { type: "update_started" }, { type: "update_finished", ok: false, error: "boom" }, ]); expect( diff --git a/src/tui/components/status-bar.tsx b/src/tui/components/status-bar.tsx index 4782332a..c98525d6 100644 --- a/src/tui/components/status-bar.tsx +++ b/src/tui/components/status-bar.tsx @@ -12,7 +12,11 @@ import type { TuiState } from "../tui-state.js"; import { getAppVersion } from "../../version.js"; import { Chip, tracked } from "./chip.js"; import { sessionTitleLine } from "./session-title.js"; -import { planUpdateBanner, UpdateBanner } from "./update-banner.js"; +import { + planUpdateBanner, + UpdateBanner, + type UpdateBannerPhase, +} from "./update-banner.js"; interface StatusBarProps { state: TuiState; @@ -68,13 +72,17 @@ export function StatusBar({ const title = currentSessionTitle(state); const { columns } = useTerminalSize(); // The banner outlives the modal (`updateBanner` survives - // `update_dismissed`) and yields only to an update actually running - // or finished — `failed` keeps it up, because the banner is then the - // one remaining way to retry. - const banner = - state.updateStatus === "idle" || state.updateStatus === "failed" - ? state.updateBanner - : null; + // `update_dismissed`) and then narrates the whole lifecycle: the + // offer while nothing runs, "do not close" while the installer works, + // the restart hint once it lands. `failed` renders as a fresh offer, + // because the button is then the one remaining way to retry. + const banner = state.updateBanner; + const bannerPhase: UpdateBannerPhase = + state.updateStatus === "running" + ? "running" + : state.updateStatus === "done" + ? "done" + : "offer"; // `chipBudget` reserves cells for a session tag whether or not one is // drawn — safe slack for the download chip, but it starves the banner // out of a fresh 70-column session where the corner is visibly empty. @@ -85,7 +93,7 @@ export function StatusBar({ (state.session.sessionId ? 0 : SESSION_TAG), ); const bannerPlan = banner - ? planUpdateBanner(banner.latest, bannerBudget) + ? planUpdateBanner(banner.latest, bannerBudget, bannerPhase) : null; return ( @@ -128,7 +136,11 @@ export function StatusBar({ the bar knows its row width; content-sized bars (no `width`) collapse the spacer to two plain cells. */} - + ) : null} diff --git a/src/tui/components/update-banner.test.tsx b/src/tui/components/update-banner.test.tsx index 607a3aa3..567d517e 100644 --- a/src/tui/components/update-banner.test.tsx +++ b/src/tui/components/update-banner.test.tsx @@ -7,7 +7,7 @@ const strip = (s: string): string => s.replace(/\u001b\[[0-9;]*m/g, ""); describe("UpdateBanner", () => { it("says the whole sentence when the row has room", () => { - const view = render(); + const view = render(); const frame = strip(view.lastFrame() ?? ""); expect(frame).toContain("new version v9.9.9 available"); expect(frame).toContain("Update"); @@ -15,29 +15,70 @@ describe("UpdateBanner", () => { it("sheds the sentence, then the version, as the row fills up", () => { const medium = strip( - render().lastFrame() ?? "", + render().lastFrame() ?? "", ); expect(medium).toContain("v9.9.9"); expect(medium).not.toContain("new version"); expect(medium).toContain("Update"); const tight = strip( - render().lastFrame() ?? "", + render().lastFrame() ?? "", ); expect(tight).toContain("Update"); expect(tight).not.toContain("9.9.9"); }); it("disappears rather than wrapping the one-row bar", () => { - const view = render(); + const view = render(); expect(strip(view.lastFrame() ?? "").trim()).toBe(""); }); - it("never plans a form wider than its budget", () => { - for (const latest of ["1.0.0", "10.20.30", "0.5.5-rc.1"]) { - for (let budget = 0; budget <= 60; budget += 1) { - const plan = planUpdateBanner(latest, budget); - if (plan) expect(plan.width).toBeLessThanOrEqual(budget); + it("tells the operator not to close the terminal while installing", () => { + const frame = strip( + render( + , + ).lastFrame() ?? "", + ); + expect(frame).toContain("updating to v9.9.9"); + expect(frame).toContain("do not close"); + expect(frame).not.toContain("Update"); + }); + + it("asks for a restart once the install has landed", () => { + const frame = strip( + render( + , + ).lastFrame() ?? "", + ); + expect(frame).toContain("restart to apply"); + expect(frame).not.toContain("Update"); + }); + + it("degrades the running strip rather than wrapping it", () => { + const medium = strip( + render( + , + ).lastFrame() ?? "", + ); + expect(medium).toContain("do not close"); + expect(medium).not.toContain("9.9.9"); + + const tight = strip( + render( + , + ).lastFrame() ?? "", + ); + expect(tight).toContain("updating"); + expect(tight).not.toContain("do not close"); + }); + + it("never plans a form wider than its budget, in any phase", () => { + for (const phase of ["offer", "running", "done"] as const) { + for (const latest of ["1.0.0", "10.20.30", "0.5.5-rc.1"]) { + for (let budget = 0; budget <= 60; budget += 1) { + const plan = planUpdateBanner(latest, budget, phase); + if (plan) expect(plan.width).toBeLessThanOrEqual(budget); + } } } }); diff --git a/src/tui/components/update-banner.tsx b/src/tui/components/update-banner.tsx index 41284221..2d5d0c6d 100644 --- a/src/tui/components/update-banner.tsx +++ b/src/tui/components/update-banner.tsx @@ -2,21 +2,22 @@ import { Text } from "ink"; import type { ReactElement } from "react"; import { MouseTarget, useMouseCommands } from "../mouse/mouse-context.js"; import { isPrimaryPress } from "../mouse/mouse-event.js"; +import { MOUSE_LAYER_MODAL } from "../mouse/mouse-registry.js"; import { theme } from "../theme/theme.js"; /** - * The persistent "a newer release exists" strip at the right end of the - * status bar. + * The persistent update strip at the right end of the status bar. * * The startup {@link UpdateModal} already offers the update once; this - * banner is what remains after the operator skips it. It has to survive - * the whole session without stealing attention from the work — so it - * sits in the one corner the eye only visits deliberately, and it never - * blinks, animates, or claims a key. What it *does* claim is contrast: - * the strip renders inverse-video, swapping ink and ground, which is - * distinguishable on every palette by construction — whatever the - * terminal's background is, the banner is its opposite. No hand-picked - * colour can promise that across twelve palettes and user terminals. + * banner is what remains after the operator skips it — and what narrates + * the install once they accept. It has to survive the whole session + * without stealing attention from the work, so it sits in the one corner + * the eye only visits deliberately, and it never blinks, animates, or + * claims a key. What it *does* claim is contrast: the strip renders + * inverse-video, swapping ink and ground, which is distinguishable on + * every palette by construction — whatever the terminal's background is, + * the banner is its opposite. No hand-picked colour can promise that + * across twelve palettes and user terminals. * * `Update` is the click target and runs the same path as the modal's * `y` (`onUpdateConfirmed` → `runUpdate`), including its refusal while @@ -25,28 +26,61 @@ import { theme } from "../theme/theme.js"; */ export interface UpdateBannerProps { latest: string; + /** + * Where the update is in its life. `offer` shows the sentence and the + * button; `running` swaps them for "updating — do not close" (the + * installer is replacing the binary and the one useful instruction is + * to leave it alone); `done` says a restart applies it. The bar maps + * `updateStatus` onto this — the failed state renders as a fresh + * `offer`, because the button is then the way to retry. + */ + phase: UpdateBannerPhase; /** * Columns the banner may use. Ink wraps rather than clips, so an * over-wide banner would fold the one-row status bar into a - * paragraph; the banner degrades instead — full sentence, then bare - * version, then the button alone, then nothing. + * paragraph; the banner degrades instead — full sentence, then a + * terse one, then (for `offer`) the button alone, then nothing. */ budget: number; } +export type UpdateBannerPhase = "offer" | "running" | "done"; + /** The click target. Fixed label, so its width is a constant. */ const BUTTON = " Update "; -/** Cell between the label and the button. */ -const GAP = 1; - export interface UpdateBannerPlan { - /** Inverse-video label before the button; `null` for button-only. */ + /** Inverse-video label; `null` for the button-only offer form. */ label: string | null; + /** Whether the `Update` button renders (offer phase only). */ + button: boolean; /** Total cells the banner occupies, button included. */ width: number; } +/** Longest-first label ladder for each phase. */ +function labelLadder(phase: UpdateBannerPhase, latest: string): string[] { + switch (phase) { + case "offer": + return [` new version v${latest} available `, ` v${latest} `]; + case "running": + // "do not close" is the payload: the installer is mid-way through + // replacing the binary, and killing the terminal now is the one + // thing the operator can do to make it worse. + return [ + ` updating to v${latest} — do not close `, + ` updating — do not close `, + ` updating… `, + ]; + case "done": + return [ + ` updated to v${latest} — restart to apply `, + ` restart to apply `, + ` updated `, + ]; + } +} + /** * Which form fits the budget. Exported so the status bar can subtract * the banner's real width from the download chip's budget instead of @@ -55,23 +89,26 @@ export interface UpdateBannerPlan { export function planUpdateBanner( latest: string, budget: number, + phase: UpdateBannerPhase = "offer", ): UpdateBannerPlan | null { - const full = ` new version v${latest} available `; - const short = ` v${latest} `; - for (const label of [full, short]) { - const width = label.length + GAP + BUTTON.length; - if (width <= budget) return { label, width }; + const button = phase === "offer"; + const buttonWidth = button ? BUTTON.length : 0; + for (const label of labelLadder(phase, latest)) { + const width = label.length + buttonWidth; + if (width <= budget) return { label, button, width }; } - if (BUTTON.length <= budget) return { label: null, width: BUTTON.length }; + if (button && BUTTON.length <= budget) + return { label: null, button, width: BUTTON.length }; return null; } export function UpdateBanner({ latest, + phase, budget, }: UpdateBannerProps): ReactElement | null { const mouse = useMouseCommands(); - const plan = planUpdateBanner(latest, budget); + const plan = planUpdateBanner(latest, budget, phase); if (!plan) return null; // Inverse accent: the palette's accent as ground, the terminal's own // background as ink. Louder than the inverse label beside it, so the @@ -85,20 +122,30 @@ export function UpdateBanner({ // Box to own a measurable region, and Ink refuses a Box inside Text. return ( <> - {plan.label ? {`${plan.label} `} : null} - {mouse ? ( - { - if (!isPrimaryPress(hit.event)) return false; - mouse.callbacks.onUpdateConfirmed?.(); - return true; - }} - > - {button} - - ) : ( - button - )} + {plan.label ? {plan.label} : null} + {plan.button ? ( + mouse ? ( + { + if (!isPrimaryPress(hit.event)) return false; + mouse.callbacks.onUpdateConfirmed?.(); + return true; + }} + > + {button} + + ) : ( + button + ) + ) : null} ); } diff --git a/src/tui/tui-command.ts b/src/tui/tui-command.ts index 963a0ac8..6fcdf588 100644 --- a/src/tui/tui-command.ts +++ b/src/tui/tui-command.ts @@ -13,6 +13,7 @@ import { checkLlamaServer } from "../llm/llama-server-health.js"; import { describeLlamaHealthFailure } from "../llm/describe-llama-health-failure.js"; import { createAgentRuntime, type AgentRuntime } from "../runtime/bootstrap.js"; import { getAppVersion } from "../version.js"; +import type { TuiAction } from "./tui-action.js"; import type { LogRecord, LogSink } from "../tracing/structured-logger.js"; import type { MetricSample, MetricSink } from "../tracing/metrics-collector.js"; import { isKnownLocalModelId } from "../local-llm/index.js"; @@ -666,11 +667,12 @@ export async function tuiCommand(args: string[]): Promise { ? // The testing ground must never reach install.sh: the // point of `--fake-update` is to look at the surfaces, and // "accept" on a dev build would install the real latest - // release over whatever is being worked on. - bus.emit({ - type: "system_message", - text: `--fake-update: accepted (v${parsed.fakeUpdateVersion}); install skipped in fake mode`, - }) + // release over whatever is being worked on. Instead, walk + // the same events the real installer emits so the whole + // lifecycle — "do not close" strip, feed lines, restart + // prompt — is on show. The restart re-execs this same dev + // command, which is a no-op by construction. + simulateFakeUpdate(bus, parsed.fakeUpdateVersion) : orchestrator.runUpdate(), onUpdateRestart: () => { restartRequested = true; @@ -834,6 +836,31 @@ export async function tuiCommand(args: string[]): Promise { return orchestrator.exitCode; } +/** + * `--fake-update` accept path: emit the exact event sequence + * `runUpdate` emits, on a human-watchable timeline, without ever + * touching the installer. Ends in `update_finished ok`, so the "press + * any key to restart" prompt is exercised too — the restart re-execs + * the same `tui --fake-update` command, landing back at the offer. + */ +function simulateFakeUpdate( + bus: ReturnType, + version: string, +): void { + bus.emit({ type: "update_started" }); + const script: readonly [number, TuiAction][] = [ + [400, { type: "runtime_info", line: `[update] (fake) downloading atomic-agent v${version}…` }], + [1500, { type: "runtime_info", line: "[update] (fake) verifying checksum…" }], + [2200, { type: "runtime_info", line: "[update] (fake) installing — nothing on this machine is being replaced" }], + [3000, { type: "update_finished", ok: true, version }], + ]; + for (const [delay, action] of script) { + // Unref'd so a Ctrl+C mid-"install" never has the process lingering + // on demo timers. + setTimeout(() => bus.emit(action), delay).unref(); + } +} + /** * Ctrl+N / `/window`: launch a second agent in a new OS terminal window. * Fire-and-forget — the result is reported into the chat log either way, From 1b677b8642f015f8d8a5917a81c6916b1c1df914 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:05:08 +0300 Subject: [PATCH 22/36] fix(llm): provider-availability failures no longer block fallover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two kinds of "this provider is unusable" failure were filed under categories that shouldAdvance refuses to advance on, so a permanently broken link pinned the whole fallback chain and the user got a diagnosis for a problem they did not have. - llama-server 4xx: every non-null status under 500 mapped to grammar. A 404 (the configured localModels.url does not serve completions) or a 405 read as "Turn failed [grammar]" and stopped the chain. The endpoint/auth/availability statuses (401 402 403 404 405 408 409 429) now classify as transport; the request-shape statuses (400 413 422 and any other unlisted 4xx) stay grammar. - SubscriptionCliNotInstalledError / SubscriptionCliAuthError were plain Errors, so they fell through to the catch-all tool arm. A missing or signed-out claude/codex CLI now classifies as transport. Routing the llama 404 to transport also unblocks the llama-unreachable hint in format-agent-error-for-chat, which is gated on the transport category — the one failure where "check your llama URL" is the right advice was the one that never got it. Doc comments in failure-category.ts, classify-failure.ts and should-advance.ts updated to the new rule. --- src/llm/fallback/should-advance.test.ts | 59 ++++++++++++++++- src/llm/fallback/should-advance.ts | 14 +++-- src/llm/reliability/classify-failure.test.ts | 66 +++++++++++++++++++- src/llm/reliability/classify-failure.ts | 40 ++++++++++++ src/llm/reliability/failure-category.ts | 15 +++-- src/tui/format-agent-error-for-chat.test.ts | 47 ++++++++++++++ 6 files changed, 230 insertions(+), 11 deletions(-) diff --git a/src/llm/fallback/should-advance.test.ts b/src/llm/fallback/should-advance.test.ts index 98f1291e..259f3d2e 100644 --- a/src/llm/fallback/should-advance.test.ts +++ b/src/llm/fallback/should-advance.test.ts @@ -9,6 +9,10 @@ import { ToolExecutionError, CancelledError, } from "../reliability/llm-failures.js"; +import { + SubscriptionCliAuthError, + SubscriptionCliNotInstalledError, +} from "../provider/subscription-cli/subscription-cli-errors.js"; describe("shouldAdvance", () => { it("advances immediately on a cloud 429", () => { @@ -75,14 +79,65 @@ describe("shouldAdvance", () => { }); }); - it("does NOT advance on a local llama 4xx (grammar category)", () => { - // LlamaServerError with a 4xx status classifies as grammar. + it("does NOT advance on a local llama 400 (request-shape, grammar category)", () => { + // A 400 is the server rejecting THIS request; the next link rejects + // it the same way, so falling over buys nothing. expect(shouldAdvance(new LlamaServerError("x", 400, "http://local"))).toEqual({ advance: false, immediate: false, }); }); + it("advances via threshold on a local llama 404 — the URL serves no completions", () => { + // The endpoint is permanently wrong (bad `localModels.url`, or a + // server that is not a llama-server). Not an immediate signal: 404 is + // not in the 429/408/5xx provider-down set, so it advances once the + // consecutive-failure threshold trips. + expect(shouldAdvance(new LlamaServerError("x", 404, "http://local"))).toEqual({ + advance: true, + immediate: false, + }); + }); + + it("advances via threshold on a local llama 405", () => { + expect(shouldAdvance(new LlamaServerError("x", 405, "http://local"))).toEqual({ + advance: true, + immediate: false, + }); + }); + + it("advances immediately on a local llama 429", () => { + // Transport category plus an unambiguous provider-down status — the + // `isImmediateSignal` status read already handled 429; it was simply + // unreachable while 4xx classified as grammar. + expect(shouldAdvance(new LlamaServerError("x", 429, "http://local"))).toEqual({ + advance: true, + immediate: true, + }); + }); + + it("advances immediately on a local llama 408", () => { + expect(shouldAdvance(new LlamaServerError("x", 408, "http://local"))).toEqual({ + advance: true, + immediate: true, + }); + }); + + it("advances on a subscription-CLI binary that is not installed", () => { + // No HTTP status to read, so it advances via the threshold — but it + // must advance: a missing `claude` binary otherwise pins the chain to + // a provider that can never serve a turn. + expect( + shouldAdvance(new SubscriptionCliNotInstalledError("claude", "Install it.")), + ).toEqual({ advance: true, immediate: false }); + }); + + it("advances on a subscription-CLI provider that is signed out", () => { + expect( + shouldAdvance(new SubscriptionCliAuthError("codex", "Run /login.")), + ).toEqual({ advance: true, immediate: false }); + }); + it("advances via threshold on a TransportError carrying null status", () => { expect(shouldAdvance(new TransportError("x", null, "u"))).toEqual({ advance: true, diff --git a/src/llm/fallback/should-advance.ts b/src/llm/fallback/should-advance.ts index e7cf240c..75070d76 100644 --- a/src/llm/fallback/should-advance.ts +++ b/src/llm/fallback/should-advance.ts @@ -29,15 +29,21 @@ const NO: AdvanceDecision = { advance: false, immediate: false }; * - `transport` → advance (provider unreachable). Note every cloud * `OpenAiHttpError` classifies as `transport` regardless of status, so * a 404 model-not-found or a 401 dead key advances too — a different - * link may have the model or a working key. Untyped socket failures + * link may have the model or a working key. The local path now agrees: + * a llama-server 404/405 (the configured URL does not serve + * completions) or 401/403/429 advances for the same reason. A + * subscription-CLI provider whose binary is missing or is signed out + * is the same story with no HTTP in it. Untyped socket failures * (undici's `TypeError: fetch failed` and friends, from surfaces that * do not wrap their own errors) land here too — see `isNetworkError`. * - `model` → advance. This is a *defective completion* from a reachable * provider (truncated / empty / no_stop), not "model not found"; the * same prompt would reproduce it here, so another link is worth a try. - * - `grammar` / `tool` / `cancelled` → do not advance. A grammar/4xx - * failure is request-shape and repeats identically on every provider; - * a tool failure is our own bug; a cancellation is user intent. + * - `grammar` / `tool` / `cancelled` → do not advance. A grammar failure + * is request-shape — an unparseable completion, or the narrow band of + * llama-server 4xx that rejects the request itself (400/413/422) — + * and repeats identically on every provider; a tool failure is our own + * bug; a cancellation is user intent. * * Immediate signals are read off the typed status carried by * `OpenAiHttpError` / `LlamaServerError` / `TransportError`: an explicit diff --git a/src/llm/reliability/classify-failure.test.ts b/src/llm/reliability/classify-failure.test.ts index 963847e2..d5c8748b 100644 --- a/src/llm/reliability/classify-failure.test.ts +++ b/src/llm/reliability/classify-failure.test.ts @@ -8,6 +8,11 @@ import { ToolExecutionError, TransportError, } from "./llm-failures.js"; +import { + SubscriptionCliAuthError, + SubscriptionCliInvocationError, + SubscriptionCliNotInstalledError, +} from "../provider/subscription-cli/subscription-cli-errors.js"; import { classifyFailure } from "./classify-failure.js"; describe("classifyFailure", () => { @@ -29,7 +34,7 @@ describe("classifyFailure", () => { expect(classifyFailure(err)).toBe("transport"); }); - it("maps LlamaServerError 4xx to grammar", () => { + it("maps LlamaServerError request-shape 4xx to grammar", () => { const err = new LlamaServerError("bad grammar", 400, "http://x"); expect(classifyFailure(err)).toBe("grammar"); }); @@ -87,3 +92,62 @@ describe("classifyFailure — raw network failures", () => { expect(classifyFailure(err)).toBe("cancelled"); }); }); + +describe("classifyFailure — llama-server HTTP statuses", () => { + // A 4xx that describes the *endpoint* must not be filed as `grammar`: + // that category blocks fallover (`shouldAdvance`) and tells the user + // their grammar is broken when the real answer is "that URL is not a + // llama-server". A 4xx that rejects the request itself stays grammar. + const cases: Array<[number | null, string]> = [ + [null, "transport"], + [400, "grammar"], + [401, "transport"], + [402, "transport"], + [403, "transport"], + [404, "transport"], + [405, "transport"], + [408, "transport"], + [409, "transport"], + [413, "grammar"], + [422, "grammar"], + [429, "transport"], + [500, "transport"], + [503, "transport"], + ]; + + for (const [status, expected] of cases) { + it(`maps status ${status ?? "null"} to ${expected}`, () => { + const err = new LlamaServerError("boom", status, "http://x"); + expect(classifyFailure(err)).toBe(expected); + }); + } + + it("leaves an unlisted 4xx on the request-shape side", () => { + // Conservative default: only the statuses we can name as + // endpoint/auth/availability earn a fallover. + expect(classifyFailure(new LlamaServerError("x", 418, "http://x"))).toBe( + "grammar", + ); + }); +}); + +describe("classifyFailure — subscription-CLI providers", () => { + it("maps a missing CLI binary to transport, not tool", () => { + // "claude is not on PATH" is a dead provider link, not a bug in our + // tool layer — the chain must be free to try the next provider. + const err = new SubscriptionCliNotInstalledError("claude", "Install it."); + expect(classifyFailure(err)).toBe("transport"); + }); + + it("maps a signed-out CLI to transport, not tool", () => { + const err = new SubscriptionCliAuthError("codex", "Run /login."); + expect(classifyFailure(err)).toBe("transport"); + }); + + it("still treats a failed CLI invocation as a tool failure", () => { + // The binary ran and came back unhappy: that is not evidence the + // link is unusable, so it keeps the non-advancing category. + const err = new SubscriptionCliInvocationError("claude exited 1", 1); + expect(classifyFailure(err)).toBe("tool"); + }); +}); diff --git a/src/llm/reliability/classify-failure.ts b/src/llm/reliability/classify-failure.ts index 08764071..7fa5f4f6 100644 --- a/src/llm/reliability/classify-failure.ts +++ b/src/llm/reliability/classify-failure.ts @@ -1,10 +1,31 @@ import { LlamaServerError } from "../llama-server-client.js"; import { OpenAiHttpError } from "../provider/openai/openai-http.js"; +import { + SubscriptionCliAuthError, + SubscriptionCliNotInstalledError, +} from "../provider/subscription-cli/subscription-cli-errors.js"; import { ToolCallParseError } from "../grammar/tool-call-grammar.js"; import type { LlmFailureCategory } from "./failure-category.js"; import { LlmFailure } from "./llm-failures.js"; import { isNetworkError } from "./network-error.js"; +/** + * llama-server 4xx statuses that describe the *endpoint*, not the request + * we sent it: the URL does not serve completions (404 — a wrong + * `localModels.url`, or a server that is not a llama-server at all), the + * method is not allowed (405), a proxy in front of it wants credentials + * (401/402/403), or the server is busy / timing us out / conflicting + * (408/409/429). None of these repeat identically on a different provider, + * so they are `transport`: the link is unusable, try the next one. + * + * Everything else in the 4xx range stays `grammar` — 400/413/422 are the + * server telling us THIS request was malformed or too large, which the + * next link would reject the same way. + */ +const LLAMA_ENDPOINT_UNAVAILABLE_STATUSES = new Set([ + 401, 402, 403, 404, 405, 408, 409, 429, +]); + /** * Classify any thrown value into the canonical failure taxonomy. * @@ -15,6 +36,13 @@ import { isNetworkError } from "./network-error.js"; * grammar parser errors, abort signals, and anything else treated as * a tool-layer problem by default). * + * The governing rule across every branch: a failure that means "this + * provider is unusable" must not land in a category that blocks + * fallover. `shouldAdvance` only advances on `transport` / `model`, so + * filing an unusable link under `grammar` or `tool` pins the chain to a + * permanently broken provider and hands the user a diagnosis for a + * problem they do not have. + * * The `isNetworkError` branch sits between the abort check and that * default: an untyped socket failure (MCP streamable-http, embeddings, * a vendor SDK with its own `fetch`) is a `transport` problem even @@ -29,6 +57,7 @@ export function classifyFailure(err: unknown): LlmFailureCategory { if (err instanceof LlamaServerError) { if (err.status === null) return "transport"; if (err.status >= 500) return "transport"; + if (LLAMA_ENDPOINT_UNAVAILABLE_STATUSES.has(err.status)) return "transport"; return "grammar"; } // Cloud provider failures are provider-boundary problems whatever the @@ -36,6 +65,17 @@ export function classifyFailure(err: unknown): LlmFailureCategory { // retry budget was already spent inside the HTTP client, matching the // TransportError contract. if (err instanceof OpenAiHttpError) return "transport"; + // A CLI-backed provider whose binary is missing or signed out is the + // same shape of problem as an unreachable HTTP endpoint: this link + // cannot serve the turn, and no other link is implicated. The default + // `tool` arm below would both mislabel it ("Turn failed [tool]" for a + // binary the user never installed) and stop the chain dead. + if ( + err instanceof SubscriptionCliNotInstalledError || + err instanceof SubscriptionCliAuthError + ) { + return "transport"; + } if (isAbortError(err)) return "cancelled"; // Checked after the abort branch on purpose: an aborted request can // surface as ECONNRESET, and a user pressing Esc is not a fallover. diff --git a/src/llm/reliability/failure-category.ts b/src/llm/reliability/failure-category.ts index c2608bee..01e6bd16 100644 --- a/src/llm/reliability/failure-category.ts +++ b/src/llm/reliability/failure-category.ts @@ -1,11 +1,18 @@ /** * Canonical taxonomy of failures surfaced by the agent runtime. * - * - `transport`: llama-server unreachable, network error, HTTP 5xx. + * - `transport`: the provider link is unusable — llama-server + * unreachable, network error, HTTP 5xx, a llama-server + * 4xx that describes the endpoint rather than the request + * (401/402/403/404/405/408/409/429), any cloud HTTP + * failure, or a CLI-backed provider whose binary is + * missing or signed out. Everything here is worth + * retrying on the next link in the fallback chain. * - `grammar`: the completion payload could not be parsed into a valid - * tool call; also covers HTTP 4xx from llama-server which - * generally means the server rejected the grammar or - * request shape. + * tool call; also covers the llama-server 4xx statuses + * that reject THIS request as malformed or oversized + * (400/413/422 and any other unlisted 4xx), which the + * next provider would reject identically. * - `model`: the completion itself is defective (truncated, empty, * or generated without a stop token). Retrying the same * prompt is unlikely to help, so the runtime does not. diff --git a/src/tui/format-agent-error-for-chat.test.ts b/src/tui/format-agent-error-for-chat.test.ts index 49bc435e..cd9a77a6 100644 --- a/src/tui/format-agent-error-for-chat.test.ts +++ b/src/tui/format-agent-error-for-chat.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; +import { LlamaServerError } from "../llm/llama-server-client.js"; +import { classifyFailure } from "../llm/reliability/classify-failure.js"; import { formatAgentErrorForChat } from "./format-agent-error-for-chat.js"; describe("formatAgentErrorForChat", () => { @@ -47,3 +49,48 @@ describe("formatAgentErrorForChat", () => { ).toBe("Turn failed [model]: empty completion"); }); }); + +describe("formatAgentErrorForChat — classified llama failures", () => { + // Mirrors the real pipeline: `agent-loop` classifies the thrown error + // and the reducer hands that category straight to the formatter. The + // hint is gated on `transport`, so the one failure where "check your + // llama URL" is exactly right — a 404 from a wrong `localModels.url` — + // used to be the one failure that never got it. + const local = { + activeProviderIsLocal: true, + llamaUrl: "http://127.0.0.1:19091", + }; + + it("carries the unreachable hint for a llama 404 on a local provider", () => { + const err = new LlamaServerError( + "llama-server returned http 404", + 404, + local.llamaUrl, + ); + const text = formatAgentErrorForChat( + classifyFailure(err), + err.message, + local, + ); + expect(text).toContain("Turn failed [transport]"); + expect(text).toContain( + "llama-server is not reachable at http://127.0.0.1:19091", + ); + }); + + it("keeps a llama 400 as a grammar failure with no URL advice", () => { + const err = new LlamaServerError( + "llama-server returned http 400", + 400, + local.llamaUrl, + ); + const text = formatAgentErrorForChat( + classifyFailure(err), + err.message, + local, + ); + expect(text).toBe( + "Turn failed [grammar]: llama-server returned http 400", + ); + }); +}); From ca4b1c31af774da04e954b2e3a83be4e66b3cb75 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:07:20 +0300 Subject: [PATCH 23/36] fix(llm): a streaming llama response is bounded by idle time, not total time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `localModels.requestTimeoutMs` (default 300s) was applied as a wall-clock deadline over an entire streaming generation. `createRequestController` armed one `setTimeout` when the request was sent and nothing refreshed it, so `completeStream` aborted its own healthy stream 300s in — a reasoning model on CPU, or a llama-server across a LAN, is killed mid-token with every byte already produced discarded. Nothing downstream recovers it: `isRetryableLlamaError` refuses to replay a `timedOut` error, and `timedOutOf` in `should-advance` deliberately keeps a self-inflicted timeout off the immediate-fallover path. The turn just dies. The cloud path never behaved this way. `openAiFetch` clears its identical timer in `finally` when the fetch promise settles, i.e. at response headers, so for OpenAI-compatible providers the same knob bounds only connect. Local and cloud read one config key two incompatible ways. `createRequestController` now returns `keepAlive()`, which re-arms the deadline and marks it an idle budget; `completeStream` calls it once at headers and again on every chunk. `complete()` is untouched — a unary request has exactly one event to wait for and no idle signal to refresh against, so a total budget is the only one it can have. `timedOut()` now reports which deadline fired so the two carry honest, opposite advice: "lower completionMaxTokens" is meaningless for a stall where nothing arrived at all. An idle stall still sets `LlamaServerError.timedOut`, leaving retry and fallover exactly where they were — llama.cpp sends headers before it evaluates the prompt, so minutes of silence during a long CPU prompt-eval is not evidence the provider is dead. --- src/llm/llama-server-client.test.ts | 240 +++++++++++++++++++++++++++- src/llm/llama-server-client.ts | 119 ++++++++++++-- 2 files changed, 341 insertions(+), 18 deletions(-) diff --git a/src/llm/llama-server-client.test.ts b/src/llm/llama-server-client.test.ts index 0bfcc8d8..a1f1fe10 100644 --- a/src/llm/llama-server-client.test.ts +++ b/src/llm/llama-server-client.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect, vi } from "vitest"; import { LlamaServerClient, LlamaServerError, @@ -474,6 +474,244 @@ describe("LlamaServerClient.completeStream", () => { }); }); +/** + * The streaming deadline is an *idle* deadline: `requestTimeoutMs` bounds + * how long the server may stay silent, not how long the answer may be. + * It used to bound the whole generation, so a healthy reasoning model on + * CPU — or any llama-server on the far side of a LAN — was killed at + * exactly the budget with every token already produced thrown away, and + * neither the retry policy (`timedOut` is not retryable) nor the fallback + * chain (a self-inflicted timeout is not an immediate signal) recovered + * it. The cloud path never had this problem: `openAiFetch` clears its + * timer as soon as the fetch promise settles, i.e. at response headers. + */ +describe("LlamaServerClient.completeStream deadlines", () => { + interface PushableStream { + response: Response; + push: (text: string) => void; + close: () => void; + } + + /** + * An SSE body the test drives by hand. Aborting the request signal + * errors the body mid-read, which is what undici does when the + * controller fires while the response is still streaming — the + * behaviour the production bug depends on. + */ + function pushableSse(signal: AbortSignal | null | undefined): PushableStream { + const encoder = new TextEncoder(); + let ctrl!: ReadableStreamDefaultController; + const stream = new ReadableStream({ + start(c) { + ctrl = c; + }, + }); + let finished = false; + signal?.addEventListener("abort", () => { + if (finished) return; + finished = true; + ctrl.error( + Object.assign(new Error("The operation was aborted"), { + name: "AbortError", + }), + ); + }); + return { + response: new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + push: (text: string) => { + if (!finished) ctrl.enqueue(encoder.encode(text)); + }, + close: () => { + if (finished) return; + finished = true; + ctrl.close(); + }, + }; + } + + function streamingClient(requestTimeoutMs: number): { + client: LlamaServerClient; + opened: () => PushableStream; + } { + let handle: PushableStream | null = null; + const client = new LlamaServerClient({ + baseUrl: "http://127.0.0.1:9999", + requestTimeoutMs, + fetchImpl: createMockFetch(async (_url, init) => { + handle = pushableSse(init.signal); + return handle.response; + }), + completionRetries: 1, + completionRetryBackoffMs: 0, + sleep: async () => {}, + }); + return { + client, + opened: () => { + if (!handle) throw new Error("stream not opened yet"); + return handle; + }, + }; + } + + afterEach(() => { + vi.useRealTimers(); + }); + + it("keeps streaming past requestTimeoutMs while chunks keep arriving", async () => { + // The regression test. Six chunks 999ms apart is 5,994ms of healthy + // generation under a 1,000ms budget — six times over the old + // wall-clock cap, and every one of those gaps is under it. + vi.useFakeTimers(); + const { client, opened } = streamingClient(1_000); + const iterator = client.completeStream({ prompt: "hi" }); + const deltas: string[] = []; + let final: { content: string } | null = null; + const consumed = (async () => { + while (true) { + const next = await iterator.next(); + if (next.done) { + final = next.value; + return; + } + if (next.value.delta) deltas.push(next.value.delta); + } + })(); + + // Let the generator open the request and park on its first read(). + await vi.advanceTimersByTimeAsync(0); + for (let i = 0; i < 6; i += 1) { + opened().push(`data: {"content":"t${i}","stop":false}\n\n`); + await vi.advanceTimersByTimeAsync(999); + } + opened().push('data: {"content":"","stop":true}\n\n'); + await vi.advanceTimersByTimeAsync(0); + opened().close(); + await vi.advanceTimersByTimeAsync(0); + await consumed; + + expect(deltas.join("")).toBe("t0t1t2t3t4t5"); + expect(final).not.toBeNull(); + expect(final!.content).toBe("t0t1t2t3t4t5"); + }); + + it("aborts a stream that goes silent for longer than requestTimeoutMs", async () => { + vi.useFakeTimers(); + const { client, opened } = streamingClient(1_000); + const iterator = client.completeStream({ prompt: "hi" }); + const deltas: string[] = []; + const failure = (async (): Promise => { + try { + while (true) { + const next = await iterator.next(); + if (next.done) return null; + if (next.value.delta) deltas.push(next.value.delta); + } + } catch (err) { + return err; + } + })(); + + await vi.advanceTimersByTimeAsync(0); + opened().push('data: {"content":"partial","stop":false}\n\n'); + await vi.advanceTimersByTimeAsync(500); + // …and then the server goes quiet for a full budget. + await vi.advanceTimersByTimeAsync(1_001); + const err = await failure; + + expect(deltas.join("")).toBe("partial"); + expect(err).toBeInstanceOf(LlamaServerError); + const llamaErr = err as LlamaServerError; + expect(llamaErr.status).toBeNull(); + // Still `timedOut` — see the field's doc comment. llama-server sends + // headers before it evaluates the prompt, so silence is not proof the + // provider is dead, and flipping this would turn a slow local model + // into an immediate fallover. + expect(llamaErr.timedOut).toBe(true); + expect(llamaErr.message).toContain("sent no data for 1000ms"); + // The old advice is wrong for a stall: nothing was too long. + expect(llamaErr.message).not.toContain("lower completionMaxTokens"); + }); + + it("still enforces a total deadline on the unary complete() path", async () => { + // Pinned deliberately. A non-streaming request has exactly one event + // to wait for, so it has no idle signal to refresh against — the + // wall-clock budget is all it can have. + vi.useFakeTimers(); + const client = new LlamaServerClient({ + baseUrl: "http://127.0.0.1:9999", + requestTimeoutMs: 1_000, + fetchImpl: createMockFetch( + (_url, init) => + new Promise((_resolve, reject) => { + init.signal?.addEventListener("abort", () => { + reject( + Object.assign(new Error("aborted"), { name: "AbortError" }), + ); + }); + }), + ), + completionRetries: 1, + completionRetryBackoffMs: 0, + sleep: async () => {}, + }); + const failure = client.complete({ prompt: "hi" }).then( + () => null, + (err: unknown) => err, + ); + await vi.advanceTimersByTimeAsync(1_001); + const err = (await failure) as LlamaServerError; + + expect(err).toBeInstanceOf(LlamaServerError); + expect(err.timedOut).toBe(true); + expect(err.message).toContain("exceeded requestTimeoutMs (1000ms)"); + }); + + it("lets an external abort cancel mid-stream without reporting a timeout", async () => { + // Esc in the TUI. The abort must not be laundered into our own + // idle-timeout error: `timedOut` stays false, so the fallback chain + // and `toLlmFailure` (which reads `ctx.signal.aborted`) still see a + // cancellation rather than a provider failure. + vi.useFakeTimers(); + const { client, opened } = streamingClient(60_000); + const abort = new AbortController(); + const iterator = client.completeStream({ + prompt: "hi", + signal: abort.signal, + }); + const deltas: string[] = []; + const failure = (async (): Promise => { + try { + while (true) { + const next = await iterator.next(); + if (next.done) return null; + if (next.value.delta) deltas.push(next.value.delta); + } + } catch (err) { + return err; + } + })(); + + await vi.advanceTimersByTimeAsync(0); + opened().push('data: {"content":"half","stop":false}\n\n'); + await vi.advanceTimersByTimeAsync(10); + abort.abort(); + await vi.advanceTimersByTimeAsync(0); + const err = await failure; + + expect(deltas.join("")).toBe("half"); + expect(err).toBeInstanceOf(LlamaServerError); + const llamaErr = err as LlamaServerError; + expect(llamaErr.timedOut).toBe(false); + expect(llamaErr.message).toMatch(/abort/i); + expect(llamaErr.message).not.toContain("requestTimeoutMs"); + expect(llamaErr.message).not.toContain("sent no data"); + }); +}); + describe("extractLlamaErrorDetail", () => { it("pulls the message from { error: { message } }", () => { expect( diff --git a/src/llm/llama-server-client.ts b/src/llm/llama-server-client.ts index 3abe229a..34460e03 100644 --- a/src/llm/llama-server-client.ts +++ b/src/llm/llama-server-client.ts @@ -39,6 +39,17 @@ const ENV_TOP_P = parseFloatEnv(process.env.ATOMIC_AGENT_LLAMA_TOP_P); const ENV_TOP_K = parseIntEnv(process.env.ATOMIC_AGENT_LLAMA_TOP_K); const ENV_SEED = parseIntEnv(process.env.ATOMIC_AGENT_LLAMA_SEED); +/** + * Which of our own deadlines fired. + * + * - `total` — the whole request was given `requestTimeoutMs` and never + * produced a response. The only signal a unary request has. + * - `idle` — a *stream* went `requestTimeoutMs` without sending a byte. + * A healthy generation refreshes this budget on every chunk, so it + * means the server went quiet, not that the answer was long. + */ +export type LlamaTimeoutKind = "total" | "idle"; + export class LlamaServerError extends Error { constructor( message: string, @@ -46,11 +57,21 @@ export class LlamaServerError extends Error { public readonly url: string, /** * True when *our own* `requestTimeoutMs` controller fired rather than - * the transport failing. Both surface as `status === null`, but a + * the transport failing — for either deadline, `total` or `idle` + * (see `LlamaTimeoutKind`). Both surface as `status === null`, but a * timeout is a "the model is slower than the budget" signal, not a * transient blip — replaying it just burns another full timeout of * GPU time (3 attempts x 300s = 15 silent minutes). See * `isRetryableLlamaError`. + * + * An idle stall stays flagged here on purpose. It is tempting to read + * "the server went quiet" as harder evidence of a dead provider than + * "the server is slow", and so let `shouldAdvance` fall over on the + * first occurrence — but llama.cpp streams response headers before it + * evaluates the prompt, so a long CPU prompt-eval is genuinely silent + * for minutes while nothing is wrong. Keeping the flag leaves the + * fallover behaviour exactly where it was: advance on the + * consecutive-failure threshold, never immediately. */ public readonly timedOut = false, /** @@ -254,13 +275,14 @@ export class LlamaServerClient { response: Response; controller: AbortController; cleanup: () => void; - timedOut: () => boolean; + timedOut: () => LlamaTimeoutKind | null; + keepAlive: () => void; }; try { opened = await this.runWithRetry( url, async () => { - const { controller, cleanup, timedOut } = + const { controller, cleanup, timedOut, keepAlive } = this.createRequestController(request.signal); try { const response = await this.fetchImpl(url, { @@ -272,7 +294,7 @@ export class LlamaServerClient { if (!response.ok || !response.body) { throw await buildHttpError(response, url); } - return { response, controller, cleanup, timedOut }; + return { response, controller, cleanup, timedOut, keepAlive }; } catch (err) { cleanup(); throw this.wrapTransportError(err, url, timedOut()); @@ -287,7 +309,7 @@ export class LlamaServerClient { cause: err, }); } - const { response, cleanup, timedOut } = opened; + const { response, cleanup, timedOut, keepAlive } = opened; let finalResult: CompletionResult = { content: "", reasoningContent: "", @@ -314,12 +336,23 @@ export class LlamaServerClient { const reader = response.body .pipeThrough(new TextDecoderStream()) .getReader(); + // Headers are in; from here the deadline bounds *silence*, not the + // length of the answer. Re-arming once here also hands the body a + // full budget rather than whatever the connect phase left over — + // llama.cpp answers with headers immediately and only then evaluates + // the prompt, so the first token can legitimately be minutes away. + keepAlive(); let buffer = ""; let accumulated = ""; let accumulatedReasoning = ""; while (true) { const { value, done } = await reader.read(); if (done) break; + // A byte arrived: the server is alive, so start the clock over. + // Deliberately not called on `done` — that breaks straight out of + // the loop into `finally { cleanup() }` with nothing awaited in + // between, so there is no window left for the timer to fire. + keepAlive(); buffer += value; let eventEnd = buffer.indexOf("\n\n"); while (eventEnd !== -1) { @@ -367,32 +400,69 @@ export class LlamaServerClient { * The returned `cleanup` clears the timeout and detaches the external * listener — call it in `finally` so a long-lived stream does not leak * the listener. + * + * The deadline starts as a **total** budget, which is all a unary + * request can be given: it has exactly one event to wait for. A + * streaming caller converts it into an **idle** budget by calling + * `keepAlive()` on every byte it receives — see `completeStream`. + * Without that, `requestTimeoutMs` was a wall-clock cap on the whole + * generation and killed healthy long answers at exactly the budget, + * discarding every token already produced. */ private createRequestController(externalSignal?: AbortSignal): { controller: AbortController; cleanup: () => void; - /** True once the per-request timeout (not the caller) fired the abort. */ - timedOut: () => boolean; + /** + * Which of our own deadlines fired the abort, or `null` when the + * abort came from the caller / nothing fired at all. + */ + timedOut: () => LlamaTimeoutKind | null; + /** + * Restart the deadline and mark it an idle budget. A no-op once the + * request is already aborted, so a late call cannot resurrect a + * controller the caller or the timer has finished with. + */ + keepAlive: () => void; } { const controller = new AbortController(); - let expired = false; - const timer = setTimeout(() => { - expired = true; - controller.abort(); - }, this.requestTimeoutMs); - const timedOut = (): boolean => expired; + let expired: LlamaTimeoutKind | null = null; + let kind: LlamaTimeoutKind = "total"; + const arm = (): ReturnType => + setTimeout(() => { + expired = kind; + controller.abort(); + }, this.requestTimeoutMs); + let timer = arm(); + const timedOut = (): LlamaTimeoutKind | null => expired; + const keepAlive = (): void => { + if (expired !== null || controller.signal.aborted) return; + clearTimeout(timer); + kind = "idle"; + timer = arm(); + }; if (!externalSignal) { - return { controller, cleanup: () => clearTimeout(timer), timedOut }; + return { + controller, + cleanup: () => clearTimeout(timer), + timedOut, + keepAlive, + }; } if (externalSignal.aborted) { controller.abort(); - return { controller, cleanup: () => clearTimeout(timer), timedOut }; + return { + controller, + cleanup: () => clearTimeout(timer), + timedOut, + keepAlive, + }; } const onAbort = (): void => controller.abort(); externalSignal.addEventListener("abort", onAbort, { once: true }); return { controller, timedOut, + keepAlive, cleanup: () => { clearTimeout(timer); externalSignal.removeEventListener("abort", onAbort); @@ -408,10 +478,25 @@ export class LlamaServerClient { private wrapTransportError( err: unknown, url: string, - timedOut: boolean, + timedOut: LlamaTimeoutKind | null, ): LlamaServerError { if (err instanceof LlamaServerError) return err; - if (timedOut) { + // An idle stall and a blown total budget need opposite advice. + // "Lower completionMaxTokens" is meaningless when the server sent + // nothing at all — the answer was not too long, it never came. + if (timedOut === "idle") { + return new LlamaServerError( + `llama-server sent no data for ${this.requestTimeoutMs}ms mid-stream — ` + + `the server stopped responding after starting the reply; check that ` + + `llama-server is still running, or raise localModels.requestTimeoutMs`, + null, + url, + true, + undefined, + { cause: err }, + ); + } + if (timedOut === "total") { return new LlamaServerError( `llama-server request exceeded requestTimeoutMs (${this.requestTimeoutMs}ms) — ` + `raise localModels.requestTimeoutMs or lower completionMaxTokens`, From 2a32511af0de052fa001e4dc1a6d66bed924b543 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:36:29 +0300 Subject: [PATCH 24/36] fix(runtime): stop probing the local llama backend while a cloud provider is active MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cloud-backed session opened with `/health` + `/props` against `http://127.0.0.1:8080`, warned that nothing answered, kept a 3 s footer poller running against it, and refreshed a local `ModelProfileManager` on every turn — all for a backend the session never talks to. The warnings read as an active-backend failure on a run whose real provider was healthy the whole time (issue #112, Yabloko Labs §9). `activeTextProviderIsLlamaServer` moves from `src/tui/local-turn-gate.ts` to `src/llm/provider/registry/active-text-provider.ts` (re-exported from its old home, so its callers and tests are untouched). It is a pure function of `resolveLlmConfig`, which does no I/O, so the answer is available at the top of `buildRuntime` — hundreds of lines before `ProviderRegistry.fromConfig` resolves the active provider. Detection stays KIND-based and conservative: an id resolving to no entry counts as local. Gated on that predicate: - bootstrap's `/health` line, the `/props` profile probe, and the context-window advice that only names llama-server flags; - the sidecar's `start_session` health probe and its `llm_unavailable` event; - the agent loop's turn-start `refresh()` and between-steps `refreshIfStale()`; - the TUI footer poller's `/health` tick and `/props` label fetch, and `select-context-usage`'s use of the poller's `n_ctx` — a stale local reading must not scale a cloud model's gauge. The `ModelProfileManager` is still constructed on a cloud boot (construction is pure field assignment): deleting it would leave a mid-turn fallover to a `llama-server` link running on a frozen `plain-instruct` profile with no way back. Only its probing is deferred, into `DeferredLocalBackendProbes`, which replays health + `/props` + slot discovery + the context advice exactly once for whichever path reaches local inference first — a provider switch (the loop's turn-start gate) or a cloud→local fallover (the fallback seam's new `prepareLink` hook, called with the chosen link before its completion is sent). Local embeddings are untouched: they hang off their own flags and their own port, and are still probed under a cloud text provider. Fixes #112 --- src/agent/agent-loop-local-gate.test.ts | 192 ++++++++++++ src/agent/agent-loop.ts | 42 ++- src/llm/local-backend-gate.test.ts | 78 +++++ src/llm/local-backend-gate.ts | 97 ++++++ .../provider/registry/active-text-provider.ts | 36 +++ src/llm/provider/registry/index.ts | 4 + src/runtime/bootstrap.ts | 182 ++++++++--- src/runtime/llm-fallback-seam.test.ts | 112 +++++++ src/runtime/llm-fallback-seam.ts | 16 + src/runtime/local-probe-gating.test.ts | 294 ++++++++++++++++++ src/sidecar/local-probe-gating.test.ts | 210 +++++++++++++ src/sidecar/main.ts | 33 +- src/tui/llm-health/llm-health-poller.test.ts | 138 +++++++- src/tui/llm-health/llm-health-poller.ts | 26 ++ src/tui/local-turn-gate.ts | 20 +- src/tui/select-context-usage.ts | 11 +- 16 files changed, 1420 insertions(+), 71 deletions(-) create mode 100644 src/agent/agent-loop-local-gate.test.ts create mode 100644 src/llm/local-backend-gate.test.ts create mode 100644 src/llm/local-backend-gate.ts create mode 100644 src/llm/provider/registry/active-text-provider.ts create mode 100644 src/runtime/local-probe-gating.test.ts create mode 100644 src/sidecar/local-probe-gating.test.ts diff --git a/src/agent/agent-loop-local-gate.test.ts b/src/agent/agent-loop-local-gate.test.ts new file mode 100644 index 00000000..a782d85e --- /dev/null +++ b/src/agent/agent-loop-local-gate.test.ts @@ -0,0 +1,192 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { AgentLoop } from "./agent-loop.js"; +import { buildDefaultToolRegistry } from "../tools/index.js"; +import { SlotManager } from "../llm/slot-manager.js"; +import { createEmptySessionState } from "../session/session-state.js"; +import type { + CompletionResult, + LlamaServerClient, +} from "../llm/llama-server-client.js"; +import { ModelProfileManager } from "../llm/model-profile-manager.js"; +import { GEMMA4_PROPS } from "../llm/model-profile.fixtures.js"; +import { QWEN_THINK_PROFILE } from "../llm/model-profile.js"; +import { buildGrammar } from "../llm/grammar/build-grammar.js"; +import { DeferredLocalBackendProbes } from "../llm/local-backend-gate.js"; +import type { + CapabilitiesSummary, + SkillCatalogEntry, + ToolDescriptor, +} from "../prompt/stable-prefix.js"; + +/** + * Issue #112 — the loop's two `ModelProfileManager` probes are + * llama-server traffic, and a cloud turn must produce none of it. + * + * `fetchProps` is the counted local request: it IS the `/props` call, + * one layer below the HTTP client. Counts are exact — the pre-fix + * behaviour was one probe per turn plus one per stale step, so a + * `not.toHaveBeenCalled()` would not catch a regression that merely + * moved the probe. + */ + +function makeCompletion( + content: string, + modelId: string = "mock", +): CompletionResult { + return { + content, + reasoningContent: "", + stop: true, + truncated: false, + timing: { promptMs: 1, predictedMs: 1, promptTokens: 10, predictedTokens: 5 }, + cacheHitTokens: 0, + slotId: 0, + modelId, + }; +} + +const TOOLS: ToolDescriptor[] = [ + { + name: "finish", + summary: "Finish the session with a summary.", + argsSchema: '{"summary": string}', + }, +]; + +const CAPS: CapabilitiesSummary = { + platform: "darwin", + arch: "arm64", + browserChannel: "chrome", + workingDir: "/work", + hasClipboard: true, + hasWmctrl: false, + hasNotifications: true, +}; + +const SKILLS: SkillCatalogEntry[] = []; + +describe("AgentLoop — local profile probes are gated on the active route", () => { + let workingDir: string; + + beforeEach(() => { + workingDir = mkdtempSync(join(tmpdir(), "atomic-loop-gate-")); + }); + + afterEach(() => { + rmSync(workingDir, { recursive: true, force: true }); + }); + + const buildLoop = async ( + localBackend: ConstructorParameters[0]["localBackend"], + ) => { + const grammar = await buildGrammar(QWEN_THINK_PROFILE); + const fetchProps = vi.fn<[], Promise>>(); + fetchProps.mockResolvedValue(GEMMA4_PROPS); + const profileManager = new ModelProfileManager({ + llama: { fetchProps } as unknown as LlamaServerClient, + initialProfile: QWEN_THINK_PROFILE, + initialGrammar: grammar, + initialModelId: "qwen3-30b-a3b-instruct-2507", + }); + const loop = new AgentLoop({ + registry: buildDefaultToolRegistry(), + slotManager: new SlotManager(2), + grammar, + profile: QWEN_THINK_PROFILE, + profileManager, + ...(localBackend ? { localBackend } : {}), + // Pre-closed reasoning channel so the reply parses under either + // profile — what is under test is the probe count, not parsing. + // The completion echoes the model the manager already believes is + // loaded, so nothing here marks it stale: staleness has its own + // reactive-refresh tests, and letting it leak in would add probes + // that the gate is not responsible for. + llmComplete: async () => + makeCompletion( + `${JSON.stringify({ + tool: "finish", + args: { summary: "done" }, + })}`, + "qwen3-30b-a3b-instruct-2507", + ), + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + }); + return { loop, fetchProps, profileManager }; + }; + + const runTurn = async (loop: AgentLoop, id: string) => + loop.runTurn(createEmptySessionState({ id, workingDir }), { + userMessage: "go", + maxSteps: 2, + signal: new AbortController().signal, + }); + + it("makes zero /props requests on a cloud turn", async () => { + const { loop, fetchProps, profileManager } = await buildLoop({ + isActive: () => false, + ensureProbed: async () => false, + }); + + const result = await runTurn(loop, "s-cloud"); + + expect(result.reason).toBe("finish"); + expect(fetchProps).toHaveBeenCalledTimes(0); + // ...and the turn ran on the plain/non-local profile it started on, + // rather than one detected from a llama-server that is not serving. + expect(profileManager.getProfile().id).toBe(QWEN_THINK_PROFILE.id); + }); + + it("probes exactly once per local turn", async () => { + const { loop, fetchProps, profileManager } = await buildLoop({ + isActive: () => true, + ensureProbed: async () => false, + }); + + await runTurn(loop, "s-local"); + + // One turn-start refresh. The between-steps `refreshIfStale` is a + // no-op on a manager that is not stale, exactly as before #112. + expect(fetchProps).toHaveBeenCalledTimes(1); + expect(profileManager.getProfile().id).toBe("gemma4-think"); + }); + + it("behaves as before the gate when no gate is wired (legacy deps)", async () => { + const { loop, fetchProps } = await buildLoop(undefined); + await runTurn(loop, "s-legacy"); + expect(fetchProps).toHaveBeenCalledTimes(1); + }); + + it("lazily restores local state on the first turn after a switch back to local", async () => { + // Boot was cloud, so the probes were deferred; the operator has + // since switched the active provider to a llama-server link. + let active = false; + const restore = vi.fn(async () => {}); + const gate = new DeferredLocalBackendProbes( + { isActive: () => active, restore }, + /* probedAtBoot */ false, + ); + const { loop, fetchProps } = await buildLoop(gate); + + await runTurn(loop, "s-still-cloud"); + expect(restore).toHaveBeenCalledTimes(0); + expect(fetchProps).toHaveBeenCalledTimes(0); + + active = true; + await runTurn(loop, "s-switched"); + // The restore ran instead of the loop's own refresh — it already + // carries a fresh `/props`, so the turn does not probe twice. + expect(restore).toHaveBeenCalledTimes(1); + expect(fetchProps).toHaveBeenCalledTimes(0); + + // Every later local turn is back on the ordinary refresh. + await runTurn(loop, "s-local-again"); + expect(restore).toHaveBeenCalledTimes(1); + expect(fetchProps).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index 3bdf8f3c..d3176001 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -10,6 +10,7 @@ import { type ModelProfile, } from "../llm/model-profile.js"; import type { ModelProfileManager } from "../llm/model-profile-manager.js"; +import type { LocalBackendGate } from "../llm/local-backend-gate.js"; import type { ToolRegistry } from "../tools/tool-registry.js"; import { CancelledError, @@ -108,6 +109,19 @@ export interface AgentLoopDependencies { * for the lifetime of the loop (test-mode wiring). */ profileManager?: ModelProfileManager; + /** + * Gate for the `profileManager` probes above (issue #112). The manager + * talks to the local llama-server, so on a cloud turn its refreshes + * are pure `/props` noise against a backend nothing is routed to — + * `isActive()` false skips them. `ensureProbed()` covers the reverse + * case: the operator switched back to a local provider after a cloud + * boot that deferred the probes, and this turn is the first local one. + * It returns `true` when it just ran them, which already includes a + * fresh `/props` — the loop then skips its own refresh rather than + * probing twice. Absent (test / legacy wiring) means "always local", + * preserving the pre-#112 behaviour. + */ + localBackend?: LocalBackendGate; /** Skill catalog (name + description only), rebuilt on install/uninstall. */ skillCatalog: readonly SkillCatalogEntry[]; /** @@ -394,6 +408,15 @@ export interface RunTurnResult { export class AgentLoop { constructor(private readonly deps: AgentLoopDependencies) {} + /** + * Whether the local llama-server is the route this turn takes. No gate + * wired (test / legacy deps) reads as `true` so the profile manager + * behaves exactly as it did before issue #112. + */ + private localBackendActive(): boolean { + return this.deps.localBackend?.isActive() ?? true; + } + /** * Drive one macro-turn: * user message → 0..N tool steps → `reply` (or `finish` / max_steps). @@ -480,9 +503,13 @@ export class AgentLoop { // Proactively sync with the live `llama-server` before the first // step. Catches the case where the operator swapped the model // between turns — without this, step 0 would still build the prompt - // with the previous model's template. - if (this.deps.profileManager) { - await this.deps.profileManager.refresh(); + // with the previous model's template. Skipped whole on a cloud turn + // (issue #112): there is no llama-server behind the prompt to sync + // with, and the probe would fail against a backend nobody is using. + if (this.deps.profileManager && this.localBackendActive()) { + if (!(await this.deps.localBackend?.ensureProbed())) { + await this.deps.profileManager.refresh(); + } } let reason: AgentLoopReason = "max_steps"; @@ -545,8 +572,13 @@ export class AgentLoop { // Reactive refresh between steps: if the previous completion // observed a foreign `modelId`, rebuild profile + grammar so the // next prompt matches what `llama-server` is actually serving. - if (this.deps.profileManager) { - await this.deps.profileManager.refreshIfStale(); + // Same cloud-turn gate as the turn-start refresh (issue #112) — a + // mid-turn fallover onto a local link is warmed by the fallback + // seam instead, at the point the link is picked. + if (this.deps.profileManager && this.localBackendActive()) { + if (!(await this.deps.localBackend?.ensureProbed())) { + await this.deps.profileManager.refreshIfStale(); + } } this.deps.onEvent?.({ type: "step_started", stepIndex: i }); const started = Date.now(); diff --git a/src/llm/local-backend-gate.test.ts b/src/llm/local-backend-gate.test.ts new file mode 100644 index 00000000..4f865314 --- /dev/null +++ b/src/llm/local-backend-gate.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; + +import { DeferredLocalBackendProbes } from "./local-backend-gate.js"; + +describe("DeferredLocalBackendProbes", () => { + it("never restores when boot already probed (local-from-boot run)", async () => { + const restore = vi.fn(async () => {}); + const gate = new DeferredLocalBackendProbes( + { isActive: () => true, restore }, + true, + ); + expect(await gate.ensureProbed()).toBe(false); + expect(await gate.ensureProbed()).toBe(false); + expect(restore).toHaveBeenCalledTimes(0); + }); + + it("restores exactly once, and only the winner may skip its own refresh", async () => { + const restore = vi.fn(async () => {}); + const gate = new DeferredLocalBackendProbes( + { isActive: () => true, restore }, + false, + ); + expect(await gate.ensureProbed()).toBe(true); + expect(await gate.ensureProbed()).toBe(false); + expect(await gate.ensureProbed()).toBe(false); + expect(restore).toHaveBeenCalledTimes(1); + }); + + it("a concurrent caller waits for the restore but does not claim it", async () => { + // Turn start racing a mid-turn fallover: both must see warm state + // when they proceed, and only one may report "a fresh /props landed". + let release!: () => void; + const started = vi.fn(); + const gate = new DeferredLocalBackendProbes( + { + isActive: () => true, + restore: () => + new Promise((resolve) => { + started(); + release = resolve; + }), + }, + false, + ); + + const first = gate.ensureProbed(); + const second = gate.ensureProbed(); + expect(started).toHaveBeenCalledTimes(1); + release(); + + expect(await first).toBe(true); + expect(await second).toBe(false); + }); + + it("latches after a throwing restore so the probes cannot re-arm every step", async () => { + const restore = vi.fn(async () => { + throw new Error("llama-server is down"); + }); + const gate = new DeferredLocalBackendProbes( + { isActive: () => true, restore }, + false, + ); + await expect(gate.ensureProbed()).rejects.toThrow("llama-server is down"); + expect(await gate.ensureProbed()).toBe(false); + expect(restore).toHaveBeenCalledTimes(1); + }); + + it("reads `isActive` per call so a hot switch is observed", () => { + let active = false; + const gate = new DeferredLocalBackendProbes( + { isActive: () => active, restore: async () => {} }, + false, + ); + expect(gate.isActive()).toBe(false); + active = true; + expect(gate.isActive()).toBe(true); + }); +}); diff --git a/src/llm/local-backend-gate.ts b/src/llm/local-backend-gate.ts new file mode 100644 index 00000000..172c0779 --- /dev/null +++ b/src/llm/local-backend-gate.ts @@ -0,0 +1,97 @@ +/** + * Gate for every probe aimed at the managed/external llama-server text + * backend (issue #112). + * + * A cloud-backed session used to open with `/health` + `/props` against + * `http://127.0.0.1:8080`, warn that nothing answered, and then run the + * whole session on a cloud provider that was healthy all along — the + * warning reads as an active-backend failure on the one screen where the + * operator has the least context to judge it. + * + * Boot skips those probes when the active text provider is not a + * `llama-server` link. That leaves the local state cold, which is only + * safe if it can be warmed again before local inference. Two paths reach + * local inference after a cloud boot and both call {@link + * LocalBackendGate.ensureProbed} first: + * + * - the operator switching the active text provider to a llama-server + * link (the agent loop's turn-start refresh); and + * - the fallback chain falling over from a cloud link to a + * `llama-server` link mid-turn (`createFallbackCompleter` / + * `createFallbackStreamer` prepare each link before the attempt). + */ + +export interface LocalBackendGate { + /** Is the active text provider a `llama-server` link right now? */ + isActive(): boolean; + /** + * Run the probes boot skipped — `/health` logging, the `/props` + * profile + slot refresh, and the context-window advice — exactly + * once. + * + * Returns `true` only for the call that performed them, so a caller + * whose next act would be its own `/props` refresh can skip it: one + * just landed. `false` means boot already probed (a local-from-boot + * run) or another caller got there first, and the caller owns its + * usual refresh. + */ + ensureProbed(): Promise; +} + +export interface LocalBackendGateDeps { + /** Live predicate — re-read per call so a hot switch is observed. */ + isActive: () => boolean; + /** The deferred boot probes, in boot order. Must not throw. */ + restore: () => Promise; +} + +/** + * `LocalBackendGate` with a one-shot restore. Not reset when the + * operator switches back to cloud: the state the restore rebuilds + * (profile, grammar, slot pool) stays valid and the profile manager + * keeps it fresh from then on, so re-arming would only buy a second + * round of the same probes. + */ +export class DeferredLocalBackendProbes implements LocalBackendGate { + private restored: boolean; + private inFlight: Promise | null = null; + + /** + * @param probedAtBoot `true` when bootstrap already ran the probes + * (the active provider was local at boot), which makes `ensureProbed` + * a pure no-op for the life of the runtime. + */ + constructor( + private readonly deps: LocalBackendGateDeps, + probedAtBoot: boolean, + ) { + this.restored = probedAtBoot; + } + + isActive(): boolean { + return this.deps.isActive(); + } + + async ensureProbed(): Promise { + if (this.restored) return false; + // A concurrent caller (turn start racing a mid-turn fallover) waits + // on the same restore but reports `false`: it did not produce the + // fresh `/props` and must not claim the winner's right to skip. + if (this.inFlight !== null) { + await this.inFlight; + return false; + } + this.inFlight = this.deps.restore(); + try { + await this.inFlight; + } finally { + // Latched even on failure. `restore` swallows its own errors, but + // a hard throw must not re-arm the probes on every step — the + // profile manager's refresh already owns retrying `/props`, and a + // dead backend fails the completion itself a moment later. + this.restored = true; + this.inFlight = null; + } + return true; + } +} diff --git a/src/llm/provider/registry/active-text-provider.ts b/src/llm/provider/registry/active-text-provider.ts new file mode 100644 index 00000000..b35a6994 --- /dev/null +++ b/src/llm/provider/registry/active-text-provider.ts @@ -0,0 +1,36 @@ +import type { ResolvedLlmConfig } from "./provider-registry.js"; + +/** + * KIND-based local detection, mirroring `selectComposerBackend`: any + * `llama-server` entry is the local route, because `LlamaServerProvider` + * accepts a custom id (`options.id`) — keying on the literal + * `local-llama` id would leave a renamed entry ungated. An id that + * resolves to no entry reads as local too, matching the composer's + * no-active-row rule (and the no-`llm`-block default, which + * `resolveLlmConfig` synthesizes as a `llama-server` entry anyway). + * + * The conservative direction matters: every caller uses this to decide + * whether the local llama backend is worth probing, and an unrecognised + * id costs one probe against a backend nobody is using — while the + * opposite mistake runs inference on an unprobed profile. + * + * Lives beside `resolveLlmConfig` rather than under `src/tui/` because + * `resolveLlmConfig` is a pure function of config with no I/O: the + * answer is available at the very top of `buildRuntime`, long before a + * `ProviderRegistry` exists. `src/tui/local-turn-gate.ts` re-exports it + * for its original callers. + */ +export function providerIdIsLlamaServer( + llm: ResolvedLlmConfig, + providerId: string, +): boolean { + const entry = llm.providers.find((p) => p.id === providerId); + return entry === undefined || entry.kind === "llama-server"; +} + +/** {@link providerIdIsLlamaServer} for the active text provider. */ +export function activeTextProviderIsLlamaServer( + llm: ResolvedLlmConfig, +): boolean { + return providerIdIsLlamaServer(llm, llm.activeTextProvider); +} diff --git a/src/llm/provider/registry/index.ts b/src/llm/provider/registry/index.ts index 33eb85fa..fda0c992 100644 --- a/src/llm/provider/registry/index.ts +++ b/src/llm/provider/registry/index.ts @@ -7,5 +7,9 @@ export { type UserModelConfigEntry, type ResolvedLlmConfig, } from "./provider-registry.js"; +export { + activeTextProviderIsLlamaServer, + providerIdIsLlamaServer, +} from "./active-text-provider.js"; export { registerBuiltInProviderKinds } from "./register-built-in-providers.js"; export { resolveActiveToolTransport } from "./resolve-tool-transport.js"; diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index 143bcc05..b32a0764 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -71,6 +71,11 @@ import { resolveLlmConfig, } from "../llm/provider/index.js"; import { resolveActiveToolTransport } from "../llm/provider/registry/resolve-tool-transport.js"; +import { + activeTextProviderIsLlamaServer, + providerIdIsLlamaServer, +} from "../llm/provider/registry/active-text-provider.js"; +import { DeferredLocalBackendProbes } from "../llm/local-backend-gate.js"; import { catalogForProvider } from "../llm/provider/catalog-for-provider.js"; import { CostAccumulator } from "../llm/provider/cost-accumulator.js"; import { @@ -898,34 +903,59 @@ export async function createAgentRuntime( approvalRequired: true, }; - if ( - !options.overrides?.skipLlamaHealthCheck && - !options.overrides?.deferLlamaHealthCheck && - !options.overrides?.llamaComplete - ) { - // One attempt, not the retry ladder: this probe exists to log a line, - // and with llama down the default ladder (5 attempts, exponential - // backoff) stalled every boot for 15.5 s before the loop then failed - // fast anyway. The first real completion is the retry. - const health = await checkLlamaServer({ retries: 0 }); - if (!health.reachable) { - logger.warn("llama-server health check failed", { - error: health.error, - url: config.localModels.url, - }); - if (config.localModels.mode === "managed") { - logger.warn(managedLocalLlmHealthFailureHint(config.localModels.managed.port), { - mode: "managed", + // Issue #112. Every local text probe below hangs off this one answer, + // and it is available here — hundreds of lines before + // `ProviderRegistry.fromConfig` resolves the active provider — + // because `resolveLlmConfig` is a pure function of config with no + // I/O. A cloud-backed boot must not open `/health` or `/props` + // against a llama-server nobody is routed to: the warnings it prints + // read as an active-backend failure while the real provider is fine. + const localTextActiveAtBoot = activeTextProviderIsLlamaServer( + resolveLlmConfig(config), + ); + + // The boot-time local `/health` line. Skipped whole when the route is + // cloud — including the "deferred" notice, which is advice about a + // backend this session never talks to. + const runBootHealthProbe = async (): Promise => { + if ( + !options.overrides?.skipLlamaHealthCheck && + !options.overrides?.deferLlamaHealthCheck && + !options.overrides?.llamaComplete + ) { + // One attempt, not the retry ladder: this probe exists to log a line, + // and with llama down the default ladder (5 attempts, exponential + // backoff) stalled every boot for 15.5 s before the loop then failed + // fast anyway. The first real completion is the retry. + const health = await checkLlamaServer({ retries: 0 }); + if (!health.reachable) { + logger.warn("llama-server health check failed", { + error: health.error, + url: config.localModels.url, + }); + if (config.localModels.mode === "managed") { + logger.warn(managedLocalLlmHealthFailureHint(config.localModels.managed.port), { + mode: "managed", + }); + } + } else { + logger.info("llama-server reachable", { + url: config.localModels.url, + latencyMs: health.latencyMs, }); } - } else { - logger.info("llama-server reachable", { + } else if (options.overrides?.deferLlamaHealthCheck) { + logger.info("llama-server health check deferred; runtime will refresh on first turn", { url: config.localModels.url, - latencyMs: health.latencyMs, }); } - } else if (options.overrides?.deferLlamaHealthCheck) { - logger.info("llama-server health check deferred; runtime will refresh on first turn", { + }; + + if (localTextActiveAtBoot) { + await runBootHealthProbe(); + } else { + logger.info("local llama probes skipped; active text provider is not local", { + activeTextProvider: resolveLlmConfig(config).activeTextProvider, url: config.localModels.url, }); } @@ -936,6 +966,7 @@ export async function createAgentRuntime( llama, logger, config.localModels.url, + localTextActiveAtBoot, ); const slotManager = new SlotManager(totalSlots ?? undefined); if (totalSlots !== null) { @@ -958,19 +989,31 @@ export async function createAgentRuntime( // generation budget makes every step come back `truncated` — the model // burns its remaining tokens and never closes a tool-call array. Loud // at startup because the failure mode downstream is silent. - const minUsableCtx = minUsableContextWindow( - config.localModels.completionMaxTokens, - ); - if (profile.contextWindow && profile.contextWindow < minUsableCtx) { - logger.warn("context window too small for the agent prompt", { - contextWindow: profile.contextWindow, - required: minUsableCtx, - completionMaxTokens: config.localModels.completionMaxTokens, - hint: - config.localModels.mode === "managed" - ? "raise localModels.managed.contextSize, lower localModels.completionMaxTokens, or pick a model that fits VRAM" - : "start llama-server with a larger --ctx-size, or lower localModels.completionMaxTokens", - }); + // + // Advice about the LOCAL server's `--ctx-size` only, so it rides the + // same gate as the probe that produced the number (issue #112): on a + // cloud route there is no `/props` reading to judge, and the hints it + // prints name flags a cloud provider does not have. + const warnOnSmallContextWindow = ( + candidate: ReturnType, + ): void => { + const minUsableCtx = minUsableContextWindow( + config.localModels.completionMaxTokens, + ); + if (candidate.contextWindow && candidate.contextWindow < minUsableCtx) { + logger.warn("context window too small for the agent prompt", { + contextWindow: candidate.contextWindow, + required: minUsableCtx, + completionMaxTokens: config.localModels.completionMaxTokens, + hint: + config.localModels.mode === "managed" + ? "raise localModels.managed.contextSize, lower localModels.completionMaxTokens, or pick a model that fits VRAM" + : "start llama-server with a larger --ctx-size, or lower localModels.completionMaxTokens", + }); + } + }; + if (localTextActiveAtBoot) { + warnOnSmallContextWindow(profile); } const browserBackend: BrowserBackend = @@ -1290,6 +1333,42 @@ export async function createAgentRuntime( const getLiveProfile = () => profileManager?.getProfile() ?? profile; + // Issue #112. The manager above is built either way — construction is + // pure field assignment, no I/O — because deleting it on a cloud boot + // would leave a mid-turn fallover to a `llama-server` link running on + // a frozen `plain-instruct` profile with no way back. What is gated is + // its *probing*: the boot probes are deferred here and replayed once, + // lazily, by whichever path reaches local inference first (a provider + // switch, via the agent loop's turn-start gate, or a cloud→local + // fallover, via the fallback seam's `prepareLink`). + const localBackend = new DeferredLocalBackendProbes( + { + isActive: () => + activeTextProviderIsLlamaServer(resolveLlmConfig(getConfig())), + restore: async () => { + logger.info("restoring local llama backend state", { + url: config.localModels.url, + }); + try { + await runBootHealthProbe(); + // `refresh()` is the deferred `/props`: profile, grammar and + // the slot pool (via `onTotalSlots`) in one round trip. It + // swallows its own failures and keeps the prior profile. + await profileManager?.refresh(); + warnOnSmallContextWindow(getLiveProfile()); + } catch (err) { + // The seam awaits this before a fallover attempt: a throw here + // would fail the link and advance the chain over a diagnostic. + // The completion itself is the real verdict on the backend. + logger.warn("local llama backend restore failed; continuing", { + error: err instanceof Error ? err.message : String(err), + }); + } + }, + }, + localTextActiveAtBoot, + ); + const providerRegistry = await ProviderRegistry.fromConfig(config, { config, llamaClient: llama, @@ -1591,6 +1670,18 @@ export async function createAgentRuntime( const { provider, transport } = resolveActiveLlmSlice(providerId); return { provider, transport }; }, + // Issue #112. The one place that knows a cloud→local fallover is + // about to happen: the chain has already picked the link and the + // completion has not been sent. A `llama-server` link reached from a + // cloud boot runs on deferred state (plain profile, one-slot pool, + // no `/props`), so warm it here rather than infer against it. + // No-op on every other attempt — one boolean after the first call. + prepareLink: async (providerId) => { + if (!providerIdIsLlamaServer(resolveLlmConfig(getConfig()), providerId)) { + return; + } + await localBackend.ensureProbed(); + }, recordUnaryUsage, recordStreamUsage, }; @@ -1981,6 +2072,9 @@ export async function createAgentRuntime( profile, contextWindow: resolveCatalogContextWindow, ...(profileManager ? { profileManager } : {}), + // Gates the two `/props` refreshes the loop owns, and carries the + // lazy restore for a switch back to a local provider (issue #112). + localBackend, ...(config.memory.profile.enabled ? { profileFactsProvider: () => profileStore.list() } : {}), @@ -2801,7 +2895,23 @@ async function resolveModelProfile( llama: LlamaServerClient, logger: StructuredLogger, llamaUrl: string, + /** + * `false` when the active text provider is not a `llama-server` link: + * the `/props` probe is skipped entirely and the run starts on the + * plain profile (issue #112). Deliberately silent — the cloud route is + * not a failed probe, and the "using plain fallback" warning below + * would say it was. A later switch or fallover to a local link warms + * the real profile through `DeferredLocalBackendProbes`. + */ + probeLocal: boolean, ): Promise { + if (!probeLocal) { + return { + profile: PLAIN_INSTRUCT_PROFILE, + modelAlias: null, + totalSlots: null, + }; + } if (overrides?.llamaPropsError) { logger.warn("model profile probe failed; using plain fallback", { error: overrides.llamaPropsError.message, diff --git a/src/runtime/llm-fallback-seam.test.ts b/src/runtime/llm-fallback-seam.test.ts index 51c0ee82..cad1284a 100644 --- a/src/runtime/llm-fallback-seam.test.ts +++ b/src/runtime/llm-fallback-seam.test.ts @@ -299,3 +299,115 @@ describe("per-link prompt substitution (grammarPrompt)", () => { expect(localPrompts).toEqual(["shared prompt"]); }); }); + +/** + * Issue #112. Boot skips the local `/health` + `/props` probes while a + * cloud provider is active, which leaves a `llama-server` link running + * on a deferred profile, a one-slot pool and no health reading. A + * cloud→local FALLOVER reaches that link without any config change and + * without the agent loop's turn-start refresh (it saw a cloud route when + * the turn began), so the seam is the last point at which the state can + * still be warmed. These tests pin the ordering: `prepareLink` for the + * link that is about to serve, before its completion is sent. + */ +describe("prepareLink — warming a link before it serves (issue #112)", () => { + function tracingDeps( + providers: Map, + trace: string[], + ): FallbackSeamDeps { + const deps = seamDeps(providers); + deps.prepareLink = async (providerId) => { + trace.push(`prepare:${providerId}`); + }; + return deps; + } + + it("prepares the local link before the fallover attempt is sent", async () => { + const trace: string[] = []; + const providers = new Map([ + [ + "cloud", + fakeProvider("cloud", "native_tools", async () => { + trace.push("serve:cloud"); + throw new OpenAiHttpError( + "rate limited", + 429, + "http://cloud", + false, + null, + "cloud", + ); + }), + ], + [ + "local", + fakeProvider("local", "grammar", async () => { + trace.push("serve:local"); + return answer("local"); + }), + ], + ]); + const result = await createFallbackCompleter( + tracingDeps(providers, trace), + )(baseParams); + + expect(result.modelId).toBe("local-model"); + // The load-bearing ordering: `prepare:local` sits BEFORE + // `serve:local`. Without the hook the local link would answer with + // its profile, grammar and slot pool never probed. + expect(trace).toEqual([ + "prepare:cloud", + "serve:cloud", + "prepare:local", + "serve:local", + ]); + }); + + it("streaming: prepares the local link before the stream is opened", async () => { + const trace: string[] = []; + const providers = new Map([ + [ + "cloud", + fakeProvider("cloud", "native_tools", async () => { + trace.push("serve:cloud"); + throw new OpenAiHttpError( + "rate limited", + 429, + "http://cloud", + false, + null, + "cloud", + ); + }), + ], + [ + "local", + fakeProvider("local", "grammar", async () => { + trace.push("serve:local"); + return answer("local"); + }), + ], + ]); + const streamer = createFallbackStreamer(tracingDeps(providers, trace)); + const gen = streamer(baseParams); + let next = await gen.next(); + while (!next.done) next = await gen.next(); + + expect(next.value.servedTransport).toBe("grammar"); + expect(trace).toEqual([ + "prepare:cloud", + "serve:cloud", + "prepare:local", + "serve:local", + ]); + }); + + it("is optional — an unwired seam behaves exactly as before", async () => { + const providers = new Map([ + ["cloud", fakeProvider("cloud", "native_tools", async () => answer("cloud"))], + ["local", fakeProvider("local", "grammar", async () => answer("local"))], + ]); + const result = await createFallbackCompleter(seamDeps(providers))(baseParams); + expect(result.modelId).toBe("cloud-model"); + }); +}); diff --git a/src/runtime/llm-fallback-seam.ts b/src/runtime/llm-fallback-seam.ts index 2d47e002..8a6211b0 100644 --- a/src/runtime/llm-fallback-seam.ts +++ b/src/runtime/llm-fallback-seam.ts @@ -34,6 +34,20 @@ export interface FallbackSeamDeps { fallbackChain: ProviderFallbackChain; /** Resolve the served link's provider + transport for `providerId`. */ resolveSlice: (providerId: string) => ResolvedLinkSlice; + /** + * Awaited once per attempt, before the completion is sent, with the + * link the chain picked. Exists for the state a link may need warmed + * before it can serve: a `llama-server` link reached by fallover from + * a cloud primary boots with its `/health` + `/props` probes deferred + * (issue #112), and this is the last point at which they can still + * run. Kept as a hook rather than folded into `resolveSlice` because + * that seam is synchronous, and rather than into the attempt body + * because only bootstrap knows what "warm" means for a link kind. + * + * Must not throw for a reachable link: a rejection here fails the + * attempt and advances the chain, same as a failed completion. + */ + prepareLink?: (providerId: string) => Promise; /** Fold a unary completion's usage into cost + meter (no-op when absent). */ recordUnaryUsage: (params: LlmStreamParams, result: CompletionResult) => void; /** Fold a streamed completion's usage into the meter. */ @@ -79,6 +93,7 @@ export function createFallbackCompleter( runWithFallback( deps.fallbackChain, async (providerId) => { + await deps.prepareLink?.(providerId); const { provider, transport } = deps.resolveSlice(providerId); const base = { prompt: promptFor(params, transport), @@ -142,6 +157,7 @@ export function createFallbackStreamer( primed: PrimedStream; transport: ToolCallTransport; }> => { + await deps.prepareLink?.(providerId); const { provider, transport } = deps.resolveSlice(providerId); const base = { prompt: promptFor(params, transport), diff --git a/src/runtime/local-probe-gating.test.ts b/src/runtime/local-probe-gating.test.ts new file mode 100644 index 00000000..acee236d --- /dev/null +++ b/src/runtime/local-probe-gating.test.ts @@ -0,0 +1,294 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { createAgentRuntime } from "./bootstrap.js"; +import { + getUserConfigPath, + resetConfigCache, + USER_CONFIG_DEFAULTS, + writeUserConfigFileSync, +} from "../config/index.js"; +import type { UserConfigFile } from "../config/index.js"; +import { GEMMA4_PROPS } from "../llm/model-profile.fixtures.js"; +import { DEFAULT_EMBEDDING_MODEL_ID } from "../local-llm/index.js"; +import { FakeBrowserBackend } from "../http/test-harness.js"; +import type { LogRecord } from "../tracing/structured-logger.js"; + +/** + * Issue #112 — a cloud-backed session must not probe the local + * llama-server. + * + * The assertions are exact request COUNTS against the two local ports, + * not "was it called": the bug was a fixed number of probes (`/health` + * once, `/props` once) firing on a route that never uses them, and a + * boolean would pass again the moment one of them came back. + */ + +const TEXT_PORT = "127.0.0.1:8080"; +const EMBED_PORT = "127.0.0.1:19092"; + +interface LocalTraffic { + /** Every URL the process asked for, in order. */ + urls: string[]; + countTo(hostPort: string, path?: string): number; + reset(): void; +} + +/** + * Count every outbound request. Answers the local endpoints with real + * llama.cpp shapes so the *local* control cases probe successfully — + * a stub that failed every probe would make "zero requests" and "all + * requests failed" indistinguishable. + */ +function installCountingFetch(): LocalTraffic { + const urls: string[] = []; + const impl: typeof fetch = async (input) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : (input as Request).url; + urls.push(url); + if (url.includes(TEXT_PORT) || url.includes(EMBED_PORT)) { + if (url.includes("/completion")) { + // Answers with a 200 the client accepts as a completion (the + // content parses to nothing useful, which is fine — the lazy + // restore assertions are about the probes, not the reply). + return new Response(JSON.stringify({ content: "", stop: true }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.includes("/props")) { + return new Response(JSON.stringify(GEMMA4_PROPS), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.includes("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + } + // Everything else (analytics, update check, provider catalogues) + // is answered flatly so no test ever reaches the network. + return new Response("{}", { + status: 404, + headers: { "content-type": "application/json" }, + }); + }; + vi.stubGlobal("fetch", impl); + return { + urls, + countTo: (hostPort, path) => + urls.filter((u) => u.includes(hostPort) && (!path || u.includes(path))) + .length, + reset: () => { + urls.length = 0; + }, + }; +} + +/** A cloud text provider that needs no network to construct. */ +const CLOUD_PROVIDER = { + id: "cloudy", + kind: "openai-compatible" as const, + baseUrl: "https://cloud.invalid", + defaultChatModel: "cloudy-1", + apiKey: "sk-test", +}; + +function writeConfig(stateDir: string, over: Partial): void { + writeUserConfigFileSync(getUserConfigPath(stateDir), { + ...USER_CONFIG_DEFAULTS, + // Keep the runtime off the network for everything unrelated. + analytics: { enabled: false }, + ...over, + }); + resetConfigCache(); +} + +/** The embedding half of the registry, left on the local default. */ +const LOCAL_EMBED_PROVIDER = { + id: "local-llama-embed", + kind: "llama-server" as const, + url: "http://127.0.0.1:19092", +}; + +const cloudLlm = { + activeTextProvider: "cloudy", + activeEmbeddingProvider: "local-llama-embed", + providers: [CLOUD_PROVIDER, LOCAL_EMBED_PROVIDER], + toolTransport: "auto" as const, +}; + +describe("issue #112 — local probe gating at CLI bootstrap", () => { + let stateDir: string; + let workingDir: string; + let traffic: LocalTraffic; + + beforeEach(() => { + stateDir = mkdtempSync(join(tmpdir(), "atomic-gate-")); + workingDir = mkdtempSync(join(tmpdir(), "atomic-gate-cwd-")); + mkdirSync(join(workingDir, ".atomic-agent", "skills"), { recursive: true }); + process.env.ATOMIC_AGENT_STATE_DIR = stateDir; + process.env.ATOMIC_AGENT_GRAMMARS_DIR = join(process.cwd(), "grammars"); + resetConfigCache(); + traffic = installCountingFetch(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + rmSync(stateDir, { recursive: true, force: true }); + rmSync(workingDir, { recursive: true, force: true }); + delete process.env.ATOMIC_AGENT_STATE_DIR; + delete process.env.ATOMIC_AGENT_GRAMMARS_DIR; + resetConfigCache(); + }); + + const boot = async (logs: LogRecord[] = []) => + createAgentRuntime({ + workingDir, + approvalLevel: 5, + handlers: { logSinks: [(record) => logs.push(record)] }, + overrides: { + browserBackend: new FakeBrowserBackend(), + // Unary seam only: the streaming path's SSE shape is beside the + // point here, and the two share `prepareLink`. + disableStreaming: true, + }, + }); + + it("makes zero local text requests with a cloud text provider", async () => { + writeConfig(stateDir, { llm: cloudLlm }); + const logs: LogRecord[] = []; + const runtime = await boot(logs); + try { + expect(traffic.countTo(TEXT_PORT)).toBe(0); + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(0); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(0); + // ...and says nothing alarming about the backend it skipped. + const complaints = logs.filter( + (r) => + (r.level === "warn" || r.level === "error") && + /llama|context window/i.test(r.message), + ); + expect(complaints).toEqual([]); + } finally { + await runtime.shutdown(); + } + }); + + it("still probes when the active text provider IS a llama-server link", async () => { + // The control for the case above: same code path, local route. + writeConfig(stateDir, {}); + const runtime = await boot(); + try { + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(1); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(1); + } finally { + await runtime.shutdown(); + } + }); + + it("gates on provider KIND, not the `local-llama` id", async () => { + // A llama-server entry under a custom id is still the local route. + writeConfig(stateDir, { + llm: { + activeTextProvider: "my-box", + activeEmbeddingProvider: "local-llama-embed", + providers: [ + { id: "my-box", kind: "llama-server", url: "http://127.0.0.1:8080" }, + LOCAL_EMBED_PROVIDER, + ], + toolTransport: "auto", + }, + }); + const runtime = await boot(); + try { + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(1); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(1); + } finally { + await runtime.shutdown(); + } + }); + + it("lazily restores the local backend when the operator switches to it", async () => { + // The whole point of deferring rather than deleting: the state boot + // skipped has to come back before local inference, not at the next + // process start. + writeConfig(stateDir, { + llm: { + ...cloudLlm, + providers: [ + CLOUD_PROVIDER, + { id: "local-llama", kind: "llama-server", url: "http://127.0.0.1:8080" }, + LOCAL_EMBED_PROVIDER, + ], + }, + }); + const runtime = await boot(); + try { + expect(traffic.countTo(TEXT_PORT)).toBe(0); + + // Exactly what the LLM tab does: registry first, then config. + await runtime.providerRegistry.setActive("local-llama"); + writeConfig(stateDir, { + llm: { + ...cloudLlm, + activeTextProvider: "local-llama", + providers: [ + CLOUD_PROVIDER, + { id: "local-llama", kind: "llama-server", url: "http://127.0.0.1:8080" }, + LOCAL_EMBED_PROVIDER, + ], + }, + }); + + const session = runtime.createSession(); + await runtime + .executeTurn(session, "hello", { + maxSteps: 1, + signal: new AbortController().signal, + }) + .catch(() => undefined); + + // Health, profile/`/props` and the slot pool are warm before the + // first local completion — none of which boot had done. + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(1); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(1); + } finally { + await runtime.shutdown(); + } + }); + + it("keeps probing local embeddings while the text route is cloud", async () => { + writeConfig(stateDir, { + llm: cloudLlm, + localModels: { + ...USER_CONFIG_DEFAULTS.localModels, + embeddings: { + ...USER_CONFIG_DEFAULTS.localModels.embeddings, + enabled: true, + modelId: DEFAULT_EMBEDDING_MODEL_ID, + }, + }, + memory: { + ...USER_CONFIG_DEFAULTS.memory, + embeddings: { ...USER_CONFIG_DEFAULTS.memory.embeddings, enabled: true }, + }, + }); + const runtime = await boot(); + try { + expect(traffic.countTo(EMBED_PORT, "/health")).toBe(1); + expect(traffic.countTo(TEXT_PORT)).toBe(0); + } finally { + await runtime.shutdown(); + } + }); +}); diff --git a/src/sidecar/local-probe-gating.test.ts b/src/sidecar/local-probe-gating.test.ts new file mode 100644 index 00000000..a0b1e851 --- /dev/null +++ b/src/sidecar/local-probe-gating.test.ts @@ -0,0 +1,210 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { bootstrapSidecar } from "./main.js"; +import { + getUserConfigPath, + resetConfigCache, + USER_CONFIG_DEFAULTS, + writeUserConfigFileSync, +} from "../config/index.js"; +import type { UserConfigFile } from "../config/index.js"; +import { GEMMA4_PROPS } from "../llm/model-profile.fixtures.js"; +import type { SidecarMessage } from "./sidecar-events.js"; + +/** + * Issue #112 — `start_session` ran an unconditional local `/health` + * probe after `buildRuntime` and emitted `llm_unavailable` when nothing + * answered. On a cloud-backed session that event tells the desktop shell + * the backend is down for a session that is about to run perfectly. + * + * The sidecar is driven the way the host drives it: an NDJSON request + * pushed at the real stdin stream, the response read off stdout. + */ + +const TEXT_PORT = "127.0.0.1:8080"; + +describe("sidecar start_session — local probe gating", () => { + let stateDir: string; + let workingDir: string; + let previousStateDir: string | undefined; + let urls: string[]; + let stdout: string[]; + + beforeEach(() => { + stateDir = mkdtempSync(join(tmpdir(), "atomic-sidecar-gate-")); + workingDir = mkdtempSync(join(tmpdir(), "atomic-sidecar-cwd-")); + mkdirSync(join(workingDir, ".atomic-agent", "skills"), { recursive: true }); + previousStateDir = process.env.ATOMIC_AGENT_STATE_DIR; + process.env.ATOMIC_AGENT_STATE_DIR = stateDir; + process.env.ATOMIC_AGENT_GRAMMARS_DIR = join(process.cwd(), "grammars"); + resetConfigCache(); + + urls = []; + vi.stubGlobal("fetch", async (input: unknown) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : (input as Request).url; + urls.push(url); + if (url.includes(TEXT_PORT) && url.includes("/props")) { + return new Response(JSON.stringify(GEMMA4_PROPS), { status: 200 }); + } + if (url.includes(TEXT_PORT) && url.includes("/health")) { + return new Response(JSON.stringify({ status: "ok" }), { status: 200 }); + } + return new Response("{}", { status: 404 }); + }); + + // The sidecar speaks NDJSON on the real stdout; capture it instead + // of letting it interleave with the reporter's output. + stdout = []; + vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + stdout.push(String(chunk)); + return true; + }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + rmSync(stateDir, { recursive: true, force: true }); + rmSync(workingDir, { recursive: true, force: true }); + if (previousStateDir === undefined) { + delete process.env.ATOMIC_AGENT_STATE_DIR; + } else { + process.env.ATOMIC_AGENT_STATE_DIR = previousStateDir; + } + delete process.env.ATOMIC_AGENT_GRAMMARS_DIR; + resetConfigCache(); + }); + + const writeConfig = (llm: UserConfigFile["llm"]): void => { + writeUserConfigFileSync(getUserConfigPath(stateDir), { + ...USER_CONFIG_DEFAULTS, + analytics: { enabled: false }, + ...(llm ? { llm } : {}), + }); + resetConfigCache(); + }; + + /** + * Boot the sidecar, push one `start_session`, wait for its response. + * + * `bootstrapSidecar` attaches to the process-wide stdin/stdout, and a + * listener left behind would make the NEXT test's request run through + * two sidecars at once (two runtimes seeding the same skills dir, and + * whichever answered first winning the response). So the listeners it + * adds are recorded and removed on the way out. + */ + const startSession = async (): Promise<{ + messages: SidecarMessage[]; + shutdown: () => Promise; + }> => { + const before = new Map([ + ["stdin:data", process.stdin.listeners("data").slice()], + ["stdin:end", process.stdin.listeners("end").slice()], + ["stdout:error", process.stdout.listeners("error").slice()], + ["stdout:close", process.stdout.listeners("close").slice()], + ]); + const detach = (): void => { + for (const [key, kept] of before) { + const [target, event] = key.split(":") as ["stdin" | "stdout", string]; + const emitter = target === "stdin" ? process.stdin : process.stdout; + for (const listener of emitter.listeners(event)) { + if (!kept.includes(listener)) { + emitter.removeListener(event, listener as () => void); + } + } + } + }; + const { shutdown } = await bootstrapSidecar(); + process.stdin.emit( + "data", + `${JSON.stringify({ + kind: "request", + id: "req-1", + type: "start_session", + payload: { workingDir }, + })}\n`, + ); + const deadline = Date.now() + 10_000; + const parsed = (): SidecarMessage[] => + stdout + .join("") + .split("\n") + .filter((line) => line.trim().length > 0) + .flatMap((line) => { + try { + return [JSON.parse(line) as SidecarMessage]; + } catch { + return []; + } + }); + while ( + Date.now() < deadline && + !parsed().some((m) => m.kind === "response") + ) { + await new Promise((r) => setTimeout(r, 20)); + } + return { + messages: parsed(), + shutdown: async () => { + detach(); + await shutdown(); + }, + }; + }; + + it("emits no local health probe or llm_unavailable on a cloud route", async () => { + writeConfig({ + activeTextProvider: "cloudy", + activeEmbeddingProvider: "local-llama-embed", + providers: [ + { + id: "cloudy", + kind: "openai-compatible", + baseUrl: "https://cloud.invalid", + defaultChatModel: "cloudy-1", + apiKey: "sk-test", + }, + { id: "local-llama-embed", kind: "llama-server", url: "http://127.0.0.1:19092" }, + ], + toolTransport: "auto", + }); + + const { messages, shutdown } = await startSession(); + try { + expect( + messages.some((m) => m.kind === "response" && m.ok), + ).toBe(true); + expect(urls.filter((u) => u.includes(TEXT_PORT))).toEqual([]); + expect( + messages.filter( + (m) => m.kind === "event" && m.type === "llm_unavailable", + ), + ).toEqual([]); + } finally { + await shutdown(); + } + }); + + it("still probes on a local route (the control)", async () => { + writeConfig(undefined); + const { messages, shutdown } = await startSession(); + try { + expect(messages.some((m) => m.kind === "response" && m.ok)).toBe(true); + // Boot's `/health` + `/props`, then start_session's own `/health`. + expect( + urls.filter((u) => u.includes(TEXT_PORT) && u.includes("/health")) + .length, + ).toBe(2); + } finally { + await shutdown(); + } + }); +}); diff --git a/src/sidecar/main.ts b/src/sidecar/main.ts index 30d54dda..2bdb96ad 100644 --- a/src/sidecar/main.ts +++ b/src/sidecar/main.ts @@ -5,6 +5,8 @@ import { MessageRouter } from "./message-router.js"; import { StdioProtocol } from "./stdio-protocol.js"; import { getConfig } from "../config/index.js"; import { checkLlamaServer } from "../llm/llama-server-health.js"; +import { activeTextProviderIsLlamaServer } from "../llm/provider/registry/active-text-provider.js"; +import { resolveLlmConfig } from "../llm/provider/registry/provider-registry.js"; import { createAgentRuntime } from "../runtime/bootstrap.js"; import type { AgentRuntime } from "../runtime/bootstrap.js"; import type { AgentLoopEvent } from "../agent/agent-loop.js"; @@ -250,18 +252,25 @@ export async function bootstrapSidecar(): Promise<{ const runtime = await buildRuntime(workingDir); // Status probe for the desktop shell — one attempt; the retry // ladder only delayed the `llm_unavailable` event by 15.5 s. - const health = await checkLlamaServer({ retries: 0 }); - if (!health.reachable) { - const hint = - config.localModels.mode === "managed" - ? "run atomic-agent models start" - : "check localModels.url or ATOMIC_AGENT_LLAMA_URL"; - protocol.emitEvent("llm_unavailable", { - url: config.localModels.url, - error: health.error, - mode: config.localModels.mode, - hint, - }); + // + // Only when the local backend is the route (issue #112). Config is + // re-read rather than closed over: the shell can rewrite it + // between sessions, and `llm_unavailable` about a llama-server the + // session never talks to is a failure report for a healthy run. + if (activeTextProviderIsLlamaServer(resolveLlmConfig(getConfig()))) { + const health = await checkLlamaServer({ retries: 0 }); + if (!health.reachable) { + const hint = + config.localModels.mode === "managed" + ? "run atomic-agent models start" + : "check localModels.url or ATOMIC_AGENT_LLAMA_URL"; + protocol.emitEvent("llm_unavailable", { + url: config.localModels.url, + error: health.error, + mode: config.localModels.mode, + hint, + }); + } } const session = runtime.createSession({ ...(request.payload.metadata diff --git a/src/tui/llm-health/llm-health-poller.test.ts b/src/tui/llm-health/llm-health-poller.test.ts index 8ec18ff1..070b0fbc 100644 --- a/src/tui/llm-health/llm-health-poller.test.ts +++ b/src/tui/llm-health/llm-health-poller.test.ts @@ -1,4 +1,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + getUserConfigPath, + resetConfigCache, + USER_CONFIG_DEFAULTS, + writeUserConfigFileSync, +} from "../../config/index.js"; +import type { UserConfigFile } from "../../config/index.js"; import * as healthModule from "../../llm/llama-server-health.js"; import type { HealthResult } from "../../llm/llama-server-health.js"; @@ -435,4 +446,129 @@ describe("LlmHealthPoller", () => { capture.actions.map((a) => a.type).filter((t) => t.includes("rss")), ).toEqual([]); }); -}); \ No newline at end of file +}); +/** + * Issue #112 — the footer poller is the noisiest local prober in the + * process: `/health` every 3 s plus a one-shot `/props`, from the moment + * the TUI mounts, whatever the route. On a cloud session that is a + * request every three seconds against a server nobody is running, and a + * `down` badge plus an `n_ctx` reading about a backend that is not + * serving the turn. + */ +describe("LlmHealthPoller — gated on the active text provider", () => { + const stateDir = mkdtempSync(join(tmpdir(), "atomic-poller-gate-")); + let previousStateDir: string | undefined; + let spy: ReturnType; + + const writeLlm = (llm: UserConfigFile["llm"]): void => { + writeUserConfigFileSync(getUserConfigPath(stateDir), { + ...USER_CONFIG_DEFAULTS, + ...(llm ? { llm } : {}), + }); + resetConfigCache(); + }; + + beforeEach(() => { + previousStateDir = process.env.ATOMIC_AGENT_STATE_DIR; + process.env.ATOMIC_AGENT_STATE_DIR = stateDir; + spy = vi.spyOn(healthModule, "checkLlamaServer"); + }); + + afterEach(() => { + vi.restoreAllMocks(); + if (previousStateDir === undefined) { + delete process.env.ATOMIC_AGENT_STATE_DIR; + } else { + process.env.ATOMIC_AGENT_STATE_DIR = previousStateDir; + } + rmSync(getUserConfigPath(stateDir), { force: true }); + resetConfigCache(); + }); + + it("probes nothing at TUI startup when a cloud provider is active", async () => { + writeLlm({ + activeTextProvider: "cloudy", + activeEmbeddingProvider: "local-llama-embed", + providers: [ + { + id: "cloudy", + kind: "openai-compatible", + baseUrl: "https://cloud.invalid", + defaultChatModel: "cloudy-1", + apiKey: "sk-test", + }, + { id: "local-llama-embed", kind: "llama-server", url: "http://127.0.0.1:19092" }, + ], + toolTransport: "auto", + }); + const props = vi.fn(stubProps); + const capture = makeCapture(); + const poller = new LlmHealthPoller(capture, "http://127.0.0.1:8080", 10, props); + poller.start(); + await sleep(40); + poller.stop(); + + // Exact counts: several tick windows elapsed, and every one of them + // must have cost zero requests. + expect(spy).toHaveBeenCalledTimes(0); + expect(props).toHaveBeenCalledTimes(0); + expect(capture.actions).toEqual([]); + }); + + it("probes on the same schedule as before when the route is local", async () => { + // The control: same poller, same timings, default (llama-server) + // config — one `/health` per tick and the one-shot `/props`. + writeLlm(undefined); + spy.mockResolvedValue({ + reachable: true, + status: 200, + error: null, + latencyMs: 1, + } satisfies HealthResult); + const props = vi.fn(stubProps); + const capture = makeCapture(); + const poller = new LlmHealthPoller(capture, "http://127.0.0.1:8080", 10_000, props); + poller.start(); + await sleep(30); + poller.stop(); + + expect(spy).toHaveBeenCalledTimes(1); + expect(props).toHaveBeenCalledTimes(1); + }); + + it("resumes within one tick after a hot switch back to a local provider", async () => { + writeLlm({ + activeTextProvider: "cloudy", + activeEmbeddingProvider: "local-llama-embed", + providers: [ + { + id: "cloudy", + kind: "openai-compatible", + baseUrl: "https://cloud.invalid", + defaultChatModel: "cloudy-1", + apiKey: "sk-test", + }, + { id: "local-llama-embed", kind: "llama-server", url: "http://127.0.0.1:19092" }, + ], + toolTransport: "auto", + }); + spy.mockResolvedValue({ + reachable: true, + status: 200, + error: null, + latencyMs: 1, + } satisfies HealthResult); + const capture = makeCapture(); + const poller = new LlmHealthPoller(capture, "http://127.0.0.1:8080", 10, stubProps); + poller.start(); + await sleep(40); + expect(spy).toHaveBeenCalledTimes(0); + + // The operator picks the local backend again. No start/stop call + // reaches the poller — it re-reads config on its own tick. + writeLlm(undefined); + await sleep(40); + poller.stop(); + expect(spy.mock.calls.length).toBeGreaterThan(0); + }); +}); diff --git a/src/tui/llm-health/llm-health-poller.ts b/src/tui/llm-health/llm-health-poller.ts index a90f364c..11ebcc37 100644 --- a/src/tui/llm-health/llm-health-poller.ts +++ b/src/tui/llm-health/llm-health-poller.ts @@ -1,5 +1,7 @@ import { getConfig } from "../../config/index.js"; import { checkLlamaServer } from "../../llm/llama-server-health.js"; +import { activeTextProviderIsLlamaServer } from "../../llm/provider/registry/active-text-provider.js"; +import { resolveLlmConfig } from "../../llm/provider/registry/provider-registry.js"; import { llamaEndpointUrl } from "../../llm/llama-endpoint-url.js"; import type { TuiAction } from "../tui-action.js"; @@ -114,11 +116,35 @@ export class LlmHealthPoller { async refreshModelLabel(): Promise { this.modelFetchedForUrl = false; if (this.stopped) return; + // Same gate as `tick`: the Models tab can restart a managed daemon + // while the route is cloud, and the label this would fetch belongs + // to a backend that is not serving the session. + if (!this.localTextActive()) return; await this.fetchModelLabel(); } + /** + * Whether the local backend this poller watches is the route the + * operator is actually on. Read per tick from config rather than + * latched at construction: the active provider changes from the LLM + * tab, the composer switch and the provider wizard, and a poller that + * had to be told about each of them would miss the one that was added + * last (issue #112). + */ + private localTextActive(): boolean { + return activeTextProviderIsLlamaServer(resolveLlmConfig(getConfig())); + } + private async tick(): Promise { if (this.probing || this.stopped) return; + // Cloud route: probe nothing and emit nothing. The alternative — + // poll and hide — still costs a request every 3 s against a server + // nobody is running, and leaves a `down` reading in state that the + // context gauge (`select-context-usage`) would read as the active + // model's window. The interval keeps ticking so a switch back to a + // local provider resumes within one period, with no start/stop + // wiring on every switch path. + if (!this.localTextActive()) return; this.probing = true; if (!this.hasSettledResult) { this.emitter.emit({ diff --git a/src/tui/local-turn-gate.ts b/src/tui/local-turn-gate.ts index ded9348f..ce7f8d38 100644 --- a/src/tui/local-turn-gate.ts +++ b/src/tui/local-turn-gate.ts @@ -6,8 +6,8 @@ import { } from "../local-llm/index.js"; import { resolveFallbackChain } from "../llm/fallback/index.js"; import { + activeTextProviderIsLlamaServer, resolveLlmConfig, - type ResolvedLlmConfig, } from "../llm/provider/registry/index.js"; import { formatBytes } from "./hooks/use-transfer-rate.js"; import type { LocalModelsPullState } from "./local-models/local-models-panel-state.js"; @@ -53,20 +53,12 @@ export type LocalTurnGateDecision = | { kind: "block"; text: string }; /** - * KIND-based local detection, mirroring `selectComposerBackend`: any - * `llama-server` entry is the local route, because `LlamaServerProvider` - * accepts a custom id (`options.id`) — keying on the literal - * `local-llama` id would leave a renamed entry ungated. An active id - * that resolves to no entry reads as local too, matching the composer's - * no-active-row rule (and the no-`llm`-block default, which - * `resolveLlmConfig` synthesizes as a `llama-server` entry anyway). + * Moved beside `resolveLlmConfig` (issue #112): `src/runtime/` and + * `src/sidecar/` gate their local probes on the same predicate and must + * not import from `src/tui/`. Re-exported here so the gate's original + * callers keep working. */ -export function activeTextProviderIsLlamaServer( - llm: ResolvedLlmConfig, -): boolean { - const active = llm.providers.find((p) => p.id === llm.activeTextProvider); - return active === undefined || active.kind === "llama-server"; -} +export { activeTextProviderIsLlamaServer }; /** * Read the live facts from config + disk. Cheap on the happy path: the diff --git a/src/tui/select-context-usage.ts b/src/tui/select-context-usage.ts index ef5a8e5d..20bb4dec 100644 --- a/src/tui/select-context-usage.ts +++ b/src/tui/select-context-usage.ts @@ -65,7 +65,11 @@ const CONVERSATION_CAP_FLOOR = 512; * 2. The health poller's reading of the same endpoint. Not redundant: * `localModels.mode: "managed"` *defers* the boot probe, so a local * turn can build its prompt with no window while the poller already - * has one. + * has one. Only consulted while a local backend is the active route + * (issue #112) — after a local→cloud switch the poller's last local + * reading is still in state, and drawing the cloud model's gauge + * against a llama-server `n_ctx` is a fabrication with a number + * attached. * 3. The active cloud provider's catalogue. Read here, at render time, * rather than resolved once into a `ProviderRow`: the live catalogue * arrives from an async fetch at start-up, so anything baked into a @@ -80,9 +84,10 @@ const CONVERSATION_CAP_FLOOR = 512; function resolveWindow(state: TuiState): number | null { const fromPrompt = state.contextUsage.contextWindow; if (fromPrompt !== null && fromPrompt > 0) return fromPrompt; - const fromPoller = state.llmHealth.contextWindow; - if (fromPoller !== null && fromPoller > 0) return fromPoller; const active = state.providersPanel.rows.find((row) => row.isActiveText); + const localActive = active === undefined || active.kind === "llama-server"; + const fromPoller = state.llmHealth.contextWindow; + if (localActive && fromPoller !== null && fromPoller > 0) return fromPoller; if (!active?.chatModel) return null; const lookup = catalogEntryLookupForKind(active.kind); const entry = lookup?.(active.chatModel); From 33af1f60d50c3f14012eed43a942dca132568c3b Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:37:39 +0300 Subject: [PATCH 25/36] fix(agent): route the llama status split through classifyFailure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toLlmFailure carried a second, hardcoded copy of the llama-server taxonomy (status null or >= 500 => transport, everything else => grammar). That copy, not classifyFailure, decided the category the user actually reads: executeStep wraps every escaping error through toLlmFailure and rethrows the wrapper, and classifyFailure short-circuits on `err instanceof LlmFailure`, so the new endpoint/availability status set was never consulted on the path that produces the chat message. Concretely, a raw LlamaServerError(404) still surfaced as `Turn failed [grammar]: llama-server returned http 404` with no unreachable hint, and the Sentry clusters CLI-B7 / CLI-BE (category=grammar, cause_type=LlamaServerError) are exactly the signature of the GrammarError constructed here — they would have kept firing at the same rate. Delete the duplicate and ask classifyFailure instead: transport keeps TransportError(message, status, url), anything else keeps the historical GrammarError(message, ""). The cause chain is unchanged in both arms, so the scrubber's causeType still resolves. classifyFailure cannot answer cancelled/model/tool for a LlamaServerError, and an aborted step is already claimed by the ctx.signal.aborted check above this arm, so no other category is laundered into a TransportError. The fallover half was already correct: runWithFallback catches the raw LlamaServerError before executeStep's wrapper, so only the user-facing category was stuck on the old taxonomy. --- src/agent/step-executor.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/agent/step-executor.ts b/src/agent/step-executor.ts index 480efcfd..fc03e2da 100644 --- a/src/agent/step-executor.ts +++ b/src/agent/step-executor.ts @@ -1967,7 +1967,24 @@ function toLlmFailure(err: unknown, ctx: StepContext): LlmFailure { ); } if (err instanceof LlamaServerError) { - if (err.status === null || err.status >= 500) { + // Delegate the status split to `classifyFailure` rather than + // restating it. This arm used to carry its own hardcoded copy + // (`status === null || >= 500` ⇒ transport, everything else ⇒ + // grammar), and because `executeStep` rethrows *this* wrapper — and + // `classifyFailure`'s first line short-circuits on `LlmFailure` — + // the copy, not the classifier, decided the category the user reads. + // The two diverged the moment the taxonomy moved: a 404 from a wrong + // `localModels.url` still surfaced as `Turn failed [grammar]` with no + // unreachable hint. One taxonomy, one place. + // + // `classifyFailure` cannot return `cancelled`/`model`/`tool` for a + // `LlamaServerError` (its own arm returns only `transport` or + // `grammar`, and it is reached before the abort/network branches), + // and an aborted step has already been claimed by the + // `ctx.signal.aborted` check above — so nothing is laundered here. + // Only `transport` becomes a `TransportError`; every other answer + // keeps the historical `GrammarError`. + if (classifyFailure(err) === "transport") { return new TransportError(err.message, err.status, err.url, { cause: err }); } return new GrammarError(err.message, "", { cause: err }); From 778f2ee4e253d2d3d602f533e345e051ddac5022 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:37:46 +0300 Subject: [PATCH 26/36] test(tui): drive the llama chat-error assertion through the real pipeline The block added in this PR claimed to mirror production but composed formatAgentErrorForChat(classifyFailure(err), err.message, local) by hand, omitting the toLlmFailure link that was exactly what was broken. It was green while a 404 still reached the user as a grammar failure. Rewritten to run the whole path: AgentLoop executes a step whose llmComplete throws a raw LlamaServerError, executeStep normalises it through toLlmFailure, the loop's catch classifies that wrapper and emits loop_failed { category, error }, and the assertion formats exactly those fields the way agent-event-reducer's loop_failed case does. With the toLlmFailure change reverted, the 404 and 405 cases fail with `Turn failed [grammar]: llama-server returned http 404`, reproducing the production defect. The 400 case is the regression guard for the half that is intentionally unchanged. --- src/tui/format-agent-error-for-chat.test.ts | 160 +++++++++++++++----- 1 file changed, 125 insertions(+), 35 deletions(-) diff --git a/src/tui/format-agent-error-for-chat.test.ts b/src/tui/format-agent-error-for-chat.test.ts index cd9a77a6..1a9908d0 100644 --- a/src/tui/format-agent-error-for-chat.test.ts +++ b/src/tui/format-agent-error-for-chat.test.ts @@ -1,7 +1,18 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AgentLoop } from "../agent/agent-loop.js"; +import { buildDefaultToolRegistry } from "../tools/index.js"; +import { SlotManager } from "../llm/slot-manager.js"; +import { createEmptySessionState } from "../session/session-state.js"; import { LlamaServerError } from "../llm/llama-server-client.js"; -import { classifyFailure } from "../llm/reliability/classify-failure.js"; +import type { + CapabilitiesSummary, + SkillCatalogEntry, + ToolDescriptor, +} from "../prompt/stable-prefix.js"; import { formatAgentErrorForChat } from "./format-agent-error-for-chat.js"; describe("formatAgentErrorForChat", () => { @@ -50,47 +61,126 @@ describe("formatAgentErrorForChat", () => { }); }); -describe("formatAgentErrorForChat — classified llama failures", () => { - // Mirrors the real pipeline: `agent-loop` classifies the thrown error - // and the reducer hands that category straight to the formatter. The - // hint is gated on `transport`, so the one failure where "check your - // llama URL" is exactly right — a 404 from a wrong `localModels.url` — - // used to be the one failure that never got it. - const local = { - activeProviderIsLocal: true, - llamaUrl: "http://127.0.0.1:19091", - }; - - it("carries the unreachable hint for a llama 404 on a local provider", () => { - const err = new LlamaServerError( - "llama-server returned http 404", - 404, - local.llamaUrl, +const LOCAL = { + activeProviderIsLocal: true, + llamaUrl: "http://127.0.0.1:19091", +}; + +const TOOLS: ToolDescriptor[] = [ + { + name: "finish", + summary: "Finish the session with a summary.", + argsSchema: '{"summary": string}', + }, +]; + +const CAPS: CapabilitiesSummary = { + platform: "darwin", + arch: "arm64", + browserChannel: "chrome", + workingDir: "/work", + hasClipboard: true, + hasWmctrl: false, + hasNotifications: true, +}; + +const SKILLS: SkillCatalogEntry[] = []; + +describe("formatAgentErrorForChat — llama failures through the real pipeline", () => { + // Drives the WHOLE production path, not a hand-composed imitation of + // it: `AgentLoop` runs a step whose `llmComplete` throws a raw + // `LlamaServerError`; `executeStep` normalises it through + // `toLlmFailure`; the loop's catch calls `classifyFailure` on THAT + // wrapper and emits `loop_failed { category, error }`; the TUI reducer + // (`agent-event-reducer.ts`, "loop_failed" case) hands exactly those two + // fields plus the local-provider context to the formatter. + // + // The `toLlmFailure` link is the point of the exercise. It used to carry + // its own hardcoded copy of the llama status split, so a 404 reached the + // user as `Turn failed [grammar]` however `classifyFailure` was written — + // and a test that called `formatAgentErrorForChat(classifyFailure(err), …)` + // directly stayed green while production stayed broken. Route the + // assertion through the loop and that gap cannot hide. + let workingDir: string; + + beforeEach(() => { + workingDir = mkdtempSync(join(tmpdir(), "atomic-agent-chat-error-")); + }); + + afterEach(() => { + rmSync(workingDir, { recursive: true, force: true }); + }); + + /** + * Run one turn whose only LLM call throws `LlamaServerError(status)`, + * and render the resulting `loop_failed` exactly as the reducer does. + */ + async function chatTextForLlamaStatus(status: number): Promise { + const failures: Array<{ category: string; message: string }> = []; + const loop = new AgentLoop({ + registry: buildDefaultToolRegistry(), + slotManager: new SlotManager(2), + grammar: 'root ::= "ok"', + llmComplete: async () => { + throw new LlamaServerError( + `llama-server returned http ${status}`, + status, + LOCAL.llamaUrl, + ); + }, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + onEvent: (event) => { + if (event.type === "loop_failed") { + failures.push({ + category: event.category, + message: event.error.message, + }); + } + }, + }); + const result = await loop.runTurn( + createEmptySessionState({ id: `s-llama-${status}`, workingDir }), + { + userMessage: "go", + maxSteps: 3, + signal: new AbortController().signal, + }, ); - const text = formatAgentErrorForChat( - classifyFailure(err), - err.message, - local, + expect(result.reason).toBe("failed"); + expect(failures).toHaveLength(1); + return formatAgentErrorForChat( + failures[0]!.category, + failures[0]!.message, + LOCAL, ); + } + + it("carries the unreachable hint for a llama 404 on a local provider", async () => { + // The one failure where "check your llama URL" is exactly the right + // advice — a wrong `localModels.url`, or a server that is not a + // llama-server — was the one failure that never got it. + const text = await chatTextForLlamaStatus(404); expect(text).toContain("Turn failed [transport]"); expect(text).toContain( "llama-server is not reachable at http://127.0.0.1:19091", ); }); - it("keeps a llama 400 as a grammar failure with no URL advice", () => { - const err = new LlamaServerError( - "llama-server returned http 400", - 400, - local.llamaUrl, - ); - const text = formatAgentErrorForChat( - classifyFailure(err), - err.message, - local, - ); - expect(text).toBe( - "Turn failed [grammar]: llama-server returned http 400", + it("carries the unreachable hint for a llama 405 on a local provider", async () => { + const text = await chatTextForLlamaStatus(405); + expect(text).toContain("Turn failed [transport]"); + expect(text).toContain( + "llama-server is not reachable at http://127.0.0.1:19091", ); }); + + it("keeps a llama 400 as a grammar failure with no URL advice", async () => { + // Regression guard for the half that is intentionally unchanged: a + // 400 is the server rejecting THIS request, and the next link would + // reject it identically. + const text = await chatTextForLlamaStatus(400); + expect(text).toBe("Turn failed [grammar]: llama-server returned http 400"); + }); }); From 1aa7141e0cb7f740da5913ac8a7b744801ec4233 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:37:53 +0300 Subject: [PATCH 27/36] docs(fallback): say what AdvanceDecision.immediate actually governs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AdvanceDecision doc has long said an immediate signal "should switch on the FIRST occurrence, bypassing the consecutive-failure threshold", and the test names this PR added inherited that wording ("advances via threshold on a local llama 404", "advances immediately on a local llama 429"). ProviderFallbackChain.advanceFrom returns the next link on the FIRST fallover-worthy failure whenever decision.advance is true, immediate or not. What immediate changes is registerFailure: it arms the breaker cooldown right away instead of waiting for failureThreshold consecutive failures. The threshold governs how long a failed link stays quarantined across later turns, not the in-turn switch. Assertions are unchanged — only the doc comment and the six new test names/comments that misstated the mechanism. --- src/llm/fallback/should-advance.test.ts | 27 ++++++++++++++++--------- src/llm/fallback/should-advance.ts | 10 +++++++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/llm/fallback/should-advance.test.ts b/src/llm/fallback/should-advance.test.ts index 259f3d2e..78941d38 100644 --- a/src/llm/fallback/should-advance.test.ts +++ b/src/llm/fallback/should-advance.test.ts @@ -88,10 +88,14 @@ describe("shouldAdvance", () => { }); }); - it("advances via threshold on a local llama 404 — the URL serves no completions", () => { + it("advances on a local llama 404 without arming the breaker — the URL serves no completions", () => { // The endpoint is permanently wrong (bad `localModels.url`, or a - // server that is not a llama-server). Not an immediate signal: 404 is - // not in the 429/408/5xx provider-down set, so it advances once the + // server that is not a llama-server), so the chain must move off this + // link — and it moves on THIS failure: `advanceFrom` returns the next + // provider whenever `advance` is true, immediate or not. + // `immediate: false` is about the breaker, not the switch: 404 is not + // in the 429/408/5xx provider-down set, so the cooldown that keeps the + // link quarantined across later turns is armed only once the // consecutive-failure threshold trips. expect(shouldAdvance(new LlamaServerError("x", 404, "http://local"))).toEqual({ advance: true, @@ -99,24 +103,25 @@ describe("shouldAdvance", () => { }); }); - it("advances via threshold on a local llama 405", () => { + it("advances on a local llama 405 without arming the breaker", () => { expect(shouldAdvance(new LlamaServerError("x", 405, "http://local"))).toEqual({ advance: true, immediate: false, }); }); - it("advances immediately on a local llama 429", () => { + it("advances on a local llama 429 and arms the breaker on the first failure", () => { // Transport category plus an unambiguous provider-down status — the // `isImmediateSignal` status read already handled 429; it was simply - // unreachable while 4xx classified as grammar. + // unreachable while 4xx classified as grammar. The extra `immediate` + // buys the cooldown straight away, not an earlier switch. expect(shouldAdvance(new LlamaServerError("x", 429, "http://local"))).toEqual({ advance: true, immediate: true, }); }); - it("advances immediately on a local llama 408", () => { + it("advances on a local llama 408 and arms the breaker on the first failure", () => { expect(shouldAdvance(new LlamaServerError("x", 408, "http://local"))).toEqual({ advance: true, immediate: true, @@ -124,9 +129,11 @@ describe("shouldAdvance", () => { }); it("advances on a subscription-CLI binary that is not installed", () => { - // No HTTP status to read, so it advances via the threshold — but it - // must advance: a missing `claude` binary otherwise pins the chain to - // a provider that can never serve a turn. + // It must advance: a missing `claude` binary otherwise pins the chain + // to a provider that can never serve a turn — and the switch happens + // on this first failure. There is no HTTP status to read, so + // `immediate` is false and the breaker cooldown waits for the + // consecutive-failure threshold. expect( shouldAdvance(new SubscriptionCliNotInstalledError("claude", "Install it.")), ).toEqual({ advance: true, immediate: false }); diff --git a/src/llm/fallback/should-advance.ts b/src/llm/fallback/should-advance.ts index 75070d76..1e76ba66 100644 --- a/src/llm/fallback/should-advance.ts +++ b/src/llm/fallback/should-advance.ts @@ -11,8 +11,14 @@ import { TransportError } from "../reliability/llm-failures.js"; * `false` means the error is deterministic (same request fails the * same way everywhere) or is a cancellation — propagate it untouched. * - `immediate`: an unambiguous provider-down signal (429 / 408 / 5xx / - * network-null) that should switch on the FIRST occurrence, bypassing - * the consecutive-failure threshold. + * network-null). This does NOT control whether the chain switches — + * `ProviderFallbackChain.advanceFrom` returns the next link on the + * FIRST fallover-worthy failure whenever `advance` is true, immediate + * or not. What it controls is the breaker: `registerFailure` arms the + * cooldown right away on an immediate signal, instead of waiting for + * `failureThreshold` consecutive failures. So `immediate` decides how + * long the failed link stays quarantined across later turns, not the + * in-turn switch. */ export interface AdvanceDecision { advance: boolean; From 9d2bb40247243e1ed1b27cc055338c0c844ee9fa Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:49:02 +0300 Subject: [PATCH 28/36] fix(llm): put an upper bound back on a single streaming response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Making `requestTimeoutMs` an idle deadline removed the last cap on one local completion: a server emitting one byte every (budget - 1)ms refreshes the deadline forever, and nothing else on the turn path stops it — `src/agent` arms no timers, there is no `AbortSignal.timeout` anywhere on the path, and `ctx.signal` is user-driven only. A wedged or hostile llama-server could pin a slot, a session and, under headless `run`, the process indefinitely. Adds `localModels.streamTotalTimeoutMs` (env `ATOMIC_AGENT_LLAMA_STREAM_TOTAL_TIMEOUT_MS`, default 6 h) alongside the existing knob. It is a backstop, not a budget: 72x `REQUEST_TIMEOUT_MS`, and well clear of the worst honest local generation — the default `completionMaxTokens` of 8192 decoded at 0.4 tok/s is ~5.7 h. --- src/config/config-schema.ts | 31 +++++++++++++++++++++++++++++++ src/config/load-config.ts | 4 ++++ 2 files changed, 35 insertions(+) diff --git a/src/config/config-schema.ts b/src/config/config-schema.ts index bf541c99..1de6b5ee 100644 --- a/src/config/config-schema.ts +++ b/src/config/config-schema.ts @@ -175,7 +175,23 @@ export interface AtomicAgentConfig { /** Upper bound on `n_predict` for each completion when the caller omits `maxTokens`. */ completionMaxTokens: number; healthTimeoutMs: number; + /** + * For a unary `complete()`, the whole-request budget. For + * `completeStream()`, an **idle** budget: how long llama-server may + * stay silent between bytes. A healthy generation refreshes it on + * every chunk, so it never caps how long an answer may be — see + * `streamTotalTimeoutMs` for that. + */ requestTimeoutMs: number; + /** + * Absolute cap on one streaming response, measured from the moment + * response headers arrive. `requestTimeoutMs` only bounds silence, + * so without this a server dribbling one byte just under the idle + * budget would pin a slot, a session and — in headless `run` — a + * process forever. Deliberately far above any honest local + * generation; it is a backstop, not a budget. + */ + streamTotalTimeoutMs: number; healthRetries: number; healthRetryBackoffMs: number; /** @@ -2103,6 +2119,21 @@ export const ENV_DEFAULTS = { STATE_DIR: "~/.atomic-agent", HEALTH_TIMEOUT_MS: 3000, REQUEST_TIMEOUT_MS: 300_000, + /** + * 6 hours. The backstop on a single streaming response — see + * `AtomicAgentConfig.localModels.streamTotalTimeoutMs`. + * + * Chosen to clear the worst *honest* local generation by a wide + * margin: the default `completionMaxTokens` of 8 192 tokens decoded at + * 0.4 tok/s — slower than any CPU setup people actually sit through — + * is about 5.7 h. It is also 72x `REQUEST_TIMEOUT_MS`, so the idle + * deadline gets dozens of chances to fire first; if this one fires, + * the server was streaming continuously for six hours without + * finishing, which no local model this project targets does by + * accident. Raise it with `ATOMIC_AGENT_LLAMA_STREAM_TOTAL_TIMEOUT_MS` + * if you really do run a 131 072-token completion on a slow box. + */ + STREAM_TOTAL_TIMEOUT_MS: 6 * 60 * 60 * 1_000, HEALTH_RETRIES: 5, HEALTH_BACKOFF_MS: 500, COMPLETION_RETRIES: 3, diff --git a/src/config/load-config.ts b/src/config/load-config.ts index 43dec648..a602d16b 100644 --- a/src/config/load-config.ts +++ b/src/config/load-config.ts @@ -162,6 +162,10 @@ export function loadConfig(): AtomicAgentConfig { "ATOMIC_AGENT_LLAMA_REQUEST_TIMEOUT_MS", ENV_DEFAULTS.REQUEST_TIMEOUT_MS, ), + streamTotalTimeoutMs: readInt( + "ATOMIC_AGENT_LLAMA_STREAM_TOTAL_TIMEOUT_MS", + ENV_DEFAULTS.STREAM_TOTAL_TIMEOUT_MS, + ), healthRetries: readInt( "ATOMIC_AGENT_LLAMA_HEALTH_RETRIES", ENV_DEFAULTS.HEALTH_RETRIES, From f3f103b4fb4d9fb28b7e9a521274d1ad6d95b8a3 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 02:49:11 +0300 Subject: [PATCH 29/36] fix(llm): tell a pre-first-token stall apart from a mid-reply stall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups to the idle deadline, in one file: 1. Enforce `streamTotalTimeoutMs`. `createRequestController` now also returns `startStreamDeadline()`, a second timer armed once at response headers and never refreshed, reported as its own `stream-total` kind with its own wording ("data kept arriving, so this is the absolute cap … not a stall"). A live stream therefore holds two pending timers; `cleanup()` clears both. 2. A stall *before the first token* no longer claims the server "stopped responding after starting the reply". llama.cpp sends headers and only then evaluates the prompt, so that silence is the ordinary look of a long CPU prompt eval — exactly the population this change exists to protect. `keepAlive()` now takes the kind to record: `first-token` at headers, `idle` once a byte has actually arrived. The new wording says the request was accepted but no first token came, and points at `requestTimeoutMs` or the prompt/context size. 3. Tests for both, plus the two behaviours that were load-bearing but unpinned: the `keepAlive()` at headers (deleting it now fails two tests instead of none) and its no-op-once-aborted guard (a byte still in the decode pipe when the abort lands must not re-arm the timer and rewrite which deadline gets reported). --- src/llm/llama-server-client.test.ts | 151 +++++++++++++++++++++++++++- src/llm/llama-server-client.ts | 148 ++++++++++++++++++++++----- 2 files changed, 269 insertions(+), 30 deletions(-) diff --git a/src/llm/llama-server-client.test.ts b/src/llm/llama-server-client.test.ts index a1f1fe10..53bfcfb6 100644 --- a/src/llm/llama-server-client.test.ts +++ b/src/llm/llama-server-client.test.ts @@ -490,6 +490,8 @@ describe("LlamaServerClient.completeStream deadlines", () => { response: Response; push: (text: string) => void; close: () => void; + /** Error the body by hand — for streams that ignore the abort. */ + fail: () => void; } /** @@ -497,8 +499,15 @@ describe("LlamaServerClient.completeStream deadlines", () => { * errors the body mid-read, which is what undici does when the * controller fires while the response is still streaming — the * behaviour the production bug depends on. + * + * `errorOnAbort: false` models the narrow window in which the abort + * has landed but bytes already sitting in the decode pipe are still + * delivered; the test then errors the body itself with `fail()`. */ - function pushableSse(signal: AbortSignal | null | undefined): PushableStream { + function pushableSse( + signal: AbortSignal | null | undefined, + errorOnAbort = true, + ): PushableStream { const encoder = new TextEncoder(); let ctrl!: ReadableStreamDefaultController; const stream = new ReadableStream({ @@ -507,7 +516,7 @@ describe("LlamaServerClient.completeStream deadlines", () => { }, }); let finished = false; - signal?.addEventListener("abort", () => { + const fail = (): void => { if (finished) return; finished = true; ctrl.error( @@ -515,7 +524,8 @@ describe("LlamaServerClient.completeStream deadlines", () => { name: "AbortError", }), ); - }); + }; + if (errorOnAbort) signal?.addEventListener("abort", fail); return { response: new Response(stream, { status: 200, @@ -529,10 +539,14 @@ describe("LlamaServerClient.completeStream deadlines", () => { finished = true; ctrl.close(); }, + fail, }; } - function streamingClient(requestTimeoutMs: number): { + function streamingClient( + requestTimeoutMs: number, + options: { streamTotalTimeoutMs?: number; errorOnAbort?: boolean } = {}, + ): { client: LlamaServerClient; opened: () => PushableStream; } { @@ -540,8 +554,11 @@ describe("LlamaServerClient.completeStream deadlines", () => { const client = new LlamaServerClient({ baseUrl: "http://127.0.0.1:9999", requestTimeoutMs, + ...(options.streamTotalTimeoutMs === undefined + ? {} + : { streamTotalTimeoutMs: options.streamTotalTimeoutMs }), fetchImpl: createMockFetch(async (_url, init) => { - handle = pushableSse(init.signal); + handle = pushableSse(init.signal, options.errorOnAbort ?? true); return handle.response; }), completionRetries: 1, @@ -710,6 +727,130 @@ describe("LlamaServerClient.completeStream deadlines", () => { expect(llamaErr.message).not.toContain("requestTimeoutMs"); expect(llamaErr.message).not.toContain("sent no data"); }); + + it("reports a stall before the first token as a prompt eval, not a dead server", async () => { + // llama.cpp sends response headers and *then* evaluates the prompt, + // so this is the exact shape of the population this change exists to + // protect: a healthy server grinding a long context on CPU. Telling + // that user the server "stopped responding after starting the reply" + // would just be a different piece of wrong advice. + // + // This is also the test that covers the `keepAlive()` call at + // headers: delete it and the deadline is still the connect-phase + // `total` budget, so the error comes back with the unary wording. + vi.useFakeTimers(); + const { client } = streamingClient(1_000); + const iterator = client.completeStream({ prompt: "hi" }); + const failure = (async (): Promise => { + try { + while (true) { + const next = await iterator.next(); + if (next.done) return null; + } + } catch (err) { + return err; + } + })(); + + // Headers land, and then the body sends nothing at all. + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(1_001); + const err = (await failure) as LlamaServerError; + + expect(err).toBeInstanceOf(LlamaServerError); + expect(err.status).toBeNull(); + expect(err.timedOut).toBe(true); + expect(err.message).toContain("sent no first token within 1000ms"); + expect(err.message).toContain("still be evaluating the prompt"); + // The two wordings this one must not be confused with. + expect(err.message).not.toContain("stopped responding"); + expect(err.message).not.toContain("after starting the reply"); + expect(err.message).not.toContain("exceeded requestTimeoutMs"); + }); + + it("caps one streaming response with streamTotalTimeoutMs even while chunks keep arriving", async () => { + // The idle deadline is not an upper bound: a server emitting one + // byte every (budget - 1)ms refreshes it forever. Without this cap a + // wedged or hostile llama-server pins a slot, a session and — under + // headless `run` — the process, with nothing else on the turn path + // to stop it (`ctx.signal` is user-driven only). + vi.useFakeTimers(); + const { client, opened } = streamingClient(1_000, { + streamTotalTimeoutMs: 5_000, + }); + const iterator = client.completeStream({ prompt: "hi" }); + const deltas: string[] = []; + const failure = (async (): Promise => { + try { + while (true) { + const next = await iterator.next(); + if (next.done) return null; + if (next.value.delta) deltas.push(next.value.delta); + } + } catch (err) { + return err; + } + })(); + + await vi.advanceTimersByTimeAsync(0); + // 900ms apart: every gap is inside the 1,000ms idle budget, so the + // idle deadline can never fire. Only the cap can. + for (let i = 0; i < 20; i += 1) { + opened().push('data: {"content":"t","stop":false}\n\n'); + await vi.advanceTimersByTimeAsync(900); + } + const err = (await failure) as LlamaServerError; + + expect(err).toBeInstanceOf(LlamaServerError); + expect(err.status).toBeNull(); + expect(err.timedOut).toBe(true); + // It streamed healthily right up to the cap. + expect(deltas.length).toBeGreaterThanOrEqual(5); + expect(err.message).toContain("streamTotalTimeoutMs (5000ms)"); + expect(err.message).toContain("ATOMIC_AGENT_LLAMA_STREAM_TOTAL_TIMEOUT_MS"); + // Not a stall, and the user must not be sent looking for one. + expect(err.message).not.toContain("sent no data for"); + expect(err.message).not.toContain("sent no first token"); + }); + + it("does not let a byte still in flight rewrite which deadline fired", async () => { + // `keepAlive()` is a no-op once a deadline has fired or the caller + // has aborted. The window is narrow but real: the abort lands while + // bytes already sitting in the decode pipe are still delivered, and + // the read loop calls `keepAlive()` on each of them. Without the + // guard those late bytes re-arm the timer, which fires a second time + // and overwrites the recorded reason — so the user is told the + // server stalled mid-reply when what actually happened is that it + // never produced a first token. + vi.useFakeTimers(); + const { client, opened } = streamingClient(1_000, { errorOnAbort: false }); + const iterator = client.completeStream({ prompt: "hi" }); + const failure = (async (): Promise => { + try { + while (true) { + const next = await iterator.next(); + if (next.done) return null; + } + } catch (err) { + return err; + } + })(); + + await vi.advanceTimersByTimeAsync(0); + // Silence past the budget: the first-token deadline fires and aborts. + await vi.advanceTimersByTimeAsync(1_001); + // …and only now does the byte that was already in flight land. + opened().push('data: {"content":"late","stop":false}\n\n'); + await vi.advanceTimersByTimeAsync(0); + // Long enough for a re-armed deadline to fire a second time. + await vi.advanceTimersByTimeAsync(2_000); + opened().fail(); + const err = (await failure) as LlamaServerError; + + expect(err).toBeInstanceOf(LlamaServerError); + expect(err.message).toContain("sent no first token within 1000ms"); + expect(err.message).not.toContain("sent no data for"); + }); }); describe("extractLlamaErrorDetail", () => { diff --git a/src/llm/llama-server-client.ts b/src/llm/llama-server-client.ts index 34460e03..d0eedcf5 100644 --- a/src/llm/llama-server-client.ts +++ b/src/llm/llama-server-client.ts @@ -44,11 +44,24 @@ const ENV_SEED = parseIntEnv(process.env.ATOMIC_AGENT_LLAMA_SEED); * * - `total` — the whole request was given `requestTimeoutMs` and never * produced a response. The only signal a unary request has. - * - `idle` — a *stream* went `requestTimeoutMs` without sending a byte. - * A healthy generation refreshes this budget on every chunk, so it - * means the server went quiet, not that the answer was long. + * - `first-token` — a *stream*'s headers arrived and then nothing did, + * for `requestTimeoutMs`. llama.cpp answers with headers immediately + * and only then evaluates the prompt, so this usually means the + * prompt eval is still running, **not** that the server is broken. + * - `idle` — a *stream* that had already sent at least one byte went + * `requestTimeoutMs` without sending another. A healthy generation + * refreshes this budget on every chunk, so it means the server went + * quiet mid-reply, not that the answer was long. + * - `stream-total` — a stream kept sending but never finished within + * `streamTotalTimeoutMs`. The backstop that keeps a wedged or + * hostile server from pinning a slot forever by dribbling one byte + * just under the idle budget. */ -export type LlamaTimeoutKind = "total" | "idle"; +export type LlamaTimeoutKind = + | "total" + | "first-token" + | "idle" + | "stream-total"; export class LlamaServerError extends Error { constructor( @@ -57,8 +70,8 @@ export class LlamaServerError extends Error { public readonly url: string, /** * True when *our own* `requestTimeoutMs` controller fired rather than - * the transport failing — for either deadline, `total` or `idle` - * (see `LlamaTimeoutKind`). Both surface as `status === null`, but a + * the transport failing — for any of our deadlines (see + * `LlamaTimeoutKind`). Both surface as `status === null`, but a * timeout is a "the model is slower than the budget" signal, not a * transient blip — replaying it just burns another full timeout of * GPU time (3 attempts x 300s = 15 silent minutes). See @@ -158,6 +171,12 @@ export interface LlamaServerClientOptions { baseUrl?: string; apiKey?: string | null; requestTimeoutMs?: number; + /** + * Overrides `config.localModels.streamTotalTimeoutMs`, the absolute + * cap on a single streaming response. Streaming only — a unary + * request is already bounded by `requestTimeoutMs`. + */ + streamTotalTimeoutMs?: number; fetchImpl?: typeof fetch; /** * Overrides the retry budget for `complete()` and the initial fetch @@ -187,6 +206,7 @@ export class LlamaServerClient { private readonly baseUrlOverride: string | undefined; private readonly apiKey: string | null; private readonly requestTimeoutMs: number; + private readonly streamTotalTimeoutMs: number; private readonly fetchImpl: typeof fetch; private readonly completionRetriesOverride: number | undefined; private readonly completionRetryBackoffMsOverride: number | undefined; @@ -198,6 +218,8 @@ export class LlamaServerClient { this.apiKey = options.apiKey ?? config.localModels.apiKey; this.requestTimeoutMs = options.requestTimeoutMs ?? config.localModels.requestTimeoutMs; + this.streamTotalTimeoutMs = + options.streamTotalTimeoutMs ?? config.localModels.streamTotalTimeoutMs; this.fetchImpl = options.fetchImpl ?? fetch; this.completionRetriesOverride = options.completionRetries; this.completionRetryBackoffMsOverride = options.completionRetryBackoffMs; @@ -276,13 +298,14 @@ export class LlamaServerClient { controller: AbortController; cleanup: () => void; timedOut: () => LlamaTimeoutKind | null; - keepAlive: () => void; + keepAlive: (next: Exclude) => void; + startStreamDeadline: () => void; }; try { opened = await this.runWithRetry( url, async () => { - const { controller, cleanup, timedOut, keepAlive } = + const { controller, cleanup, timedOut, keepAlive, startStreamDeadline } = this.createRequestController(request.signal); try { const response = await this.fetchImpl(url, { @@ -294,7 +317,14 @@ export class LlamaServerClient { if (!response.ok || !response.body) { throw await buildHttpError(response, url); } - return { response, controller, cleanup, timedOut, keepAlive }; + return { + response, + controller, + cleanup, + timedOut, + keepAlive, + startStreamDeadline, + }; } catch (err) { cleanup(); throw this.wrapTransportError(err, url, timedOut()); @@ -309,7 +339,8 @@ export class LlamaServerClient { cause: err, }); } - const { response, cleanup, timedOut, keepAlive } = opened; + const { response, cleanup, timedOut, keepAlive, startStreamDeadline } = + opened; let finalResult: CompletionResult = { content: "", reasoningContent: "", @@ -341,18 +372,27 @@ export class LlamaServerClient { // full budget rather than whatever the connect phase left over — // llama.cpp answers with headers immediately and only then evaluates // the prompt, so the first token can legitimately be minutes away. - keepAlive(); + // Until a byte actually arrives the deadline reports `first-token`: + // a silence *before* the reply starts is most likely a long prompt + // eval, and telling that user their server "stopped responding" is + // the same bad advice this change exists to remove. + keepAlive("first-token"); + // And an idle budget alone is not an upper bound — arm the absolute + // cap so a server dribbling one byte per (budget - 1)ms cannot pin + // this slot, session and process forever. + startStreamDeadline(); let buffer = ""; let accumulated = ""; let accumulatedReasoning = ""; while (true) { const { value, done } = await reader.read(); if (done) break; - // A byte arrived: the server is alive, so start the clock over. + // A byte arrived: the server is alive, so start the clock over — + // and from now on a silence really is a mid-reply stall. // Deliberately not called on `done` — that breaks straight out of // the loop into `finally { cleanup() }` with nothing awaited in // between, so there is no window left for the timer to fire. - keepAlive(); + keepAlive("idle"); buffer += value; let eventEnd = buffer.indexOf("\n\n"); while (eventEnd !== -1) { @@ -408,6 +448,12 @@ export class LlamaServerClient { * Without that, `requestTimeoutMs` was a wall-clock cap on the whole * generation and killed healthy long answers at exactly the budget, * discarding every token already produced. + * + * An idle budget alone is not an upper bound: a server emitting one + * byte just under it streams forever. `startStreamDeadline()` arms the + * second, never-refreshed timer that puts a ceiling back on — see + * `streamTotalTimeoutMs`. So a live stream holds two pending timers, + * and `cleanup` clears both. */ private createRequestController(externalSignal?: AbortSignal): { controller: AbortController; @@ -418,11 +464,20 @@ export class LlamaServerClient { */ timedOut: () => LlamaTimeoutKind | null; /** - * Restart the deadline and mark it an idle budget. A no-op once the - * request is already aborted, so a late call cannot resurrect a - * controller the caller or the timer has finished with. + * Restart the deadline and record what a subsequent expiry means: + * `first-token` once headers are in, `idle` once the body has + * actually produced something. A no-op once the request is already + * aborted or a deadline has already fired, so a byte that was still + * in the decode pipe when the abort landed cannot re-arm the timer + * or rewrite which deadline gets reported. */ - keepAlive: () => void; + keepAlive: (next: Exclude) => void; + /** + * Arm the absolute streaming cap. Idempotent, and a no-op once the + * request is aborted. Called once, at response headers, so the cap + * measures the body and not the connect phase. + */ + startStreamDeadline: () => void; } { const controller = new AbortController(); let expired: LlamaTimeoutKind | null = null; @@ -433,28 +488,43 @@ export class LlamaServerClient { controller.abort(); }, this.requestTimeoutMs); let timer = arm(); + let streamTimer: ReturnType | null = null; const timedOut = (): LlamaTimeoutKind | null => expired; - const keepAlive = (): void => { + const keepAlive = (next: Exclude): void => { if (expired !== null || controller.signal.aborted) return; clearTimeout(timer); - kind = "idle"; + kind = next; timer = arm(); }; + const startStreamDeadline = (): void => { + if (expired !== null || controller.signal.aborted) return; + if (streamTimer !== null) return; + streamTimer = setTimeout(() => { + expired = "stream-total"; + controller.abort(); + }, this.streamTotalTimeoutMs); + }; + const clearTimers = (): void => { + clearTimeout(timer); + if (streamTimer !== null) clearTimeout(streamTimer); + }; if (!externalSignal) { return { controller, - cleanup: () => clearTimeout(timer), + cleanup: clearTimers, timedOut, keepAlive, + startStreamDeadline, }; } if (externalSignal.aborted) { controller.abort(); return { controller, - cleanup: () => clearTimeout(timer), + cleanup: clearTimers, timedOut, keepAlive, + startStreamDeadline, }; } const onAbort = (): void => controller.abort(); @@ -463,8 +533,9 @@ export class LlamaServerClient { controller, timedOut, keepAlive, + startStreamDeadline, cleanup: () => { - clearTimeout(timer); + clearTimers(); externalSignal.removeEventListener("abort", onAbort); }, }; @@ -481,9 +552,36 @@ export class LlamaServerClient { timedOut: LlamaTimeoutKind | null, ): LlamaServerError { if (err instanceof LlamaServerError) return err; - // An idle stall and a blown total budget need opposite advice. - // "Lower completionMaxTokens" is meaningless when the server sent - // nothing at all — the answer was not too long, it never came. + // Each deadline needs its own advice. "Lower completionMaxTokens" + // is meaningless when the server sent nothing at all — the answer + // was not too long, it never came — and "the server stopped + // responding" is wrong when it never started, which for llama.cpp + // is the ordinary look of a long prompt eval. + if (timedOut === "first-token") { + return new LlamaServerError( + `llama-server accepted the request but sent no first token within ${this.requestTimeoutMs}ms — ` + + `it may still be evaluating the prompt; raise localModels.requestTimeoutMs, ` + + `or shorten the prompt/context if it is too large for this machine to evaluate in time`, + null, + url, + true, + undefined, + { cause: err }, + ); + } + if (timedOut === "stream-total") { + return new LlamaServerError( + `llama-server streamed for longer than streamTotalTimeoutMs (${this.streamTotalTimeoutMs}ms) without finishing — ` + + `data kept arriving, so this is the absolute cap on one streaming reply, not a stall; ` + + `raise localModels.streamTotalTimeoutMs (ATOMIC_AGENT_LLAMA_STREAM_TOTAL_TIMEOUT_MS) ` + + `or lower completionMaxTokens`, + null, + url, + true, + undefined, + { cause: err }, + ); + } if (timedOut === "idle") { return new LlamaServerError( `llama-server sent no data for ${this.requestTimeoutMs}ms mid-stream — ` + From d34383199f708f3e26877ea745463b142001997c Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:27:10 +0300 Subject: [PATCH 30/36] fix(llm): latch the deferred probes on a sync throw, and signal a served local link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the same class. F3. `ensureProbed()` assigned `this.inFlight = this.deps.restore()` outside the `try`, so a SYNCHRONOUS throw from `restore` escaped before the assignment: `restored` stayed `false`, `inFlight` stayed `null`, and the probes re-armed on every later call — three `ensureProbed()` calls ran `restore()` three times, contradicting the class's own "latched even on failure" contract. The existing test throws from an `async` function, whose rejection arrives after the assignment, so it passed either way. Moving the call inside the `try` makes the `finally` latch both shapes. Bootstrap's `restore` is `async` with a catch-all so this was latent there, but the class is exported. F1 (groundwork). `noteLinkServed()` / `takeLinkServed()` carry the fact that a `llama-server` link served an attempt while the active text provider was something else — a cloud->local fallover. Take-and-clear, so a recovered cloud primary quiets the local probes again after one turn. The two members are optional on the interface, so legacy and test wiring that implements only `isActive` + `ensureProbed` still type-checks. --- src/llm/local-backend-gate.test.ts | 38 ++++++++++++++++++++++ src/llm/local-backend-gate.ts | 51 +++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/src/llm/local-backend-gate.test.ts b/src/llm/local-backend-gate.test.ts index 4f865314..7e1ce54f 100644 --- a/src/llm/local-backend-gate.test.ts +++ b/src/llm/local-backend-gate.test.ts @@ -65,6 +65,44 @@ describe("DeferredLocalBackendProbes", () => { expect(restore).toHaveBeenCalledTimes(1); }); + it("latches after a SYNCHRONOUSLY throwing restore too", async () => { + // The async-throw test above passes even with the `restore()` call + // outside the try: the rejection is produced after `inFlight` has + // been assigned. A sync throw escapes before the assignment, so the + // latch never armed and every later call re-ran the probes — three + // `ensureProbed()` calls, three `restore()` calls. + const restore = vi.fn((): Promise => { + throw new Error("config read blew up"); + }); + const gate = new DeferredLocalBackendProbes( + { isActive: () => true, restore }, + false, + ); + await expect(gate.ensureProbed()).rejects.toThrow("config read blew up"); + expect(await gate.ensureProbed()).toBe(false); + expect(await gate.ensureProbed()).toBe(false); + expect(restore).toHaveBeenCalledTimes(1); + }); + + it("take-and-clear reports whether a local link served since the last read", () => { + const gate = new DeferredLocalBackendProbes( + { isActive: () => false, restore: async () => {} }, + false, + ); + // Nothing served yet: a pure cloud turn must not refresh anything. + expect(gate.takeLinkServed()).toBe(false); + + gate.noteLinkServed(); + expect(gate.takeLinkServed()).toBe(true); + // Cleared — one refresh per fallover, not one per turn forever. + expect(gate.takeLinkServed()).toBe(false); + + gate.noteLinkServed(); + gate.noteLinkServed(); + expect(gate.takeLinkServed()).toBe(true); + expect(gate.takeLinkServed()).toBe(false); + }); + it("reads `isActive` per call so a hot switch is observed", () => { let active = false; const gate = new DeferredLocalBackendProbes( diff --git a/src/llm/local-backend-gate.ts b/src/llm/local-backend-gate.ts index 172c0779..4881858e 100644 --- a/src/llm/local-backend-gate.ts +++ b/src/llm/local-backend-gate.ts @@ -19,6 +19,17 @@ * - the fallback chain falling over from a cloud link to a * `llama-server` link mid-turn (`createFallbackCompleter` / * `createFallbackStreamer` prepare each link before the attempt). + * + * The second path is not a one-off: a rate-limited or down cloud primary + * falls over on *every* turn, and `appendLocal` defaults to `true`, so + * that is the shape of the default config under an outage. Restoring + * once is not enough there — the operator can still swap the model + * behind `llama-server` mid-outage, and the active provider stays cloud + * the whole time, so the loop's own turn-start refresh never re-opens. + * {@link LocalBackendGate.noteLinkServed} / {@link + * LocalBackendGate.takeLinkServed} carry that fact from the seam to the + * loop so the profile keeps tracking the live server for as long as the + * local link keeps serving — and stops within one turn of it stopping. */ export interface LocalBackendGate { @@ -36,6 +47,27 @@ export interface LocalBackendGate { * usual refresh. */ ensureProbed(): Promise; + /** + * Record that a `llama-server` link just served — or is about to serve + * — an attempt while the *active* text provider is something else: a + * cloud→local fallover. + * + * The agent loop's turn-start refresh keys off the active provider, + * which stays cloud for the whole outage, so without this signal the + * profile and grammar would stay pinned to whatever the first fallover + * probed (issue #112 review, F1). Optional so legacy / test wiring that + * implements only the two original members still type-checks. + */ + noteLinkServed?(): void; + /** + * Take-and-clear the {@link noteLinkServed} flag: `true` when a local + * link served since the last call. Read once per turn by the agent + * loop, which then refreshes the profile even though the active + * provider is cloud. Clearing is what keeps this self-limiting — once + * the cloud primary recovers, exactly one more turn refreshes and then + * the local probes go quiet again. + */ + takeLinkServed?(): boolean; } export interface LocalBackendGateDeps { @@ -55,6 +87,7 @@ export interface LocalBackendGateDeps { export class DeferredLocalBackendProbes implements LocalBackendGate { private restored: boolean; private inFlight: Promise | null = null; + private linkServed = false; /** * @param probedAtBoot `true` when bootstrap already ran the probes @@ -81,8 +114,14 @@ export class DeferredLocalBackendProbes implements LocalBackendGate { await this.inFlight; return false; } - this.inFlight = this.deps.restore(); try { + // Inside the `try` so a SYNCHRONOUS throw from `restore` latches + // too. With the call outside it, the throw escaped before + // `inFlight` was even assigned and `restored` stayed `false` — the + // probes then re-armed on every single call, contradicting the + // contract below. Bootstrap's `restore` is `async` with a + // catch-all, so this was latent there, but the class is exported. + this.inFlight = this.deps.restore(); await this.inFlight; } finally { // Latched even on failure. `restore` swallows its own errors, but @@ -94,4 +133,14 @@ export class DeferredLocalBackendProbes implements LocalBackendGate { } return true; } + + noteLinkServed(): void { + this.linkServed = true; + } + + takeLinkServed(): boolean { + const served = this.linkServed; + this.linkServed = false; + return served; + } } From f618af958614082aef4f79e7b9ebbec701fc7255 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:27:10 +0300 Subject: [PATCH 31/36] fix(agent): keep the local profile live through a sustained cloud->local fallover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1, the one confirmed regression against `main`. `fallback.appendLocal` defaults to `true`, so a rate-limited or down cloud primary falls over to the llama-server link on every turn under the DEFAULT config shape — this is not an opt-in path. The loop's turn-start gate reads `activeTextProvider`, which stays cloud for the whole outage, so after the first fallover latched `ensureProbed()` nothing refreshed the profile again: profile and GBNF grammar stayed pinned to whatever the first fallover probed, and `observeCompletionModelId`'s staleness flag had no consumer. `main` refreshed unconditionally at every turn start and did not have this hole. Measured over three turns with the model hot-swapped behind llama-server between turns 2 and 3: before: turn1 props=1 | turn2 props=0 | turn3 props=0 (pinned) main: turn1 props=1 | turn2 props=1 | turn3 props=1 after: turn1 props=1 | turn2 props=1 | turn3 props=1 Two arms, neither of which probes on a turn that never touches a local link: - bootstrap's `prepareLink` calls `noteLinkServed()` for a `llama-server` link, and the loop's turn-start refresh gains a second arm that fires on `takeLinkServed()` even while the active provider is cloud. Take-and-clear: one trailing refresh after the outage ends, then silence. - `prepareLink` no longer returns empty-handed once the gate has latched. It falls through to `profileManager.refreshIfStale()`, which on a cloud-active turn is the only surviving consumer of the staleness flag, and sits strictly closer to the request than the loop's between-steps refresh it replaces. The zero-request criterion is unchanged: both arms are reached only via a `llama-server` link, so a cloud bootstrap, a cloud turn and a sidecar cloud start still make zero local requests. Pinned by a test that drives the REAL `createFallbackCompleter` seam and the REAL `DeferredLocalBackendProbes` through a real `AgentLoop`, with `prepareLink` wired verbatim from `bootstrap.ts`, a cloud primary that 429s and a fake llama-server whose model is swapped between turns 2 and 3. It asserts the cumulative `/props` counts (1, 2, 3), that `/health` is still replayed only once, and — the load-bearing one — the profile id each local completion was actually built with: ["gemma4-think", "gemma4-think", "qwen-think"]. Reverting either arm reproduces the measured [1, 1, 1]. --- src/agent/agent-loop-local-gate.test.ts | 265 +++++++++++++++++++++++- src/agent/agent-loop.ts | 28 ++- src/runtime/bootstrap.ts | 13 +- 3 files changed, 299 insertions(+), 7 deletions(-) diff --git a/src/agent/agent-loop-local-gate.test.ts b/src/agent/agent-loop-local-gate.test.ts index a782d85e..9c06df77 100644 --- a/src/agent/agent-loop-local-gate.test.ts +++ b/src/agent/agent-loop-local-gate.test.ts @@ -12,10 +12,19 @@ import type { LlamaServerClient, } from "../llm/llama-server-client.js"; import { ModelProfileManager } from "../llm/model-profile-manager.js"; -import { GEMMA4_PROPS } from "../llm/model-profile.fixtures.js"; +import { GEMMA4_PROPS, QWEN3_PROPS } from "../llm/model-profile.fixtures.js"; import { QWEN_THINK_PROFILE } from "../llm/model-profile.js"; import { buildGrammar } from "../llm/grammar/build-grammar.js"; import { DeferredLocalBackendProbes } from "../llm/local-backend-gate.js"; +import { ProviderFallbackChain } from "../llm/fallback/index.js"; +import { DEFAULT_FALLBACK_TIMING } from "../llm/fallback/fallback-config.js"; +import { providerIdIsLlamaServer } from "../llm/provider/registry/active-text-provider.js"; +import type { ResolvedLlmConfig } from "../llm/provider/registry/provider-registry.js"; +import type { LlmProvider } from "../llm/provider/llm-provider.js"; +import type { StreamChunk } from "../llm/provider/completion-types.js"; +import { OpenAiHttpError } from "../llm/provider/openai/openai-http.js"; +import { openAiToolCallAdapter } from "../llm/provider/openai/openai-tool-call-adapter.js"; +import { createFallbackCompleter } from "../runtime/llm-fallback-seam.js"; import type { CapabilitiesSummary, SkillCatalogEntry, @@ -190,3 +199,257 @@ describe("AgentLoop — local profile probes are gated on the active route", () expect(fetchProps).toHaveBeenCalledTimes(1); }); }); + +/** + * Issue #112 review, F1 — the SUSTAINED cloud→local fallover. + * + * `fallback.appendLocal` defaults to `true`, so a rate-limited or down + * cloud primary falls over to the llama-server link on every turn under + * the default config shape. The active text provider stays cloud for the + * whole outage, which is what the loop's turn-start gate reads — so once + * `ensureProbed()` had latched on the first fallover, nothing refreshed + * the profile ever again and the prompt kept being built with the first + * model's template. `main` refreshed unconditionally at every turn start + * and did not have that hole. + * + * These tests drive the REAL `createFallbackCompleter` seam and the REAL + * `DeferredLocalBackendProbes` through a real `AgentLoop`, with + * `prepareLink` wired exactly as `bootstrap.ts` wires it, and hot-swap + * the model behind the fake llama-server between turns 2 and 3. + */ +describe("AgentLoop — sustained cloud→local fallover keeps the profile live", () => { + let workingDir: string; + + beforeEach(() => { + workingDir = mkdtempSync(join(tmpdir(), "atomic-loop-fallover-")); + }); + + afterEach(() => { + rmSync(workingDir, { recursive: true, force: true }); + }); + + /** Cloud primary + llama-server tail, as `resolveFallbackChain` builds it. */ + const LLM: ResolvedLlmConfig = { + activeTextProvider: "cloud", + activeEmbeddingProvider: "cloud", + providers: [ + { id: "cloud", kind: "openai" }, + { id: "local", kind: "llama-server" }, + ] as ResolvedLlmConfig["providers"], + toolTransport: "auto", + }; + + function fakeProvider( + id: string, + transport: "grammar" | "native_tools", + serve: () => Promise, + ): LlmProvider { + return { + id, + name: id, + capabilities: { + vision: false, + visionSource: "absent", + toolTransport: transport, + contextWindow: 128_000, + supportsParallelTools: transport === "native_tools", + supportsSlotAffinity: transport === "grammar", + supportsPromptCache: false, + reasoningFormat: "none", + }, + toolCallAdapter: + transport === "native_tools" ? openAiToolCallAdapter : null, + streamConsumer: null, + complete: serve, + async *completeStream() { + const result = await serve(); + yield { + delta: result.content, + reasoningDelta: "", + done: true, + } as StreamChunk; + return result; + }, + async describeImage() { + throw new Error("no vision"); + }, + async health() { + return { reachable: true, status: 200, error: null, latencyMs: 1 }; + }, + async close() {}, + } as unknown as LlmProvider; + } + + const buildFalloverLoop = async () => { + const grammar = await buildGrammar(QWEN_THINK_PROFILE); + // What the fake llama-server currently has loaded. Swapped mid-test. + let loaded = { + props: GEMMA4_PROPS as Record, + modelId: "gemma-4-it", + }; + const fetchProps = vi.fn(async () => loaded.props); + const healthProbes = vi.fn(); + + const profileManager = new ModelProfileManager({ + llama: { fetchProps } as unknown as LlamaServerClient, + initialProfile: QWEN_THINK_PROFILE, + initialGrammar: grammar, + // Boot was cloud, so nothing probed: the manager runs on the + // synthesized default until something warms it. + initialModelId: null, + }); + + const gate = new DeferredLocalBackendProbes( + { + // The active provider is cloud for the whole outage — the + // fallover never changes it. This is the exact condition that + // used to freeze the profile. + isActive: () => false, + restore: async () => { + healthProbes(); + await profileManager.refresh(); + }, + }, + /* probedAtBoot */ false, + ); + + // The profile the prompt was actually built with, per local + // completion. This is the assertion that matters: a stale profile + // here means a stale chat template and a stale GBNF grammar. + const servedWithProfile: string[] = []; + const localServe = async (): Promise => { + servedWithProfile.push(profileManager.getProfile().id); + return makeCompletion( + `${JSON.stringify({ + tool: "finish", + args: { summary: "done" }, + })}`, + loaded.modelId, + ); + }; + const providers = new Map([ + [ + "cloud", + fakeProvider("cloud", "native_tools", async () => { + throw new OpenAiHttpError( + "rate limited", + 429, + "http://cloud", + false, + null, + "cloud", + ); + }), + ], + ["local", fakeProvider("local", "grammar", localServe)], + ]); + + const llmComplete = createFallbackCompleter({ + fallbackChain: new ProviderFallbackChain({ + resolve: () => ({ + chain: ["cloud", "local"], + timing: DEFAULT_FALLBACK_TIMING, + }), + }), + resolveSlice: (providerId) => { + const provider = providers.get(providerId)!; + return { provider, transport: provider.capabilities.toolTransport }; + }, + // Verbatim from `bootstrap.ts`'s `fallbackSeamDeps`. + prepareLink: async (providerId) => { + if (!providerIdIsLlamaServer(LLM, providerId)) return; + gate.noteLinkServed(); + if (await gate.ensureProbed()) return; + await profileManager.refreshIfStale(); + }, + recordUnaryUsage: () => {}, + recordStreamUsage: () => {}, + }); + + const loop = new AgentLoop({ + registry: buildDefaultToolRegistry(), + slotManager: new SlotManager(2), + grammar, + profile: QWEN_THINK_PROFILE, + profileManager, + localBackend: gate, + llmComplete, + toolDescriptors: TOOLS, + capabilities: CAPS, + skillCatalog: SKILLS, + }); + + return { + loop, + fetchProps, + healthProbes, + profileManager, + servedWithProfile, + swapModel: () => { + loaded = { + props: QWEN3_PROPS as Record, + modelId: "qwen3-30b-a3b-instruct-2507", + }; + }, + }; + }; + + const runTurn = async (loop: AgentLoop, id: string) => + loop.runTurn(createEmptySessionState({ id, workingDir }), { + userMessage: "go", + maxSteps: 2, + signal: new AbortController().signal, + }); + + it("re-probes on every turn the local link serves, and follows a hot swap on turn 3", async () => { + const t = await buildFalloverLoop(); + const propsAfter: number[] = []; + + // Turn 1 — cloud 429s, the chain falls over, the seam restores. + await runTurn(t.loop, "s1"); + propsAfter.push(t.fetchProps.mock.calls.length); + expect(t.healthProbes).toHaveBeenCalledTimes(1); + + // Turn 2 — still cloud-active, still falling over. The turn-start + // refresh must run again: `ensureProbed()` has latched, so before + // this fix nothing did. + await runTurn(t.loop, "s2"); + propsAfter.push(t.fetchProps.mock.calls.length); + + // The operator swaps the model behind llama-server mid-outage. + t.swapModel(); + + // Turn 3 — the swap must be picked up BEFORE the prompt is built. + await runTurn(t.loop, "s3"); + propsAfter.push(t.fetchProps.mock.calls.length); + + // One `/props` per turn, matching `main`'s unconditional turn-start + // refresh. Cumulative: 1, 2, 3. + expect(propsAfter).toEqual([1, 2, 3]); + // The restore is still one-shot — turns 2 and 3 refresh, they do not + // replay `/health`. + expect(t.healthProbes).toHaveBeenCalledTimes(1); + + // The load-bearing assertion. Turn 3's completion was built with the + // profile of the model llama-server is NOW serving. Pinned to + // `gemma4-think` before this fix. + expect(t.servedWithProfile).toEqual([ + "gemma4-think", + "gemma4-think", + "qwen-think", + ]); + expect(t.profileManager.getModelId()).toBe("qwen3-30b-a3b-instruct-2507"); + }); + + it("goes quiet again within one turn of the cloud primary recovering", async () => { + // The other half of take-and-clear: the refresh must not become a + // permanent per-turn `/props` just because one fallover happened. + const gate = new DeferredLocalBackendProbes( + { isActive: () => false, restore: async () => {} }, + /* probedAtBoot */ false, + ); + gate.noteLinkServed(); + expect(gate.takeLinkServed()).toBe(true); + expect(gate.takeLinkServed()).toBe(false); + }); +}); diff --git a/src/agent/agent-loop.ts b/src/agent/agent-loop.ts index d3176001..b7ea8bb1 100644 --- a/src/agent/agent-loop.ts +++ b/src/agent/agent-loop.ts @@ -506,8 +506,22 @@ export class AgentLoop { // with the previous model's template. Skipped whole on a cloud turn // (issue #112): there is no llama-server behind the prompt to sync // with, and the probe would fail against a backend nobody is using. - if (this.deps.profileManager && this.localBackendActive()) { - if (!(await this.deps.localBackend?.ensureProbed())) { + // + // ...unless the previous turn was actually SERVED by a local link + // through the fallback chain. `appendLocal` defaults to `true`, so a + // rate-limited cloud primary falls over to llama-server on every + // turn while the active provider stays cloud; without this second + // arm the profile and grammar would stay pinned to whatever the + // first fallover probed for the whole outage. Take-and-clear, so a + // recovered primary quiets the probes again after one turn. + const localLinkServedLastTurn = + this.deps.localBackend?.takeLinkServed?.() ?? false; + if (this.deps.profileManager) { + if (this.localBackendActive()) { + if (!(await this.deps.localBackend?.ensureProbed())) { + await this.deps.profileManager.refresh(); + } + } else if (localLinkServedLastTurn) { await this.deps.profileManager.refresh(); } } @@ -572,9 +586,13 @@ export class AgentLoop { // Reactive refresh between steps: if the previous completion // observed a foreign `modelId`, rebuild profile + grammar so the // next prompt matches what `llama-server` is actually serving. - // Same cloud-turn gate as the turn-start refresh (issue #112) — a - // mid-turn fallover onto a local link is warmed by the fallback - // seam instead, at the point the link is picked. + // Same cloud-turn gate as the turn-start refresh (issue #112). + // Nothing is lost on a cloud turn that falls over: the fallback + // seam's `prepareLink` runs this same `refreshIfStale` for a + // `llama-server` link at the point the link is picked, which is + // strictly later than here and strictly closer to the request — + // the completion that flagged the manager stale may not even have + // happened yet when this line runs. if (this.deps.profileManager && this.localBackendActive()) { if (!(await this.deps.localBackend?.ensureProbed())) { await this.deps.profileManager.refreshIfStale(); diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index b32a0764..26a8aa5e 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -1680,7 +1680,18 @@ export async function createAgentRuntime( if (!providerIdIsLlamaServer(resolveLlmConfig(getConfig()), providerId)) { return; } - await localBackend.ensureProbed(); + // Tell the agent loop a local link is serving, so its turn-start + // refresh re-opens for as long as the outage lasts even though the + // ACTIVE provider stays cloud (issue #112 review, F1). + localBackend.noteLinkServed(); + // `true` means the restore just ran, which already carries a fresh + // `/props`. Otherwise the local state is warm but possibly stale — + // and on a cloud-active turn the loop's own `refreshIfStale` is + // gated off, so this is the only place left that can consume the + // staleness `observeCompletionModelId` flagged. Cheap: a no-op + // unless a completion actually reported a different model. + if (await localBackend.ensureProbed()) return; + await profileManager?.refreshIfStale(); }, recordUnaryUsage, recordStreamUsage, From 2b7ae94b7df965f8b913c0c737997ec8664ea70e Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:27:31 +0300 Subject: [PATCH 32/36] test(tui): cover the resolveWindow local-route guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F2. The `localActive &&` clause in `resolveWindow` had zero coverage: dropping it survived all 244 files / 2638 tests of `src/tui`, and the PR body credited it with an acceptance criterion on the strength of that run. The existing poller-fallback test uses a state with no provider rows, where `active === undefined` reads as local, so it exercises the other side of the branch. Two tests. The killer: a cloud row is the active text route, `llmHealth.contextWindow` is 4096 (a llama-server `n_ctx` left in state after a local->cloud switch — `agent-event-reducer` deliberately PRESERVES `contextWindow` when an `llm_model_updated` omits it, which is exactly the shape `notifyCatalogModel` emits, so the stale value is reachable by design), and the row's model is in no catalogue. Guard present: `null`. Guard removed: 4096, a local slot size drawn as a cloud model's context gauge. Plus the control: the guard must not cost the local route its window. --- src/tui/select-context-usage.test.ts | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/tui/select-context-usage.test.ts b/src/tui/select-context-usage.test.ts index 8b50671e..cc7acbe3 100644 --- a/src/tui/select-context-usage.test.ts +++ b/src/tui/select-context-usage.test.ts @@ -176,6 +176,52 @@ describe("resolving the model's context window", () => { expect(view?.contextWindow).toBe(32_768); }); + /** + * Issue #112. The poller's window is a llama-server `n_ctx`, and after + * a local→cloud switch the last local reading is still sitting in + * `llmHealth`: `agent-event-reducer` deliberately PRESERVES + * `contextWindow` when an `llm_model_updated` omits it (which is + * exactly the shape `notifyCatalogModel` emits), so the stale value is + * reachable by design, not only by a lost race. + * + * Without the `localActive &&` guard in `resolveWindow` this returns + * 4096 — a local server's slot size drawn as a cloud model's context + * gauge. The row's model is deliberately uncatalogued so the guard is + * the ONLY thing standing between the poller's number and the result. + */ + it("ignores the local poller's n_ctx while a cloud provider is the active route", () => { + const base = createInitialTuiState(fakeSession()); + const cloudRow = providerRow({ + kind: "openai-compatible", + chatModel: "vendor/never-heard-of-it", + }); + const view = selectContextUsage({ + ...base, + contextUsage: usage({ contextWindow: null }), + llmHealth: { ...base.llmHealth, contextWindow: 4096 }, + providersPanel: { ...base.providersPanel, rows: [cloudRow] }, + }); + expect(view?.contextWindow).toBeNull(); + expect(view?.percent).toBeNull(); + }); + + it("still uses the poller's n_ctx when the active row IS the local backend", () => { + // The control: the guard must not cost the local route its window. + const base = createInitialTuiState(fakeSession()); + const localRow = providerRow({ + id: "local-llama", + kind: "llama-server", + chatModel: null, + }); + const view = selectContextUsage({ + ...base, + contextUsage: usage({ contextWindow: null }), + llmHealth: { ...base.llmHealth, contextWindow: 4096 }, + providersPanel: { ...base.providersPanel, rows: [localRow] }, + }); + expect(view?.contextWindow).toBe(4096); + }); + it("falls back to the active provider's catalogue on a cloud turn", () => { const view = selectContextUsage(withRow(usage(), providerRow())); // `openai/gpt-5.5-2026-04-23`, from the bundled aimlapi catalogue. From a247b9feead1dccdfadc8161bdc68b360d254608 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:27:31 +0300 Subject: [PATCH 33/36] fix(tui): gate the poller's updateUrl emit on the active route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F4. `updateUrl()` emitted `llm_model_updated {model: null, contextWindow: null}` unconditionally. No request is made — the follow-up `tick()` returns early on a cloud route — so the zero-request criterion was never at risk, but "on a cloud route it emits nothing at all" was inaccurate: `/llama ` on a cloud session blanked the tray label and window that the active provider had put there. The new poller test asserted `actions == []` for `start()` only. The bookkeeping reset still runs unconditionally (so a later switch back to local re-discovers the model); only the emit is gated, behind the same `localTextActive()` predicate as `tick` and `refreshModelLabel`. Test covers `updateUrl` + `refreshModelLabel` on a cloud route, with a local control asserting the URL change is still announced. --- src/tui/llm-health/llm-health-poller.test.ts | 78 ++++++++++++++++++++ src/tui/llm-health/llm-health-poller.ts | 8 ++ 2 files changed, 86 insertions(+) diff --git a/src/tui/llm-health/llm-health-poller.test.ts b/src/tui/llm-health/llm-health-poller.test.ts index 070b0fbc..2c5af199 100644 --- a/src/tui/llm-health/llm-health-poller.test.ts +++ b/src/tui/llm-health/llm-health-poller.test.ts @@ -460,6 +460,27 @@ describe("LlmHealthPoller — gated on the active text provider", () => { let previousStateDir: string | undefined; let spy: ReturnType; + /** A cloud primary with a local EMBEDDING entry — the shape #112 is about. */ + const CLOUD_LLM: UserConfigFile["llm"] = { + activeTextProvider: "cloudy", + activeEmbeddingProvider: "local-llama-embed", + providers: [ + { + id: "cloudy", + kind: "openai-compatible", + baseUrl: "https://cloud.invalid", + defaultChatModel: "cloudy-1", + apiKey: "sk-test", + }, + { + id: "local-llama-embed", + kind: "llama-server", + url: "http://127.0.0.1:19092", + }, + ], + toolTransport: "auto", + }; + const writeLlm = (llm: UserConfigFile["llm"]): void => { writeUserConfigFileSync(getUserConfigPath(stateDir), { ...USER_CONFIG_DEFAULTS, @@ -536,6 +557,63 @@ describe("LlmHealthPoller — gated on the active text provider", () => { expect(props).toHaveBeenCalledTimes(1); }); + /** + * Issue #112 review, F4. `updateUrl` reset its bookkeeping and then + * emitted `llm_model_updated {model: null, contextWindow: null}` + * unconditionally — no request, but still a statement about the + * session's model, published on a route where this poller's backend is + * not the one serving. `/llama ` on a cloud session would blank + * the tray label the active provider had put there. + */ + it("emits nothing from updateUrl while a cloud provider is active", async () => { + writeLlm(CLOUD_LLM); + const props = vi.fn(stubProps); + const capture = makeCapture(); + const poller = new LlmHealthPoller( + capture, + "http://127.0.0.1:8080", + 10_000, + props, + ); + poller.start(); + poller.updateUrl("http://127.0.0.1:9090"); + await poller.refreshModelLabel(); + await sleep(20); + poller.stop(); + + expect(capture.actions).toEqual([]); + expect(spy).toHaveBeenCalledTimes(0); + expect(props).toHaveBeenCalledTimes(0); + }); + + it("still announces a URL change on a local route (control)", async () => { + writeLlm(undefined); + spy.mockResolvedValue({ + reachable: true, + status: 200, + error: null, + latencyMs: 1, + } satisfies HealthResult); + const capture = makeCapture(); + const poller = new LlmHealthPoller( + capture, + "http://127.0.0.1:8080", + 10_000, + stubProps, + ); + poller.updateUrl("http://127.0.0.1:9090"); + await sleep(20); + poller.stop(); + + expect( + capture.actions.filter((a) => a.type === "llm_model_updated"), + ).toContainEqual({ + type: "llm_model_updated", + model: null, + contextWindow: null, + }); + }); + it("resumes within one tick after a hot switch back to a local provider", async () => { writeLlm({ activeTextProvider: "cloudy", diff --git a/src/tui/llm-health/llm-health-poller.ts b/src/tui/llm-health/llm-health-poller.ts index 11ebcc37..74d3e66d 100644 --- a/src/tui/llm-health/llm-health-poller.ts +++ b/src/tui/llm-health/llm-health-poller.ts @@ -91,6 +91,14 @@ export class LlmHealthPoller { this.url = nextUrl; this.hasSettledResult = false; this.modelFetchedForUrl = false; + // Same gate as `tick` / `refreshModelLabel` (issue #112). The reset + // above is bookkeeping and always runs, but the *emit* is a claim + // about the session's model — and on a cloud route it would clear + // the label and window for a backend the session is not talking to. + // The follow-up `tick()` already returns early on a cloud route, so + // only this emit was ungated; leaving it in made "on a cloud route + // the poller emits nothing at all" untrue for `/llama `. + if (!this.localTextActive()) return; this.emitter.emit({ type: "llm_model_updated", model: null, From b5f86f0394e186d55c48266f0497dad0ed15e97c Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:27:32 +0300 Subject: [PATCH 34/36] docs: correct two claims the review found overstated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F6. `prepareLink`'s docstring implied per-link warming. Bootstrap's implementation warms the one local backend the runtime owns — the `ModelProfileManager` over the shared `LlamaServerClient`, which reads `localModels.url` per request — so a second `llama-server` entry pointed at a different host is announced through the hook but probed against the configured URL. That is a pre-existing `ModelProfileManager` limitation (a singleton over one client, not a per-link cache), now stated where the hook is defined. F5. `activeTextProviderIsLlamaServer` is KIND-based while `LocalModelsOrchestrator.autoStartIfReady` is id-based (`!== "local-llama"`). Calling that "the same conservative default" overstated it: for a `llama-server` entry under a custom id the two disagree and the managed daemon is not auto-started. Left as-is on purpose — it errs toward less local activity and leaves no acceptance criterion unmet — but recorded as a divergence rather than an equivalence. --- src/llm/provider/registry/active-text-provider.ts | 11 +++++++++++ src/runtime/llm-fallback-seam.ts | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/llm/provider/registry/active-text-provider.ts b/src/llm/provider/registry/active-text-provider.ts index b35a6994..bf20c162 100644 --- a/src/llm/provider/registry/active-text-provider.ts +++ b/src/llm/provider/registry/active-text-provider.ts @@ -14,6 +14,17 @@ import type { ResolvedLlmConfig } from "./provider-registry.js"; * id costs one probe against a backend nobody is using — while the * opposite mistake runs inference on an unprobed profile. * + * Not the only "is the route local?" predicate in the tree, and the two + * are NOT equivalent: `LocalModelsOrchestrator.autoStartIfReady` asks the + * same question by **id** (`activeTextProvider !== "local-llama"`). For a + * `llama-server` entry under a custom id the two disagree — this one + * calls it local, the orchestrator does not, so the managed daemon is + * not auto-started for it. That is pre-existing and errs toward less + * local activity (a custom-id local entry is almost always an external + * server the operator runs themselves), so it is left alone here rather + * than folded into this change; it is a divergence, not a shared + * default. + * * Lives beside `resolveLlmConfig` rather than under `src/tui/` because * `resolveLlmConfig` is a pure function of config with no I/O: the * answer is available at the very top of `buildRuntime`, long before a diff --git a/src/runtime/llm-fallback-seam.ts b/src/runtime/llm-fallback-seam.ts index 8a6211b0..69559d7d 100644 --- a/src/runtime/llm-fallback-seam.ts +++ b/src/runtime/llm-fallback-seam.ts @@ -46,6 +46,17 @@ export interface FallbackSeamDeps { * * Must not throw for a reachable link: a rejection here fails the * attempt and advances the chain, same as a failed completion. + * + * The `providerId` says WHICH link is about to serve, not where it + * lives. Bootstrap's implementation warms the one local backend the + * runtime owns — the `ModelProfileManager` built over the shared + * `LlamaServerClient`, which reads `localModels.url` per request — so + * a second `llama-server` entry pointed at a different host is + * announced here but warmed against the configured URL. That is a + * pre-existing `ModelProfileManager` limitation (it is a singleton + * over one client, not a per-link cache), not something this hook + * introduces; multi-endpoint local links would need a manager per + * link before it could mean anything more. */ prepareLink?: (providerId: string) => Promise; /** Fold a unary completion's usage into cost + meter (no-op when absent). */ From 34a9fe2edc7e137b70fd67bd9188777edf700330 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:31:03 +0300 Subject: [PATCH 35/36] test(runtime): re-assert the zero-request criterion across cloud TURNS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1's fix opens a second arm on the loop's turn-start refresh — a `llama-server` link that SERVED the previous turn reopens it even while the active provider is cloud — so the criterion the whole PR rests on had to be re-proved past boot. Boots the real runtime on the default fallover shape (`appendLocal` defaults to true and a `llama-server` text entry is configured, so the chain really is `[cloudy, local-llama]`), runs three turns, and asserts `127.0.0.1:8080` sees zero requests after each one. The counting `fetchImpl` now answers the cloud provider with a real chat completion, because a cloud link that fails would never reach the chain's local tail and the assertion would be vacuous; the run is checked for exactly three cloud completions. Measured outside the suite as well: those three completions are the ONLY outbound requests the process makes. --- src/runtime/local-probe-gating.test.ts | 79 ++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/runtime/local-probe-gating.test.ts b/src/runtime/local-probe-gating.test.ts index acee236d..016e720d 100644 --- a/src/runtime/local-probe-gating.test.ts +++ b/src/runtime/local-probe-gating.test.ts @@ -75,6 +75,33 @@ function installCountingFetch(): LocalTraffic { }); } } + // A WORKING cloud provider, so a "cloud turn" is a turn the cloud + // link actually SERVES. Answering it flatly would make the turn fail + // before the fallback chain's local tail is ever consulted, which is + // the one thing a zero-request assertion over cloud turns must not + // do. + if (url.includes("cloud.invalid") && url.includes("completions")) { + return new Response( + JSON.stringify({ + id: "c1", + object: "chat.completion", + created: 1, + model: "cloudy-1", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: '{"tool":"finish","args":{"summary":"ok"}}', + }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + } // Everything else (analytics, update check, provider catalogues) // is answered flatly so no test ever reaches the network. return new Response("{}", { @@ -184,6 +211,58 @@ describe("issue #112 — local probe gating at CLI bootstrap", () => { } }); + /** + * Issue #112 review, F1. The gating is no longer a single latch: the + * loop refreshes the profile again whenever a `llama-server` link + * SERVED the previous turn, so that a sustained cloud->local fallover + * does not freeze the profile. That arm must stay shut on a session + * where the local link never serves — including turns 2 and 3, which a + * boot-only assertion would never reach. + * + * The config is the default fallover shape: `appendLocal` defaults to + * `true` and a `llama-server` text entry is configured, so the chain + * really is `[cloudy, local-llama]`; the cloud link simply keeps + * answering, and nothing behind it is touched. + */ + it("makes zero local text requests across three cloud TURNS, not just at boot", async () => { + writeConfig(stateDir, { + llm: { + ...cloudLlm, + providers: [ + CLOUD_PROVIDER, + { id: "local-llama", kind: "llama-server", url: "http://127.0.0.1:8080" }, + LOCAL_EMBED_PROVIDER, + ], + }, + }); + const runtime = await boot(); + try { + expect(traffic.countTo(TEXT_PORT)).toBe(0); + const session = runtime.createSession(); + for (let i = 0; i < 3; i += 1) { + await runtime + .executeTurn(session, `hello ${i}`, { + maxSteps: 2, + signal: new AbortController().signal, + }) + .catch(() => undefined); + // Reported with the turn index so a failure names the turn. + expect({ turn: i, local: traffic.countTo(TEXT_PORT) }).toEqual({ + turn: i, + local: 0, + }); + } + // ...and the turns really were served by the cloud link. + expect( + traffic.urls.filter( + (u) => u.includes("cloud.invalid") && u.includes("completions"), + ).length, + ).toBe(3); + } finally { + await runtime.shutdown(); + } + }); + it("still probes when the active text provider IS a llama-server link", async () => { // The control for the case above: same code path, local route. writeConfig(stateDir, {}); From 6debbbf0f4ca80a073e4b0e50e2185499524d4b0 Mon Sep 17 00:00:00 2001 From: Valera Brizhatiuk <19537764+plombeer31@users.noreply.github.com> Date: Thu, 3 Sep 2026 03:36:47 +0300 Subject: [PATCH 36/36] refactor(llm): make the seam's local-link preparer a testable unit, and pin the fallover end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to F1's fix. Two mutations of the new `prepareLink` body — deleting `noteLinkServed()`, and deleting the `refreshIfStale()` fall-through — survived the whole suite, because the body was an inline closure inside `buildRuntime` that only a booted runtime driving a real fallover could reach, and no such test existed. A third (bootstrap not wiring `prepareLink` into its seam deps at all) survived too. `createLocalLinkPreparer` lifts the three decisions out of `buildRuntime` into `local-backend-gate.ts`, next to the gate they drive, with unit tests for each: the non-local early return, the served mark, restore without a double refresh, the fall-through on every later attempt, and the local-from-boot run where there is nothing to restore but the staleness flag still needs a consumer. The agent-loop fallover test now drives that same function instead of a verbatim copy of it. And the end-to-end case the PR body had listed as tested only at the seam: a booted runtime, a cloud primary flipped to 429, and the fallback chain routing three turns onto the llama-server link. Asserts `/health` = 1 and `/props` = 1 with `/props` landing BEFORE the local `/completion`, then `/props` = 2 and 3 on the following turns with `/health` still 1. All four mutations are now killed. --- src/agent/agent-loop-local-gate.test.ts | 19 +++--- src/llm/local-backend-gate.test.ts | 75 ++++++++++++++++++++++- src/llm/local-backend-gate.ts | 44 ++++++++++++++ src/runtime/bootstrap.ts | 30 ++++------ src/runtime/local-probe-gating.test.ts | 80 +++++++++++++++++++++++++ 5 files changed, 221 insertions(+), 27 deletions(-) diff --git a/src/agent/agent-loop-local-gate.test.ts b/src/agent/agent-loop-local-gate.test.ts index 9c06df77..8c8674de 100644 --- a/src/agent/agent-loop-local-gate.test.ts +++ b/src/agent/agent-loop-local-gate.test.ts @@ -15,7 +15,10 @@ import { ModelProfileManager } from "../llm/model-profile-manager.js"; import { GEMMA4_PROPS, QWEN3_PROPS } from "../llm/model-profile.fixtures.js"; import { QWEN_THINK_PROFILE } from "../llm/model-profile.js"; import { buildGrammar } from "../llm/grammar/build-grammar.js"; -import { DeferredLocalBackendProbes } from "../llm/local-backend-gate.js"; +import { + createLocalLinkPreparer, + DeferredLocalBackendProbes, +} from "../llm/local-backend-gate.js"; import { ProviderFallbackChain } from "../llm/fallback/index.js"; import { DEFAULT_FALLBACK_TIMING } from "../llm/fallback/fallback-config.js"; import { providerIdIsLlamaServer } from "../llm/provider/registry/active-text-provider.js"; @@ -355,13 +358,13 @@ describe("AgentLoop — sustained cloud→local fallover keeps the profile live" const provider = providers.get(providerId)!; return { provider, transport: provider.capabilities.toolTransport }; }, - // Verbatim from `bootstrap.ts`'s `fallbackSeamDeps`. - prepareLink: async (providerId) => { - if (!providerIdIsLlamaServer(LLM, providerId)) return; - gate.noteLinkServed(); - if (await gate.ensureProbed()) return; - await profileManager.refreshIfStale(); - }, + // The REAL preparer `bootstrap.ts` wires into its seam deps, with + // the same three collaborators — not a re-implementation of it. + prepareLink: createLocalLinkPreparer({ + gate, + isLocalLink: (providerId) => providerIdIsLlamaServer(LLM, providerId), + refreshIfStale: () => profileManager.refreshIfStale(), + }), recordUnaryUsage: () => {}, recordStreamUsage: () => {}, }); diff --git a/src/llm/local-backend-gate.test.ts b/src/llm/local-backend-gate.test.ts index 7e1ce54f..61001ec0 100644 --- a/src/llm/local-backend-gate.test.ts +++ b/src/llm/local-backend-gate.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; -import { DeferredLocalBackendProbes } from "./local-backend-gate.js"; +import { + createLocalLinkPreparer, + DeferredLocalBackendProbes, +} from "./local-backend-gate.js"; describe("DeferredLocalBackendProbes", () => { it("never restores when boot already probed (local-from-boot run)", async () => { @@ -114,3 +117,73 @@ describe("DeferredLocalBackendProbes", () => { expect(gate.isActive()).toBe(true); }); }); + +describe("createLocalLinkPreparer", () => { + /** + * The three decisions bootstrap's `prepareLink` makes. Covered here + * because deleting any one of them from an inline closure inside + * `buildRuntime` used to survive every test in the tree. + */ + const build = (opts: { + isLocalLink?: (id: string) => boolean; + probedAtBoot?: boolean; + } = {}) => { + const restore = vi.fn(async () => {}); + const refreshIfStale = vi.fn(async () => {}); + const gate = new DeferredLocalBackendProbes( + { isActive: () => false, restore }, + opts.probedAtBoot ?? false, + ); + const prepare = createLocalLinkPreparer({ + gate, + isLocalLink: opts.isLocalLink ?? ((id) => id === "local"), + refreshIfStale, + }); + return { gate, prepare, restore, refreshIfStale }; + }; + + it("does nothing at all for a link that is not llama-server", async () => { + const { prepare, gate, restore, refreshIfStale } = build(); + await prepare("cloudy"); + expect(restore).toHaveBeenCalledTimes(0); + expect(refreshIfStale).toHaveBeenCalledTimes(0); + // The zero-request criterion in one assertion: a cloud attempt does + // not even record that a local link served. + expect(gate.takeLinkServed()).toBe(false); + }); + + it("marks the link served so the loop's turn-start refresh reopens", async () => { + const { prepare, gate } = build(); + await prepare("local"); + expect(gate.takeLinkServed()).toBe(true); + }); + + it("restores on the first local attempt and does not also refresh", async () => { + const { prepare, restore, refreshIfStale } = build(); + await prepare("local"); + expect(restore).toHaveBeenCalledTimes(1); + // The restore's own `/props` just landed; refreshing again would + // probe twice for one attempt. + expect(refreshIfStale).toHaveBeenCalledTimes(0); + }); + + it("falls through to refreshIfStale on every later local attempt", async () => { + // The reactive path the loop's between-steps refresh cannot serve + // while the active provider is cloud. + const { prepare, restore, refreshIfStale } = build(); + await prepare("local"); + await prepare("local"); + await prepare("local"); + expect(restore).toHaveBeenCalledTimes(1); + expect(refreshIfStale).toHaveBeenCalledTimes(2); + }); + + it("refreshes from the very first attempt on a local-from-boot run", async () => { + // Boot already probed, so there is nothing to restore — but the + // staleness flag still needs a consumer. + const { prepare, restore, refreshIfStale } = build({ probedAtBoot: true }); + await prepare("local"); + expect(restore).toHaveBeenCalledTimes(0); + expect(refreshIfStale).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/llm/local-backend-gate.ts b/src/llm/local-backend-gate.ts index 4881858e..9f6c35c1 100644 --- a/src/llm/local-backend-gate.ts +++ b/src/llm/local-backend-gate.ts @@ -144,3 +144,47 @@ export class DeferredLocalBackendProbes implements LocalBackendGate { return served; } } + +export interface LocalLinkPreparerDeps { + gate: LocalBackendGate; + /** Is `providerId` a `llama-server` link? Live, re-read per attempt. */ + isLocalLink: (providerId: string) => boolean; + /** + * `ModelProfileManager.refreshIfStale`, bound. A no-op unless a + * completion reported a model the manager does not believe is loaded. + */ + refreshIfStale: () => Promise; +} + +/** + * The fallback seam's `prepareLink`, as bootstrap wires it. A named + * function rather than an inline closure in `buildRuntime` so the three + * decisions it makes are testable on their own — inline, the only way to + * reach them was to boot a whole runtime and drive a real fallover. + * + * For a `llama-server` link, in order: + * + * 1. `noteLinkServed()` — tell the loop a local link is serving, so its + * turn-start refresh reopens for the duration of the outage even + * though the ACTIVE provider stays cloud. + * 2. `ensureProbed()` — replay the probes boot deferred. `true` means + * they just ran and a fresh `/props` already landed; nothing more to + * do for this attempt. + * 3. otherwise `refreshIfStale()` — on a cloud-active turn the loop's + * own between-steps refresh is gated off, which leaves this the only + * consumer of the staleness `observeCompletionModelId` flagged, and + * it sits closer to the request than the line it replaces. + * + * Every other link kind returns on the first line: one predicate call, + * and no local request on a turn the local backend never serves. + */ +export function createLocalLinkPreparer( + deps: LocalLinkPreparerDeps, +): (providerId: string) => Promise { + return async (providerId: string): Promise => { + if (!deps.isLocalLink(providerId)) return; + deps.gate.noteLinkServed?.(); + if (await deps.gate.ensureProbed()) return; + await deps.refreshIfStale(); + }; +} diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index 26a8aa5e..df87191b 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -75,7 +75,10 @@ import { activeTextProviderIsLlamaServer, providerIdIsLlamaServer, } from "../llm/provider/registry/active-text-provider.js"; -import { DeferredLocalBackendProbes } from "../llm/local-backend-gate.js"; +import { + createLocalLinkPreparer, + DeferredLocalBackendProbes, +} from "../llm/local-backend-gate.js"; import { catalogForProvider } from "../llm/provider/catalog-for-provider.js"; import { CostAccumulator } from "../llm/provider/cost-accumulator.js"; import { @@ -1676,23 +1679,14 @@ export async function createAgentRuntime( // cloud boot runs on deferred state (plain profile, one-slot pool, // no `/props`), so warm it here rather than infer against it. // No-op on every other attempt — one boolean after the first call. - prepareLink: async (providerId) => { - if (!providerIdIsLlamaServer(resolveLlmConfig(getConfig()), providerId)) { - return; - } - // Tell the agent loop a local link is serving, so its turn-start - // refresh re-opens for as long as the outage lasts even though the - // ACTIVE provider stays cloud (issue #112 review, F1). - localBackend.noteLinkServed(); - // `true` means the restore just ran, which already carries a fresh - // `/props`. Otherwise the local state is warm but possibly stale — - // and on a cloud-active turn the loop's own `refreshIfStale` is - // gated off, so this is the only place left that can consume the - // staleness `observeCompletionModelId` flagged. Cheap: a no-op - // unless a completion actually reported a different model. - if (await localBackend.ensureProbed()) return; - await profileManager?.refreshIfStale(); - }, + prepareLink: createLocalLinkPreparer({ + gate: localBackend, + isLocalLink: (providerId) => + providerIdIsLlamaServer(resolveLlmConfig(getConfig()), providerId), + refreshIfStale: async () => { + await profileManager?.refreshIfStale(); + }, + }), recordUnaryUsage, recordStreamUsage, }; diff --git a/src/runtime/local-probe-gating.test.ts b/src/runtime/local-probe-gating.test.ts index 016e720d..0a1d6fca 100644 --- a/src/runtime/local-probe-gating.test.ts +++ b/src/runtime/local-probe-gating.test.ts @@ -33,7 +33,10 @@ interface LocalTraffic { /** Every URL the process asked for, in order. */ urls: string[]; countTo(hostPort: string, path?: string): number; + firstIndexOf(hostPort: string, path: string): number; reset(): void; + /** Flip the fake cloud provider to rate-limiting, to force a fallover. */ + rateLimitCloud(): void; } /** @@ -44,6 +47,7 @@ interface LocalTraffic { */ function installCountingFetch(): LocalTraffic { const urls: string[] = []; + let cloudRateLimited = false; const impl: typeof fetch = async (input) => { const url = typeof input === "string" @@ -81,6 +85,14 @@ function installCountingFetch(): LocalTraffic { // the one thing a zero-request assertion over cloud turns must not // do. if (url.includes("cloud.invalid") && url.includes("completions")) { + if (cloudRateLimited) { + // The shape that makes `runWithFallback` advance to the next + // link rather than fail the turn. + return new Response(JSON.stringify({ error: { message: "slow down" } }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } return new Response( JSON.stringify({ id: "c1", @@ -115,9 +127,14 @@ function installCountingFetch(): LocalTraffic { countTo: (hostPort, path) => urls.filter((u) => u.includes(hostPort) && (!path || u.includes(path))) .length, + firstIndexOf: (hostPort, path) => + urls.findIndex((u) => u.includes(hostPort) && u.includes(path)), reset: () => { urls.length = 0; }, + rateLimitCloud: () => { + cloudRateLimited = true; + }, }; } @@ -346,6 +363,69 @@ describe("issue #112 — local probe gating at CLI bootstrap", () => { } }); + /** + * Issue #112 review, F1 + F7 — the fallover, end to end through a + * booted runtime rather than at the seam. + * + * Boot is cloud, so the local link starts with no `/health`, no + * `/props`, a `plain-instruct` profile and a one-slot pool. Then the + * cloud primary starts returning 429 and `appendLocal` (default + * `true`) routes every turn onto the llama-server link. Two things + * have to hold, and neither was covered end-to-end before: the link is + * warmed BEFORE its first completion, and it keeps being refreshed on + * later turns even though the active provider never stops being cloud. + */ + it("a cloud->local FALLOVER warms the link before it serves, turn after turn", async () => { + writeConfig(stateDir, { + llm: { + ...cloudLlm, + providers: [ + CLOUD_PROVIDER, + { id: "local-llama", kind: "llama-server", url: "http://127.0.0.1:8080" }, + LOCAL_EMBED_PROVIDER, + ], + }, + }); + const runtime = await boot(); + try { + expect(traffic.countTo(TEXT_PORT)).toBe(0); + traffic.rateLimitCloud(); + const session = runtime.createSession(); + + const turn = async (n: number) => + runtime + .executeTurn(session, `hello ${n}`, { + maxSteps: 1, + signal: new AbortController().signal, + }) + .catch(() => undefined); + + await turn(0); + // The deferred boot probes were replayed by the seam... + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(1); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(1); + // ...and BEFORE the local link was asked to serve. Ordering is the + // whole point: a `/props` that lands after the completion has + // already been sent on a plain profile buys nothing. + const props = traffic.firstIndexOf(TEXT_PORT, "/props"); + const completion = traffic.firstIndexOf(TEXT_PORT, "/completion"); + expect(completion).toBeGreaterThan(-1); + expect(props).toBeLessThan(completion); + + // Turns 2 and 3: the profile keeps tracking the live server. The + // active provider is still cloud, so before the F1 fix these were + // both zero and the profile stayed frozen for the whole outage. + await turn(1); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(2); + await turn(2); + expect(traffic.countTo(TEXT_PORT, "/props")).toBe(3); + // The restore stays one-shot: `/health` is not replayed per turn. + expect(traffic.countTo(TEXT_PORT, "/health")).toBe(1); + } finally { + await runtime.shutdown(); + } + }); + it("keeps probing local embeddings while the text route is cloud", async () => { writeConfig(stateDir, { llm: cloudLlm,