diff --git a/docs/decisions/2026-11-15-diagnose-cli-chat-failures.md b/docs/decisions/2026-11-15-diagnose-cli-chat-failures.md new file mode 100644 index 0000000..e2d568e --- /dev/null +++ b/docs/decisions/2026-11-15-diagnose-cli-chat-failures.md @@ -0,0 +1,32 @@ +--- +date: 2026-11-15 +title: "Diagnose CLI chat failures with finite tags" +--- + +# 2026-11-15 — Diagnose CLI chat failures with finite tags + +## Context + +ATOMIC-CHAT-SERVER-1S reported a generic CLI chat error in the SSE reader. The reporter +scrubs free-form exception text, so it could not distinguish a provider-reported generation +error from an incomplete stream. The client also preserves the message from a non-2xx HTTP +response for the caller. A message lookup could mistake that remote text for a local error. + +## Decision + +Have the terminal client attach a bounded diagnostic code when it rejects a parsed endpoint, +finds an absent response body, reads a generation error in SSE, or reaches the end of an +incomplete stream. The reporter reads that metadata from the error class. Existing allowlisted +transport codes still identify failures in the response reader, and the reporter still scrubs +remote HTTP text. + +## Consequences + +A tagged event identifies the client branch that produced it. Invalid URL syntax, malformed +SSE and arbitrary HTTP messages can still reach the reporter without a terminal chat code. +CLI output, exit status and the reporting rules use their existing behaviour. + +Owner: team + +References: [Sentry ATOMIC-CHAT-SERVER-1S](https://atomic-chat.sentry.io/issues/7848662602/), +`src/terminal-service/client.ts`, `src/telemetry/native-diagnostics.ts`. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 4b06e66..f359d2c 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -242,3 +242,4 @@ add a new one that says which record it supersedes. - [2026-11-07 — Reuse the API listener when connecting an agent](2026-11-07-reuse-api-for-connections.md) - [2026-11-07 — Package a runtime that does not inherit Windows sockets](2026-11-07-prevent-windows-socket-inheritance.md) +- **2026-11-15** — [Diagnose CLI chat failures with finite tags](2026-11-15-diagnose-cli-chat-failures.md) — tag client-detected failures without interpreting remote HTTP text as a client diagnostic. diff --git a/src/errors/index.ts b/src/errors/index.ts index 693f5f2..516e72a 100644 --- a/src/errors/index.ts +++ b/src/errors/index.ts @@ -1,3 +1,4 @@ /** `AtcError`, its codes, exit codes and the one rendering of any failure. Depends on nothing. */ export * from './errors.js' export * from './ownership.js' +export { TerminalChatError } from './terminal-chat.js' diff --git a/src/errors/terminal-chat.ts b/src/errors/terminal-chat.ts new file mode 100644 index 0000000..e5cbece --- /dev/null +++ b/src/errors/terminal-chat.ts @@ -0,0 +1,24 @@ +const MESSAGES = { + CLI_CHAT_ENDPOINT_INVALID: 'Use an HTTP endpoint without credentials, a query, or a fragment.', + CLI_CHAT_GENERATION_ERROR: 'The server reported a generation error. Inspect the server log.', + CLI_CHAT_EMPTY_RESPONSE: 'The server returned an empty response.', + CLI_CHAT_STREAM_TRUNCATED: 'The response ended before completion. The partial reply is kept.', +} as const + +type TerminalChatCode = keyof typeof MESSAGES + +/** A failure detected by the terminal client itself. The code is reporting metadata. */ +export class TerminalChatError extends Error { + readonly #code: TerminalChatCode + + constructor(code: TerminalChatCode) { + super(MESSAGES[code]) + this.#code = code + } + + static diagnosticCode(error: unknown): TerminalChatCode | undefined { + if (!(error instanceof Error) || !(#code in error)) return undefined + const code = error.#code + return Object.hasOwn(MESSAGES, code) ? code : undefined + } +} diff --git a/src/telemetry/native-diagnostics.ts b/src/telemetry/native-diagnostics.ts index 0e298ca..2411cbf 100644 --- a/src/telemetry/native-diagnostics.ts +++ b/src/telemetry/native-diagnostics.ts @@ -1,3 +1,4 @@ +import { TerminalChatError } from '../errors/index.js' import { sentryAgentDiagnosticTags } from './sentry-agent-diagnostics.js' /** Fixed runtime messages become finite tags; arbitrary exception text stays private. */ const SEGMENTER_FAILURES = new Map([ @@ -15,5 +16,8 @@ function sentryBaseNativeFailureTags(error: unknown): Record { } export function nativeFailureTags(error: unknown): Record { - return { ...sentryBaseNativeFailureTags(error), ...sentryAgentDiagnosticTags(error) } + const tags = { ...sentryBaseNativeFailureTags(error), ...sentryAgentDiagnosticTags(error) } + const chatCode = TerminalChatError.diagnosticCode(error) + if (chatCode) tags['diagnostic_code'] = chatCode + return tags } diff --git a/src/terminal-service/client.test.ts b/src/terminal-service/client.test.ts index 960e2ec..43c0fb7 100644 --- a/src/terminal-service/client.test.ts +++ b/src/terminal-service/client.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { failureReport } from '../telemetry/index.js' import { endpointUrl, sameEndpoint, listChatModels, streamChat } from './client.js' function response(chunks: string[]) { @@ -129,6 +130,54 @@ describe('a refused chat request', () => { }) }) +describe('CLI chat failure reports', () => { + const context = { component: 'cli', operation: 'chat' } + + async function chatFailure(reply: () => Response, endpoint = 'http://host/v1'): Promise { + return streamChat({ + endpoint, + model: 'm', + messages: [], + request: vi.fn(async () => reply()), + signal: new AbortController().signal, + onText: () => {}, + }).catch((cause: unknown) => cause) + } + + it.each([ + { + code: 'CLI_CHAT_ENDPOINT_INVALID', + endpoint: 'file:///private-fixture', + reply: () => new Response(null), + }, + { code: 'CLI_CHAT_EMPTY_RESPONSE', reply: () => new Response(null) }, + { + code: 'CLI_CHAT_GENERATION_ERROR', + reply: () => new Response('data: {"error":{"message":"private-fixture"}}\n\n'), + }, + { code: 'CLI_CHAT_STREAM_TRUNCATED', reply: () => new Response(delta('partial reply')) }, + ])('reports $code only from the local client branch', async ({ code, endpoint, reply }) => { + const error = await chatFailure(reply, endpoint) + const report = failureReport(error, context) + expect(report).toMatchObject({ + level: 'error', + error: { name: 'Error', message: 'Unexpected failure in chat' }, + tags: { ...context, diagnostic_code: code }, + }) + expect(JSON.stringify(report)).not.toContain('private-fixture') + }) + + it('keeps a matching message from a remote HTTP error out of the local diagnostic tags', async () => { + const message = 'The response ended before completion. The partial reply is kept.' + const error = await chatFailure(() => Response.json({ error: { message } }, { status: 502 })) + + expect(error).toMatchObject({ message }) + const report = failureReport(error, context) + expect(report).toMatchObject({ error: { message: 'Unexpected failure in chat' }, tags: context }) + expect(report?.tags).not.toHaveProperty('diagnostic_code') + }) +}) + it('matches loopback aliases without treating a different listener or API path as local', () => { const local = 'http://127.0.0.1:63163/v1' expect(sameEndpoint('http://localhost:63163/v1/', local)).toBe(true) diff --git a/src/terminal-service/client.ts b/src/terminal-service/client.ts index 9bb0a09..0ca30d4 100644 --- a/src/terminal-service/client.ts +++ b/src/terminal-service/client.ts @@ -1,4 +1,6 @@ /** Public OpenAI-compatible calls shared by the terminal and plain chat command. */ +import { TerminalChatError } from '../errors/index.js' + export type ChatContentPart = | { type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } } @@ -17,7 +19,7 @@ export interface ChatMessage { export function endpointUrl(value: string): string { const url = new URL(value) if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) - throw new Error('Use an HTTP endpoint without credentials, a query, or a fragment.') + throw new TerminalChatError('CLI_CHAT_ENDPOINT_INVALID') return url.toString().replace(/\/$/, '') } /** Loopback aliases share a listener, but a different port or API path is a different server. */ @@ -80,7 +82,7 @@ export async function streamChat(options: { : `Request returned HTTP ${response.status}. Check the model, endpoint, and access settings.` ) } - if (!response.body) throw new Error('The server returned an empty response.') + if (!response.body) throw new TerminalChatError('CLI_CHAT_EMPTY_RESPONSE') const reader = response.body.getReader(), decoder = new TextDecoder() let buffer = '', @@ -107,13 +109,13 @@ export async function streamChat(options: { error?: { message?: string } choices?: { delta?: { content?: string }; finish_reason?: string | null }[] } - if (chunk.error) throw new Error('The server reported a generation error. Inspect the server log.') + if (chunk.error) throw new TerminalChatError('CLI_CHAT_GENERATION_ERROR') const text = chunk.choices?.[0]?.delta?.content if (text) options.onText(text) if (chunk.choices?.[0]?.finish_reason) done = true } if (next.done) { - if (!done) throw new Error('The response ended before completion. The partial reply is kept.') + if (!done) throw new TerminalChatError('CLI_CHAT_STREAM_TRUNCATED') break } }