Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions docs/decisions/2026-11-15-diagnose-cli-chat-failures.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
---
date: 2026-11-15
title: "Diagnose CLI chat failures with finite tags"
---

# 2026-11-15 — Diagnose CLI chat failures with finite tags

## Context

ATOMIC-CHAT-SERVER-1S reported a generic CLI chat error in the SSE reader. The reporter
scrubs free-form exception text, so it could not distinguish a provider-reported generation
error from an incomplete stream. The client also preserves the message from a non-2xx HTTP
response for the caller. A message lookup could mistake that remote text for a local error.

## Decision

Have the terminal client attach a bounded diagnostic code when it rejects a parsed endpoint,
finds an absent response body, reads a generation error in SSE, or reaches the end of an
incomplete stream. The reporter reads that metadata from the error class. Existing allowlisted
transport codes still identify failures in the response reader, and the reporter still scrubs
remote HTTP text.

## Consequences

A tagged event identifies the client branch that produced it. Invalid URL syntax, malformed
SSE and arbitrary HTTP messages can still reach the reporter without a terminal chat code.
CLI output, exit status and the reporting rules use their existing behaviour.

Owner: team

References: [Sentry ATOMIC-CHAT-SERVER-1S](https://atomic-chat.sentry.io/issues/7848662602/),
`src/terminal-service/client.ts`, `src/telemetry/native-diagnostics.ts`.
1 change: 1 addition & 0 deletions docs/decisions/INDEX.md
Original file line number Diff line number Diff line change
Expand Up @@ -242,3 +242,4 @@ add a new one that says which record it supersedes.
- [2026-11-07 — Reuse the API listener when connecting an agent](2026-11-07-reuse-api-for-connections.md)

- [2026-11-07 — Package a runtime that does not inherit Windows sockets](2026-11-07-prevent-windows-socket-inheritance.md)
- **2026-11-15** — [Diagnose CLI chat failures with finite tags](2026-11-15-diagnose-cli-chat-failures.md) — tag client-detected failures without interpreting remote HTTP text as a client diagnostic.
1 change: 1 addition & 0 deletions src/errors/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/** `AtcError`, its codes, exit codes and the one rendering of any failure. Depends on nothing. */
export * from './errors.js'
export * from './ownership.js'
export { TerminalChatError } from './terminal-chat.js'
24 changes: 24 additions & 0 deletions src/errors/terminal-chat.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
const MESSAGES = {
CLI_CHAT_ENDPOINT_INVALID: 'Use an HTTP endpoint without credentials, a query, or a fragment.',
CLI_CHAT_GENERATION_ERROR: 'The server reported a generation error. Inspect the server log.',
CLI_CHAT_EMPTY_RESPONSE: 'The server returned an empty response.',
CLI_CHAT_STREAM_TRUNCATED: 'The response ended before completion. The partial reply is kept.',
} as const

type TerminalChatCode = keyof typeof MESSAGES

/** A failure detected by the terminal client itself. The code is reporting metadata. */
export class TerminalChatError extends Error {
readonly #code: TerminalChatCode

constructor(code: TerminalChatCode) {
super(MESSAGES[code])
this.#code = code
}

static diagnosticCode(error: unknown): TerminalChatCode | undefined {
if (!(error instanceof Error) || !(#code in error)) return undefined
const code = error.#code
return Object.hasOwn(MESSAGES, code) ? code : undefined
}
}
6 changes: 5 additions & 1 deletion src/telemetry/native-diagnostics.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { TerminalChatError } from '../errors/index.js'
import { sentryAgentDiagnosticTags } from './sentry-agent-diagnostics.js'
/** Fixed runtime messages become finite tags; arbitrary exception text stays private. */
const SEGMENTER_FAILURES = new Map([
Expand All @@ -15,5 +16,8 @@ function sentryBaseNativeFailureTags(error: unknown): Record<string, string> {
}

export function nativeFailureTags(error: unknown): Record<string, string> {
return { ...sentryBaseNativeFailureTags(error), ...sentryAgentDiagnosticTags(error) }
const tags = { ...sentryBaseNativeFailureTags(error), ...sentryAgentDiagnosticTags(error) }
const chatCode = TerminalChatError.diagnosticCode(error)
if (chatCode) tags['diagnostic_code'] = chatCode
return tags
}
49 changes: 49 additions & 0 deletions src/terminal-service/client.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
import { failureReport } from '../telemetry/index.js'
import { endpointUrl, sameEndpoint, listChatModels, streamChat } from './client.js'

function response(chunks: string[]) {
Expand Down Expand Up @@ -129,6 +130,54 @@ describe('a refused chat request', () => {
})
})

describe('CLI chat failure reports', () => {
const context = { component: 'cli', operation: 'chat' }

async function chatFailure(reply: () => Response, endpoint = 'http://host/v1'): Promise<unknown> {
return streamChat({
endpoint,
model: 'm',
messages: [],
request: vi.fn<typeof fetch>(async () => reply()),
signal: new AbortController().signal,
onText: () => {},
}).catch((cause: unknown) => cause)
}

it.each([
{
code: 'CLI_CHAT_ENDPOINT_INVALID',
endpoint: 'file:///private-fixture',
reply: () => new Response(null),
},
{ code: 'CLI_CHAT_EMPTY_RESPONSE', reply: () => new Response(null) },
{
code: 'CLI_CHAT_GENERATION_ERROR',
reply: () => new Response('data: {"error":{"message":"private-fixture"}}\n\n'),
},
{ code: 'CLI_CHAT_STREAM_TRUNCATED', reply: () => new Response(delta('partial reply')) },
])('reports $code only from the local client branch', async ({ code, endpoint, reply }) => {
const error = await chatFailure(reply, endpoint)
const report = failureReport(error, context)
expect(report).toMatchObject({
level: 'error',
error: { name: 'Error', message: 'Unexpected failure in chat' },
tags: { ...context, diagnostic_code: code },
})
expect(JSON.stringify(report)).not.toContain('private-fixture')
})

it('keeps a matching message from a remote HTTP error out of the local diagnostic tags', async () => {
const message = 'The response ended before completion. The partial reply is kept.'
const error = await chatFailure(() => Response.json({ error: { message } }, { status: 502 }))

expect(error).toMatchObject({ message })
const report = failureReport(error, context)
expect(report).toMatchObject({ error: { message: 'Unexpected failure in chat' }, tags: context })
expect(report?.tags).not.toHaveProperty('diagnostic_code')
})
})

it('matches loopback aliases without treating a different listener or API path as local', () => {
const local = 'http://127.0.0.1:63163/v1'
expect(sameEndpoint('http://localhost:63163/v1/', local)).toBe(true)
Expand Down
10 changes: 6 additions & 4 deletions src/terminal-service/client.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
/** Public OpenAI-compatible calls shared by the terminal and plain chat command. */
import { TerminalChatError } from '../errors/index.js'

export type ChatContentPart =
| { type: 'text'; text: string }
| { type: 'image_url'; image_url: { url: string; detail?: 'auto' | 'low' | 'high' } }
Expand All @@ -17,7 +19,7 @@ export interface ChatMessage {
export function endpointUrl(value: string): string {
const url = new URL(value)
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash)
throw new Error('Use an HTTP endpoint without credentials, a query, or a fragment.')
throw new TerminalChatError('CLI_CHAT_ENDPOINT_INVALID')
return url.toString().replace(/\/$/, '')
}
/** Loopback aliases share a listener, but a different port or API path is a different server. */
Expand Down Expand Up @@ -80,7 +82,7 @@ export async function streamChat(options: {
: `Request returned HTTP ${response.status}. Check the model, endpoint, and access settings.`
)
}
if (!response.body) throw new Error('The server returned an empty response.')
if (!response.body) throw new TerminalChatError('CLI_CHAT_EMPTY_RESPONSE')
const reader = response.body.getReader(),
decoder = new TextDecoder()
let buffer = '',
Expand All @@ -107,13 +109,13 @@ export async function streamChat(options: {
error?: { message?: string }
choices?: { delta?: { content?: string }; finish_reason?: string | null }[]
}
if (chunk.error) throw new Error('The server reported a generation error. Inspect the server log.')
if (chunk.error) throw new TerminalChatError('CLI_CHAT_GENERATION_ERROR')
const text = chunk.choices?.[0]?.delta?.content
if (text) options.onText(text)
if (chunk.choices?.[0]?.finish_reason) done = true
}
if (next.done) {
if (!done) throw new Error('The response ended before completion. The partial reply is kept.')
if (!done) throw new TerminalChatError('CLI_CHAT_STREAM_TRUNCATED')
break
}
}
Expand Down
Loading