diff --git a/apps/buddy/service/src/images/ImageGenerationGateway.ts b/apps/buddy/service/src/images/ImageGenerationGateway.ts index aeb04d1e..c5c36546 100644 --- a/apps/buddy/service/src/images/ImageGenerationGateway.ts +++ b/apps/buddy/service/src/images/ImageGenerationGateway.ts @@ -2,6 +2,7 @@ import type { Api, ImageContent, Model } from '@earendil-works/pi-ai' export const IMAGE_GENERATION_ERROR_CODES = [ 'IMAGE_GENERATION_FAILED', + 'IMAGE_GENERATION_INCOMPLETE', 'IMAGE_GENERATION_INVALID_RESPONSE', 'IMAGE_GENERATION_RESPONSE_TOO_LARGE', 'IMAGE_GENERATION_UNSUPPORTED', diff --git a/apps/buddy/service/src/images/OpenAiImageGenerationService.ts b/apps/buddy/service/src/images/OpenAiImageGenerationService.ts index 82d1993b..8a3993c9 100644 --- a/apps/buddy/service/src/images/OpenAiImageGenerationService.ts +++ b/apps/buddy/service/src/images/OpenAiImageGenerationService.ts @@ -50,6 +50,7 @@ export class OpenAiImageGenerationService implements ImageGenerationGateway { input.signal.throwIfAborted() throw new ImageGenerationError('PROVIDER_AUTHENTICATION_FAILED', { cause: error }) } + input.signal.throwIfAborted() const apiKey = auth?.auth.apiKey if (!auth || !apiKey) throw new ImageGenerationError('PROVIDER_AUTHENTICATION_FAILED') @@ -63,16 +64,25 @@ export class OpenAiImageGenerationService implements ImageGenerationGateway { method: 'POST', signal: input.signal, }) - const body = await readBoundedResponse(response) if (!response.ok) { + const body = await readBoundedResponse(response, input.signal) throw new ImageGenerationError(normalizeProviderError(response.status), { diagnostic: readProviderErrorDiagnostic(response, body), }) } - const output = codex - ? parseSseOutput(body) - : parseJsonOutput(body) + let output: ImageGenerationResult + try { + output = codex + ? await readSseOutput(response, input.signal) + : parseJsonOutput(await readBoundedResponse(response, input.signal), response) + } + catch (error) { + input.signal.throwIfAborted() + if (error instanceof ImageGenerationError) + throw error + throw new ImageGenerationError('IMAGE_GENERATION_INCOMPLETE', { cause: error, diagnostic: readProviderErrorDiagnostic(response, null) }) + } if (output.images.length === 0) throw new ImageGenerationError('IMAGE_GENERATION_FAILED') return output @@ -168,33 +178,53 @@ function extractChatGptAccountId(token: string): string { } } -async function readBoundedResponse(response: Response): Promise { - const declaredLength = Number(response.headers.get('content-length')) - if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) { - await response.body?.cancel() - throw new ImageGenerationError('IMAGE_GENERATION_RESPONSE_TOO_LARGE') +async function* readResponseChunks(response: Response, signal: AbortSignal): AsyncGenerator { + if (!response.body) { + signal.throwIfAborted() + return } - if (!response.body) - return '' const reader = response.body.getReader() const decoder = new TextDecoder() - const parts: string[] = [] let receivedBytes = 0 - while (true) { - const chunk = await reader.read() - if (chunk.done) - return parts.join('') + decoder.decode() - receivedBytes += chunk.value.byteLength - if (receivedBytes > MAX_RESPONSE_BYTES) { - await reader.cancel() + const cancel = () => { + void reader.cancel().catch(() => {}) + } + signal.addEventListener('abort', cancel, { once: true }) + try { + signal.throwIfAborted() + const declaredLength = Number(response.headers.get('content-length')) + if (Number.isFinite(declaredLength) && declaredLength > MAX_RESPONSE_BYTES) throw new ImageGenerationError('IMAGE_GENERATION_RESPONSE_TOO_LARGE') + while (true) { + signal.throwIfAborted() + const chunk = await reader.read() + signal.throwIfAborted() + if (chunk.done) { + yield decoder.decode() + return + } + receivedBytes += chunk.value.byteLength + if (receivedBytes > MAX_RESPONSE_BYTES) + throw new ImageGenerationError('IMAGE_GENERATION_RESPONSE_TOO_LARGE') + yield decoder.decode(chunk.value, { stream: true }) } - parts.push(decoder.decode(chunk.value, { stream: true })) } + finally { + signal.removeEventListener('abort', cancel) + cancel() + reader.releaseLock() + } +} + +async function readBoundedResponse(response: Response, signal: AbortSignal): Promise { + const parts: string[] = [] + for await (const chunk of readResponseChunks(response, signal)) + parts.push(chunk) + return parts.join('') } -function parseJsonOutput(body: string): ImageGenerationResult { +function parseJsonOutput(body: string, response: Response): ImageGenerationResult { let parsed: unknown try { parsed = JSON.parse(body) @@ -203,21 +233,25 @@ function parseJsonOutput(body: string): ImageGenerationResult { throw new ImageGenerationError('IMAGE_GENERATION_INVALID_RESPONSE', { cause: error }) } const record = readRecord(parsed) + if (record?.error || record?.status !== 'completed') { + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { + diagnostic: readProviderErrorDiagnostic(response, record), + }) + } return { images: extractImages(record?.output), responseId: readString(record, 'id'), } } -function parseSseOutput(body: string): ImageGenerationResult { +async function readSseOutput(response: Response, signal: AbortSignal): Promise { const outputItems: unknown[] = [] let responseId: string | null = null - for (const line of body.split(/\r?\n/)) { - if (!line.startsWith('data:')) - continue - const data = line.slice(5).trim() - if (!data || data === '[DONE]') + for await (const data of readSseData(readResponseChunks(response, signal))) { + if (!data.trim()) continue + if (data.trim() === '[DONE]') + return { images: extractImages(outputItems), responseId } let event: unknown try { event = JSON.parse(data) @@ -226,15 +260,65 @@ function parseSseOutput(body: string): ImageGenerationResult { throw new ImageGenerationError('IMAGE_GENERATION_INVALID_RESPONSE', { cause: error }) } const eventRecord = readRecord(event) - const response = readRecord(eventRecord?.response) - responseId = readString(response, 'id') ?? responseId - if (Array.isArray(response?.output)) - outputItems.push(...response.output) - const item = readRecord(eventRecord?.item) - if (item) - outputItems.push(item) + if (!eventRecord) + throw new ImageGenerationError('IMAGE_GENERATION_INVALID_RESPONSE') + const result = readRecord(eventRecord.response) + responseId = readString(result, 'id') ?? responseId + const type = eventRecord.type + if (type === 'error' || type === 'response.failed' || type === 'response.incomplete' || type === 'response.cancelled' || eventRecord.error || result?.error) { + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { + diagnostic: readProviderErrorDiagnostic(response, result ?? eventRecord), + }) + } + if (type === 'response.output_item.done') + outputItems.push(eventRecord.item) + if (type === 'response.completed' || type === 'response.done') { + if (!result || (result.status !== undefined && result.status !== 'completed')) { + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { + diagnostic: readProviderErrorDiagnostic(response, result), + }) + } + if (Array.isArray(result.output)) + outputItems.push(...result.output) + return { images: extractImages(outputItems), responseId } + } + } + throw new ImageGenerationError('IMAGE_GENERATION_INCOMPLETE', { diagnostic: readProviderErrorDiagnostic(response, null) }) +} + +async function* readSseData(chunks: AsyncIterable): AsyncGenerator { + let line = '' + let data: string[] = [] + let skipLineFeed = false + const accept = (value: string) => { + if (value === 'data') + data.push('') + else if (value.startsWith('data:')) + data.push(value.slice(5).replace(/^ /, '')) + } + for await (let chunk of chunks) { + if (!chunk) + continue + if (skipLineFeed && chunk.startsWith('\n')) + chunk = chunk.slice(1) + skipLineFeed = chunk.endsWith('\r') + const lines = chunk.split(/\r\n|[\r\n]/) + lines[0] = line + lines[0] + line = lines.pop()! + for (const value of lines) { + if (value === '') { + if (data.length > 0) + yield data.join('\n') + data = [] + } + else { + accept(value) + } + } } - return { images: extractImages(outputItems), responseId } + accept(line) + if (data.length > 0) + yield data.join('\n') } function extractImages(output: unknown): ImageGenerationResult['images'] { @@ -243,7 +327,7 @@ function extractImages(output: unknown): ImageGenerationResult['images'] { const seen = new Set() return output.flatMap((item) => { const record = readRecord(item) - if (record?.type !== 'image_generation_call' || record.status === 'failed') + if (record?.type !== 'image_generation_call' || record.status !== 'completed') return [] const result = readString(record, 'result') if (!result || seen.has(result)) @@ -278,13 +362,15 @@ function normalizeProviderError(status: number) { function readProviderErrorDiagnostic( response: Response, - body: string, + body: unknown, ): ImageGenerationErrorDiagnostic | undefined { - let error: Record | null = null + let record = readRecord(body) try { - error = readRecord(readRecord(JSON.parse(body))?.error) + if (typeof body === 'string') + record = readRecord(JSON.parse(body)) } catch {} + const error = readRecord(record?.error) ?? record const providerCode = readDiagnosticValue(error?.code) const providerParameter = readDiagnosticValue(error?.param) ?? readDiagnosticValue(error?.parameter) diff --git a/apps/buddy/service/src/images/__tests__/OpenAiImageGenerationService.spec.ts b/apps/buddy/service/src/images/__tests__/OpenAiImageGenerationService.spec.ts index d6101b29..b7c06d6e 100644 --- a/apps/buddy/service/src/images/__tests__/OpenAiImageGenerationService.spec.ts +++ b/apps/buddy/service/src/images/__tests__/OpenAiImageGenerationService.spec.ts @@ -8,8 +8,202 @@ import { } from '../OpenAiImageGenerationService' const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]) +const imageItem = { id: 'image-call-1', result: png.toString('base64'), status: 'completed', type: 'image_generation_call' } describe('openAiImageGenerationService', () => { + it.each(['response.completed', 'response.done', '[DONE]'])('settles %s without waiting for the connection or cancellation cleanup to close', async (type) => { + let controller!: ReadableStreamDefaultController + let cancelled = false + const response = new Response(new ReadableStream({ + start: value => controller = value, + cancel() { + cancelled = true + return new Promise(() => {}) + }, + })) + const service = streamedService(response) + const pending = service.generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: new AbortController().signal }) + let settled = false + void pending.then(() => settled = true, () => settled = true) + controller.enqueue(new TextEncoder().encode([ + sse({ type: 'response.created', response: { id: 'response-1', status: 'in_progress' } }), + sse({ type: 'response.output_item.done', item: imageItem }), + type === '[DONE]' ? 'data: [DONE]\n\n' : sse({ type, response: { id: 'response-1', status: 'completed', output: [imageItem] } }), + ].join(''))) + try { + await vi.waitFor(() => expect(settled).toBe(true)) + await expect(pending).resolves.toEqual({ images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' }) + expect(cancelled).toBe(true) + expect(response.body?.locked).toBe(false) + } + finally { + if (!cancelled) + controller.close() + await pending.catch(() => {}) + } + }) + + it.each(['\n', '\r\n', '\r'])('parses byte-split %j frames, multiline data and the final EOF frame without duplicating images', async (newline) => { + const text = [ + ': 心跳', + '', + `data: ${JSON.stringify({ type: 'response.output_item.done', item: imageItem })}`, + '', + 'event: response.completed', + 'data: {"type":"response.completed",', + `data: "response":${JSON.stringify({ id: 'response-1', status: 'completed', output: [imageItem], metadata: '中文' })}}`, + ].join(newline) + const bytes = new TextEncoder().encode(text) + let offset = 0 + const response = new Response(new ReadableStream({ + pull(controller) { + if (offset < bytes.length) + controller.enqueue(bytes.slice(offset, ++offset)) + else + controller.close() + }, + })) + await expect(streamedService(response).generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: new AbortController().signal })).resolves.toEqual({ + images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], + responseId: 'response-1', + }) + expect(response.body?.locked).toBe(false) + }) + + it.each([ + { type: 'error', code: 'provider_failure', message: 'private provider message' }, + { type: 'error', error: { code: 'provider_failure', message: 'private provider message' } }, + ...['response.failed', 'response.incomplete', 'response.cancelled', 'response.done'].map(type => ({ + type, + response: { status: 'failed', error: { code: 'provider_failure', message: 'private provider message' }, output: [imageItem] }, + })), + ])('rejects $type immediately even after receiving a completed image', async (event) => { + let cancelled = false + const response = new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode([ + sse({ type: 'response.output_item.done', item: imageItem }), + sse(event), + ].join(''))) + }, + cancel: () => { cancelled = true }, + }), { headers: { 'x-request-id': 'req_image_fixture' } }) + await expect(streamedService(response).generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: new AbortController().signal })).rejects.toMatchObject({ + code: 'IMAGE_GENERATION_FAILED', + diagnostic: { providerCode: 'provider_failure', requestId: 'req_image_fixture' }, + message: 'Lexora Buddy image generation failed', + }) + expect(cancelled).toBe(true) + expect(response.body?.locked).toBe(false) + }) + + it.each(['eof', 'disconnect'])('reports an unknown result on %s before a response terminal event', async (ending) => { + let sent = false + const response = new Response(new ReadableStream({ + pull(controller) { + if (!sent) { + sent = true + controller.enqueue(new TextEncoder().encode(sse({ type: 'response.output_item.done', item: imageItem }))) + } + else if (ending === 'disconnect') { + controller.error(new Error('private transport failure')) + } + else { + controller.close() + } + }, + }), { headers: { 'x-request-id': 'req_image_fixture' } }) + await expect(streamedService(response).generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: new AbortController().signal })).rejects.toMatchObject({ + code: 'IMAGE_GENERATION_INCOMPLETE', + diagnostic: { requestId: 'req_image_fixture' }, + }) + expect(response.body?.locked).toBe(false) + }) + + it('reports an unknown result and keeps the request id when a JSON response disconnects', async () => { + let sent = false + const response = new Response(new ReadableStream({ + pull(controller) { + if (!sent) { + sent = true + controller.enqueue(new TextEncoder().encode('{"status":"completed","output":[')) + } + else { + controller.error(new Error('private transport failure')) + } + }, + }), { headers: { 'x-request-id': 'req_image_fixture' } }) + await expect(streamedService(response).generate({ inputImages: [], model: openAiModel(), prompt: 'fixture', signal: new AbortController().signal })).rejects.toMatchObject({ + code: 'IMAGE_GENERATION_INCOMPLETE', + diagnostic: { requestId: 'req_image_fixture' }, + message: 'Lexora Buddy image generation failed', + }) + expect(response.body?.locked).toBe(false) + }) + + it.each([openAiModel(), codexModel()])('cancels a blocked $api body read without waiting for transport cleanup', async (model) => { + let cancelled = false + const response = new Response(new ReadableStream({ + cancel() { + cancelled = true + return new Promise(() => {}) + }, + })) + const abort = new AbortController() + const pending = streamedService(response).generate({ inputImages: [], model, prompt: 'fixture', signal: abort.signal }) + const assertion = expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + await vi.waitFor(() => expect(response.body?.locked).toBe(true)) + abort.abort() + await assertion + expect(cancelled).toBe(true) + expect(response.body?.locked).toBe(false) + }) + + it('allows a long-running stream to complete after an hour without a total timeout', async () => { + vi.useFakeTimers() + let controller!: ReadableStreamDefaultController + const response = new Response(new ReadableStream({ start: value => controller = value })) + const abort = new AbortController() + const pending = streamedService(response).generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: abort.signal }) + let settled = false + void pending.then(() => settled = true, () => settled = true) + try { + await vi.waitFor(() => expect(response.body?.locked).toBe(true)) + controller.enqueue(new TextEncoder().encode(sse({ type: 'response.in_progress', response: { id: 'response-1', status: 'in_progress' } }))) + await vi.advanceTimersByTimeAsync(3_600_000) + expect(settled).toBe(false) + controller.enqueue(new TextEncoder().encode(sse({ type: 'response.completed', response: { id: 'response-1', status: 'completed', output: [imageItem] } }))) + await expect(pending).resolves.toMatchObject({ images: [{ mimeType: 'image/png' }], responseId: 'response-1' }) + } + finally { + abort.abort() + await pending.catch(() => {}) + vi.useRealTimers() + } + }) + + it.each([ + ['malformed frame', 'data: {invalid}\n\n', 'IMAGE_GENERATION_INVALID_RESPONSE'], + ['empty terminal', sse({ type: 'response.completed', response: { status: 'completed', output: [] } }), 'IMAGE_GENERATION_FAILED'], + ['unfinished image', sse({ type: 'response.completed', response: { status: 'completed', output: [{ ...imageItem, status: 'in_progress' }] } }), 'IMAGE_GENERATION_FAILED'], + ])('rejects %s and releases the stream', async (_name, body, code) => { + let cancelled = false + const response = new Response(new ReadableStream({ + start: controller => controller.enqueue(new TextEncoder().encode(body)), + cancel: () => { cancelled = true }, + })) + await expect(streamedService(response).generate({ inputImages: [], model: codexModel(), prompt: 'fixture', signal: new AbortController().signal })).rejects.toMatchObject({ code }) + expect(cancelled).toBe(true) + }) + + it('rejects a failed JSON response even when it contains an earlier image result', async () => { + const response = Response.json({ status: 'failed', output: [imageItem], error: { code: 'provider_failure', message: 'private failure' } }) + await expect(streamedService(response).generate({ inputImages: [], model: openAiModel(), prompt: 'fixture', signal: new AbortController().signal })).rejects.toMatchObject({ + code: 'IMAGE_GENERATION_FAILED', + diagnostic: { providerCode: 'provider_failure' }, + }) + }) + it.each(['openai', 'builtin-personal'])('uses the selected instance %s for OpenAI Responses image generation', async (providerId) => { const getAuth = vi.fn(async () => ({ auth: { apiKey: `fixture-key-${providerId}` }, @@ -182,6 +376,38 @@ describe('openAiImageGenerationService', () => { })).rejects.toMatchObject({ code: 'PROVIDER_AUTHENTICATION_FAILED' }) }) + it('does not submit an image request when cancellation occurs during authentication', async () => { + const abort = new AbortController() + const request = vi.fn() + const service = new OpenAiImageGenerationService({ + fetch: request, + modelRuntime: { getAuth: async () => { + abort.abort() + return { auth: { apiKey: 'fixture-key' } } + } }, + }) + await expect(service.generate({ inputImages: [], model: openAiModel(), prompt: 'fixture', signal: abort.signal })).rejects.toMatchObject({ name: 'AbortError' }) + expect(request).not.toHaveBeenCalled() + }) + + it('releases a response arriving after cancellation without reading it', async () => { + const abort = new AbortController() + let cancelled = false + const response = new Response(new ReadableStream({ + cancel: () => { cancelled = true }, + })) + const service = new OpenAiImageGenerationService({ + fetch: async () => { + abort.abort() + return response + }, + modelRuntime: { getAuth: async () => ({ auth: { apiKey: 'fixture-key' } }) }, + }) + await expect(service.generate({ inputImages: [], model: openAiModel(), prompt: 'fixture', signal: abort.signal })).rejects.toMatchObject({ name: 'AbortError' }) + expect(cancelled).toBe(true) + expect(response.body?.locked).toBe(false) + }) + it('stops reading an undeclared response once the bounded size is exceeded', async () => { let cancelled = false let pulls = 0 @@ -247,3 +473,14 @@ function jwt(payload: Record): string { 'signature', ].join('.') } + +function streamedService(response: Response) { + return new OpenAiImageGenerationService({ + fetch: async () => response, + modelRuntime: { getAuth: async () => ({ auth: { apiKey: jwt({ 'https://api.openai.com/auth': { chatgpt_account_id: 'account-1' } }) } }) }, + }) +} + +function sse(event: unknown): string { + return `data: ${JSON.stringify(event)}\n\n` +} diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts new file mode 100644 index 00000000..f7467648 --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts @@ -0,0 +1,190 @@ +import type { Api, AssistantMessage, Model } from '@earendil-works/pi-ai' +import type { ServerResponse } from 'node:http' +import type { BuddyProjectedEvent } from '../../agent/events/projectPiEvent' +import type { CreatedBuddySession } from '../../agent/sessions/createBuddySession' +import type { ImageOperationEvent } from '../ImageOperationLifecycle' +import { Buffer } from 'node:buffer' +import { mkdtemp, realpath, rm } from 'node:fs/promises' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { createAssistantMessageEventStream, InMemoryCredentialStore, InMemoryModelsStore } from '@earendil-works/pi-ai' +import { ModelRuntime } from '@earendil-works/pi-coding-agent' +import { describe, expect, it, vi } from 'vitest' +import { createPiEventProjectionState, projectPiEvent } from '../../agent/events/projectPiEvent' +import { createIsolatedBuddySession } from '../../agent/sessions/__tests__/isolatedBuddySession' +import { createImageGenerationExtension } from '../imageGenerationExtension' +import { ImageGenerationService } from '../ImageGenerationService' +import { IMAGE_GENERATION_TOOL_NAME } from '../imageGenerationToolContract' +import { OpenAiImageGenerationService } from '../OpenAiImageGenerationService' + +describe('image generation through the Pi session', () => { + it.each(['incomplete', 'cancel', 'json-disconnect'] as const)('settles a mixed image batch and releases its HTTP requests on %s', async (ending) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'buddy-image-session-'))) + const codex = ending !== 'json-disconnect' + const requests: string[] = [] + const closed: string[] = [] + let pendingResponse: ServerResponse | undefined + const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]) + const server = createServer((request, response) => { + let body = '' + request.setEncoding('utf8') + request.on('data', chunk => body += chunk) + request.on('end', () => { + const prompt = JSON.parse(body).input[0].content[0].text as string + requests.push(prompt) + response.on('close', () => closed.push(prompt)) + response.writeHead(200, { 'content-type': codex ? 'text/event-stream' : 'application/json', 'x-request-id': 'req_image_fixture' }) + if (prompt === 'success') { + const result = { + id: 'response-1', + status: 'completed', + output: [{ type: 'image_generation_call', status: 'completed', result: png.toString('base64') }], + } + if (codex) + response.write(`data: ${JSON.stringify({ type: 'response.completed', response: result })}\n\n`) + else + response.end(JSON.stringify(result)) + } + else if (prompt === 'failure') { + const result = { status: 'failed', error: { code: 'provider_failure' } } + if (codex) + response.write(`data: ${JSON.stringify({ type: 'response.failed', response: result })}\n\n`) + else + response.end(JSON.stringify(result)) + } + else { + pendingResponse = response + response.write(codex ? ': waiting\n\n' : '{"status":"completed","output":[') + if (ending === 'incomplete') + response.end() + } + }) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + let created: CreatedBuddySession | undefined + let service: ImageGenerationService | undefined + let pending: Promise | undefined + try { + const credentials = new InMemoryCredentialStore() + const runtime = await ModelRuntime.create({ credentials, modelsStore: new InMemoryModelsStore(), modelsPath: null, refreshOnCreate: false }) + const address = server.address() + if (!address || typeof address === 'string') + throw new Error('Expected a loopback server') + const provider = codex ? 'openai-codex' : 'openai' + const api = codex ? 'openai-codex-responses' : 'openai-responses' + const model = { ...runtime.getModels(provider).find(model => model.api === api)!, baseUrl: `http://127.0.0.1:${address.port}` } + const token = `fixture.${Buffer.from(JSON.stringify({ 'https://api.openai.com/auth': { chatgpt_account_id: 'fixture-account' } })).toString('base64url')}.fixture` + await credentials.modify(model.provider, async () => codex + ? { type: 'oauth', access: token, refresh: 'fixture-refresh', expires: Date.now() + 3_600_000 } + : { type: 'api_key', key: 'fixture-key' }) + let turns = 0 + vi.spyOn(runtime, 'streamSimple').mockImplementation(target => assistantResponse(target, turns++ === 0)) + const source = { materializeConversationImages: async () => ({ images: [], records: [] }) } + const published: string[] = [] + service = new ImageGenerationService({ + conversationId: 'conversation-1', + cwd: root, + grants: [], + attachmentService: source, + artifactService: { + ...source, + async registerGeneratedImages(input) { + expect(input.images).toEqual([{ bytes: new Uint8Array(png), mimeType: 'image/png' }]) + published.push(input.outputPath) + return [{ id: 'artifact-1' }] + }, + }, + imageGenerationGateway: new OpenAiImageGenerationService({ + modelRuntime: runtime, + fetch: async (url, init) => { + expect(url).toBe(`${model.baseUrl}${codex ? '/codex' : ''}/responses`) + const response = await fetch(url, init) + if (ending === 'json-disconnect' && JSON.parse(String(init?.body)).input[0].content[0].text === 'pending') + pendingResponse!.destroy() + return response + }, + }), + }) + const operations: ImageOperationEvent[] = [] + service.onDidChange(event => operations.push(event)) + created = await createIsolatedBuddySession({ + agentDir: join(root, 'agent'), + canonicalRoot: root, + cwd: root, + conversationsDirectory: join(root, 'conversations'), + conversationId: 'conversation-1', + branchId: 'branch-1', + approvalPolicy: 'policy', + executionProfile: 'workspace_write', + model, + modelRuntime: runtime, + thinkingLevel: 'off', + resources: { skillReadRoots: [], skillReferences: [], approvedSkills: [], context: { agentsFiles: [], diagnostics: [] }, directoryContext: '', revision: 'empty' }, + inProcessExtensions: [createImageGenerationExtension({ getRunId: () => 'run-1', service })], + }) + created.session.setActiveToolsByName([IMAGE_GENERATION_TOOL_NAME]) + const state = createPiEventProjectionState({ canonicalRoot: root }) + const projected: BuddyProjectedEvent[] = [] + created.session.subscribe(event => projected.push(...projectPiEvent(event, state).events)) + pending = created.session.prompt('Generate the fixture images') + void pending.catch(() => {}) + if (ending === 'cancel') { + await vi.waitFor(() => { + expect(requests).toContain('pending') + expect(operations.filter(event => event.phase === 'settled')).toHaveLength(2) + }) + await created.session.abort() + } + await pending + await service.dispose() + expect(requests.toSorted()).toEqual(['failure', 'pending', 'success']) + await vi.waitFor(() => expect(closed.toSorted()).toEqual(requests.toSorted())) + expect(published).toEqual(['success.png']) + expect(operations.filter(event => event.phase === 'settled').map(event => event.outcome).toSorted()).toEqual( + ending === 'cancel' ? ['cancelled', 'completed', 'failed'] : ['completed', 'failed', 'failed'], + ) + expect(projected.filter(event => event.type === 'tool.completed')).toEqual(expect.arrayContaining([ + expect.objectContaining({ payload: expect.objectContaining({ toolCallId: 'image-success', isError: false, presentation: expect.objectContaining({ card: 'image', status: 'completed', generatedCount: 1 }) }) }), + expect.objectContaining({ payload: expect.objectContaining({ toolCallId: 'image-failure', isError: true, presentation: expect.objectContaining({ card: 'image', status: 'failed', generatedCount: 0 }) }) }), + expect.objectContaining({ payload: expect.objectContaining({ toolCallId: 'image-pending', isError: true, presentation: expect.objectContaining({ card: 'image', status: 'failed' }) }) }), + ])) + expect(projected.filter(event => event.type === 'output.produced')).toHaveLength(1) + expect(state.toolCalls.size).toBe(0) + if (ending !== 'cancel') { + expect(created.session.messages.filter(message => message.role === 'toolResult').at(-1)).toMatchObject({ + isError: true, + details: { code: 'IMAGE_GENERATION_INCOMPLETE', diagnostic: { requestId: 'req_image_fixture' } }, + content: [{ type: 'text', text: expect.stringContaining('do not automatically repeat') }], + }) + } + } + finally { + server.closeAllConnections() + await created?.session.abort() + await pending?.catch(() => {}) + await service?.dispose() + await created?.shutdown('quit') + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) + await rm(root, { recursive: true, force: true }) + } + }) +}) + +function assistantResponse(model: Model, tools: boolean) { + const message: AssistantMessage = { + api: model.api, + model: model.id, + provider: model.provider, + role: 'assistant', + timestamp: Date.now(), + stopReason: tools ? 'toolUse' : 'stop', + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + content: tools + ? ['success', 'failure', 'pending'].map(prompt => ({ type: 'toolCall', id: `image-${prompt}`, name: IMAGE_GENERATION_TOOL_NAME, arguments: { prompt, outputPath: `${prompt}.png` } })) + : [{ type: 'text', text: 'Finished' }], + } + const stream = createAssistantMessageEventStream() + queueMicrotask(() => stream.push({ type: 'done', reason: tools ? 'toolUse' : 'stop', message })) + return stream +} diff --git a/apps/buddy/service/src/images/imageGenerationExtension.ts b/apps/buddy/service/src/images/imageGenerationExtension.ts index 4a220a9b..2589ecdb 100644 --- a/apps/buddy/service/src/images/imageGenerationExtension.ts +++ b/apps/buddy/service/src/images/imageGenerationExtension.ts @@ -103,9 +103,12 @@ function imageToolFailure( const diagnosticText = diagnostic ? Object.entries(diagnostic).map(([key, value]) => `${key}=${value}`).join(', ') : '' + const recovery = code === 'IMAGE_GENERATION_INCOMPLETE' + ? '. The provider result is unknown; do not automatically repeat this image request.' + : '' return { content: [{ - text: `Lexora Buddy image generation failed: ${code}${diagnosticText ? ` (${diagnosticText})` : ''}`, + text: `Lexora Buddy image generation failed: ${code}${diagnosticText ? ` (${diagnosticText})` : ''}${recovery}`, type: 'text' as const, }], details: {