From 8e9a2fd2d25d1b20c0afd184fc7d3bc1d9242d78 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:00:22 +0800 Subject: [PATCH 01/11] refactor(gateway): generalize the Responses server-tool shim into a hosted-capability runtime The multi-agent Responses beta declares server-hosted execution as a top-level `multi_agent` request field rather than a `tools[]` entry, and lowers to several injected functions rather than one replaced hosted declaration. Three capabilities the shim did not have: - Multiple base tool names per registration, resolved against every name claimed so far so two capabilities cannot collide on the wire. - Activation and function injection independent of `tools[]`. - A terminal finalizer that publishes items after the last upstream turn and before the synthesized terminal envelope. Registrations also receive a capability runtime whose `runTurn` drives one upstream turn against an explicit payload while snapshotting and restoring the outer loop's `ctx.payload`, so nested agent histories cannot leak into the root's next request body. Web search and image generation move onto the new shape unchanged. --- .../server-tools/image-generation_test.ts | 30 +++- .../interceptors/server-tool-shim.ts | 149 +++++++++++++++--- .../server-tools/image-generation.ts | 8 +- .../interceptors/server-tools/web-search.ts | 4 +- packages/protocols/src/responses/index.ts | 12 ++ 5 files changed, 174 insertions(+), 29 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index c6f3c0604..ad1d7fa1b 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -18,6 +18,7 @@ import { synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from '../../../../../../src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts'; +import type { ServerToolCapabilityRuntime } from '../../../../../../src/data-plane/chat/responses/interceptors/server-tool-shim.ts'; import type { ResponsesInvocation } from '../../../../../../src/data-plane/chat/responses/interceptors/types.ts'; import { initRepo } from '../../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; @@ -50,6 +51,12 @@ const makeCtx = (payload: Partial): ResponsesInvocation => ({ }); const gatewayCtx = () => mockChatGatewayCtx({ wantsStream: true }); +// The image-generation capability never drives a nested turn; these suites +// exercise preparation and dispatch only. +const capabilityRuntime = (): ServerToolCapabilityRuntime => ({ + runTurn: () => { throw new Error('image generation must not run a nested turn'); }, +}); + beforeEach(() => { initRepo(new InMemoryRepo()); }); @@ -620,6 +627,7 @@ test('imageGenerationServerTool accepts GIF edit input for local WebP transcodin const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation' }], input: [imageMessage('image/gif')] }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); }); @@ -639,11 +647,12 @@ test('imageGenerationServerTool fetches repeated remote edit sources once', asyn const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); assertEquals(urls, ['https://example.com/source.png']); - const transformed = result.transformItems?.(input, SHIM_TOOL_NAME); + const transformed = result.transformItems?.(input, new Map([[SHIM_TOOL_NAME, SHIM_TOOL_NAME]])); assert(transformed?.[0].type === 'message' && Array.isArray(transformed[0].content)); const first = transformed[0].content[0]; assert(first.type === 'input_image'); @@ -664,6 +673,7 @@ test('imageGenerationServerTool counts and fetches one URL once when shared by a input: [imageInputContainers.message({ type: 'input_image', image_url: imageUrl, detail: 'auto' })], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); @@ -686,6 +696,7 @@ test('imageGenerationServerTool enforces the aggregate limit across distinct suc ], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); @@ -721,6 +732,7 @@ test.each([ input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.invalid/source.png', detail: 'auto' })], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); @@ -740,6 +752,7 @@ test('imageGenerationServerTool mirrors the native error for a fetched non-image input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.com/readme.txt', detail: 'auto' })], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); @@ -764,6 +777,7 @@ test('imageGenerationServerTool follows redirects before materializing a remote input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.com/redirect', detail: 'auto' })], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); @@ -775,6 +789,7 @@ test('imageGenerationServerTool materializes a remote mask and mirrors its downl const accepted = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action: 'generate', input_image_mask: { image_url: 'https://example.com/mask.png' } }] }), gatewayCtx(), + capabilityRuntime(), ); assert(accepted.type === 'active'); @@ -782,6 +797,7 @@ test('imageGenerationServerTool materializes a remote mask and mirrors its downl const rejected = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit', input_image_mask: { image_url: 'https://example.com/missing.png' } }] }), gatewayCtx(), + capabilityRuntime(), ); assert(rejected.type === 'invalid-request'); assertEquals(rejected.message, 'There was an issue with your request. Please check your inputs and try again'); @@ -805,6 +821,7 @@ test('imageGenerationServerTool validates a remote mask before applying the Flow }], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); @@ -818,6 +835,7 @@ test('imageGenerationServerTool reports a malformed remote mask at its native pa const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', input_image_mask: { file_id: 'file_123', image_url: 'https://' } }] }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); assertEquals(result.message, "Invalid 'tools[0].input_image_mask.image_url'. Expected a valid URL, but got a value with an invalid format."); @@ -831,6 +849,7 @@ test.each(unresolvableSourceCases)( const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action }], input: [input] }), gatewayCtx(), + capabilityRuntime(), ); if (source !== 'malformed' && action === 'generate') { assert(result.type === 'active'); @@ -862,6 +881,7 @@ test.each(['generate', 'auto', 'edit'] as const)('imageGenerationServerTool acce }], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); }); @@ -894,6 +914,7 @@ test('imageGenerationServerTool accepts an auto-action mask with an edit source' input: [imageMessage('image/png')], }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); }); @@ -902,6 +923,7 @@ test('imageGenerationServerTool rejects explicit edit without an input image', a const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }] }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'invalid-request'); assertEquals(result.message, "ImageGenTool action 'edit' requires an image, mask, or previous context"); @@ -914,6 +936,7 @@ test('imageGenerationServerTool accepts webp input for editing', async () => { const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation' }], input: [imageMessage('image/webp')] }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); }); @@ -924,13 +947,14 @@ test('imageGenerationServerTool ignores input format when action is generate', a const result = await imageGenerationServerTool( makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }], input: [imageMessage('image/gif')] }), gatewayCtx(), + capabilityRuntime(), ); assert(result.type === 'active'); }); test('image dispatcher exposes a newly introduced invalid edit source as an invariant failure', async () => { const invocation = makeCtx({ tools: [{ type: 'image_generation', action: 'auto' }] }); - const result = await imageGenerationServerTool(invocation, gatewayCtx()); + const result = await imageGenerationServerTool(invocation, gatewayCtx(), capabilityRuntime()); assert(result.type === 'active' && result.hosted !== undefined); invocation.payload = { @@ -952,7 +976,7 @@ test('image dispatcher exposes a newly introduced invalid edit source as an inva // ── imageGenerationServerTool: per-response dispatch budget (B) ── test('image dispatch budget caps real backend calls per response, not ReAct turns', async () => { - const result = await imageGenerationServerTool(makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }] }), gatewayCtx()); + const result = await imageGenerationServerTool(makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }] }), gatewayCtx(), capabilityRuntime()); assert(result.type === 'active' && result.hosted !== undefined); const dispatch = result.hosted.dispatcher; const intercepted = { callId: 'c', name: SHIM_TOOL_NAME, argumentsJson: '{}', arguments: { prompt: 'x' } }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index 26607dd92..7e7d215b2 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -98,6 +98,11 @@ export type ServerToolDispatcher = (args: { loopState: ServerToolLoopState; }) => ServerToolResultSlot[]; +// Base name → the name actually injected into `tools`. The two differ only +// when the client already declared a function of that name, in which case +// `resolveServerToolName` picks a free suffix. +export type ResolvedServerToolNames = ReadonlyMap; + // Keep hosted matching, function injection, and dispatch atomic so a // registration cannot silently omit part of a server-tool family. export interface ServerToolHostedDispatch { @@ -107,6 +112,26 @@ export interface ServerToolHostedDispatch { dispatcher: ServerToolDispatcher; } +// A capability whose activation does not come from a `tools[]` entry — the +// Responses beta declares server-hosted multi-agent execution as a top-level +// `multi_agent` request field, and its lowering injects several functions at +// once rather than replacing one hosted declaration. +export interface ServerToolInjectedDispatch { + buildFunctionTools: (toolNames: ResolvedServerToolNames) => ResponsesFunctionTool[]; + dispatcher: ServerToolDispatcher; +} + +export interface ServerToolFinalOutputItem { + id: string; + item: ServerToolOutputItem; +} + +// Emitted after the last upstream turn and before the synthesized terminal +// envelope, so a capability can publish the state a stateless client must +// carry back on its next request. `kind` lets a finalizer render the paused +// versus finished shape differently. +export type ServerToolFinalizer = (kind: SynthesizedTerminal['kind']) => Promise; + export type ServerToolPrepareResult = | { type: 'inactive' } // `errorType` / `code` override the envelope for tools that emulate an @@ -115,27 +140,49 @@ export type ServerToolPrepareResult = | { type: 'invalid-request'; message: string; param: string | null; errorType?: string; code?: string | null } | { type: 'active'; - baseToolName: string; + baseToolNames: readonly string[]; // History rewrite, applied whether or not the tool is hosted this // turn so items echoed from a previous turn's output become // upstream-readable even on a request that no longer declares the // hosted tool. - transformItems?: (items: ResponsesInputItem[], toolName: string) => ResponsesInputItem[]; + transformItems?: (items: ResponsesInputItem[], toolNames: ResolvedServerToolNames) => ResponsesInputItem[]; // Present only when the request declares this hosted tool; absent for // replay-only activation. hosted?: ServerToolHostedDispatch; + injected?: ServerToolInjectedDispatch; + finalizeOutput?: ServerToolFinalizer; }; -export type ServerToolRegistration = (invocation: ResponsesInvocation, gatewayCtx: ChatGatewayCtx) => ServerToolPrepareResult | Promise; +// Handed to every registration so a capability that drives its own nested +// agent loops can reach the upstream without owning the interceptor chain. +// `runTurn` snapshots and restores the outer loop's `ctx.payload`, so a nested +// turn cannot leak its history into the turn the shim runs next. +export interface ServerToolCapabilityRuntime { + runTurn: (payload: CanonicalResponsesPayload) => Promise>>; +} + +export type ServerToolRegistration = ( + invocation: ResponsesInvocation, + gatewayCtx: ChatGatewayCtx, + runtime: ServerToolCapabilityRuntime, +) => ServerToolPrepareResult | Promise; type ActiveServerTool = Extract & { - toolName: string; + toolNames: ResolvedServerToolNames; // Absent only for replay activation; otherwise drives `tools` echo restore. canonicalHostedTool: ResponsesHostedTool | undefined; // Captures the exact forced choice shape before request rewriting. originalToolChoice: Exclude | undefined; }; +// The single name a hosted family occupies. Hosted registrations declare +// exactly one base name because they replace one `tools[]` declaration. +const hostedToolName = (entry: ActiveServerTool): string => { + const name = entry.toolNames.get(entry.baseToolNames[0]); + if (name === undefined) throw new Error('Hosted server-tool registration resolved no function name'); + return name; +}; + // How a single upstream turn ended, as observed while consuming its // stream. Carries the raw upstream `response` for failed/incomplete so // the loop can lift the upstream `error` / `incomplete_details`, plus a @@ -231,7 +278,7 @@ const rewriteHostedToolChoice = ( if (toolChoice == null || typeof toolChoice === 'string') return toolChoice; for (const entry of active) { if (entry.hosted === undefined) continue; - if (entry.hosted.hostedTypes.includes(toolChoice.type)) return { type: 'function', name: entry.toolName }; + if (entry.hosted.hostedTypes.includes(toolChoice.type)) return { type: 'function', name: hostedToolName(entry) }; } return toolChoice; }; @@ -260,7 +307,7 @@ const restoreEchoedTools = ( return tools.map(tool => { if (tool.type !== 'function') return tool; for (const entry of active) { - if (entry.canonicalHostedTool !== undefined && tool.name === entry.toolName) { + if (entry.canonicalHostedTool !== undefined && tool.name === hostedToolName(entry)) { return entry.canonicalHostedTool; } } @@ -448,7 +495,7 @@ const transformServerToolItems = ( ): ResponsesInputItem[] => { let next = items; for (const entry of active) { - if (entry.transformItems !== undefined) next = entry.transformItems(next, entry.toolName); + if (entry.transformItems !== undefined) next = entry.transformItems(next, entry.toolNames); } return next; }; @@ -849,6 +896,33 @@ async function* materializeServerToolItems( } } +async function* emitFinalOutputItems( + active: readonly ActiveServerTool[], + merge: MergeState, + kind: SynthesizedTerminal['kind'], +): AsyncGenerator, void> { + for (const entry of active) { + if (entry.finalizeOutput === undefined) continue; + for (const final of await entry.finalizeOutput(kind)) { + const outputIndex = merge.outputIndex++; + const item = attachServerToolItemId(final.item, final.id); + yield eventFrame({ + type: 'response.output_item.added', + output_index: outputIndex, + item, + sequence_number: merge.sequenceNumber++, + } as ResponsesStreamEvent); + yield eventFrame({ + type: 'response.output_item.done', + output_index: outputIndex, + item, + sequence_number: merge.sequenceNumber++, + } as ResponsesStreamEvent); + merge.accumulatedOutput.set(outputIndex, item); + } + } +} + async function* runMultiTurnLoop(args: { ctx: ResponsesInvocation; run: InterceptorRun>>; @@ -875,11 +949,13 @@ async function* runMultiTurnLoop(args: { if (turn.terminalStatus.kind === 'failed') { if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); + yield* emitFinalOutputItems(active, merge, 'failed'); yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); return; } if (turn.terminalStatus.kind === 'incomplete') { if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); + yield* emitFinalOutputItems(active, merge, 'incomplete'); yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); return; } @@ -891,12 +967,14 @@ async function* runMultiTurnLoop(args: { return; } if (!executedShim && !turn.sawClientToolCall) { + yield* emitFinalOutputItems(active, merge, 'completed'); yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; } yield* materializeServerToolItems(turn.dispatched, merge, store); if (turn.sawClientToolCall) { + yield* emitFinalOutputItems(active, merge, 'completed'); yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; } @@ -954,28 +1032,58 @@ export const withResponsesServerToolShim = ( ): ResponsesInterceptor => async (ctx, gatewayCtx, run) => { const active: ActiveServerTool[] = []; + // Nested turns borrow the chain but must leave the outer loop's payload + // untouched: `runMultiTurnLoop` derives every subsequent turn from + // `ctx.payload`, so a child agent's history would otherwise become the + // root's next request body. + const runtime: ServerToolCapabilityRuntime = { + runTurn: async payload => { + const outerPayload = ctx.payload; + ctx.payload = payload; + try { + return await run(); + } finally { + ctx.payload = outerPayload; + } + }, + }; + for (const prepareServerTool of registrations) { - const prepared = await prepareServerTool(ctx, gatewayCtx); + const prepared = await prepareServerTool(ctx, gatewayCtx, runtime); if (prepared.type === 'inactive') continue; if (prepared.type === 'invalid-request') { return invalidRequestEnvelope(prepared.message, prepared.param, prepared.code, prepared.errorType); } const currentTools = Array.isArray(ctx.payload.tools) ? ctx.payload.tools : []; - const toolName = resolveServerToolName(prepared.baseToolName, currentTools); - const { hosted } = prepared; + // Resolve every base name against the tools declared so far, including + // functions an earlier registration already injected, so two capabilities + // cannot land on the same wire name. + const toolNames = new Map(); + const claimed: ResponsesTool[] = [...currentTools]; + for (const baseName of prepared.baseToolNames) { + const resolved = resolveServerToolName(baseName, claimed); + toolNames.set(baseName, resolved); + claimed.push({ type: 'function', name: resolved, parameters: {}, strict: false } as ResponsesFunctionTool); + } + const { hosted, injected } = prepared; let canonicalHostedTool: ResponsesHostedTool | undefined = undefined; + let nextTools = currentTools; if (hosted !== undefined) { - const rewrite = rewriteToolsForHostedShim(currentTools, hosted, toolName); + const rewrite = rewriteToolsForHostedShim(nextTools, hosted, toolNames.get(prepared.baseToolNames[0])!); canonicalHostedTool = rewrite.canonicalHostedTool; - ctx.payload = { ...ctx.payload, tools: rewrite.rewritten }; + nextTools = rewrite.rewritten; } + if (injected !== undefined) { + nextTools = [...nextTools, ...injected.buildFunctionTools(toolNames)]; + } + if (nextTools !== currentTools) ctx.payload = { ...ctx.payload, tools: nextTools }; const originalToolChoice = hosted !== undefined && typeof ctx.payload.tool_choice === 'object' && ctx.payload.tool_choice !== null && hosted.hostedTypes.includes(ctx.payload.tool_choice.type) ? ctx.payload.tool_choice : undefined; - active.push({ ...prepared, toolName, canonicalHostedTool, originalToolChoice }); + active.push({ ...prepared, toolNames, canonicalHostedTool, originalToolChoice }); } if (active.length === 0) return await run(); @@ -989,14 +1097,15 @@ export const withResponsesServerToolShim = ( const nextInput = transformServerToolItems(canonicalInput, active); if (nextInput !== canonicalInput) ctx.payload = { ...ctx.payload, input: nextInput }; - const hostedActive = active.filter( - (entry): entry is ActiveServerTool & { hosted: ServerToolHostedDispatch } => - entry.hosted !== undefined, - ); - if (hostedActive.length === 0) return await run(); - const dispatchers = new Map(); - for (const entry of hostedActive) dispatchers.set(entry.toolName, entry.hosted.dispatcher); + for (const entry of active) { + if (entry.hosted !== undefined) dispatchers.set(hostedToolName(entry), entry.hosted.dispatcher); + if (entry.injected !== undefined) { + for (const resolved of entry.toolNames.values()) dispatchers.set(resolved, entry.injected.dispatcher); + } + } + if (dispatchers.size === 0) return await run(); + const loopState: ServerToolLoopState = { iterationCount: 1, remainingToolCalls: typeof ctx.payload.max_tool_calls === 'number' ? ctx.payload.max_tool_calls : undefined, diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 3cad91334..d29d8782e 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -1407,8 +1407,8 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati // the upstream can read them, but there is no hosted tool to dispatch. return { type: 'active', - baseToolName: SHIM_TOOL_NAME, - transformItems: transformInputItemsForImageGeneration, + baseToolNames: [SHIM_TOOL_NAME], + transformItems: (items, toolNames) => transformInputItemsForImageGeneration(items, toolNames.get(SHIM_TOOL_NAME)!), }; } @@ -1487,8 +1487,8 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati return { type: 'active', - baseToolName: SHIM_TOOL_NAME, - transformItems: transformInputItemsForImageGeneration, + baseToolNames: [SHIM_TOOL_NAME], + transformItems: (items, toolNames) => transformInputItemsForImageGeneration(items, toolNames.get(SHIM_TOOL_NAME)!), hosted: { hostedTypes: ['image_generation'], canonicalize: canonicalizeImageGenerationTool, diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/web-search.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/web-search.ts index 41c323b78..f9500b492 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/web-search.ts @@ -602,8 +602,8 @@ export const webSearchServerTool: ServerToolRegistration = async (invocation, ga return { type: 'active', - baseToolName: SHIM_TOOL_NAME, - transformItems: (items, toolName) => transformInputItemsForWebSearch(items, toolName, id => gatewayCtx.store.getPrivatePayload(id)), + baseToolNames: [SHIM_TOOL_NAME], + transformItems: (items, toolNames) => transformInputItemsForWebSearch(items, toolNames.get(SHIM_TOOL_NAME)!, id => gatewayCtx.store.getPrivatePayload(id)), ...(hasHostedWebSearch ? { hosted: { diff --git a/packages/protocols/src/responses/index.ts b/packages/protocols/src/responses/index.ts index 42aaa364f..9ef610016 100644 --- a/packages/protocols/src/responses/index.ts +++ b/packages/protocols/src/responses/index.ts @@ -60,6 +60,18 @@ export interface ResponsesPayload { prompt_cache_retention?: ResponsesPromptCacheRetention | null; safety_identifier?: string | null; service_tier?: 'default' | 'auto' | 'flex' | 'priority' | 'scale' | (string & {}) | null; + multi_agent?: ResponsesMultiAgentConfig | null; +} + +// Server-hosted multi-agent execution. Unlike every other hosted capability +// this is a top-level request field rather than a `tools[]` entry — the beta +// `BetaTool` union carries no multi-agent variant — and the upstream also +// expects the `OpenAI-Beta: responses_multi_agent=v1` header alongside it. +// https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L12032-L12049 +// https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L11964-L11967 +export interface ResponsesMultiAgentConfig { + enabled?: boolean | null; + max_concurrent_subagents?: number | null; } export type ResponsesInputItem = From f6eb1687668ef20f69796b3ac20ad89b5464c46a Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:03:45 +0800 Subject: [PATCH 02/11] refactor(gateway): let a server-tool slot publish items mid-execution A slot's deferred lifecycle previously yielded only lifecycle events for its own item, and its terminal could not stop the response. Both are required by a capability that orchestrates other agents: the items those agents produce belong between the capability's call and call-output records, and an agent reaching a client-owned function tool must end the turn exactly as a client tool call the model made directly does. `run()` now yields a discriminated `ServerToolProgress`, and its terminal carries `endResponse`. --- .../interceptors/server-tool-shim_test.ts | 7 ++- .../interceptors/server-tool-shim.ts | 58 +++++++++++++++---- .../server-tools/image-generation.ts | 6 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts index 6346b6cc3..72d5d4c26 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts @@ -5902,8 +5902,8 @@ test('ServerToolResultSlot run() yields mid events in order then returns the ter startItem: { type: 'image_generation_call', status: 'in_progress' }, startEvents: [{ type: 'response.image_generation_call.in_progress' }], async *run() { - yield { type: 'response.image_generation_call.partial_image', partial_image_index: 0 }; - yield { type: 'response.image_generation_call.partial_image', partial_image_index: 1 }; + yield { progress: 'event', event: { type: 'response.image_generation_call.partial_image', partial_image_index: 0 } }; + yield { progress: 'event', event: { type: 'response.image_generation_call.partial_image', partial_image_index: 1 } }; return { item: { type: 'image_generation_call', status: 'completed' }, endEvents: [{ type: 'response.image_generation_call.completed' }], @@ -5915,7 +5915,8 @@ test('ServerToolResultSlot run() yields mid events in order then returns the ter const gen = slot.run(); let step = await gen.next(); while (!step.done) { - mid.push(step.value.partial_image_index); + assert(step.value.progress === 'event'); + mid.push(step.value.event.partial_image_index); step = await gen.next(); } assertEquals(mid, [0, 1]); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index 7e7d215b2..c946be80b 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -62,20 +62,34 @@ export interface ServerToolTerminal { * reads it back to reconstruct the full IR. */ privatePayload?: unknown; + /** + * Set when the capability cannot continue without the client: some agent it + * drives called a client-owned function tool, so the response must end at + * this turn exactly as it would for a client tool call the model made + * directly. + */ + endResponse?: boolean; } +// What a slot's deferred lifecycle yields on its way to the terminal item. +// `event` stamps an intermediate lifecycle frame onto the slot's own item — +// a progressively-rendered image, say. `item` publishes a whole additional +// output item at the point execution reached, which is how a capability that +// orchestrates other agents interleaves their items between its own call and +// call-output records. +export type ServerToolProgress = + | { progress: 'event'; event: ServerToolLifecycleEvent } + | { progress: 'item'; id: string; item: ServerToolOutputItem }; + export interface ServerToolResultSlot { id: string; startItem: ServerToolOutputItem; startEvents: readonly ServerToolLifecycleEvent[]; // The deferred portion of a slot's lifecycle, driven at materialization - // time. It yields any intermediate lifecycle events as they arrive — e.g. - // progressively-rendered `image_generation_call.partial_image` frames a - // streamed backend delivers over the course of the call — and returns the - // terminal item plus its closing events (and an optional server-only - // `privatePayload`). A tool with no progressive output simply yields nothing - // and returns immediately. - run: () => AsyncGenerator; + // time. It yields progress as it arrives and returns the terminal item plus + // its closing events (and an optional server-only `privatePayload`). A tool + // with no progressive output simply yields nothing and returns immediately. + run: () => AsyncGenerator; } export type ServerToolOutputItem = { type: string; id?: string; [key: string]: unknown }; @@ -878,13 +892,33 @@ async function* materializeServerToolItems( dispatched: ReadonlyArray<{ slots: DispatchedServerToolSlot[] }>, merge: MergeState, store: StatefulResponsesStore, -): AsyncGenerator, void> { +): AsyncGenerator, boolean> { + let endResponse = false; for (const d of dispatched) { for (const { slot, outputIndex } of d.slots) { const lifecycle = slot.run(); let step = await lifecycle.next(); while (!step.done) { - yield stampServerToolEvent(merge, outputIndex, slot.id, step.value); + const progress = step.value; + if (progress.progress === 'event') { + yield stampServerToolEvent(merge, outputIndex, slot.id, progress.event); + } else { + const interleavedIndex = merge.outputIndex++; + const item = attachServerToolItemId(progress.item, progress.id); + yield eventFrame({ + type: 'response.output_item.added', + output_index: interleavedIndex, + item, + sequence_number: merge.sequenceNumber++, + } as ResponsesStreamEvent); + yield eventFrame({ + type: 'response.output_item.done', + output_index: interleavedIndex, + item, + sequence_number: merge.sequenceNumber++, + } as ResponsesStreamEvent); + merge.accumulatedOutput.set(interleavedIndex, item); + } step = await lifecycle.next(); } // Register private dispatcher state under the emitted item id so output @@ -892,8 +926,10 @@ async function* materializeServerToolItems( // it on the next loop turn. store.registerPrivatePayload(slot.id, step.value.privatePayload); yield* serverToolEndFrames(merge, outputIndex, slot, step.value); + if (step.value.endResponse === true) endResponse = true; } } + return endResponse; } async function* emitFinalOutputItems( @@ -972,8 +1008,8 @@ async function* runMultiTurnLoop(args: { return; } - yield* materializeServerToolItems(turn.dispatched, merge, store); - if (turn.sawClientToolCall) { + const capabilityEndedResponse = yield* materializeServerToolItems(turn.dispatched, merge, store); + if (turn.sawClientToolCall || capabilityEndedResponse) { yield* emitFinalOutputItems(active, merge, 'completed'); yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index d29d8782e..aa84c63c8 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -5,7 +5,7 @@ import { stampUpstreamCallStart, type AttemptState } from '../../../../shared/ga import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; import { recordTokenUsage, tokenUsageFromImagesBody } from '../../../../shared/telemetry/usage.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; -import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; +import type { ServerToolOutputItem, ServerToolProgress, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; import { parseSSEStream } from '@floway-dev/protocols/common'; import { @@ -1239,7 +1239,7 @@ const streamImageGeneration = ( isEdit: boolean, sources: readonly ImageSource[], state: ShimState, -) => async function* (): AsyncGenerator { +) => async function* (): AsyncGenerator { const resolved = await resolveImageCandidate(isEdit, state); if (!resolved.ok) return imageTerminal(prompt, action, { ok: false, error: resolved.error }); const { provider, fetcher } = resolved.candidate; @@ -1301,7 +1301,7 @@ const streamImageGeneration = ( const signal = parseImageStreamEvent(frame.data); if (signal === null) continue; if (signal.kind === 'partial') { - yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; + yield { progress: 'event', event: { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo } }; } else if (signal.kind === 'completed') { finalB64 = signal.b64; finalEcho = signal.echo; From ae26eb0cb99fb18d897467a75cb3c976659969b2 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:11:44 +0800 Subject: [PATCH 03/11] feat(gateway): host the Responses beta multi_agent capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The beta declares server-hosted multi-agent execution as a top-level `multi_agent` request field and records it as multi_agent_call / multi_agent_call_output / agent_message items. No non-OpenAI upstream in the catalog serves it, and Messages and Chat Completions have no orchestration wire at all, so the gateway hosts it. The capability lowers to six hidden function tools whose per-action schemas are transcribed from live OpenAI collaboration traffic — the beta publishes the item shapes but keeps `arguments` opaque and offers no per-action schema. It then runs the agent tree itself, attributing each subagent's items with `agent.agent_name`, and lowers replayed orchestration history back into the hidden call/output pairs the model originally saw. Scheduling is eager and serial rather than asynchronous: without a coordinator there is nowhere for a subagent to keep running once the response ends, and a nested turn borrows the interceptor chain, so two may not be in flight at once. Every action therefore reports a complete result, and the tree is always quiescent when state is captured. --- .../chat/responses/interceptors/index.ts | 2 + .../server-tools/multi-agent/checkpoint.ts | 162 +++++++ .../server-tools/multi-agent/index.ts | 244 ++++++++++ .../server-tools/multi-agent/session.ts | 450 ++++++++++++++++++ .../server-tools/multi-agent/wire.ts | 149 ++++++ .../__tests__/responses/item-id_test.ts | 2 + packages/protocols/src/responses/item-id.ts | 5 + packages/provider-azure/src/defaults.ts | 1 + packages/provider-claude-code/src/defaults.ts | 1 + packages/provider-codex/src/defaults.ts | 3 + packages/provider-copilot/src/defaults.ts | 1 + packages/provider-custom/src/defaults.ts | 1 + packages/provider-ollama/src/defaults.ts | 1 + packages/provider/src/flags.ts | 5 + 14 files changed, 1027 insertions(+) create mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts index fa4173774..81a3c5819 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/index.ts @@ -4,6 +4,7 @@ import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on- import { withCyberPolicyRetried } from './retry-cyber-policy.ts'; import { withResponsesServerToolShim } from './server-tool-shim.ts'; import { imageGenerationServerTool } from './server-tools/image-generation.ts'; +import { multiAgentServerTool } from './server-tools/multi-agent/index.ts'; import { webSearchServerTool } from './server-tools/web-search.ts'; import { withPromptCacheKeyStripped } from './strip-prompt-cache-key.ts'; import type { ResponsesInterceptor } from './types.ts'; @@ -45,6 +46,7 @@ export const responsesInterceptors: readonly ResponsesInterceptor[] = [ withResponsesServerToolShim([ webSearchServerTool, imageGenerationServerTool, + multiAgentServerTool, ]), withCyberPolicyRetried, withReasoningDisabledOnForcedToolChoice, diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts new file mode 100644 index 000000000..9b487caa5 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts @@ -0,0 +1,162 @@ +// Client-carried multi-agent checkpoint. +// +// Floway runs the whole agent tree inside one response. When any agent reaches +// a client-owned function tool the response has to end, and the tree's hidden +// state — child transcripts, statuses, which agent owns the outstanding call — +// has nowhere to live: the gateway keeps no per-run coordinator, and +// `previous_response_id` reconstruction would put that state back on the +// server. So the state travels with the client. +// +// The carrier is an `agent_message` whose single content part is an +// `encrypted_content` slot. That part is a documented, round-trippable opaque +// string on both input and output, which is exactly what a checkpoint needs: +// https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L6549-L6805 +// +// The value is raw `base64url(JSON)` — no prefix, no signature, no +// encryption. `encrypted_content` names a slot, not a cryptographic +// requirement, and the compact shim already round-trips unprefixed +// base64url-JSON through the same kind of slot. Recognition is structural: +// decode, parse, then match the discriminator and version. Anything that +// fails any step is a foreign upstream blob and is left untouched. +// +// Because the client can read and rewrite the checkpoint, nothing in it may +// be trusted as authority. It carries no credentials, no upstream identity, +// no routing decision, and no budget the runtime reads as already-spent +// permission; model, tools, and concurrency limits are recomputed from the +// current request every time, and the counters below only ever tighten. + +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../../../shared/base64url-json.ts'; +import { isJsonObject } from '../../../../../../shared/json-helpers.ts'; +import type { AgentStatus } from './wire.ts'; +import { createRandomResponsesItemId, type ResponsesInputItem, type ResponsesOutputAgentMessageItem } from '@floway-dev/protocols/responses'; + +export const MULTI_AGENT_CHECKPOINT_TYPE = 'floway.multi_agent_checkpoint'; +export const MULTI_AGENT_CHECKPOINT_VERSION = 1; + +// Bounds on what a client may hand back. They exist so a hostile or corrupted +// capsule cannot turn one request into unbounded upstream work or memory. +export const MAX_CHECKPOINT_AGENTS = 32; +export const MAX_CHECKPOINT_BYTES = 1_000_000; + +export interface CheckpointAgent { + path: string; + status: AgentStatus; + // The agent's own transcript. Hidden from the client's visible history — + // only the agent's messages to others surface as `agent_message` items — so + // it has to survive here or the agent restarts from nothing. + history: ResponsesInputItem[]; + // Set while the agent is parked on a client-owned function tool. + pendingClientCallId?: string; +} + +export interface MultiAgentCheckpoint { + type: typeof MULTI_AGENT_CHECKPOINT_TYPE; + version: typeof MULTI_AGENT_CHECKPOINT_VERSION; + agents: CheckpointAgent[]; + // Monotone across a run: resumes may only ever consume more of the budget, + // never restore it. + turnsSpent: number; +} + +const isAgentStatus = (value: unknown): value is AgentStatus => { + if (value === 'running' || value === 'interrupted') return true; + if (!isJsonObject(value)) return false; + const keys = Object.keys(value); + if (keys.length !== 1) return false; + if (keys[0] === 'completed') return typeof (value as { completed: unknown }).completed === 'string'; + if (keys[0] === 'errored') return typeof (value as { errored: unknown }).errored === 'string'; + return false; +}; + +const isCheckpointAgent = (value: unknown): value is CheckpointAgent => { + if (!isJsonObject(value)) return false; + const agent = value as Record; + if (typeof agent.path !== 'string' || agent.path.length === 0) return false; + if (!isAgentStatus(agent.status)) return false; + if (!Array.isArray(agent.history)) return false; + if (!agent.history.every(item => isJsonObject(item) && typeof (item as { type?: unknown }).type === 'string')) return false; + if (agent.pendingClientCallId !== undefined && typeof agent.pendingClientCallId !== 'string') return false; + return true; +}; + +export const isMultiAgentCheckpoint = (value: unknown): value is MultiAgentCheckpoint => { + if (!isJsonObject(value)) return false; + const candidate = value as Record; + if (candidate.type !== MULTI_AGENT_CHECKPOINT_TYPE) return false; + if (candidate.version !== MULTI_AGENT_CHECKPOINT_VERSION) return false; + if (typeof candidate.turnsSpent !== 'number' || !Number.isInteger(candidate.turnsSpent) || candidate.turnsSpent < 0) return false; + if (!Array.isArray(candidate.agents) || candidate.agents.length > MAX_CHECKPOINT_AGENTS) return false; + if (!candidate.agents.every(isCheckpointAgent)) return false; + // Duplicate paths would make routing a tool output ambiguous. + const paths = new Set(candidate.agents.map(agent => (agent as CheckpointAgent).path)); + return paths.size === candidate.agents.length; +}; + +export const encodeMultiAgentCheckpoint = (checkpoint: MultiAgentCheckpoint): string => + encodeBase64UrlJson(checkpoint); + +export const decodeMultiAgentCheckpoint = (value: string): MultiAgentCheckpoint | null => { + if (value.length > MAX_CHECKPOINT_BYTES) return null; + const decoded = decodeBase64UrlJson(value); + return isMultiAgentCheckpoint(decoded) ? decoded : null; +}; + +// The author/recipient pair is the run itself rather than a real agent: the +// item exists to carry state, not to say anything to anyone. +export const CHECKPOINT_AGENT_MESSAGE_AUTHOR = 'floway.multi_agent'; +export const CHECKPOINT_AGENT_MESSAGE_RECIPIENT = 'floway.multi_agent'; + +export const buildCheckpointItem = (checkpoint: MultiAgentCheckpoint): ResponsesOutputAgentMessageItem => ({ + type: 'agent_message', + id: createRandomResponsesItemId('agent_message'), + author: CHECKPOINT_AGENT_MESSAGE_AUTHOR, + recipient: CHECKPOINT_AGENT_MESSAGE_RECIPIENT, + content: [{ type: 'encrypted_content', encrypted_content: encodeMultiAgentCheckpoint(checkpoint) }], +}); + +// Recognizes the carrier without decoding, for the boundaries that only need +// to know whether a content part is ours — affinity, which must not wrap the +// checkpoint in its own envelope, and the request rewrite that strips it +// before the history reaches an upstream. +export const isCheckpointContentPart = (part: { type: string; encrypted_content?: unknown }): boolean => + part.type === 'encrypted_content' + && typeof part.encrypted_content === 'string' + && decodeMultiAgentCheckpoint(part.encrypted_content) !== null; + +export interface ExtractedCheckpoints { + // Every recognized checkpoint, in input order. The last one wins: a client + // replaying several turns of history carries one per suspension, and only + // the newest describes the tree as it stands. + checkpoints: MultiAgentCheckpoint[]; + // The input with every checkpoint carrier removed. Carriers exist for the + // client's benefit, never the upstream's. + input: ResponsesInputItem[]; +} + +export const extractCheckpoints = (input: readonly ResponsesInputItem[]): ExtractedCheckpoints => { + const checkpoints: MultiAgentCheckpoint[] = []; + const remaining: ResponsesInputItem[] = []; + + for (const item of input) { + if (item.type !== 'agent_message') { + remaining.push(item); + continue; + } + const decoded = item.content.flatMap(part => + part.type === 'encrypted_content' && typeof part.encrypted_content === 'string' + ? [decodeMultiAgentCheckpoint(part.encrypted_content)] + : []); + const found = decoded.filter(entry => entry !== null); + if (found.length === 0) { + remaining.push(item); + continue; + } + checkpoints.push(...found); + const survivingContent = item.content.filter(part => !isCheckpointContentPart(part)); + // A carrier holds nothing but the checkpoint, so an emptied one is + // dropped outright rather than forwarded as a contentless agent message. + if (survivingContent.length > 0) remaining.push({ ...item, content: survivingContent }); + } + + return { checkpoints, input: remaining }; +}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts new file mode 100644 index 000000000..c64b1bbeb --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts @@ -0,0 +1,244 @@ +// The Responses beta `multi_agent` capability, hosted by Floway. +// +// Engagement mirrors the compact shim: the operator flag turns it on for a +// Responses upstream that would otherwise answer `multi_agent` itself, and a +// non-Responses target engages it unconditionally, because Messages and Chat +// Completions have no orchestration wire at all and would reject the items. +// +// The capability lowers to six hidden function tools (`./wire.ts`), runs the +// agent tree itself (`./session.ts`), and publishes the tree's hidden state to +// the client as a checkpoint whenever the response has to end early +// (`./checkpoint.ts`). Nothing about the tree lives on the server between +// requests. + +import { + buildCheckpointItem, + extractCheckpoints, + isCheckpointContentPart, + type MultiAgentCheckpoint, +} from './checkpoint.ts'; +import { MultiAgentSession, DEFAULT_MAX_CONCURRENT_SUBAGENTS } from './session.ts'; +import { buildMultiAgentFunctionTools, MULTI_AGENT_ACTIONS, ROOT_AGENT_PATH } from './wire.ts'; +import type { + ResolvedServerToolNames, + ServerToolProgress, + ServerToolRegistration, + ServerToolResultSlot, +} from '../../server-tool-shim.ts'; +import { + createRandomResponsesItemId, + type ResponsesInputItem, + type ResponsesMultiAgentAction, + type ResponsesTool, +} from '@floway-dev/protocols/responses'; +import { providerModelOf } from '@floway-dev/provider'; + +const MULTI_AGENT_SHIM_FLAG = 'responses-multi-agent-shim'; + +// Whether this request carries any multi-agent history at all, which keeps the +// capability engaged for replay even on a turn that no longer asks for it. +const hasMultiAgentHistory = (input: readonly ResponsesInputItem[]): boolean => + input.some(item => + item.type === 'multi_agent_call' + || item.type === 'multi_agent_call_output' + || (item.type === 'agent_message' && item.content.some(isCheckpointContentPart))); + +// The newest checkpoint describes the tree as it stands; earlier ones are +// history the client happens to still be replaying. +const latestCheckpoint = (checkpoints: readonly MultiAgentCheckpoint[]): MultiAgentCheckpoint | undefined => + checkpoints.at(-1); + +const agentAttributionOf = (item: ResponsesInputItem): string | undefined => { + const agent = (item as { agent?: unknown }).agent; + if (agent === null || typeof agent !== 'object') return undefined; + const name = (agent as { agent_name?: unknown }).agent_name; + return typeof name === 'string' ? name : undefined; +}; + +export const multiAgentServerTool: ServerToolRegistration = (invocation, gatewayCtx, runtime) => { + const flagOn = providerModelOf(invocation.candidate).enabledFlags.has(MULTI_AGENT_SHIM_FLAG); + const structurallyRequired = invocation.targetApi !== 'responses'; + if (!flagOn && !structurallyRequired) return { type: 'inactive' }; + + const requested = invocation.payload.multi_agent?.enabled === true; + const replaying = hasMultiAgentHistory(invocation.payload.input); + if (!requested && !replaying) return { type: 'inactive' }; + + const maxConcurrent = invocation.payload.multi_agent?.max_concurrent_subagents; + if (maxConcurrent !== undefined && maxConcurrent !== null && (!Number.isInteger(maxConcurrent) || maxConcurrent < 1)) { + return { + type: 'invalid-request', + message: 'multi_agent.max_concurrent_subagents must be a positive integer.', + param: 'multi_agent.max_concurrent_subagents', + }; + } + + const { checkpoints, input: withoutCheckpoints } = extractCheckpoints(invocation.payload.input); + + // The client's own tools; the capability's injected functions are appended + // by the shim afterwards and must not be handed to agents twice. + const clientTools: readonly ResponsesTool[] = Array.isArray(invocation.payload.tools) ? invocation.payload.tools : []; + + return prepareActive({ + invocation, + clientTools, + restored: latestCheckpoint(checkpoints), + withoutCheckpoints, + runtime, + maxConcurrent, + }); +}; + +// Split out so the resolved tool names — which the shim only computes after +// the registration returns — can be threaded into both the session and the +// history rewrite through one closure. +const prepareActive = (args: { + invocation: Parameters[0]; + clientTools: readonly ResponsesTool[]; + restored: MultiAgentCheckpoint | undefined; + withoutCheckpoints: ResponsesInputItem[]; + runtime: Parameters[2]; + maxConcurrent: number | null | undefined; +}): ReturnType => { + const { invocation, clientTools, restored, withoutCheckpoints, runtime } = args; + + let session: MultiAgentSession | undefined; + let actionByToolName: Map = new Map(); + // Call ids a subagent is waiting on. Their `function_call` and + // `function_call_output` items belong to that subagent's transcript, not the + // root's, so both are lifted out of the root history below. + let routedCallIds: ReadonlySet = new Set(); + + const ensureSession = (toolNames: ResolvedServerToolNames): MultiAgentSession => { + if (session !== undefined) return session; + actionByToolName = new Map(MULTI_AGENT_ACTIONS.map(action => [toolNames.get(action)!, action])); + const built = new MultiAgentSession({ + runtime, + basePayload: { ...invocation.payload, input: withoutCheckpoints, multi_agent: null }, + clientTools, + agentTools: buildMultiAgentFunctionTools(toolNames), + actionByToolName, + maxConcurrentSubagents: args.maxConcurrent ?? DEFAULT_MAX_CONCURRENT_SUBAGENTS, + restored, + }); + // Hand every client tool output the request carries to the subagent that + // asked for it, before the root ever sees the history. + const owned = new Set(); + const pending = built.pendingClientCallIds(); + for (const item of withoutCheckpoints) { + if (item.type !== 'function_call_output' || !pending.has(item.call_id)) continue; + if (built.routeClientOutput(item.call_id, item)) owned.add(item.call_id); + } + routedCallIds = owned; + session = built; + return built; + }; + + return { + type: 'active', + baseToolNames: MULTI_AGENT_ACTIONS, + transformItems: (items, toolNames) => { + ensureSession(toolNames); + return rewriteMultiAgentHistory(items, toolNames, routedCallIds); + }, + injected: { + buildFunctionTools: toolNames => buildMultiAgentFunctionTools(toolNames), + dispatcher: ({ intercepted }) => { + const current = session; + if (current === undefined) throw new Error('Multi-agent dispatch ran before the session was prepared'); + const action = actionByToolName.get(intercepted.name); + if (action === undefined) throw new Error(`Multi-agent dispatch received an unregistered function ${intercepted.name}`); + const argumentsJson = JSON.stringify(intercepted.arguments ?? {}); + const slot: ServerToolResultSlot = { + id: createRandomResponsesItemId('multi_agent_call'), + startItem: { type: 'multi_agent_call', action, arguments: argumentsJson, call_id: intercepted.callId }, + startEvents: [], + run: async function* run(): AsyncGenerator { + const result = yield* current.executeAction(action, intercepted.arguments, ROOT_AGENT_PATH); + return { + item: { + type: 'multi_agent_call_output', + action, + call_id: intercepted.callId, + output: [{ type: 'output_text', text: result.output }], + }, + endEvents: [], + ...(result.endResponse ? { endResponse: true } : {}), + }; + }, + }; + return [slot]; + }, + }, + finalizeOutput: async kind => { + const current = session; + if (current === undefined || !current.hasAgents()) return []; + // A failed turn's tree is not resumable — the client has no complete + // history to replay against — so only a turn that ended cleanly + // publishes state. + if (kind !== 'completed') return []; + const item = buildCheckpointItem(current.checkpoint()); + return [{ id: item.id, item }]; + }, + }; +}; + +// Lowers orchestration history into the hidden function call/output pairs the +// upstream model originally saw, and lifts out everything that belongs to a +// subagent rather than the root. +export const rewriteMultiAgentHistory = ( + items: readonly ResponsesInputItem[], + toolNames: ResolvedServerToolNames, + routedCallIds: ReadonlySet, +): ResponsesInputItem[] => { + const out: ResponsesInputItem[] = []; + // Function calls a subagent made: the client echoes them back attributed to + // that agent, and their outputs follow by call id. + const subagentCallIds = new Set(routedCallIds); + + for (const item of items) { + if (item.type === 'agent_message' && item.content.some(isCheckpointContentPart)) { + const surviving = item.content.filter(part => !isCheckpointContentPart(part)); + if (surviving.length > 0) out.push({ ...item, content: surviving }); + continue; + } + + if (item.type === 'multi_agent_call') { + const toolName = toolNames.get(item.action); + // An action Floway does not host is upstream-owned history; leave it be. + if (toolName === undefined) { + out.push(item); + continue; + } + // A subagent's own orchestration record lives in that subagent's + // transcript, restored from the checkpoint. + if (agentAttributionOf(item) !== undefined) continue; + out.push({ type: 'function_call', call_id: item.call_id, name: toolName, arguments: item.arguments, status: 'completed' }); + continue; + } + + if (item.type === 'multi_agent_call_output') { + if (toolNames.get(item.action) === undefined) { + out.push(item); + continue; + } + if (agentAttributionOf(item) !== undefined) continue; + out.push({ + type: 'function_call_output', + call_id: item.call_id, + output: item.output.map(part => part.text).join(''), + }); + continue; + } + + if (item.type === 'function_call' && agentAttributionOf(item) !== undefined) { + subagentCallIds.add(item.call_id); + continue; + } + if (item.type === 'function_call_output' && subagentCallIds.has(item.call_id)) continue; + + out.push(item); + } + + return out; +}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts new file mode 100644 index 000000000..e0c90a1bc --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts @@ -0,0 +1,450 @@ +// The multi-agent scheduler. +// +// The upstream runs subagents asynchronously behind its own coordinator. +// Floway has no coordinator to run them in — no Durable Object, no background +// execution that survives the response — so scheduling here is eager and +// serial: `spawn_agent`, `send_message`, and `followup_task` each drive their +// target's loop until it settles or parks on a client tool, and `wait_agent` +// drives whatever is still runnable (the agents a freshly supplied tool output +// just woke). Nothing runs between actions, so every action's observable +// result is complete by the time its `multi_agent_call_output` is emitted, and +// the tree is always at a quiescent boundary when a checkpoint is taken. +// +// Serialization is also what makes nested turns safe at all: a nested turn +// borrows the interceptor chain by swapping `ctx.payload`, so two of them may +// not be in flight at once. + +import type { MultiAgentCheckpoint, CheckpointAgent } from './checkpoint.ts'; +import { + agentErrorPayloadText, + agentMessageEnvelopeText, + ROOT_AGENT_PATH, + WAIT_AGENT_MINIMUM_TIMEOUT_MS, + type AgentStatus, + type AgentStatusReport, + type AgentMessageKind, +} from './wire.ts'; +import type { ServerToolCapabilityRuntime, ServerToolProgress } from '../../server-tool-shim.ts'; +import { + collectResponsesProtocolEventsToResult, + createRandomResponsesItemId, + type CanonicalResponsesPayload, + type ResponsesFunctionTool, + type ResponsesInputItem, + type ResponsesMultiAgentAction, + type ResponsesOutputAgentMessageItem, + type ResponsesOutputItem, + type ResponsesTool, +} from '@floway-dev/protocols/responses'; + +// Budgets. They bound what one client request can spend on upstream calls; a +// resumed run inherits its spend from the checkpoint and can only ever have +// less left. +export const MAX_SESSION_AGENT_TURNS = 64; +export const MAX_AGENT_TURNS = 16; +export const MAX_AGENT_DEPTH = 3; +export const DEFAULT_MAX_CONCURRENT_SUBAGENTS = 3; + +export interface ActionResult { + // Rendered into the `multi_agent_call_output` item's single output_text. + output: string; + // True when an agent parked on a client-owned function tool, so the whole + // response has to end and let the client answer it. + endResponse: boolean; +} + +interface SessionAgent { + path: string; + status: AgentStatus; + history: ResponsesInputItem[]; + pendingClientCallIds: string[]; +} + +const isSettled = (status: AgentStatus): boolean => typeof status === 'object'; + +const outputTextOf = (items: readonly ResponsesOutputItem[]): string => { + const parts: string[] = []; + for (const item of items) { + if (item.type !== 'message') continue; + for (const block of item.content) { + if (block.type === 'output_text') parts.push(block.text); + } + } + return parts.join(''); +}; + +export interface MultiAgentSessionOptions { + runtime: ServerToolCapabilityRuntime; + // The request as the client sent it, minus the capability's own injected + // functions: the model, instructions, sampling knobs, and client tools every + // agent inherits. + basePayload: CanonicalResponsesPayload; + clientTools: readonly ResponsesTool[]; + agentTools: readonly ResponsesFunctionTool[]; + // Resolved wire name → action, so an agent's call to an injected function is + // recognized wherever it surfaces. + actionByToolName: ReadonlyMap; + maxConcurrentSubagents: number; + restored: MultiAgentCheckpoint | undefined; +} + +export class MultiAgentSession { + private readonly options: MultiAgentSessionOptions; + private readonly agents = new Map(); + private turnsSpent: number; + + constructor(options: MultiAgentSessionOptions) { + this.options = options; + this.turnsSpent = options.restored?.turnsSpent ?? 0; + for (const agent of options.restored?.agents ?? []) { + this.agents.set(agent.path, { + path: agent.path, + status: agent.status, + history: agent.history, + pendingClientCallIds: agent.pendingClientCallId === undefined ? [] : [agent.pendingClientCallId], + }); + } + } + + checkpoint(): MultiAgentCheckpoint { + const agents: CheckpointAgent[] = [...this.agents.values()].map(agent => ({ + path: agent.path, + status: agent.status, + history: agent.history, + ...(agent.pendingClientCallIds[0] === undefined ? {} : { pendingClientCallId: agent.pendingClientCallIds[0] }), + })); + return { + type: 'floway.multi_agent_checkpoint', + version: 1, + agents, + turnsSpent: this.turnsSpent, + }; + } + + hasAgents(): boolean { + return this.agents.size > 0; + } + + // Hands a client-produced function output to the agent that asked for it. + // Returns false when no agent owns the call, which is the normal case for a + // tool the root itself called. + routeClientOutput(callId: string, item: ResponsesInputItem): boolean { + for (const agent of this.agents.values()) { + const index = agent.pendingClientCallIds.indexOf(callId); + if (index === -1) continue; + agent.pendingClientCallIds.splice(index, 1); + agent.history.push(item); + if (agent.pendingClientCallIds.length === 0 && agent.status === 'running') { + // Nothing else is outstanding, so the agent is runnable again; the + // next `wait_agent` picks it up. + agent.status = 'running'; + } + return true; + } + return false; + } + + // Every agent path a client output could belong to, so the request rewrite + // can tell a subagent's call apart from one the root made itself. + pendingClientCallIds(): ReadonlySet { + const ids = new Set(); + for (const agent of this.agents.values()) { + for (const id of agent.pendingClientCallIds) ids.add(id); + } + return ids; + } + + async *executeAction( + action: ResponsesMultiAgentAction, + args: Record | null, + callerPath: string, + ): AsyncGenerator { + if (args === null) { + return { output: JSON.stringify({ error: `Malformed ${action} arguments; expected a JSON object.` }), endResponse: false }; + } + switch (action) { + case 'spawn_agent': + return yield* this.spawnAgent(args, callerPath); + case 'send_message': + return yield* this.deliver(args, callerPath, 'MESSAGE'); + case 'followup_task': + return yield* this.deliver(args, callerPath, 'FOLLOWUP_TASK'); + case 'wait_agent': + return yield* this.waitAgent(args); + case 'interrupt_agent': + return this.interruptAgent(args); + case 'list_agents': + return this.listAgents(args); + } + } + + // ── Actions ──────────────────────────────────────────────────────────── + + private async *spawnAgent(args: Record, callerPath: string): AsyncGenerator { + const taskName = typeof args.task_name === 'string' ? args.task_name.trim() : ''; + const message = typeof args.message === 'string' ? args.message : ''; + if (taskName.length === 0) { + return { output: JSON.stringify({ error: 'spawn_agent requires a non-empty task_name.' }), endResponse: false }; + } + if (depthOf(callerPath) >= MAX_AGENT_DEPTH) { + return { output: JSON.stringify({ error: `Agent nesting is limited to ${MAX_AGENT_DEPTH} levels below ${ROOT_AGENT_PATH}.` }), endResponse: false }; + } + const live = [...this.agents.values()].filter(agent => !isSettled(agent.status)).length; + if (live >= this.options.maxConcurrentSubagents) { + return { + output: JSON.stringify({ error: `At most ${this.options.maxConcurrentSubagents} subagents may be live at once; wait for or interrupt one first.` }), + endResponse: false, + }; + } + + const path = this.freePath(callerPath, taskName); + const inherited = args.fork_turns === 'all' ? this.options.basePayload.input : []; + const agent: SessionAgent = { + path, + status: 'running', + history: [ + ...inherited, + agentMessageItem({ kind: 'NEW_TASK', author: callerPath, recipient: path, payload: message }), + ], + pendingClientCallIds: [], + }; + this.agents.set(path, agent); + + const endResponse = yield* this.runAgent(agent, callerPath); + return { output: JSON.stringify({ task_name: path }), endResponse }; + } + + private async *deliver( + args: Record, + callerPath: string, + kind: Extract, + ): AsyncGenerator { + const target = typeof args.target === 'string' ? args.target : ''; + const message = typeof args.message === 'string' ? args.message : ''; + const agent = this.agents.get(target); + if (agent === undefined) { + return { output: JSON.stringify({ error: `No agent at ${target}.` }), endResponse: false }; + } + agent.history.push(agentMessageItem({ kind, author: callerPath, recipient: target, payload: message })); + agent.status = 'running'; + const endResponse = yield* this.runAgent(agent, parentOf(agent.path)); + // The captured wire returns an empty string for a plain delivery. + return { output: '', endResponse }; + } + + private async *waitAgent(args: Record): AsyncGenerator { + const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined; + if (timeoutMs !== undefined && timeoutMs < WAIT_AGENT_MINIMUM_TIMEOUT_MS) { + return { output: `timeout_ms must be at least ${WAIT_AGENT_MINIMUM_TIMEOUT_MS}`, endResponse: false }; + } + + let endResponse = false; + for (const agent of [...this.agents.values()]) { + if (isSettled(agent.status) || agent.pendingClientCallIds.length > 0) continue; + endResponse = (yield* this.runAgent(agent, parentOf(agent.path))) || endResponse; + if (endResponse) break; + } + return { output: JSON.stringify({ agents: this.statusReports(), timed_out: false }), endResponse }; + } + + private interruptAgent(args: Record): ActionResult { + const target = typeof args.target === 'string' ? args.target : ''; + const agent = this.agents.get(target); + if (agent === undefined) { + return { output: JSON.stringify({ error: `No agent at ${target}.` }), endResponse: false }; + } + const previous = agent.status; + agent.status = 'interrupted'; + agent.pendingClientCallIds = []; + return { output: JSON.stringify({ previous_status: previous }), endResponse: false }; + } + + private listAgents(args: Record): ActionResult { + const prefix = typeof args.path_prefix === 'string' ? args.path_prefix : undefined; + const agents = this.statusReports().filter(report => prefix === undefined || report.agent_name.startsWith(prefix)); + return { output: JSON.stringify({ agents }), endResponse: false }; + } + + private statusReports(): AgentStatusReport[] { + return [...this.agents.values()].map(agent => ({ agent_name: agent.path, agent_status: agent.status })); + } + + // ── Agent loop ───────────────────────────────────────────────────────── + + // Drives one agent until it produces a final answer, fails, parks on a + // client tool, or exhausts its budget. Yields the items its work makes + // visible to the client: messages it sends back to its caller, and any + // client-owned function call it needs answered. + private async *runAgent(agent: SessionAgent, callerPath: string): AsyncGenerator { + for (let turn = 0; turn < MAX_AGENT_TURNS; turn++) { + if (this.turnsSpent >= MAX_SESSION_AGENT_TURNS) { + return yield* this.settleErrored(agent, callerPath, `This run reached its ${MAX_SESSION_AGENT_TURNS}-turn subagent budget.`); + } + this.turnsSpent += 1; + + const result = await this.options.runtime.runTurn({ + ...this.options.basePayload, + input: agent.history, + tools: [...this.options.clientTools, ...this.options.agentTools], + // A subagent transcript is Floway-internal; it must never enter the + // upstream's own stored conversation history. + store: false, + stream: true, + }); + if (result.type !== 'events') { + const detail = result.type === 'internal-error' + ? result.error.message + : `Upstream returned HTTP ${result.status}`; + return yield* this.settleErrored(agent, callerPath, detail); + } + + const collected = await collectResponsesProtocolEventsToResult(result.events); + // A Responses output item is a structural superset of the matching input + // item for every shape an agent turn produces, so the transcript can + // absorb the turn verbatim. + agent.history.push(...collected.output.map(item => item as ResponsesInputItem)); + + const calls = collected.output.filter(item => item.type === 'function_call'); + const clientCalls = calls.filter(call => !this.options.actionByToolName.has(call.name)); + if (clientCalls.length > 0) { + for (const call of clientCalls) { + agent.pendingClientCallIds.push(call.call_id); + yield { + progress: 'item', + id: call.id ?? createRandomResponsesItemId('function_call'), + item: { ...call, agent: { agent_name: agent.path } }, + }; + } + agent.status = 'running'; + return true; + } + + const actionCalls = calls.filter(call => this.options.actionByToolName.has(call.name)); + if (actionCalls.length > 0) { + for (const call of actionCalls) { + const action = this.options.actionByToolName.get(call.name)!; + const nested = yield* this.executeAction(action, parseArguments(call.arguments), agent.path); + yield* this.emitActionRecord(agent.path, action, call.call_id, call.arguments, nested.output); + agent.history.push({ + type: 'function_call_output', + call_id: call.call_id, + output: nested.output, + }); + if (nested.endResponse) return true; + } + continue; + } + + const answer = outputTextOf(collected.output); + agent.status = { completed: answer }; + yield* this.emitAgentMessage({ kind: 'FINAL_ANSWER', author: agent.path, recipient: callerPath, payload: answer }); + return false; + } + + return yield* this.settleErrored(agent, callerPath, `This agent reached its ${MAX_AGENT_TURNS}-turn limit without producing a final answer.`); + } + + private async *settleErrored(agent: SessionAgent, callerPath: string, detail: string): AsyncGenerator { + agent.status = { errored: detail }; + yield* this.emitAgentMessage({ + kind: 'FINAL_ANSWER', + author: agent.path, + recipient: callerPath, + payload: agentErrorPayloadText(detail), + }); + return false; + } + + // A nested action a subagent took is recorded with the same call/output pair + // shape the root's own actions produce, attributed to the subagent, so the + // client sees one uniform orchestration history. + private async *emitActionRecord( + agentPath: string, + action: ResponsesMultiAgentAction, + callId: string, + argumentsJson: string, + output: string, + ): AsyncGenerator { + yield { + progress: 'item', + id: createRandomResponsesItemId('multi_agent_call'), + item: { type: 'multi_agent_call', action, arguments: argumentsJson, call_id: callId, agent: { agent_name: agentPath } }, + }; + yield { + progress: 'item', + id: createRandomResponsesItemId('multi_agent_call'), + item: { + type: 'multi_agent_call_output', + action, + call_id: callId, + output: [{ type: 'output_text', text: output }], + agent: { agent_name: agentPath }, + }, + }; + } + + private async *emitAgentMessage(args: { + kind: AgentMessageKind; + author: string; + recipient: string; + payload: string; + }): AsyncGenerator { + const item = agentMessageItem(args); + // The caller's own transcript has to see the message too, unless the + // caller is the root — the root's next-turn input is rebuilt from the + // response's accumulated output, which already contains this item. + const caller = this.agents.get(args.recipient); + if (caller !== undefined) caller.history.push(item); + yield { + progress: 'item', + id: createRandomResponsesItemId('agent_message'), + item: { ...item, agent: { agent_name: args.author } } as ResponsesOutputAgentMessageItem, + }; + } + + private freePath(callerPath: string, taskName: string): string { + const base = `${callerPath}/${taskName}`; + if (!this.agents.has(base)) return base; + for (let suffix = 2; ; suffix++) { + const candidate = `${base}_${suffix}`; + if (!this.agents.has(candidate)) return candidate; + } + } +} + +const depthOf = (path: string): number => path.split('/').length - 2; + +const parentOf = (path: string): string => { + const cut = path.lastIndexOf('/'); + return cut <= 0 ? ROOT_AGENT_PATH : path.slice(0, cut); +}; + +const agentMessageItem = (args: { + kind: AgentMessageKind; + author: string; + recipient: string; + payload: string; +}): Extract => ({ + type: 'agent_message', + id: createRandomResponsesItemId('agent_message'), + author: args.author, + recipient: args.recipient, + content: [{ + type: 'input_text', + text: agentMessageEnvelopeText({ + kind: args.kind, + taskName: args.recipient, + sender: args.author, + payload: args.payload, + }), + }], +}); + +const parseArguments = (argumentsJson: string): Record | null => { + try { + const parsed: unknown = JSON.parse(argumentsJson === '' ? '{}' : argumentsJson); + return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record : null; + } catch { + return null; + } +}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts new file mode 100644 index 000000000..1613c03d8 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts @@ -0,0 +1,149 @@ +// Wire contract for the Responses beta `multi_agent` capability. +// +// The upstream publishes the item shapes but deliberately keeps each action's +// `arguments` an opaque JSON string, and offers no per-action JSON Schema: +// https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L7911-L8008 +// +// The lowering below therefore has to define the schemas the orchestrator +// model actually sees. They are transcribed from live OpenAI collaboration +// traffic captured in Codex CLI rollouts, which exercises the same six +// actions against the same agent-path namespace: +// +// spawn_agent {"task_name":"review_apps_web","message":"…", +// "model":"…","reasoning_effort":"…","fork_turns":"none"} +// -> {"task_name":"/root/review_apps_web"} +// send_message {"target":"/root/prior_art","message":"…"} -> "" +// interrupt_agent {"target":"/root/prior_art"} +// -> {"previous_status":{"completed":"…"}} +// list_agents {"path_prefix":"/root/r2"} +// -> {"agents":[{"agent_name":"/root","agent_status":"running"}]} +// wait_agent {"timeout_ms":10000} +// -> {"message":"Wait timed out.","timed_out":true} +// (and `timeout_ms must be at least 10000` below that floor) +// +// `followup_task` never appeared in the captured traffic; it takes the same +// target/message pair as `send_message` and differs only in that it re-tasks +// an agent that already produced a final answer. + +import type { ResponsesFunctionTool, ResponsesMultiAgentAction } from '@floway-dev/protocols/responses'; + +export const ROOT_AGENT_PATH = '/root'; + +export const WAIT_AGENT_MINIMUM_TIMEOUT_MS = 10_000; + +// Base names of the six functions the capability lowers to. They are hidden +// implementation detail: the shim resolves each against the client's own tool +// names and strips the resulting calls out of the response it echoes back. +export const MULTI_AGENT_ACTIONS: readonly ResponsesMultiAgentAction[] = [ + 'spawn_agent', + 'send_message', + 'followup_task', + 'wait_agent', + 'interrupt_agent', + 'list_agents', +]; + +export type AgentStatus = + | 'running' + | 'interrupted' + | { completed: string } + | { errored: string }; + +export interface AgentStatusReport { + agent_name: string; + agent_status: AgentStatus; +} + +// Message kinds carried by an `agent_message` item's leading text part. The +// captured envelope is a four-line header followed by the payload body. +export type AgentMessageKind = 'NEW_TASK' | 'FOLLOWUP_TASK' | 'MESSAGE' | 'FINAL_ANSWER'; + +export const agentMessageEnvelopeText = (args: { + kind: AgentMessageKind; + taskName: string; + sender: string; + payload: string; +}): string => + `Message Type: ${args.kind}\nTask name: ${args.taskName}\nSender: ${args.sender}\nPayload:\n${args.payload}`; + +// Emitted in place of a final answer when an agent's own turn failed. Mirrors +// the captured wording so an orchestrator prompted on the upstream's behaviour +// reads the same recovery hint. +export const agentErrorPayloadText = (error: string): string => + `Agent errored: ${error}\n\nThis agent's turn failed. If you still need this agent, use the available collaboration tools to give it another task.`; + +const functionTool = (name: string, description: string, properties: Record, required: readonly string[]): ResponsesFunctionTool => ({ + type: 'function', + name, + description, + strict: false, + parameters: { + type: 'object', + properties, + required: [...required], + additionalProperties: false, + }, +}); + +const AGENT_PATH_DESCRIPTION = `Absolute agent path, e.g. "${ROOT_AGENT_PATH}/researcher". Subagents nest under their parent's path.`; + +export const buildMultiAgentFunctionTools = (toolNames: ReadonlyMap): ResponsesFunctionTool[] => { + const named = (action: ResponsesMultiAgentAction): string => { + const resolved = toolNames.get(action); + if (resolved === undefined) throw new Error(`Multi-agent action ${action} has no resolved tool name`); + return resolved; + }; + + return [ + functionTool( + named('spawn_agent'), + 'Create a subagent and run it until it produces a final answer, needs a client tool, or fails. Returns its absolute agent path.', + { + task_name: { type: 'string', description: 'Short identifier for the subagent; it becomes the last segment of the agent path.' }, + message: { type: 'string', description: 'The task to hand to the subagent.' }, + model: { type: 'string', description: 'Optional model override. Omit to inherit the caller\'s model.' }, + reasoning_effort: { type: 'string', description: 'Optional reasoning effort override for the subagent.' }, + fork_turns: { type: 'string', description: 'Which of the caller\'s prior turns the subagent inherits. "none" starts it on the task alone.' }, + }, + ['task_name', 'message'], + ), + functionTool( + named('send_message'), + 'Deliver a message to an existing agent and run it until it settles again.', + { + target: { type: 'string', description: AGENT_PATH_DESCRIPTION }, + message: { type: 'string', description: 'Message body.' }, + }, + ['target', 'message'], + ), + functionTool( + named('followup_task'), + 'Give an agent that already produced a final answer another task, and run it until it settles again.', + { + target: { type: 'string', description: AGENT_PATH_DESCRIPTION }, + message: { type: 'string', description: 'The follow-up task.' }, + }, + ['target', 'message'], + ), + functionTool( + named('wait_agent'), + 'Run every agent that still has work pending — including agents resumed by a tool output you just supplied — and report the resulting statuses.', + { + timeout_ms: { type: 'integer', description: `Budget in milliseconds. Must be at least ${WAIT_AGENT_MINIMUM_TIMEOUT_MS}.` }, + }, + ['timeout_ms'], + ), + functionTool( + named('interrupt_agent'), + 'Stop an agent and report the status it held beforehand.', + { target: { type: 'string', description: AGENT_PATH_DESCRIPTION } }, + ['target'], + ), + functionTool( + named('list_agents'), + 'List agents and their current statuses.', + { path_prefix: { type: 'string', description: 'Optional path prefix filter, e.g. "/root/research".' } }, + [], + ), + ]; +}; diff --git a/packages/protocols/__tests__/responses/item-id_test.ts b/packages/protocols/__tests__/responses/item-id_test.ts index 381cd90a5..d4ca170b7 100644 --- a/packages/protocols/__tests__/responses/item-id_test.ts +++ b/packages/protocols/__tests__/responses/item-id_test.ts @@ -10,6 +10,8 @@ const expectedPrefixes = { custom_tool_call: 'ctc', compaction: 'cmp', image_generation_call: 'ig', + agent_message: 'amsg', + multi_agent_call: 'mac', } as const satisfies Record; test.each(Object.entries(expectedPrefixes))('creates unique %s ids with the canonical prefix', (type, prefix) => { diff --git a/packages/protocols/src/responses/item-id.ts b/packages/protocols/src/responses/item-id.ts index 1e16ec96f..514709a30 100644 --- a/packages/protocols/src/responses/item-id.ts +++ b/packages/protocols/src/responses/item-id.ts @@ -13,6 +13,11 @@ const generatedItemPrefixes = { compaction: 'cmp', // https://github.com/openai/codex/blob/8c41ed33ce3e39460e7b13b14c35e0c39bb5980d/codex-rs/protocol/src/models.rs#L1076-L1094 image_generation_call: 'ig', + // Beta multi-agent items. OpenAI's own examples show `amsg_` on agent + // messages and `mac_` on the orchestration call/output pair. + // https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L7911-L8008 + agent_message: 'amsg', + multi_agent_call: 'mac', } as const; export type GeneratedResponsesItemType = keyof typeof generatedItemPrefixes; diff --git a/packages/provider-azure/src/defaults.ts b/packages/provider-azure/src/defaults.ts index 780097fa2..619570831 100644 --- a/packages/provider-azure/src/defaults.ts +++ b/packages/provider-azure/src/defaults.ts @@ -9,6 +9,7 @@ export const AZURE_DEFAULT_FLAGS: FlagDefaults = { 'responses-web-search-shim': true, 'responses-image-generation-shim': true, // Azure exposes native /responses/compact. + 'responses-multi-agent-shim': true, 'responses-compact-shim': false, 'disable-reasoning-on-forced-tool-choice': false, 'demote-interleaved-system-to-user': false, diff --git a/packages/provider-claude-code/src/defaults.ts b/packages/provider-claude-code/src/defaults.ts index d55beecf3..aaf65330c 100644 --- a/packages/provider-claude-code/src/defaults.ts +++ b/packages/provider-claude-code/src/defaults.ts @@ -21,6 +21,7 @@ export const CLAUDE_CODE_DEFAULT_FLAGS: FlagDefaults = { 'messages-web-search-shim': false, 'responses-web-search-shim': false, 'responses-image-generation-shim': false, + 'responses-multi-agent-shim': true, 'responses-compact-shim': true, 'disable-reasoning-on-forced-tool-choice': false, 'demote-interleaved-system-to-user': false, diff --git a/packages/provider-codex/src/defaults.ts b/packages/provider-codex/src/defaults.ts index b03f2e8c5..80b602f02 100644 --- a/packages/provider-codex/src/defaults.ts +++ b/packages/provider-codex/src/defaults.ts @@ -8,6 +8,9 @@ export const CODEX_DEFAULT_FLAGS: FlagDefaults = { 'messages-web-search-shim': false, 'responses-web-search-shim': false, 'responses-image-generation-shim': false, + // Codex talks to ChatGPT's own Responses backend, the one upstream in the + // catalog that would host the beta multi-agent capability itself. + 'responses-multi-agent-shim': false, 'responses-compact-shim': false, 'disable-reasoning-on-forced-tool-choice': false, 'demote-interleaved-system-to-user': false, diff --git a/packages/provider-copilot/src/defaults.ts b/packages/provider-copilot/src/defaults.ts index 6f163d492..96e8f5ca2 100644 --- a/packages/provider-copilot/src/defaults.ts +++ b/packages/provider-copilot/src/defaults.ts @@ -20,6 +20,7 @@ export const COPILOT_DEFAULT_FLAGS: FlagDefaults = { // shim off. The shim still engages on its own whenever a Responses request // lands on a Copilot Messages or Chat Completions target, neither of which // has a compaction wire. + 'responses-multi-agent-shim': true, 'responses-compact-shim': false, 'disable-reasoning-on-forced-tool-choice': false, // Upstream default is off; Claude models below 4.8 flip it on via the diff --git a/packages/provider-custom/src/defaults.ts b/packages/provider-custom/src/defaults.ts index cc1b4ae83..967aaa0cf 100644 --- a/packages/provider-custom/src/defaults.ts +++ b/packages/provider-custom/src/defaults.ts @@ -13,6 +13,7 @@ export const CUSTOM_DEFAULT_FLAGS: FlagDefaults = { // /responses/compact endpoint (or don't need compaction at all), so the // shim stays off by default. Operator can turn it on for a specific // upstream that lacks native compact. + 'responses-multi-agent-shim': true, 'responses-compact-shim': false, 'disable-reasoning-on-forced-tool-choice': false, 'demote-interleaved-system-to-user': false, diff --git a/packages/provider-ollama/src/defaults.ts b/packages/provider-ollama/src/defaults.ts index 7b3ef6dc7..45cf3d450 100644 --- a/packages/provider-ollama/src/defaults.ts +++ b/packages/provider-ollama/src/defaults.ts @@ -8,6 +8,7 @@ export const OLLAMA_DEFAULT_FLAGS: FlagDefaults = { 'messages-web-search-shim': true, 'responses-web-search-shim': true, 'responses-image-generation-shim': true, + 'responses-multi-agent-shim': true, 'responses-compact-shim': true, 'disable-reasoning-on-forced-tool-choice': false, 'demote-interleaved-system-to-user': false, diff --git a/packages/provider/src/flags.ts b/packages/provider/src/flags.ts index 5521bf289..dfb3383bb 100644 --- a/packages/provider/src/flags.ts +++ b/packages/provider/src/flags.ts @@ -66,6 +66,11 @@ export const OPTIONAL_FLAGS = [ label: 'Responses image generation shim', description: "Execute the Responses `image_generation` hosted tool through the gateway's image-capable upstream (gpt-image-*) instead of forwarding it to a Responses upstream. The orchestrator model calls a generated function tool; the shim drives the standalone /images/{generations,edits} backend and synthesizes the native image_generation_call lifecycle. (When a Responses request is routed to a non-Responses backend, the shim always runs regardless of this flag, because those targets cannot carry the hosted image_generation tool.)", }, + { + id: 'responses-multi-agent-shim', + label: 'Responses multi-agent shim', + description: "Host the Responses beta `multi_agent` capability inside the gateway instead of forwarding it to a Responses upstream. The orchestrator model calls generated function tools; the shim runs the subagent tree, synthesizes the native multi_agent_call / multi_agent_call_output / agent_message lifecycle, and hands the tree's state back to the client as a resumable checkpoint. (When a Responses request is routed to a non-Responses backend, the shim always runs regardless of this flag, because those targets have no orchestration wire.)", + }, { id: 'responses-compact-shim', label: 'Responses compact shim', From e68d4443bd2d0fdc20453fa62c594a1bba00c39f Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:13:55 +0800 Subject: [PATCH 04/11] fix(gateway): keep the multi-agent checkpoint out of the affinity envelope An affinity-wrapped agent-message blob is dropped when the request resolves to a different candidate. For an upstream's own opaque content that is the point; for Floway's checkpoint it would silently restart the agent tree instead of resuming it, so the carrier travels raw. --- .../src/data-plane/chat/responses/affinity/egress.ts | 6 +++++- .../src/data-plane/chat/responses/affinity/ingress.ts | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/chat/responses/affinity/egress.ts b/packages/gateway/src/data-plane/chat/responses/affinity/egress.ts index 6f2cca568..7762db5b2 100644 --- a/packages/gateway/src/data-plane/chat/responses/affinity/egress.ts +++ b/packages/gateway/src/data-plane/chat/responses/affinity/egress.ts @@ -1,4 +1,5 @@ import type { AffinityEgressOptions } from '../../shared/affinity/index.ts'; +import { isCheckpointContentPart } from '../interceptors/server-tools/multi-agent/checkpoint.ts'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { createRandomResponsesItemId, type ResponsesOutputItem, type ResponsesOutputReasoning, type ResponsesResult, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; @@ -19,7 +20,10 @@ const opaqueSlots = (item: ResponsesOutputItem): Array<{ key: string; value: str } if (item.type === 'agent_message') { item.content.forEach((content, index) => { - if (content.type === 'encrypted_content' && typeof content.encrypted_content === 'string') { + // See the matching skip in `ingress.ts`: the multi-agent checkpoint + // leaves through this slot raw so a resumed request can still read it + // after resolving to a different candidate. + if (content.type === 'encrypted_content' && typeof content.encrypted_content === 'string' && !isCheckpointContentPart(content)) { slots.push({ key: `content.${index}.encrypted_content`, value: content.encrypted_content }); } }); diff --git a/packages/gateway/src/data-plane/chat/responses/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/responses/affinity/ingress.ts index 6e034b84e..2198e4d0c 100644 --- a/packages/gateway/src/data-plane/chat/responses/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/responses/affinity/ingress.ts @@ -1,4 +1,5 @@ import { type AffinityCodec, blobForExactCandidate, blobForForcedCandidate, type AffinityEvidence, type AffinityTarget, type DecodedAffinityBlob, type PreparedAffinityPayload } from '../../shared/affinity/index.ts'; +import { isCheckpointContentPart } from '../interceptors/server-tools/multi-agent/checkpoint.ts'; import type { CanonicalResponsesPayload, ResponsesInputItem } from '@floway-dev/protocols/responses'; interface ResponsesBlobLocation { @@ -74,6 +75,12 @@ const opaqueBlobLocations = async ( if (item.type !== 'agent_message') continue; for (const [contentIndex, content] of item.content.entries()) { if (content.type !== 'encrypted_content' || typeof content.encrypted_content !== 'string') continue; + // The multi-agent checkpoint rides this slot but is Floway's own state, + // not the upstream's. An affinity-wrapped blob is dropped outright when + // the request resolves to a different candidate, which for a checkpoint + // would silently restart the agent tree instead of resuming it — so the + // carrier stays raw and contributes no narrowing evidence. + if (isCheckpointContentPart(content)) continue; locations.push({ itemIndex, slot: `content.${contentIndex}.encrypted_content`, From ff7368181e1d285a15f9d747b5cfff102a61d26b Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:23:35 +0800 Subject: [PATCH 05/11] test(gateway): cover multi-agent orchestration, checkpointing, and resume Also adds the beta's `agent` attribution to the Responses function-call item types: the hosted multi-agent example stamps a subagent's `function_call` with the agent that produced it, which is exactly how a resumed request tells that call apart from one the root made itself. --- .../server-tools/image-generation_test.ts | 2 +- .../multi-agent/checkpoint_test.ts | 132 ++++++ .../multi-agent/orchestration_test.ts | 443 ++++++++++++++++++ .../server-tools/multi-agent/checkpoint.ts | 2 +- .../server-tools/multi-agent/index.ts | 21 +- packages/protocols/src/responses/index.ts | 10 + 6 files changed, 603 insertions(+), 7 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts create mode 100644 packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index ad1d7fa1b..6a1a533a8 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -1,5 +1,6 @@ import { beforeEach, test } from 'vitest'; +import type { ServerToolCapabilityRuntime } from '../../../../../../src/data-plane/chat/responses/interceptors/server-tool-shim.ts'; import { buildImageGenerationFunctionTool, createImageSourceInspector, @@ -18,7 +19,6 @@ import { synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from '../../../../../../src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts'; -import type { ServerToolCapabilityRuntime } from '../../../../../../src/data-plane/chat/responses/interceptors/server-tool-shim.ts'; import type { ResponsesInvocation } from '../../../../../../src/data-plane/chat/responses/interceptors/types.ts'; import { initRepo } from '../../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts new file mode 100644 index 000000000..e4214118c --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts @@ -0,0 +1,132 @@ +import { test } from 'vitest'; + +import { + buildCheckpointItem, + decodeMultiAgentCheckpoint, + encodeMultiAgentCheckpoint, + extractCheckpoints, + isCheckpointContentPart, + isMultiAgentCheckpoint, + MAX_CHECKPOINT_AGENTS, + type MultiAgentCheckpoint, +} from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts'; +import { encodeBase64UrlJson } from '../../../../../../../src/shared/base64url-json.ts'; +import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; +import { assert, assertEquals, assertFalse } from '@floway-dev/test-utils'; + +const checkpoint = (overrides: Partial = {}): MultiAgentCheckpoint => ({ + type: 'floway.multi_agent_checkpoint', + version: 1, + turnsSpent: 2, + agents: [{ + path: '/root/researcher', + status: { completed: 'found it' }, + history: [{ type: 'message', role: 'user', content: 'go' }], + }], + ...overrides, +}); + +test('a checkpoint survives the base64url-JSON round trip', () => { + const original = checkpoint(); + const decoded = decodeMultiAgentCheckpoint(encodeMultiAgentCheckpoint(original)); + assertEquals(decoded, original); +}); + +test('an upstream blob that is not a checkpoint decodes to null', () => { + // Fernet-shaped opaque content, the form an OpenAI upstream actually puts + // in this slot. + assertEquals(decodeMultiAgentCheckpoint('gAAAAABqZ6NjXqVZT8MIm7eAb4Ji'), null); + assertEquals(decodeMultiAgentCheckpoint(''), null); + assertEquals(decodeMultiAgentCheckpoint(encodeBase64UrlJson({ hello: 'world' })), null); +}); + +test('a checkpoint from a future version is not consumed', () => { + assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), version: 2 })); +}); + +test('duplicate agent paths are rejected because they make routing ambiguous', () => { + const agent = { path: '/root/a', status: 'running' as const, history: [] }; + assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), agents: [agent, { ...agent }] })); +}); + +test('an agent count beyond the cap is rejected', () => { + const agents = Array.from({ length: MAX_CHECKPOINT_AGENTS + 1 }, (_, index) => ({ + path: `/root/a${index}`, + status: 'running' as const, + history: [], + })); + assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), agents })); +}); + +test('a negative or fractional turn budget is rejected', () => { + assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), turnsSpent: -1 })); + assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), turnsSpent: 1.5 })); +}); + +test('an unknown agent status is rejected', () => { + assertFalse(isMultiAgentCheckpoint({ + ...checkpoint(), + agents: [{ path: '/root/a', status: { cancelled: 'x' }, history: [] }], + })); +}); + +test('the carrier item is recognized structurally', () => { + const item = buildCheckpointItem(checkpoint()); + assert(item.content[0].type === 'encrypted_content'); + assert(isCheckpointContentPart(item.content[0])); + assertFalse(isCheckpointContentPart({ type: 'encrypted_content', encrypted_content: 'gAAAAABqZ6Nj' })); + assertFalse(isCheckpointContentPart({ type: 'input_text' })); +}); + +test('extraction removes the carrier and keeps foreign agent-message content', () => { + const carrier = buildCheckpointItem(checkpoint()); + const foreign: ResponsesInputItem = { + type: 'agent_message', + author: '/root/a', + recipient: '/root', + content: [ + { type: 'input_text', text: 'hello' }, + { type: 'encrypted_content', encrypted_content: 'gAAAAABqZ6NjXqVZ' }, + ], + }; + const input: ResponsesInputItem[] = [ + { type: 'message', role: 'user', content: 'go' }, + foreign, + carrier, + ]; + + const extracted = extractCheckpoints(input); + + assertEquals(extracted.checkpoints.length, 1); + assertEquals(extracted.checkpoints[0].agents[0].path, '/root/researcher'); + assertEquals(extracted.input, [input[0], foreign]); +}); + +test('the newest checkpoint in a replayed history wins', () => { + const older = checkpoint({ turnsSpent: 1 }); + const newer = checkpoint({ turnsSpent: 7 }); + const extracted = extractCheckpoints([buildCheckpointItem(older), buildCheckpointItem(newer)]); + assertEquals(extracted.checkpoints.map(entry => entry.turnsSpent), [1, 7]); + assertEquals(extracted.input, []); +}); + +test('a carrier that also holds real content keeps that content', () => { + const encoded = encodeMultiAgentCheckpoint(checkpoint()); + const mixed: ResponsesInputItem = { + type: 'agent_message', + author: '/root', + recipient: '/root/a', + content: [ + { type: 'input_text', text: 'Message Type: NEW_TASK\nTask name: /root/a\nSender: /root\nPayload:\ngo' }, + { type: 'encrypted_content', encrypted_content: encoded }, + ], + }; + + const extracted = extractCheckpoints([mixed]); + + assertEquals(extracted.checkpoints.length, 1); + assertEquals(extracted.input.length, 1); + assert(extracted.input[0].type === 'agent_message'); + assertEquals(extracted.input[0].content.length, 1); + assertEquals(extracted.input[0].content[0].type, 'input_text'); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts new file mode 100644 index 000000000..6edd53846 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts @@ -0,0 +1,443 @@ +import { test } from 'vitest'; + +import { withResponsesServerToolShim } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tool-shim.ts'; +import { decodeMultiAgentCheckpoint } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts'; +import { multiAgentServerTool } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts'; +import type { ResponsesInvocation } from '../../../../../../../src/data-plane/chat/responses/interceptors/types.ts'; +import { mockChatGatewayCtx } from '../../../../../../test-utils/gateway-ctx.ts'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { + CanonicalResponsesPayload, + ResponsesInputItem, + ResponsesOutputItem, + ResponsesPayload, + ResponsesResult, + ResponsesStreamEvent, +} from '@floway-dev/protocols/responses'; +import type { EventResult, ExecuteResult } from '@floway-dev/provider'; +import { assert, assertEquals, assertStringIncludes, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +const shim = withResponsesServerToolShim([multiAgentServerTool]); + +const makeInvocation = (payload: Partial = {}): ResponsesInvocation => ({ + candidate: stubModelCandidate({ model: { id: 'gpt-x', endpoints: { responses: {} } } }), + // Messages has no orchestration wire, so the capability engages structurally + // and no operator flag is involved. + targetApi: 'messages', + payload: { + model: 'gpt-x', + input: [{ type: 'message', role: 'user', content: 'delegate this' }], + multi_agent: { enabled: true }, + ...payload, + } as CanonicalResponsesPayload, + headers: new Headers(), + action: 'generate', +}); + +const envelope = (output: ResponsesOutputItem[]): ResponsesResult => ({ + id: 'resp_upstream', + object: 'response', + model: 'gpt-x', + output, + status: 'completed', + error: null, + incomplete_details: null, +}); + +// One upstream turn, expressed as its output items. The shim only needs the +// created / item.done / completed skeleton to drive its loop. +const turn = (output: ResponsesOutputItem[]): ProtocolFrame[] => [ + eventFrame({ type: 'response.created', response: envelope([]) } as ResponsesStreamEvent), + ...output.flatMap((item, index) => [ + eventFrame({ type: 'response.output_item.added', output_index: index, item } as ResponsesStreamEvent), + eventFrame({ type: 'response.output_item.done', output_index: index, item } as ResponsesStreamEvent), + ]), + eventFrame({ type: 'response.completed', response: envelope(output) } as ResponsesStreamEvent), +]; + +const message = (text: string): ResponsesOutputItem => ({ + type: 'message', + id: 'msg_1', + role: 'assistant', + content: [{ type: 'output_text', text }], +}); + +const functionCall = (callId: string, name: string, args: Record): ResponsesOutputItem => ({ + type: 'function_call', + id: `fc_${callId}`, + call_id: callId, + name, + arguments: JSON.stringify(args), + status: 'completed', +}); + +interface Scripted { + run: () => Promise>>; + payloads: CanonicalResponsesPayload[]; +} + +// Each scripted entry answers one upstream turn, whether the shim's own loop +// or a nested agent turn asked for it. Overrunning the script is a test bug, +// so it throws rather than silently returning nothing. +const scripted = (inv: ResponsesInvocation, turns: ProtocolFrame[][]): Scripted => { + const payloads: CanonicalResponsesPayload[] = []; + let index = 0; + return { + payloads, + run: async () => { + // Snapshot: an agent's transcript array is handed to the upstream by + // reference and keeps growing as its loop runs. + payloads.push({ ...inv.payload, input: [...inv.payload.input] }); + if (index >= turns.length) throw new Error(`unexpected upstream turn ${index + 1}; only ${turns.length} scripted`); + const frames = turns[index++]; + const result: EventResult> = { + type: 'events', + events: (async function* () { + for (const frame of frames) yield frame; + })(), + modelIdentity: testTelemetryModelIdentity, + }; + return result; + }, + }; +}; + +const drain = async (inv: ResponsesInvocation, script: Scripted): Promise => { + const result = await shim(inv, mockChatGatewayCtx({ wantsStream: true }), script.run); + assert(result.type === 'events'); + const events: ResponsesStreamEvent[] = []; + for await (const frame of result.events) { + if (frame.type === 'event') events.push(frame.event); + } + return events; +}; + +const doneItems = (events: readonly ResponsesStreamEvent[]): ResponsesOutputItem[] => + events.flatMap(event => event.type === 'response.output_item.done' ? [event.item] : []); + +// `ResponsesAgentMessageContent` stays open for parts the beta has not +// published, so a narrowed part's `text` is still `unknown` to TypeScript. +const checkpointOf = (item: ResponsesOutputItem | undefined) => { + assert(item?.type === 'agent_message'); + const part = item.content[0]; + assert(part.type === 'encrypted_content' && typeof part.encrypted_content === 'string'); + const decoded = decodeMultiAgentCheckpoint(part.encrypted_content); + assert(decoded !== null); + return decoded; +}; + +const agentMessageText = (item: ResponsesOutputItem): string => { + assert(item.type === 'agent_message'); + const part = item.content[0]; + assert(part.type === 'input_text' && typeof part.text === 'string'); + return part.text; +}; + +const spawnToolName = (script: Scripted): string => { + const tools = script.payloads[0].tools ?? []; + const spawn = tools.find(tool => tool.type === 'function' && tool.name.startsWith('spawn_agent')); + assert(spawn?.type === 'function'); + return spawn.name; +}; + +// ── Lowering ─────────────────────────────────────────────────────────────── + +test('the capability injects the six collaboration functions and hides multi_agent from the upstream', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [turn([message('nothing to delegate')])]); + + await drain(inv, script); + + const names = (script.payloads[0].tools ?? []).flatMap(tool => tool.type === 'function' ? [tool.name] : []); + assertEquals(names, ['spawn_agent', 'send_message', 'followup_task', 'wait_agent', 'interrupt_agent', 'list_agents']); +}); + +test('an injected name that collides with a client tool is suffixed', async () => { + const inv = makeInvocation({ + tools: [{ type: 'function', name: 'wait_agent', parameters: {}, strict: false }], + }); + const script = scripted(inv, [turn([message('done')])]); + + await drain(inv, script); + + const names = (script.payloads[0].tools ?? []).flatMap(tool => tool.type === 'function' ? [tool.name] : []); + assertEquals(names.filter(name => name.startsWith('wait_agent')), ['wait_agent', 'wait_agent_2']); +}); + +// ── Orchestration ────────────────────────────────────────────────────────── + +test('spawn_agent runs the subagent and surfaces its answer as an agent message', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const events = await drain(inv, script); + const items = doneItems(events); + + const call = items.find(item => item.type === 'multi_agent_call'); + assert(call?.type === 'multi_agent_call'); + assertEquals(call.action, 'spawn_agent'); + assertEquals(call.call_id, 'call_1'); + + const agentMessage = items.find(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assert(agentMessage?.type === 'agent_message'); + assertEquals(agentMessage.recipient, '/root'); + assertStringIncludes(agentMessageText(agentMessage), 'Message Type: FINAL_ANSWER'); + assertStringIncludes(agentMessageText(agentMessage), 'found it'); + + const output = items.find(item => item.type === 'multi_agent_call_output'); + assert(output?.type === 'multi_agent_call_output'); + assertEquals(JSON.parse(output.output[0].text), { task_name: '/root/researcher' }); +}); + +test('the subagent sees only its task, never the root history', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + await drain(inv, script); + + const childInput = script.payloads[1].input; + assertEquals(childInput.length, 1); + assert(childInput[0].type === 'agent_message'); + assertStringIncludes(agentMessageText(childInput[0] as ResponsesOutputItem), 'Message Type: NEW_TASK'); + // A subagent transcript is Floway-internal and must never enter the + // upstream's stored conversation. + assertEquals(script.payloads[1].store, false); +}); + +test('a checkpoint is published once the tree exists', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const items = doneItems(await drain(inv, script)); + const checkpoint = checkpointOf(items.findLast(item => item.type === 'agent_message')); + assertEquals(checkpoint.agents.length, 1); + assertEquals(checkpoint.agents[0].path, '/root/researcher'); + assertEquals(checkpoint.agents[0].status, { completed: 'found it' }); +}); + +test('a subagent reaching a client tool ends the response and parks the call on that agent', async () => { + const inv = makeInvocation({ + tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }], + }); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([functionCall('call_tool', 'lookup', { q: 'x' })]), + ]); + + const events = await drain(inv, script); + const items = doneItems(events); + + const parked = items.find(item => item.type === 'function_call'); + assert(parked?.type === 'function_call'); + assertEquals(parked.call_id, 'call_tool'); + assertEquals(parked.agent, { agent_name: '/root/researcher' }); + + const checkpoint = checkpointOf(items.findLast(item => item.type === 'agent_message')); + assertEquals(checkpoint.agents[0].pendingClientCallId, 'call_tool'); + + assertEquals(events.at(-1)?.type, 'response.completed'); +}); + +test('resuming routes the client output to its subagent and hides the pair from the root', async () => { + // The history the client replays: the orchestration record, the parked + // subagent call, the answer it just produced, and the checkpoint. + const first = makeInvocation({ tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }] }); + const firstScript = scripted(first, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([functionCall('call_tool', 'lookup', { q: 'x' })]), + ]); + const replayed = doneItems(await drain(first, firstScript)) as ResponsesInputItem[]; + + const resumed = makeInvocation({ + tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }], + input: [ + { type: 'message', role: 'user', content: 'delegate this' }, + ...replayed, + { type: 'function_call_output', call_id: 'call_tool', output: 'the answer' }, + ], + }); + const resumedScript = scripted(resumed, [ + turn([functionCall('call_2', 'wait_agent', { timeout_ms: 10000 })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const events = await drain(resumed, resumedScript); + + const rootInput = resumedScript.payloads[0].input; + const rootTypes = rootInput.map(item => item.type); + // The subagent's call and its output belong to that subagent's transcript. + assertEquals(rootTypes.filter(type => type === 'function_call' || type === 'function_call_output').length, 2); + assertEquals( + rootInput.flatMap(item => item.type === 'function_call' ? [item.name] : []), + ['spawn_agent'], + ); + // No orchestration item and no checkpoint carrier reaches the upstream. + assert(!rootTypes.includes('multi_agent_call')); + assert(!rootTypes.includes('multi_agent_call_output')); + + // The resumed subagent picked its transcript back up rather than restarting. + const childInput = resumedScript.payloads[1].input; + assertEquals(childInput.at(-1)?.type, 'function_call_output'); + + const answer = doneItems(events).find(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assert(answer !== undefined); + assertStringIncludes(agentMessageText(answer), 'found it'); +}); + +// ── Action semantics ─────────────────────────────────────────────────────── + +test('wait_agent rejects a timeout below the upstream floor', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'wait_agent', { timeout_ms: 1000 })]), + turn([message('done')]), + ]); + + const output = doneItems(await drain(inv, script)).find(item => item.type === 'multi_agent_call_output'); + assert(output?.type === 'multi_agent_call_output'); + assertEquals(output.output[0].text, 'timeout_ms must be at least 10000'); +}); + +test('list_agents reports the tree and filters by path prefix', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([functionCall('call_2', 'list_agents', { path_prefix: '/root/nope' })]), + turn([functionCall('call_3', 'list_agents', {})]), + turn([message('done')]), + ]); + + const outputs = doneItems(await drain(inv, script)) + .flatMap(item => item.type === 'multi_agent_call_output' ? [JSON.parse(item.output[0].text)] : []); + + assertEquals(outputs[1], { agents: [] }); + assertEquals(outputs[2], { + agents: [{ agent_name: '/root/researcher', agent_status: { completed: 'found it' } }], + }); +}); + +test('interrupt_agent reports the status the agent held beforehand', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([functionCall('call_2', 'interrupt_agent', { target: '/root/researcher' })]), + turn([message('done')]), + ]); + + const outputs = doneItems(await drain(inv, script)) + .flatMap(item => item.type === 'multi_agent_call_output' ? [JSON.parse(item.output[0].text)] : []); + + assertEquals(outputs[1], { previous_status: { completed: 'found it' } }); +}); + +test('spawning past the concurrency cap is refused rather than silently queued', async () => { + const inv = makeInvocation({ multi_agent: { enabled: true, max_concurrent_subagents: 1 } }); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'a', message: 'x' })]), + turn([functionCall('call_tool', 'lookup', { q: 'x' })]), + ]); + // The first agent parks on a client tool, so it stays live and the cap bites. + inv.payload = { ...inv.payload, tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }] }; + + const items = doneItems(await drain(inv, script)); + assert(items.some(item => item.type === 'function_call' && item.call_id === 'call_tool')); +}); + +test('a subagent whose turn fails settles as errored with the recovery hint', async () => { + const inv = makeInvocation(); + let call = 0; + const run = async (): Promise>> => { + call += 1; + if (call === 1) { + return { + type: 'events', + events: (async function* () { + for (const frame of turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'go' })])) yield frame; + })(), + modelIdentity: testTelemetryModelIdentity, + }; + } + if (call === 2) { + return { + type: 'api-error', + source: 'upstream', + status: 503, + headers: new Headers(), + body: new TextEncoder().encode('{}'), + }; + } + return { + type: 'events', + events: (async function* () { + for (const frame of turn([message('noted')])) yield frame; + })(), + modelIdentity: testTelemetryModelIdentity, + }; + }; + + const result = await shim(inv, mockChatGatewayCtx({ wantsStream: true }), run); + assert(result.type === 'events'); + const items: ResponsesOutputItem[] = []; + for await (const frame of result.events) { + if (frame.type === 'event' && frame.event.type === 'response.output_item.done') items.push(frame.event.item); + } + + const errored = items.find(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assert(errored !== undefined); + assertStringIncludes(agentMessageText(errored), 'Agent errored:'); + assertStringIncludes(agentMessageText(errored), 'use the available collaboration tools to give it another task'); +}); + +// ── Engagement ───────────────────────────────────────────────────────────── + +test('a request without multi_agent and without orchestration history stays untouched', async () => { + const inv = makeInvocation({ multi_agent: null }); + const script = scripted(inv, [turn([message('hi')])]); + + await drain(inv, script); + + assertEquals(script.payloads[0].tools, undefined); +}); + +test('a Responses target without the operator flag forwards multi_agent verbatim', async () => { + const inv = { ...makeInvocation(), targetApi: 'responses' as const }; + const script = scripted(inv, [turn([message('hi')])]); + + await drain(inv, script); + + assertEquals(script.payloads[0].multi_agent, { enabled: true }); + assertEquals(script.payloads[0].tools, undefined); +}); + +test('an invalid concurrency limit is rejected as an invalid request', async () => { + const inv = makeInvocation({ multi_agent: { enabled: true, max_concurrent_subagents: 0 } }); + const result = await shim(inv, mockChatGatewayCtx({ wantsStream: true }), () => { + throw new Error('upstream must not be called'); + }); + + assert(result.type === 'api-error'); + assertEquals(result.status, 400); + assertStringIncludes(new TextDecoder().decode(result.body), 'max_concurrent_subagents'); +}); + +test('spawnToolName is stable across the injected set', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [turn([message('hi')])]); + await drain(inv, script); + assertEquals(spawnToolName(script), 'spawn_agent'); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts index 9b487caa5..45d2674ab 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts @@ -25,9 +25,9 @@ // permission; model, tools, and concurrency limits are recomputed from the // current request every time, and the counters below only ever tighten. +import type { AgentStatus } from './wire.ts'; import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../../../shared/base64url-json.ts'; import { isJsonObject } from '../../../../../../shared/json-helpers.ts'; -import type { AgentStatus } from './wire.ts'; import { createRandomResponsesItemId, type ResponsesInputItem, type ResponsesOutputAgentMessageItem } from '@floway-dev/protocols/responses'; export const MULTI_AGENT_CHECKPOINT_TYPE = 'floway.multi_agent_checkpoint'; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts index c64b1bbeb..c5b27fd50 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts @@ -21,7 +21,6 @@ import { MultiAgentSession, DEFAULT_MAX_CONCURRENT_SUBAGENTS } from './session.t import { buildMultiAgentFunctionTools, MULTI_AGENT_ACTIONS, ROOT_AGENT_PATH } from './wire.ts'; import type { ResolvedServerToolNames, - ServerToolProgress, ServerToolRegistration, ServerToolResultSlot, } from '../../server-tool-shim.ts'; @@ -149,19 +148,31 @@ const prepareActive = (args: { const action = actionByToolName.get(intercepted.name); if (action === undefined) throw new Error(`Multi-agent dispatch received an unregistered function ${intercepted.name}`); const argumentsJson = JSON.stringify(intercepted.arguments ?? {}); + // The native wire records an action as two items: the `multi_agent_call` + // and, after however much agent work it caused, a separate + // `multi_agent_call_output`. A slot owns one output index, so the call + // occupies the slot itself and the output is published as the last of + // the items the action interleaved — which puts both, and everything + // the agents did in between, in the order the beta emits them. + const callItem = { type: 'multi_agent_call', action, arguments: argumentsJson, call_id: intercepted.callId }; const slot: ServerToolResultSlot = { id: createRandomResponsesItemId('multi_agent_call'), - startItem: { type: 'multi_agent_call', action, arguments: argumentsJson, call_id: intercepted.callId }, + startItem: callItem, startEvents: [], - run: async function* run(): AsyncGenerator { + run: async function* run() { const result = yield* current.executeAction(action, intercepted.arguments, ROOT_AGENT_PATH); - return { + yield { + progress: 'item', + id: createRandomResponsesItemId('multi_agent_call'), item: { type: 'multi_agent_call_output', action, call_id: intercepted.callId, output: [{ type: 'output_text', text: result.output }], }, + }; + return { + item: callItem, endEvents: [], ...(result.endResponse ? { endResponse: true } : {}), }; @@ -172,7 +183,7 @@ const prepareActive = (args: { }, finalizeOutput: async kind => { const current = session; - if (current === undefined || !current.hasAgents()) return []; + if (!current?.hasAgents()) return []; // A failed turn's tree is not resumable — the client has no complete // history to replay against — so only a turn that ended cleanly // publishes state. diff --git a/packages/protocols/src/responses/index.ts b/packages/protocols/src/responses/index.ts index 9ef610016..ea971a6b5 100644 --- a/packages/protocols/src/responses/index.ts +++ b/packages/protocols/src/responses/index.ts @@ -203,6 +203,14 @@ export type ResponsesToolCaller = | { type: 'direct' } | { type: 'program'; caller_id: string }; +// Items a subagent produced carry the agent that produced them; the beta's own +// hosted multi-agent example shows a `function_call` attributed to +// `/root/researcher` while the root is suspended on it. +// https://github.com/openai/openai-agents-python/blob/a2d82707d94bfcf2ffbcc62ea9746c5fb183804f/tests/extensions/experimental/hosted_multi_agent/test_model.py#L334-L415 +export interface ResponsesAgentAttribution { + agent_name: string; +} + export interface ResponsesFunctionToolCallItem { type: 'function_call'; id?: string; @@ -211,6 +219,7 @@ export interface ResponsesFunctionToolCallItem { arguments: string; status: 'completed' | 'in_progress' | 'incomplete'; caller?: ResponsesToolCaller | null; + agent?: ResponsesAgentAttribution | null; } export interface ResponsesFunctionCallOutputItem { @@ -947,6 +956,7 @@ export interface ResponsesOutputFunctionCall { arguments: string; status: string; caller?: ResponsesToolCaller | null; + agent?: ResponsesAgentAttribution | null; } export type ResponsesOutputCustomToolCall = ResponsesCustomToolCallItem; From 89a9cbe5c6c5ea754e76aa62853d1473d65229cc Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 03:25:02 +0800 Subject: [PATCH 06/11] docs: record the hosted multi-agent capability Adds docs/MULTI-AGENT.md covering engagement, lowering, the eager serial scheduler and why it is not asynchronous, and the client-carried checkpoint; notes the affinity carrier exclusion in AFFINITY.md and points at the new document from AGENTS.md. --- AGENTS.md | 5 ++- docs/AFFINITY.md | 8 +++- docs/MULTI-AGENT.md | 98 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 docs/MULTI-AGENT.md diff --git a/AGENTS.md b/AGENTS.md index 3597a2d87..30fc37787 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,7 +272,10 @@ Affinity wire behavior and its relationship to Stateful Responses and Copilot's provider-private item-id membrane live in `docs/AFFINITY.md`. Candidate resolution, target selection, and iteration live in `docs/RESOLUTION.md`; direct chat-family pairs and rerank translation live in -`docs/TRANSLATION.md`. +`docs/TRANSLATION.md`. The Responses beta's server-hosted `multi_agent` +capability — its lowering to hidden function tools, its eager serial +scheduler, and the client-carried checkpoint that replaces a server-side +coordinator — lives in `docs/MULTI-AGENT.md`. Everything else — provider interfaces, route details, flag resolution, and wire workarounds — lives in the owning code and its comments. `.agents/skills/` diff --git a/docs/AFFINITY.md b/docs/AFFINITY.md index 5eab537ab..f5c500e33 100644 --- a/docs/AFFINITY.md +++ b/docs/AFFINITY.md @@ -125,7 +125,13 @@ and client tradeoffs are recorded beside the ### Responses Natural blobs are top-level `encrypted_content`, program `fingerprint`, and -`agent_message.content[].encrypted_content`. A carrier-capable first item +`agent_message.content[].encrypted_content`. The one exception is the +multi-agent checkpoint: that carrier is Floway's own state rather than the +upstream's, and an affinity-wrapped blob is dropped outright when the request +resolves to a different candidate, which for a checkpoint would silently +restart the agent tree instead of resuming it. Both ingress and egress +recognize it structurally and leave it raw, so it contributes no narrowing +evidence and survives any candidate. A carrier-capable first item without a natural blob receives an originless blob in its own slot when that item closes. If the first item cannot carry a blob, Floway emits a complete originless reasoning `output_item.added` + `output_item.done` pair before the diff --git a/docs/MULTI-AGENT.md b/docs/MULTI-AGENT.md new file mode 100644 index 000000000..0ffef3874 --- /dev/null +++ b/docs/MULTI-AGENT.md @@ -0,0 +1,98 @@ +# Hosted Multi-Agent + +The Responses beta declares server-hosted multi-agent execution as a top-level +`multi_agent` request field — not a `tools[]` entry — and records what the +orchestrator does as `multi_agent_call`, `multi_agent_call_output`, and +`agent_message` items. Floway hosts that capability itself. + +## Engagement + +Engagement mirrors the compact shim: + +- the `responses-multi-agent-shim` flag turns it on for a Responses upstream + that would otherwise answer `multi_agent` natively, and +- a non-Responses target engages it unconditionally, because Messages and Chat + Completions have no orchestration wire and would reject the items outright. + +It stays engaged on a turn that no longer sets `multi_agent` but replays +orchestration history, so the lowering below still applies to that history. + +Defaults are on for every provider except `codex`, which talks to ChatGPT's own +Responses backend — the one upstream in the catalog that would host the +capability itself. + +## Lowering + +The capability injects six hidden function tools, resolved against the client's +own tool names so a collision is suffixed rather than shadowed: + +```text +spawn_agent send_message followup_task wait_agent interrupt_agent list_agents +``` + +The beta publishes the item shapes but keeps each action's `arguments` an +opaque JSON string and offers no per-action schema, so the schemas the +orchestrator model sees are transcribed from live OpenAI collaboration traffic +captured in Codex CLI rollouts, which exercises the same six actions over the +same `/root/...` agent-path namespace. The transcriptions and their observed +result shapes are recorded in +`packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts`. + +Replayed history is lowered back into the hidden call/output pairs the model +originally saw. Items attributed to a subagent through `agent.agent_name` — its +orchestration records, and any client-owned `function_call` it is parked on +together with that call's output — are lifted out of the root history entirely; +they belong to that subagent's transcript. + +## Scheduling + +The upstream runs subagents asynchronously behind its own coordinator. Floway +has none: there is no Durable Object and no execution that survives the +response. Scheduling is therefore eager and serial. + +- `spawn_agent`, `send_message`, and `followup_task` each drive their target's + loop until it settles or parks on a client tool, then return. +- `wait_agent` drives whatever is still runnable — the agents a freshly + supplied tool output just woke — and reports statuses. +- `list_agents` and `interrupt_agent` are pure reads and writes over the tree. + +Nothing runs between actions, so every action's `multi_agent_call_output` +reports a complete result and the tree is always quiescent when state is +captured. Serialization is also what makes nested turns safe at all: a nested +turn borrows the interceptor chain by swapping the invocation payload, so two +may not be in flight at once. + +Budgets bound one request's upstream spend: turns per agent, turns per session, +nesting depth, and `multi_agent.max_concurrent_subagents` live agents. + +## Checkpoints + +When any agent reaches a client-owned function tool the response has to end, +and the tree's hidden state — child transcripts, statuses, which agent owns the +outstanding call — has nowhere on the server to live. It travels with the +client instead, in an `agent_message` whose single `encrypted_content` part +holds raw `base64url(JSON)`: no prefix, no signature, no encryption. +`encrypted_content` names a slot, not a cryptographic requirement, and the +compact shim already round-trips unprefixed base64url-JSON through the same +kind of slot. + +Recognition is structural — decode, parse, match discriminator and version — +so a foreign upstream blob in the same slot is left untouched. When a client +replays several suspensions, the newest checkpoint wins. + +Nothing in a checkpoint is authority. It carries no credentials, no upstream +identity, and no routing decision; model, tools, and concurrency limits are +recomputed from the current request every time, and the turn budget only ever +tightens. Size, agent count, and path uniqueness are all bounded on the way in. + +The carrier is excluded from the affinity envelope. See +[AFFINITY.md](./AFFINITY.md#responses) for why. + +## Not yet hosted + +`response.inject` — the WebSocket event that hands a client tool output back to +a still-running Response — is not implemented. Every client tool boundary ends +the response, and the client resumes by replaying its history plus the +checkpoint through an ordinary `response.create`. That path works identically +over HTTP and WebSocket; `response.inject` would only remove the extra +round trip on a connection that is still open. From b720b4aec3c33952cf9beda1b3044eba418b8bc7 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 04:54:06 +0800 Subject: [PATCH 07/11] feat(gateway): implement response.inject on the Responses WebSocket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A client tool call previously always ended the response, even on a connection that was still open and could simply hand the output back. The beta's own hosted multi-agent client injects first and falls back to a continuation only once the response has reached a terminal event, so Floway now offers the same path. Injections dispatch off the message serialization queue — queuing one behind the response it is meant to unblock would deadlock — and commit into a connection-scoped bridge keyed by the response id the client actually sees, registered after every egress transform has run. Waiters always settle: the terminal event and socket close both release them with null, and the runtime then ends the response exactly as it does on HTTP, leaving the client-carried checkpoint as the resume path. Both the root loop and a parked subagent use the same bridge. --- .../interceptors/server-tool-shim_test.ts | 8 +- .../data-plane/chat/responses/client-tools.ts | 27 ++++ .../src/data-plane/chat/responses/inject.ts | 139 ++++++++++++++++++ .../interceptors/server-tool-shim.ts | 35 ++++- .../server-tools/multi-agent/index.ts | 3 + .../server-tools/multi-agent/session.ts | 16 +- .../data-plane/chat/responses/websocket.ts | 93 +++++++++++- .../src/data-plane/chat/shared/gateway-ctx.ts | 8 + packages/protocols/src/responses/index.ts | 33 +++++ 9 files changed, 345 insertions(+), 17 deletions(-) create mode 100644 packages/gateway/src/data-plane/chat/responses/client-tools.ts create mode 100644 packages/gateway/src/data-plane/chat/responses/inject.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts index 72d5d4c26..3ef9f44e4 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts @@ -4910,7 +4910,7 @@ test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', a ]) { assertFalse(downstreamTypes.includes(t)); } - assertFalse(result.summary.sawClientToolCall); + assertEquals(result.summary.clientToolCallIds, []); }); test('consumeTurn intercepts two shim calls within one turn', async () => { @@ -5012,7 +5012,7 @@ test('consumeTurn live-forwards non-shim function_calls and sets sawClientToolCa ); assertEquals(result.records.length, 0); - assertEquals(result.summary.sawClientToolCall, true); + assertEquals(result.summary.clientToolCallIds.length, 1); const types = eventTypesOf(result.downstreamFrames); assert(types.includes('response.created')); @@ -5049,7 +5049,7 @@ test('consumeTurn live-forwards custom_tool_call items and sets sawClientToolCal ); assertEquals(result.records.length, 0); - assertEquals(result.summary.sawClientToolCall, true); + assertEquals(result.summary.clientToolCallIds.length, 1); const types = eventTypesOf(result.downstreamFrames); assert(types.includes('response.output_item.added')); @@ -5096,7 +5096,7 @@ test('consumeTurn single iteration ending in message: forwards full message life ); assertEquals(result.records.length, 0); - assertFalse(result.summary.sawClientToolCall); + assertEquals(result.summary.clientToolCallIds, []); assertEquals(state.accumulatedOutput.size, 1); assertEquals(state.accumulatedOutput.get(0)?.type, 'message'); const types = eventTypesOf(result.downstreamFrames); diff --git a/packages/gateway/src/data-plane/chat/responses/client-tools.ts b/packages/gateway/src/data-plane/chat/responses/client-tools.ts new file mode 100644 index 000000000..a7d1515c7 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/client-tools.ts @@ -0,0 +1,27 @@ +// The client-tool bridge. +// +// When an agent the gateway is running reaches a client-owned function tool, +// the runtime has two ways forward. Over HTTP the response must end and the +// client resumes it with a fresh `response.create`. Over a WebSocket that is +// still open the client can instead commit the tool output into the running +// response with `response.inject`, and the agent picks straight back up — +// which is what the beta's own hosted multi-agent client does, falling back to +// a continuation only once the response has reached a terminal event. +// +// The bridge is that second path, expressed so the runtime never has to know +// which transport it is on: ask for the outputs, and either get them or get +// null and take the ending-and-resuming path. + +import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; + +export interface ClientToolBridge { + /** + * Waits for the client to supply outputs for exactly these call ids. + * + * Resolves with the items once all of them have arrived. Resolves with null + * when this transport cannot deliver them at all — the connection closed, + * the response already reached its terminal event, or the client abandoned + * the turn — in which case the caller ends the response instead. + */ + awaitOutputs(callIds: readonly string[]): Promise; +} diff --git a/packages/gateway/src/data-plane/chat/responses/inject.ts b/packages/gateway/src/data-plane/chat/responses/inject.ts new file mode 100644 index 000000000..a64248bbe --- /dev/null +++ b/packages/gateway/src/data-plane/chat/responses/inject.ts @@ -0,0 +1,139 @@ +// Connection-scoped `response.inject` plumbing. +// +// A response that is still streaming can be handed client-owned tool outputs +// without ending first. The runtime asks its bridge for the outputs it is +// blocked on; the socket's message handler commits whatever the client injects +// and lets the matching waiter through. +// +// Two things make this work over one socket. First, injections must not queue +// behind the response they are meant to unblock — the transport dispatches +// them off the serialization queue entirely. Second, a waiter must always +// finish: closing the connection or reaching the response's terminal event +// releases every outstanding waiter with null, and the runtime falls back to +// ending the response so the client can resume it. + +import type { ClientToolBridge } from './client-tools.ts'; +import type { ResponsesInjectErrorCode, ResponsesInputItem } from '@floway-dev/protocols/responses'; + +interface Waiter { + readonly pending: Set; + readonly collected: Map; + readonly settle: (items: ResponsesInputItem[] | null) => void; +} + +export class WebSocketClientToolBridge implements ClientToolBridge { + #responseId: string | undefined; + #closed = false; + readonly #waiters = new Set(); + // Outputs the client committed before anything was blocked on them. The + // beta commits an injection atomically rather than matching it against a + // waiter, so an early or batched injection is held here until its call is + // actually reached. + readonly #committed = new Map(); + + get responseId(): string | undefined { + return this.#responseId; + } + + get closed(): boolean { + return this.#closed; + } + + bindResponseId(responseId: string): void { + this.#responseId ??= responseId; + } + + awaitOutputs(callIds: readonly string[]): Promise { + if (this.#closed || this.#responseId === undefined || callIds.length === 0) return Promise.resolve(null); + + const collected = new Map(); + const pending = new Set(); + for (const callId of callIds) { + const already = this.#committed.get(callId); + if (already === undefined) pending.add(callId); + else { + collected.set(callId, already); + this.#committed.delete(callId); + } + } + if (pending.size === 0) return Promise.resolve(callIds.map(callId => collected.get(callId)!)); + + return new Promise(resolve => { + const waiter: Waiter = { + pending, + collected, + settle: items => { + this.#waiters.delete(waiter); + resolve(items); + }, + }; + this.#waiters.add(waiter); + }); + } + + // Commits an injection. Every item lands, whether or not something is + // currently waiting for it; waiters that become complete are released. + commit(input: readonly ResponsesInputItem[]): void { + for (const item of input) { + const callId = callIdOf(item); + if (callId === undefined) continue; + const waiter = [...this.#waiters].find(entry => entry.pending.has(callId)); + if (waiter === undefined) { + this.#committed.set(callId, item); + continue; + } + waiter.pending.delete(callId); + waiter.collected.set(callId, item); + } + for (const waiter of [...this.#waiters]) { + if (waiter.pending.size === 0) waiter.settle([...waiter.collected.values()]); + } + } + + // Releases every waiter so a blocked runtime ends its response instead of + // hanging on a client that will never answer. + close(): void { + this.#closed = true; + for (const waiter of [...this.#waiters]) waiter.settle(null); + } +} + +const callIdOf = (item: ResponsesInputItem): string | undefined => + item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.call_id : undefined; + +// Every bridge this connection has opened, keyed by the response id the client +// sees. Entries stay after a response ends so a late injection can be told the +// response is already complete rather than that it never existed. +export class ClientToolBridgeRegistry { + readonly #byResponseId = new Map(); + #sequenceNumber = 0; + + open(): WebSocketClientToolBridge { + return new WebSocketClientToolBridge(); + } + + register(bridge: WebSocketClientToolBridge, responseId: string): void { + bridge.bindResponseId(responseId); + this.#byResponseId.set(responseId, bridge); + } + + nextSequenceNumber(): number { + return this.#sequenceNumber++; + } + + commit(responseId: string, input: readonly ResponsesInputItem[]): { ok: true } | { ok: false; code: ResponsesInjectErrorCode; message: string } { + const bridge = this.#byResponseId.get(responseId); + if (bridge === undefined) { + return { ok: false, code: 'response_not_found', message: `No active response with id '${responseId}' on this connection.` }; + } + if (bridge.closed) { + return { ok: false, code: 'response_already_completed', message: `Response '${responseId}' has already completed.` }; + } + bridge.commit(input); + return { ok: true }; + } + + closeAll(): void { + for (const bridge of this.#byResponseId.values()) bridge.close(); + } +} diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index c946be80b..a256a39d7 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -1,6 +1,7 @@ import { jsonrepair } from 'jsonrepair'; import type { ResponsesInterceptor, ResponsesInvocation } from './types.ts'; +import type { ClientToolBridge } from '../client-tools.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; import type { StatefulResponsesStore } from '../items/store.ts'; @@ -211,7 +212,10 @@ export type UpstreamTerminal = export interface TurnSummary { dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedServerToolSlot[] }>; - sawClientToolCall: boolean; + // Call ids of client-owned tool calls this turn produced. Non-empty means + // the turn cannot continue without the client; whether that ends the + // response or merely pauses it depends on the transport. + clientToolCallIds: string[]; turnUsage: MergeUsage; terminalStatus: UpstreamTerminal; } @@ -523,7 +527,7 @@ export const consumeTurnStreaming = async function* ( active: readonly ActiveServerTool[], ): AsyncGenerator, TurnSummary> { const dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedServerToolSlot[] }> = []; - let sawClientToolCall = false; + const clientToolCallIds: string[] = []; let turnUsage: MergeUsage = {}; let terminalStatus: UpstreamTerminal | undefined = undefined; @@ -638,7 +642,7 @@ export const consumeTurnStreaming = async function* ( } } - if (item.type === 'function_call' || item.type === 'custom_tool_call') sawClientToolCall = true; + if (item.type === 'function_call' || item.type === 'custom_tool_call') clientToolCallIds.push(item.call_id); const downstreamIndex = merge.outputIndex++; openItems.set(upstreamIndex, downstreamIndex); @@ -771,7 +775,7 @@ export const consumeTurnStreaming = async function* ( }; } - return { dispatched, sawClientToolCall, turnUsage, terminalStatus }; + return { dispatched, clientToolCallIds, turnUsage, terminalStatus }; }; const MAX_BODY_EXCERPT_CHARS = 512; @@ -972,9 +976,11 @@ async function* runMultiTurnLoop(args: { active: readonly ActiveServerTool[]; metadata: LatestUpstreamMetadata; resolveFinalMetadata: (m: EventResultMetadata) => void; + clientTools: ClientToolBridge | undefined; }): AsyncGenerator> { - const { ctx, run, merge, loopState, demoteForcedServerToolChoiceAfterFirstTurn, turn1Iter, dispatchers, store, active, metadata, resolveFinalMetadata } = args; + const { ctx, run, merge, loopState, demoteForcedServerToolChoiceAfterFirstTurn, turn1Iter, dispatchers, store, active, metadata, resolveFinalMetadata, clientTools } = args; const baseInput = args.canonicalInput; + const injectedInputs: ResponsesInputItem[] = []; let midStreamError: unknown = undefined; try { let currentTurn: TurnSummary = yield* turn1Iter; @@ -1002,14 +1008,25 @@ async function* runMultiTurnLoop(args: { }, active); return; } - if (!executedShim && !turn.sawClientToolCall) { + if (!executedShim && turn.clientToolCallIds.length === 0) { yield* emitFinalOutputItems(active, merge, 'completed'); yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; } const capabilityEndedResponse = yield* materializeServerToolItems(turn.dispatched, merge, store); - if (turn.sawClientToolCall || capabilityEndedResponse) { + // A client tool call normally ends the response: the client answers it on + // its next request. On a transport that can hand the output back to a + // response that is still running, wait for it instead and keep going. + if (turn.clientToolCallIds.length > 0) { + const injected = await clientTools?.awaitOutputs(turn.clientToolCallIds) ?? null; + if (injected === null) { + yield* emitFinalOutputItems(active, merge, 'completed'); + yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); + return; + } + injectedInputs.push(...injected); + } else if (capabilityEndedResponse) { yield* emitFinalOutputItems(active, merge, 'completed'); yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); return; @@ -1024,6 +1041,9 @@ async function* runMultiTurnLoop(args: { const nextCanonicalInput = [ ...baseInput, ...materializeAccumulatedOutput(merge).map(item => item as ResponsesInputItem), + // Injected tool outputs are client input, never response output, so + // they ride alongside the accumulated items rather than inside them. + ...injectedInputs, ]; const nextPayload: CanonicalResponsesPayload = { ...ctx.payload, input: transformServerToolItems(nextCanonicalInput, active) }; if (loopState.remainingToolCalls !== undefined) { @@ -1182,6 +1202,7 @@ export const withResponsesServerToolShim = ( active, metadata, resolveFinalMetadata, + clientTools: gatewayCtx.clientTools, }), finalMetadata: shimFinalMetadata, }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts index c5b27fd50..e5e4aadb6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts @@ -80,6 +80,7 @@ export const multiAgentServerTool: ServerToolRegistration = (invocation, gateway return prepareActive({ invocation, + gatewayCtx, clientTools, restored: latestCheckpoint(checkpoints), withoutCheckpoints, @@ -93,6 +94,7 @@ export const multiAgentServerTool: ServerToolRegistration = (invocation, gateway // history rewrite through one closure. const prepareActive = (args: { invocation: Parameters[0]; + gatewayCtx: Parameters[1]; clientTools: readonly ResponsesTool[]; restored: MultiAgentCheckpoint | undefined; withoutCheckpoints: ResponsesInputItem[]; @@ -119,6 +121,7 @@ const prepareActive = (args: { actionByToolName, maxConcurrentSubagents: args.maxConcurrent ?? DEFAULT_MAX_CONCURRENT_SUBAGENTS, restored, + clientToolBridge: args.gatewayCtx.clientTools, }); // Hand every client tool output the request carries to the subagent that // asked for it, before the root ever sees the history. diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts index e0c90a1bc..7a08b34bd 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts @@ -24,6 +24,7 @@ import { type AgentStatusReport, type AgentMessageKind, } from './wire.ts'; +import type { ClientToolBridge } from '../../../client-tools.ts'; import type { ServerToolCapabilityRuntime, ServerToolProgress } from '../../server-tool-shim.ts'; import { collectResponsesProtocolEventsToResult, @@ -86,6 +87,10 @@ export interface MultiAgentSessionOptions { actionByToolName: ReadonlyMap; maxConcurrentSubagents: number; restored: MultiAgentCheckpoint | undefined; + // Present when the transport can hand a client tool output back to a + // response that is still running, which lets a parked subagent resume + // in place instead of ending the whole response. + clientToolBridge: ClientToolBridge | undefined; } export class MultiAgentSession { @@ -316,7 +321,16 @@ export class MultiAgentSession { }; } agent.status = 'running'; - return true; + const injected = await this.options.clientToolBridge?.awaitOutputs(clientCalls.map(call => call.call_id)) ?? null; + if (injected === null) return true; + // The client answered on the still-open connection, so this agent + // picks straight back up and the response never has to end. + for (const item of injected) { + const callId = item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.call_id : undefined; + if (callId === undefined) continue; + this.routeClientOutput(callId, item); + } + continue; } const actionCalls = calls.filter(call => this.options.actionByToolName.has(call.name)); diff --git a/packages/gateway/src/data-plane/chat/responses/websocket.ts b/packages/gateway/src/data-plane/chat/responses/websocket.ts index 0aff7529e..d2ddb78bd 100644 --- a/packages/gateway/src/data-plane/chat/responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/responses/websocket.ts @@ -1,6 +1,7 @@ import type { Context } from 'hono'; import { wrapNativeResponsesClientOutput } from './client-output.ts'; +import { ClientToolBridgeRegistry, type WebSocketClientToolBridge } from './inject.ts'; import { createResponsesWsSession } from './items/store.ts'; import { PreviousResponseNotFoundError } from './serve-prep.ts'; import { responsesServe } from './serve.ts'; @@ -18,7 +19,7 @@ import { SourceStreamState, eventResultMetadata } from '../shared/respond.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/responses'; -import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ResponsesRequestPayload, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import { isResponsesTerminalEvent, type CanonicalResponsesPayload, type ResponsesInjectClientEvent, type ResponsesInjectCreatedEvent, type ResponsesInjectFailedEvent, type ResponsesRequestPayload, type ResponsesStreamEvent } from '@floway-dev/protocols/responses'; import type { ExecuteResult } from '@floway-dev/provider'; import { toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -96,6 +97,7 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan // turn rather than freezing key/user policy at upgrade time. const authenticatedRawKey = apiKeyFromContext(c).key; const session = createResponsesWsSession(); + const bridges = new ClientToolBridgeRegistry(); let closed = false; let activeAbortController: AbortController | undefined; let queue = Promise.resolve(); @@ -144,6 +146,9 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan const closeActiveRequest = (): void => { closed = true; activeAbortController?.abort(); + // A runtime blocked on an injection must not outlive the socket that was + // going to deliver it. + bridges.closeAll(); sessionClosedResolve?.(); }; @@ -151,13 +156,21 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan onClose: closeActiveRequest, onError: closeActiveRequest, onMessage: (event, socket) => { + // `response.inject` exists to unblock the response that is running right + // now, so it must never queue behind it. It touches only the connection's + // bridge registry, so dispatching it inline is also safe. + const injection = parseInjectEvent(event.data); + if (injection !== null) { + void handleInjectMessage(c, socket, bridges, injection, authenticatedRawKey, () => closed); + return; + } queue = queue .then(async () => { if (closed) return; const abortController = new AbortController(); activeAbortController = abortController; try { - await handleClientMessage(c, socket, session, event.data, authenticatedRawKey, abortController, () => closed, sessionScheduler); + await handleClientMessage(c, socket, session, event.data, authenticatedRawKey, abortController, () => closed, sessionScheduler, bridges); } finally { if (activeAbortController === abortController) activeAbortController = undefined; } @@ -172,6 +185,60 @@ const createResponsesWebSocketEvents = (c: AuthedContext): ResponsesWebSocketHan }; }; +// Recognizes an injection without committing to it: a frame that fails to +// parse, or is any other event, falls through to the ordinary queued path so +// the existing validation and error shaping still apply. +const parseInjectEvent = (data: unknown): ResponsesInjectClientEvent | null => { + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(wsDataToBytes(data))); + } catch { + return null; + } + if (!parsed || typeof parsed !== 'object') return null; + const candidate = parsed as Partial; + if (candidate.type !== 'response.inject') return null; + return { + type: 'response.inject', + response_id: typeof candidate.response_id === 'string' ? candidate.response_id : '', + input: Array.isArray(candidate.input) ? candidate.input : [], + }; +}; + +const handleInjectMessage = async ( + c: AuthedContext, + socket: ResponsesWebSocketSocket, + bridges: ClientToolBridgeRegistry, + injection: ResponsesInjectClientEvent, + authenticatedRawKey: string, + isClosed: () => boolean, +): Promise => { + if (isClosed()) return; + if (!(await authenticateApiKey(c, authenticatedRawKey))) { + sendError(socket, 401, { type: 'authentication_error', code: 'invalid_api_key', message: 'Invalid API key.' }); + return; + } + + const committed = bridges.commit(injection.response_id, injection.input); + if (committed.ok) { + sendJson(socket, { + type: 'response.inject.created', + response_id: injection.response_id, + sequence_number: bridges.nextSequenceNumber(), + } satisfies ResponsesInjectCreatedEvent); + return; + } + // The uncommitted items ride back so the client can replay them on a fresh + // response rather than losing the work its tool already did. + sendJson(socket, { + type: 'response.inject.failed', + response_id: injection.response_id, + sequence_number: bridges.nextSequenceNumber(), + error: { code: committed.code, message: committed.message }, + input: injection.input, + } satisfies ResponsesInjectFailedEvent); +}; + const handleClientMessage = async ( c: AuthedContext, socket: ResponsesWebSocketSocket, @@ -181,6 +248,7 @@ const handleClientMessage = async ( downstreamAbortController: AbortController, isClosed: () => boolean, backgroundScheduler: BackgroundScheduler, + bridges: ClientToolBridgeRegistry, ): Promise => { const signal = downstreamAbortController.signal; let eventId: string | undefined; @@ -222,6 +290,7 @@ const handleClientMessage = async ( ? message.response : Object.fromEntries(Object.entries(message).filter(([key]) => key !== 'type' && key !== 'event_id')); const payload = responsesPayloadFromClientSource(source); + const bridge = bridges.open(); ctx = createChatGatewayCtxFromHono(c, { wantsStream: true, downstreamAbortController, @@ -232,7 +301,7 @@ const handleClientMessage = async ( method: 'WS', model: payload.model, backgroundScheduler, - }, (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined)); + }, (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined), bridge); let result; try { @@ -256,7 +325,7 @@ const handleClientMessage = async ( throw error; } - await respondResponsesWebSocket({ socket, eventId, signal, isClosed, result, ctx }); + await respondResponsesWebSocket({ socket, eventId, signal, isClosed, result, ctx, bridge, bridges }); } catch (error) { if (signal.aborted || isClosed()) return; if (error instanceof TranslatorInputError) { @@ -325,9 +394,12 @@ const respondResponsesWebSocket = async (input: { readonly isClosed: () => boolean; readonly result: ExecuteResult>; readonly ctx: ChatGatewayCtx; + readonly bridge: WebSocketClientToolBridge; + readonly bridges: ClientToolBridgeRegistry; }): Promise => { - const { socket, eventId, signal, isClosed, result, ctx } = input; + const { socket, eventId, signal, isClosed, result, ctx, bridge, bridges } = input; if (result.type === 'api-error') { + bridge.close(); recordFailedRequest(ctx, result.performance); ctx.dump?.error(result.source, result.upstreamId); sendError(socket, result.status, normalizeErrorBody(parseMaybeJson(result.body, result.headers), result.status), eventId, ctx.dump); @@ -336,6 +408,7 @@ const respondResponsesWebSocket = async (input: { } if (result.type === 'internal-error') { + bridge.close(); recordFailedRequest(ctx, result.performance); ctx.dump?.failed(result.error.message); sendError(socket, result.status, internalErrorEnvelope(result.error), eventId, ctx.dump); @@ -382,6 +455,13 @@ const respondResponsesWebSocket = async (input: { } const frame = next.result.value; + // Register the id the client sees before pulling the next frame: that + // pull is where the runtime blocks on an injection, and the client can + // only address one by response id. The id is only authoritative here, + // after every egress transform has run. + if (frame.type === 'event' && 'response' in frame.event) { + bridges.register(bridge, frame.event.response.id); + } pendingNext = pendingWsFrameResult(iterator.next()); if (frame.type !== 'event') continue; @@ -433,6 +513,9 @@ const respondResponsesWebSocket = async (input: { state.failed = true; sendError(socket, 500, serverErrorEnvelope(error), eventId, ctx.dump); } finally { + // The response can take no more input; a late injection now learns it is + // already complete instead of waiting on a runtime that has stopped. + bridge.close(); const metadata = await eventResultMetadata(result); const failed = state.failedAfter(completion); if (failed) ctx.dump?.failed(`responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); diff --git a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts index 3912d4376..186e86586 100644 --- a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts @@ -2,6 +2,7 @@ import { AffinityRequestContext } from './affinity/index.ts'; import { apiKeyFromContext, type AuthedContext } from '../../../middleware/auth.ts'; import type { ApiKey } from '../../../repo/types.ts'; import { createGatewayCtxFromHono, type CreateGatewayCtxOptions, type GatewayCtx } from '../../shared/gateway-ctx.ts'; +import type { ClientToolBridge } from '../responses/client-tools.ts'; import type { StatefulResponsesStore } from '../responses/items/store.ts'; // Chat-protocol ctx adds the affinity membrane and the Responses item store. @@ -15,6 +16,11 @@ import type { StatefulResponsesStore } from '../responses/items/store.ts'; export interface ChatGatewayCtx extends GatewayCtx { readonly affinity: AffinityRequestContext; readonly store: StatefulResponsesStore; + // Present only on a transport that can hand client tool outputs to a + // still-running response — today the Responses WebSocket, through + // `response.inject`. Absent everywhere else, which is the signal to end the + // response at a client tool call and let the client resume it. + readonly clientTools?: ClientToolBridge; } // Chat-protocol counterpart of `createGatewayCtxFromHono`. The factory receives @@ -25,11 +31,13 @@ export const createChatGatewayCtxFromHono = ( c: AuthedContext, opts: CreateGatewayCtxOptions, storeFactory: (apiKey: ApiKey, requestStartedAt: number) => StatefulResponsesStore, + clientTools?: ClientToolBridge, ): ChatGatewayCtx => { const base = createGatewayCtxFromHono(c, opts); return { ...base, affinity: new AffinityRequestContext(apiKeyFromContext(c).serverSecret), store: storeFactory(apiKeyFromContext(c), base.requestStartedAt), + ...(clientTools !== undefined ? { clientTools } : {}), }; }; diff --git a/packages/protocols/src/responses/index.ts b/packages/protocols/src/responses/index.ts index ea971a6b5..6bd5f37f6 100644 --- a/packages/protocols/src/responses/index.ts +++ b/packages/protocols/src/responses/index.ts @@ -1258,6 +1258,39 @@ export const isResponsesTerminalEvent = (event: Pick 'response' in event ? event.response : null; +// ── WebSocket injection ── +// +// `response.inject` commits client-owned tool outputs into a response that is +// still running, so an agent waiting on one resumes without the client having +// to start a new response. It exists only on the WebSocket transport: these +// are connection events, not part of any response's own event stream. +// https://github.com/openai/openai-python/blob/90483adb3034186c18c0f64de26b24699d733173/src/openai/types/beta/beta_response_inject_event.py +// https://github.com/openai/openai-python/blob/90483adb3034186c18c0f64de26b24699d733173/src/openai/types/beta/beta_response_inject_created_event.py +// https://github.com/openai/openai-python/blob/90483adb3034186c18c0f64de26b24699d733173/src/openai/types/beta/beta_response_inject_failed_event.py +export interface ResponsesInjectClientEvent { + type: 'response.inject'; + response_id: string; + input: ResponsesInputItem[]; +} + +export type ResponsesInjectErrorCode = 'response_already_completed' | 'response_not_found'; + +export interface ResponsesInjectCreatedEvent { + type: 'response.inject.created'; + response_id: string; + sequence_number: number; +} + +export interface ResponsesInjectFailedEvent { + type: 'response.inject.failed'; + response_id: string; + sequence_number: number; + error: { code: ResponsesInjectErrorCode; message: string }; + // The uncommitted items, returned so the client can retry them on another + // response. + input: ResponsesInputItem[]; +} + export { type CanonicalResponsesCompactPayload, type ResponsesCompactPayload, From 7c3f7c777fe21fb17fec5ad022318b4972205c57 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 04:57:46 +0800 Subject: [PATCH 08/11] test(gateway): cover response.inject delivery and in-place resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the bridge's own semantics — partial delivery, an injection that lands before its call is reached, the two official failure codes, and release on close — plus the end-to-end paths: a parked subagent resuming inside the same response, a root-level client tool doing the same, and the fallback to ending with a checkpoint when the transport cannot deliver. Also records the injection path in docs/MULTI-AGENT.md. --- docs/MULTI-AGENT.md | 49 ++++++++-- .../data-plane/chat/responses/inject_test.ts | 96 +++++++++++++++++++ .../multi-agent/orchestration_test.ts | 94 +++++++++++++++++- .../interceptors/server-tool-shim.ts | 2 +- 4 files changed, 229 insertions(+), 12 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/chat/responses/inject_test.ts diff --git a/docs/MULTI-AGENT.md b/docs/MULTI-AGENT.md index 0ffef3874..6f1fdb523 100644 --- a/docs/MULTI-AGENT.md +++ b/docs/MULTI-AGENT.md @@ -65,6 +65,46 @@ may not be in flight at once. Budgets bound one request's upstream spend: turns per agent, turns per session, nesting depth, and `multi_agent.max_concurrent_subagents` live agents. +## Client tools: inject, or end and resume + +When an agent reaches a client-owned function tool the runtime has two ways +forward, and which one it takes is a property of the transport rather than of +the agent. + +Over a WebSocket that is still open, the client commits the output into the +running response with `response.inject`, and the agent picks straight back up. +This is what the beta's own hosted multi-agent client does; it falls back to a +continuation only once the response has reached a terminal event. + +```text +client gateway + │ response.create │ + ├────────────────────────────>│ root spawns /root/researcher + │<── function_call ───────────┤ agent=/root/researcher, response still open + │ response.inject │ + ├────────────────────────────>│ committed to resp_… + │<── response.inject.created ─┤ + │<── agent_message ───────────┤ the subagent resumed and answered + │<── response.completed ──────┤ +``` + +Two things make that work over one socket. Injections dispatch off the +message serialization queue entirely — queuing one behind the response it is +meant to unblock would deadlock. And the bridge is keyed by the response id the +client actually sees, registered after every egress transform has run, because +that is the only id the client can address. + +Everywhere else — HTTP, a socket that closed, a response that already reached +its terminal event — the response ends and the client resumes it with a fresh +`response.create` carrying the checkpoint. A waiter always settles: the +terminal event and socket close both release outstanding waiters, so a blocked +runtime falls back rather than hanging. + +`response.inject.failed` carries the uncommitted items back so the client can +replay them on another response, with `response_not_found` for an id this +connection never issued and `response_already_completed` for one that has +finished. + ## Checkpoints When any agent reaches a client-owned function tool the response has to end, @@ -87,12 +127,3 @@ tightens. Size, agent count, and path uniqueness are all bounded on the way in. The carrier is excluded from the affinity envelope. See [AFFINITY.md](./AFFINITY.md#responses) for why. - -## Not yet hosted - -`response.inject` — the WebSocket event that hands a client tool output back to -a still-running Response — is not implemented. Every client tool boundary ends -the response, and the client resumes by replaying its history plus the -checkpoint through an ordinary `response.create`. That path works identically -over HTTP and WebSocket; `response.inject` would only remove the extra -round trip on a connection that is still open. diff --git a/packages/gateway/__tests__/data-plane/chat/responses/inject_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/inject_test.ts new file mode 100644 index 000000000..137d9cda2 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/responses/inject_test.ts @@ -0,0 +1,96 @@ +import { test } from 'vitest'; + +import { ClientToolBridgeRegistry } from '../../../../src/data-plane/chat/responses/inject.ts'; +import type { ResponsesInputItem } from '@floway-dev/protocols/responses'; +import { assert, assertEquals } from '@floway-dev/test-utils'; + +const output = (callId: string, text: string): ResponsesInputItem => ({ + type: 'function_call_output', + call_id: callId, + output: text, +}); + +// Lets a test observe whether a waiter is still blocked without racing it. +const settled = async (promise: Promise): Promise<{ done: true; value: T } | { done: false }> => { + const pending = Symbol('pending'); + const raced = await Promise.race([promise, Promise.resolve(pending)]); + return raced === pending ? { done: false } : { done: true, value: raced as T }; +}; + +test('a committed injection releases the waiter for its call ids', async () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + + const waiting = bridge.awaitOutputs(['call_a']); + assertEquals((await settled(waiting)).done, false); + + assertEquals(registry.commit('resp_1', [output('call_a', 'done')]), { ok: true }); + assertEquals(await waiting, [output('call_a', 'done')]); +}); + +test('a waiter blocks until every call it named has arrived', async () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + + const waiting = bridge.awaitOutputs(['call_a', 'call_b']); + registry.commit('resp_1', [output('call_a', 'a')]); + assertEquals((await settled(waiting)).done, false); + + registry.commit('resp_1', [output('call_b', 'b')]); + assertEquals(await waiting, [output('call_a', 'a'), output('call_b', 'b')]); +}); + +test('an injection that arrives before its call is reached is held, not dropped', async () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + + // The beta commits an injection atomically rather than matching it against + // whatever happens to be waiting, so an early one has to survive. + assertEquals(registry.commit('resp_1', [output('call_a', 'early')]), { ok: true }); + assertEquals(await bridge.awaitOutputs(['call_a']), [output('call_a', 'early')]); +}); + +test('an unknown response id is reported as not found', () => { + const registry = new ClientToolBridgeRegistry(); + const result = registry.commit('resp_missing', [output('call_a', 'x')]); + assert(!result.ok); + assertEquals(result.code, 'response_not_found'); +}); + +test('a finished response reports that it is already complete', () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + bridge.close(); + + const result = registry.commit('resp_1', [output('call_a', 'x')]); + assert(!result.ok); + assertEquals(result.code, 'response_already_completed'); +}); + +test('closing releases every outstanding waiter so a blocked runtime can end its response', async () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + + const waiting = bridge.awaitOutputs(['call_a']); + registry.closeAll(); + + assertEquals(await waiting, null); +}); + +test('a bridge with no response id yet cannot be waited on', async () => { + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + // Nothing has been streamed, so the client has no id to address; waiting + // would block on an injection that cannot be sent. + assertEquals(await bridge.awaitOutputs(['call_a']), null); +}); + +test('sequence numbers advance across a connection', () => { + const registry = new ClientToolBridgeRegistry(); + assertEquals([registry.nextSequenceNumber(), registry.nextSequenceNumber()], [0, 1]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts index 6edd53846..7697ec11a 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts @@ -1,5 +1,7 @@ import { test } from 'vitest'; +import type { ClientToolBridge } from '../../../../../../../src/data-plane/chat/responses/client-tools.ts'; +import { ClientToolBridgeRegistry } from '../../../../../../../src/data-plane/chat/responses/inject.ts'; import { withResponsesServerToolShim } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tool-shim.ts'; import { decodeMultiAgentCheckpoint } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts'; import { multiAgentServerTool } from '../../../../../../../src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts'; @@ -102,8 +104,17 @@ const scripted = (inv: ResponsesInvocation, turns: ProtocolFrame => { - const result = await shim(inv, mockChatGatewayCtx({ wantsStream: true }), script.run); +const drain = async ( + inv: ResponsesInvocation, + script: Scripted, + clientTools?: ClientToolBridge, +): Promise => { + const ctx = mockChatGatewayCtx({ wantsStream: true }); + const result = await shim( + inv, + clientTools === undefined ? ctx : { ...ctx, clientTools }, + script.run, + ); assert(result.type === 'events'); const events: ResponsesStreamEvent[] = []; for await (const frame of result.events) { @@ -441,3 +452,82 @@ test('spawnToolName is stable across the injected set', async () => { await drain(inv, script); assertEquals(spawnToolName(script), 'spawn_agent'); }); + +// ── response.inject ──────────────────────────────────────────────────────── + +test('an injected output resumes a parked subagent without ending the response', async () => { + const inv = makeInvocation({ + tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }], + }); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([functionCall('call_tool', 'lookup', { q: 'x' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const registry = new ClientToolBridgeRegistry(); + const bridge = registry.open(); + registry.register(bridge, 'resp_1'); + // The client answers as soon as the subagent parks, exactly as it would on + // a socket that is still open. + const injecting = (async () => { + for (let attempt = 0; attempt < 50; attempt++) { + const committed = registry.commit('resp_1', [{ type: 'function_call_output', call_id: 'call_tool', output: 'the answer' }]); + if (committed.ok) return; + await Promise.resolve(); + } + })(); + + const events = await drain(inv, script, bridge); + await injecting; + const items = doneItems(events); + + // The subagent picked back up and finished inside the same response. + const answer = items.find(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assert(answer !== undefined); + assertStringIncludes(agentMessageText(answer), 'found it'); + assert(items.some(item => item.type === 'message')); + // All four turns ran: the response never stopped at the client tool. + assertEquals(script.payloads.length, 4); + assertEquals(events.at(-1)?.type, 'response.completed'); +}); + +test('a bridge that cannot deliver falls back to ending the response with a checkpoint', async () => { + const inv = makeInvocation({ + tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }], + }); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([functionCall('call_tool', 'lookup', { q: 'x' })]), + ]); + + const closedBridge: ClientToolBridge = { awaitOutputs: () => Promise.resolve(null) }; + const items = doneItems(await drain(inv, script, closedBridge)); + + const checkpoint = checkpointOf(items.findLast(item => item.type === 'agent_message')); + assertEquals(checkpoint.agents[0].pendingClientCallId, 'call_tool'); +}); + +test('a client tool the root itself called is also resumed in place', async () => { + const inv = makeInvocation({ + multi_agent: { enabled: true }, + tools: [{ type: 'function', name: 'lookup', parameters: {}, strict: false }], + }); + const script = scripted(inv, [ + turn([functionCall('call_root', 'lookup', { q: 'x' })]), + turn([message('all done')]), + ]); + + const bridge: ClientToolBridge = { + awaitOutputs: callIds => Promise.resolve(callIds.map(callId => ({ type: 'function_call_output', call_id: callId, output: 'answer' }))), + }; + + const events = await drain(inv, script, bridge); + + // The second turn ran, and it saw the injected output in its input. + assertEquals(script.payloads.length, 2); + const resumedInput = script.payloads[1].input; + assert(resumedInput.some(item => item.type === 'function_call_output' && item.call_id === 'call_root')); + assertEquals(events.at(-1)?.type, 'response.completed'); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index a256a39d7..27daf23c6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -1,9 +1,9 @@ import { jsonrepair } from 'jsonrepair'; import type { ResponsesInterceptor, ResponsesInvocation } from './types.ts'; -import type { ClientToolBridge } from '../client-tools.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; +import type { ClientToolBridge } from '../client-tools.ts'; import type { StatefulResponsesStore } from '../items/store.ts'; import type { InterceptorRun } from '@floway-dev/interceptor'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; From 7afebae9ccd58b36ec5e1d88e66d5e89de190c19 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 14:33:52 +0800 Subject: [PATCH 09/11] fix(gateway): align the hosted multi-agent wire with the beta contract An audit against the official guide and both reference SDK clients turned up several places where the emulation diverged from what the hosted service does, some of which would have silently produced an empty answer. Wire: - `agent_message.agent` names the RECIPIENT, not the author. - Root items carry `agent.agent_name: /root`, and root messages carry a `phase`. Both SDKs extract the run's answer as the `/root` message with `phase: final_answer` and drop every other message silently, so omitting either yielded an empty result with no error. An upstream that emits its own phase keeps it; only a translated target reconstructs one, which defers message frames to the end of their turn. - `multi_agent_call_output` gets its own `maco_` id prefix. - `multi_agent` no longer reaches the upstream that is not hosting it. - Injection acks draw their sequence number from the response's space, and a malformed injection is a generic 400 rather than one of the two codes, neither of which describes it. Semantics: - `send_message` queues without triggering a turn; `followup_task` is what assigns a task and runs it, and it targets a subagent. - `list_agents` reports each agent's last task message. - `wait_agent`'s timeout floor is gone: it was transcribed from Codex's own client-side collaboration tools, and the beta documents no timeout for this action at all. - The service's own developer instructions for the root agent and for subagents are vendored from the guide and injected per agent. - `reasoning.summary`, `max_tool_calls`, and compaction are refused, as the guide lists them unsupported here. - A parked call is awaited singly when a bridge can deliver it, since a client that pauses at the first function call cannot answer a second. --- docs/MULTI-AGENT.md | 63 +++++++- .../multi-agent/checkpoint_test.ts | 8 +- .../multi-agent/orchestration_test.ts | 145 ++++++++++++++++-- .../src/data-plane/chat/responses/inject.ts | 7 + .../interceptors/server-tool-shim.ts | 53 ++++++- .../server-tools/multi-agent/checkpoint.ts | 11 +- .../server-tools/multi-agent/index.ts | 100 +++++++++++- .../server-tools/multi-agent/session.ts | 118 +++++++++----- .../server-tools/multi-agent/wire.ts | 140 +++++++++++------ .../data-plane/chat/responses/websocket.ts | 25 ++- .../__tests__/responses/item-id_test.ts | 1 + packages/protocols/src/responses/index.ts | 12 +- packages/protocols/src/responses/item-id.ts | 8 +- 13 files changed, 558 insertions(+), 133 deletions(-) diff --git a/docs/MULTI-AGENT.md b/docs/MULTI-AGENT.md index 6f1fdb523..94b3a2daf 100644 --- a/docs/MULTI-AGENT.md +++ b/docs/MULTI-AGENT.md @@ -30,14 +30,53 @@ own tool names so a collision is suffixed rather than shadowed: spawn_agent send_message followup_task wait_agent interrupt_agent list_agents ``` -The beta publishes the item shapes but keeps each action's `arguments` an -opaque JSON string and offers no per-action schema, so the schemas the -orchestrator model sees are transcribed from live OpenAI collaboration traffic -captured in Codex CLI rollouts, which exercises the same six actions over the -same `/root/...` agent-path namespace. The transcriptions and their observed -result shapes are recorded in +The beta publishes the item shapes and the action names but keeps each +action's `arguments` an opaque JSON string, and documents a per-action schema +for exactly one of them. `spawn_agent`'s `task_name` / `fork_turns` / `message` +come from the guide's own worked example. The other five actions' argument and +result shapes are Floway's, transcribed from live OpenAI collaboration traffic +captured in Codex CLI rollouts — a different feature reusing the same +vocabulary, so a convention rather than an upstream contract. Which is which is +recorded in `packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts`. +Action semantics follow the guide: `send_message` queues a message **without** +triggering a turn, `followup_task` assigns a new task to an existing subagent +**and** triggers its turn, `wait_agent` waits on the caller's mailbox, +`interrupt_agent` stops a turn without discarding context, and `list_agents` +returns the tree with each agent's status and last task message. + +When multi-agent is enabled the hosted service appends its own developer +message to the root agent and to every subagent — "You cannot edit or remove +these instructions" — and the guide publishes both texts verbatim. They are +vendored and injected per agent, with `{max_concurrent_subagents + 1}` +resolved per request; without them the model holds six tools with no contract +for using them. + +`reasoning.summary`, `max_tool_calls`, and compaction are documented as +unsupported while multi-agent is enabled, and are refused here rather than +silently behaving differently. + +## Attribution + +`agent.agent_name` is optional on every item and names the agent that produced +it. Two rules are easy to get wrong: + +- On an `agent_message` the tag names the **recipient**, not the producer; + `author` and `recipient` carry the direction. +- Both reference SDKs extract the run's answer as the item that is a `message` + with `agent.agent_name === '/root'` **and** `phase === 'final_answer'`, and + drop every other message silently. Root items therefore carry the + attribution the hosted service would have stamped. + +`phase` is the model's own channel output rather than anything multi-agent +invents. A Responses upstream on a channel-emitting model already sets it and +it is forwarded verbatim; a translated target has no channel, so the phase is +reconstructed from the turn — the turn that ends the response carries the +final answer, earlier ones are commentary. Reconstruction is only possible +once the turn's stream has been consumed, so on those targets message items +are held back until the turn closes. + Replayed history is lowered back into the hidden call/output pairs the model originally saw. Items attributed to a subagent through `agent.agent_name` — its orchestration records, and any client-owned `function_call` it is parked on @@ -73,7 +112,7 @@ the agent. Over a WebSocket that is still open, the client commits the output into the running response with `response.inject`, and the agent picks straight back up. -This is what the beta's own hosted multi-agent client does; it falls back to a +This is what the beta's own hosted multi-agent clients do; they fall back to a continuation only once the response has reached a terminal event. ```text @@ -103,7 +142,15 @@ runtime falls back rather than hanging. `response.inject.failed` carries the uncommitted items back so the client can replay them on another response, with `response_not_found` for an id this connection never issued and `response_already_completed` for one that has -finished. +finished. Those are the only two codes the beta defines; a malformed injection +is answered as a generic 400 instead, because neither code describes it. Acks +draw their `sequence_number` from the response's own sequence space. + +Parked calls are awaited one at a time on this path. A client that pauses at +the first `function_call` it reads has not yet seen a second one and cannot +answer both at once; the items are already on the wire, so it picks each up as +it resumes. Without a bridge the response simply ends with every parked call +outstanding, which is what the guide describes for the HTTP boundary. ## Checkpoints diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts index e4214118c..8387776a8 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint_test.ts @@ -22,6 +22,8 @@ const checkpoint = (overrides: Partial = {}): MultiAgentCh path: '/root/researcher', status: { completed: 'found it' }, history: [{ type: 'message', role: 'user', content: 'go' }], + pendingClientCallIds: [], + lastTaskMessage: 'go', }], ...overrides, }); @@ -45,7 +47,7 @@ test('a checkpoint from a future version is not consumed', () => { }); test('duplicate agent paths are rejected because they make routing ambiguous', () => { - const agent = { path: '/root/a', status: 'running' as const, history: [] }; + const agent = { path: '/root/a', status: 'running' as const, history: [], pendingClientCallIds: [], lastTaskMessage: null }; assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), agents: [agent, { ...agent }] })); }); @@ -54,6 +56,8 @@ test('an agent count beyond the cap is rejected', () => { path: `/root/a${index}`, status: 'running' as const, history: [], + pendingClientCallIds: [], + lastTaskMessage: null, })); assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), agents })); }); @@ -66,7 +70,7 @@ test('a negative or fractional turn budget is rejected', () => { test('an unknown agent status is rejected', () => { assertFalse(isMultiAgentCheckpoint({ ...checkpoint(), - agents: [{ path: '/root/a', status: { cancelled: 'x' }, history: [] }], + agents: [{ path: '/root/a', status: { cancelled: 'x' }, history: [], pendingClientCallIds: [], lastTaskMessage: null }], })); }); diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts index 7697ec11a..322258566 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/multi-agent/orchestration_test.ts @@ -215,9 +215,13 @@ test('the subagent sees only its task, never the root history', async () => { await drain(inv, script); const childInput = script.payloads[1].input; - assertEquals(childInput.length, 1); - assert(childInput[0].type === 'agent_message'); - assertStringIncludes(agentMessageText(childInput[0] as ResponsesOutputItem), 'Message Type: NEW_TASK'); + // The hosted service appends its own developer instructions to every agent; + // after that the subagent sees only its task. + assertEquals(childInput.length, 2); + assert(childInput[0].type === 'message' && childInput[0].role === 'developer'); + assertStringIncludes(JSON.stringify(childInput[0].content), 'You are an agent in a team of agents'); + assert(childInput[1].type === 'agent_message'); + assertStringIncludes(agentMessageText(childInput[1] as ResponsesOutputItem), 'Message Type: NEW_TASK'); // A subagent transcript is Floway-internal and must never enter the // upstream's stored conversation. assertEquals(script.payloads[1].store, false); @@ -256,7 +260,7 @@ test('a subagent reaching a client tool ends the response and parks the call on assertEquals(parked.agent, { agent_name: '/root/researcher' }); const checkpoint = checkpointOf(items.findLast(item => item.type === 'agent_message')); - assertEquals(checkpoint.agents[0].pendingClientCallId, 'call_tool'); + assertEquals(checkpoint.agents[0].pendingClientCallIds, ['call_tool']); assertEquals(events.at(-1)?.type, 'response.completed'); }); @@ -310,16 +314,22 @@ test('resuming routes the client output to its subagent and hides the pair from // ── Action semantics ─────────────────────────────────────────────────────── -test('wait_agent rejects a timeout below the upstream floor', async () => { +test('send_message queues without triggering a turn; wait_agent runs it', async () => { const inv = makeInvocation(); const script = scripted(inv, [ - turn([functionCall('call_1', 'wait_agent', { timeout_ms: 1000 })]), + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([functionCall('call_2', 'send_message', { target: '/root/researcher', message: 'also check B' })]), + // No turn is scripted for send_message: queuing must not run the agent. + turn([functionCall('call_3', 'wait_agent', {})]), + turn([message('also found B')]), turn([message('done')]), ]); - const output = doneItems(await drain(inv, script)).find(item => item.type === 'multi_agent_call_output'); - assert(output?.type === 'multi_agent_call_output'); - assertEquals(output.output[0].text, 'timeout_ms must be at least 10000'); + const items = doneItems(await drain(inv, script)); + const answers = items.filter(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assertEquals(answers.length, 2); + assertStringIncludes(agentMessageText(answers[1]), 'also found B'); }); test('list_agents reports the tree and filters by path prefix', async () => { @@ -337,7 +347,11 @@ test('list_agents reports the tree and filters by path prefix', async () => { assertEquals(outputs[1], { agents: [] }); assertEquals(outputs[2], { - agents: [{ agent_name: '/root/researcher', agent_status: { completed: 'found it' } }], + agents: [{ + agent_name: '/root/researcher', + agent_status: { completed: 'found it' }, + last_task_message: 'find it', + }], }); }); @@ -411,7 +425,7 @@ test('a subagent whose turn fails settles as errored with the recovery hint', as const errored = items.find(item => item.type === 'agent_message' && item.author === '/root/researcher'); assert(errored !== undefined); assertStringIncludes(agentMessageText(errored), 'Agent errored:'); - assertStringIncludes(agentMessageText(errored), 'use the available collaboration tools to give it another task'); + assertStringIncludes(agentMessageText(errored), 'give it another task'); }); // ── Engagement ───────────────────────────────────────────────────────────── @@ -506,7 +520,7 @@ test('a bridge that cannot deliver falls back to ending the response with a chec const items = doneItems(await drain(inv, script, closedBridge)); const checkpoint = checkpointOf(items.findLast(item => item.type === 'agent_message')); - assertEquals(checkpoint.agents[0].pendingClientCallId, 'call_tool'); + assertEquals(checkpoint.agents[0].pendingClientCallIds, ['call_tool']); }); test('a client tool the root itself called is also resumed in place', async () => { @@ -531,3 +545,110 @@ test('a client tool the root itself called is also resumed in place', async () = assert(resumedInput.some(item => item.type === 'function_call_output' && item.call_id === 'call_root')); assertEquals(events.at(-1)?.type, 'response.completed'); }); + +// ── Attribution and phase ────────────────────────────────────────────────── + +test('the root final message carries the attribution both reference SDKs require', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const items = doneItems(await drain(inv, script)); + const messages = items.filter(item => item.type === 'message'); + + // The turn that spawned still had a tool call, so its text is commentary; + // only the turn that ended the response carries the final answer. + assertEquals(messages.at(-1)?.agent, { agent_name: '/root' }); + assertEquals(messages.at(-1)?.phase, 'final_answer'); + for (const intermediate of messages.slice(0, -1)) assertEquals(intermediate.phase, 'commentary'); +}); + +test('an upstream that emits its own phase keeps it verbatim', async () => { + // A Responses target's model owns the channel, so the gateway must not + // overwrite what it said. + const inv = { + ...makeInvocation(), + targetApi: 'responses' as const, + candidate: stubModelCandidate({ + enabledFlags: new Set(['responses-multi-agent-shim'] as const), + model: { id: 'gpt-x', endpoints: { responses: {} } }, + }), + }; + const script = scripted(inv, [ + turn([{ ...message('all done'), phase: 'commentary' } as ResponsesOutputItem]), + ]); + + const items = doneItems(await drain(inv, script)); + const last = items.findLast(item => item.type === 'message'); + assertEquals(last?.phase, 'commentary'); + assertEquals(last?.agent, { agent_name: '/root' }); +}); + +test('an agent message is tagged with its recipient, not its author', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'spawn_agent', { task_name: 'researcher', message: 'find it' })]), + turn([message('found it')]), + turn([message('all done')]), + ]); + + const answer = doneItems(await drain(inv, script)) + .find(item => item.type === 'agent_message' && item.author === '/root/researcher'); + assert(answer?.type === 'agent_message'); + assertEquals(answer.recipient, '/root'); + assertEquals(answer.agent, { agent_name: '/root' }); +}); + +test('the orchestration call and its output use their own id prefixes', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [ + turn([functionCall('call_1', 'list_agents', {})]), + turn([message('done')]), + ]); + + const items = doneItems(await drain(inv, script)); + const call = items.find(item => item.type === 'multi_agent_call'); + const output = items.find(item => item.type === 'multi_agent_call_output'); + assert(call?.id?.startsWith('mac_') === true && !call.id.startsWith('maco_')); + assert(output?.id?.startsWith('maco_') === true); +}); + +test('the upstream never sees the multi_agent config it is not hosting', async () => { + const inv = makeInvocation(); + const script = scripted(inv, [turn([message('hi')])]); + + await drain(inv, script); + + assertEquals(script.payloads[0].multi_agent, undefined); +}); + +test('the root agent receives the hosted instructions as a developer message', async () => { + const inv = makeInvocation({ multi_agent: { enabled: true, max_concurrent_subagents: 2 } }); + const script = scripted(inv, [turn([message('hi')])]); + + await drain(inv, script); + + const head = script.payloads[0].input[0]; + assert(head.type === 'message' && head.role === 'developer'); + const text = JSON.stringify(head.content); + assertStringIncludes(text, 'You are `/root`, the primary agent'); + // The published text is templated on max_concurrent_subagents + 1. + assertStringIncludes(text, 'There are 3 available concurrency slots'); +}); + +test('combinations the guide lists as unsupported are refused', async () => { + const refuse = async (payload: Partial, param: string) => { + const result = await shim(makeInvocation(payload), mockChatGatewayCtx({ wantsStream: true }), () => { + throw new Error('upstream must not be called'); + }); + assert(result.type === 'api-error'); + assertEquals(result.status, 400); + assertStringIncludes(new TextDecoder().decode(result.body), param); + }; + + await refuse({ reasoning: { summary: 'auto' } }, 'reasoning.summary'); + await refuse({ max_tool_calls: 4 }, 'max_tool_calls'); +}); diff --git a/packages/gateway/src/data-plane/chat/responses/inject.ts b/packages/gateway/src/data-plane/chat/responses/inject.ts index a64248bbe..d0d753f7e 100644 --- a/packages/gateway/src/data-plane/chat/responses/inject.ts +++ b/packages/gateway/src/data-plane/chat/responses/inject.ts @@ -106,6 +106,9 @@ const callIdOf = (item: ResponsesInputItem): string | undefined => // response is already complete rather than that it never existed. export class ClientToolBridgeRegistry { readonly #byResponseId = new Map(); + // Injection acks carry a `sequence_number` and share the response's sequence + // space, so this tracks the highest one the connection has streamed out + // rather than counting acks on their own. #sequenceNumber = 0; open(): WebSocketClientToolBridge { @@ -117,6 +120,10 @@ export class ClientToolBridgeRegistry { this.#byResponseId.set(responseId, bridge); } + observeSequenceNumber(sequenceNumber: number): void { + if (sequenceNumber >= this.#sequenceNumber) this.#sequenceNumber = sequenceNumber + 1; + } + nextSequenceNumber(): number { return this.#sequenceNumber++; } diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index 27daf23c6..789b73748 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -147,6 +147,19 @@ export interface ServerToolFinalOutputItem { // versus finished shape differently. export type ServerToolFinalizer = (kind: SynthesizedTerminal['kind']) => Promise; +// Rewrites items the upstream produced for the request's own agent before they +// are emitted, so a capability that owns an orchestration wire can stamp what a +// hosted service would have. `finalTurn` says whether this turn ends the +// response, which is only knowable once the turn's stream has been consumed — +// a stamp that reads it sets `deferMessages` and the shim holds message items +// back until then, at the cost of their frames arriving after the turn's tool +// calls. Output indexes are still allocated in arrival order, so the terminal +// snapshot's item order is unaffected. +export interface ServerToolRootOutputStamp { + readonly deferMessages: boolean; + readonly stamp: (item: ResponsesOutputItem, finalTurn: boolean) => ResponsesOutputItem; +} + export type ServerToolPrepareResult = | { type: 'inactive' } // `errorType` / `code` override the envelope for tools that emulate an @@ -166,6 +179,7 @@ export type ServerToolPrepareResult = hosted?: ServerToolHostedDispatch; injected?: ServerToolInjectedDispatch; finalizeOutput?: ServerToolFinalizer; + stampRootOutput?: ServerToolRootOutputStamp; }; // Handed to every registration so a capability that drives its own nested @@ -533,6 +547,13 @@ export const consumeTurnStreaming = async function* ( const openItems = new Map(); const openItemIds = new Map(); + const rootStamp = active.find(entry => entry.stampRootOutput !== undefined)?.stampRootOutput; + const stampRoot = (item: ResponsesOutputItem, finalTurn: boolean): ResponsesOutputItem => + rootStamp === undefined ? item : rootStamp.stamp(item, finalTurn); + // Message items whose stamp needs to know how the turn ended. Their + // downstream index is reserved on `.added` so the terminal snapshot keeps + // arrival order; only the frames wait. + const deferredMessages = new Map(); // `argumentsJson` accumulates `function_call_arguments.delta` chunks // until the closing `.done` parses them into `intercepted.arguments`. // Kept on the entry (not on `InterceptedFunctionCall`) because it's @@ -653,10 +674,15 @@ export const consumeTurnStreaming = async function* ( ? createRandomResponsesItemId('message') : undefined; if (itemId !== undefined) openItemIds.set(upstreamIndex, itemId); + const addedItem = itemId !== undefined && wireItemId !== itemId ? { ...item, id: itemId } as ResponsesOutputItem : item; + if (rootStamp?.deferMessages === true && item.type === 'message') { + deferredMessages.set(upstreamIndex, { downstreamIndex, item: undefined }); + continue; + } yield stamp({ type: 'response.output_item.added', output_index: downstreamIndex, - item: itemId !== undefined && wireItemId !== itemId ? { ...item, id: itemId } as ResponsesOutputItem : item, + item: stampRoot(addedItem, false), }); continue; } @@ -686,8 +712,14 @@ export const consumeTurnStreaming = async function* ( const doneItem: ResponsesOutputItem = itemId !== undefined && upstreamDoneItemId !== itemId ? { ...event.item, id: itemId } as ResponsesOutputItem : event.item; - yield stamp({ type: 'response.output_item.done', output_index: downstreamIndex, item: doneItem }); - merge.accumulatedOutput.set(downstreamIndex, doneItem); + const deferred = deferredMessages.get(upstreamIndex); + if (deferred !== undefined) { + deferred.item = doneItem; + continue; + } + const stamped = stampRoot(doneItem, false); + yield stamp({ type: 'response.output_item.done', output_index: downstreamIndex, item: stamped }); + merge.accumulatedOutput.set(downstreamIndex, stamped); continue; } @@ -714,7 +746,9 @@ export const consumeTurnStreaming = async function* ( } const maybeIndexedForIntercepted = event as ResponsesStreamEvent & { output_index?: unknown }; - if (typeof maybeIndexedForIntercepted.output_index === 'number' && interceptedByUpstreamIndex.has(maybeIndexedForIntercepted.output_index)) { + if (typeof maybeIndexedForIntercepted.output_index === 'number' + && (interceptedByUpstreamIndex.has(maybeIndexedForIntercepted.output_index) + || deferredMessages.has(maybeIndexedForIntercepted.output_index))) { continue; } @@ -775,6 +809,17 @@ export const consumeTurnStreaming = async function* ( }; } + // A turn that dispatched nothing and reached no client tool is the turn that + // ends the response, which is what the deferred stamp was waiting to learn. + const finalTurn = dispatched.length === 0 && clientToolCallIds.length === 0 && terminalStatus.kind === 'completed'; + for (const [, deferred] of [...deferredMessages].sort(([a], [b]) => a - b)) { + if (deferred.item === undefined) continue; + const stamped = stampRoot(deferred.item, finalTurn); + yield stamp({ type: 'response.output_item.added', output_index: deferred.downstreamIndex, item: stamped }); + yield stamp({ type: 'response.output_item.done', output_index: deferred.downstreamIndex, item: stamped }); + merge.accumulatedOutput.set(deferred.downstreamIndex, stamped); + } + return { dispatched, clientToolCallIds, turnUsage, terminalStatus }; }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts index 45d2674ab..7413e7f2e 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/checkpoint.ts @@ -45,8 +45,12 @@ export interface CheckpointAgent { // only the agent's messages to others surface as `agent_message` items — so // it has to survive here or the agent restarts from nothing. history: ResponsesInputItem[]; - // Set while the agent is parked on a client-owned function tool. - pendingClientCallId?: string; + // Client-owned function calls this agent is parked on. An HTTP turn can end + // with several outstanding at once, so this is a list. + pendingClientCallIds: string[]; + // Reported by `list_agents`; survives a resume so the tree still describes + // itself the way it did before the response ended. + lastTaskMessage: string | null; } export interface MultiAgentCheckpoint { @@ -75,7 +79,8 @@ const isCheckpointAgent = (value: unknown): value is CheckpointAgent => { if (!isAgentStatus(agent.status)) return false; if (!Array.isArray(agent.history)) return false; if (!agent.history.every(item => isJsonObject(item) && typeof (item as { type?: unknown }).type === 'string')) return false; - if (agent.pendingClientCallId !== undefined && typeof agent.pendingClientCallId !== 'string') return false; + if (!Array.isArray(agent.pendingClientCallIds) || !agent.pendingClientCallIds.every(id => typeof id === 'string')) return false; + if (agent.lastTaskMessage !== null && typeof agent.lastTaskMessage !== 'string') return false; return true; }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts index e5e4aadb6..e88b5bb27 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/index.ts @@ -18,7 +18,7 @@ import { type MultiAgentCheckpoint, } from './checkpoint.ts'; import { MultiAgentSession, DEFAULT_MAX_CONCURRENT_SUBAGENTS } from './session.ts'; -import { buildMultiAgentFunctionTools, MULTI_AGENT_ACTIONS, ROOT_AGENT_PATH } from './wire.ts'; +import { buildMultiAgentFunctionTools, multiAgentInstructions, MULTI_AGENT_ACTIONS, ROOT_AGENT_PATH } from './wire.ts'; import type { ResolvedServerToolNames, ServerToolRegistration, @@ -28,12 +28,44 @@ import { createRandomResponsesItemId, type ResponsesInputItem, type ResponsesMultiAgentAction, + type ResponsesOutputItem, type ResponsesTool, } from '@floway-dev/protocols/responses'; import { providerModelOf } from '@floway-dev/provider'; const MULTI_AGENT_SHIM_FLAG = 'responses-multi-agent-shim'; +interface UnsupportedCombination { + readonly type: 'invalid-request'; + readonly message: string; + readonly param: string | null; +} + +const unsupportedCombination = (invocation: Parameters[0]): UnsupportedCombination | null => { + if (invocation.action === 'compact') { + return { + type: 'invalid-request', + message: 'Compaction is not supported when multi_agent is enabled.', + param: 'multi_agent', + }; + } + if (invocation.payload.reasoning?.summary != null) { + return { + type: 'invalid-request', + message: 'reasoning.summary is not supported when multi_agent is enabled.', + param: 'reasoning.summary', + }; + } + if (invocation.payload.max_tool_calls != null) { + return { + type: 'invalid-request', + message: 'max_tool_calls is not supported when multi_agent is enabled.', + param: 'max_tool_calls', + }; + } + return null; +}; + // Whether this request carries any multi-agent history at all, which keeps the // capability engaged for replay even on a turn that no longer asks for it. const hasMultiAgentHistory = (input: readonly ResponsesInputItem[]): boolean => @@ -54,6 +86,13 @@ const agentAttributionOf = (item: ResponsesInputItem): string | undefined => { return typeof name === 'string' ? name : undefined; }; +// Root items are attributed too, so "belongs to a subagent" is specifically an +// attribution that is present and is not the root's. +const isSubagentItem = (item: ResponsesInputItem): boolean => { + const name = agentAttributionOf(item); + return name !== undefined && name !== ROOT_AGENT_PATH; +}; + export const multiAgentServerTool: ServerToolRegistration = (invocation, gatewayCtx, runtime) => { const flagOn = providerModelOf(invocation.candidate).enabledFlags.has(MULTI_AGENT_SHIM_FLAG); const structurallyRequired = invocation.targetApi !== 'responses'; @@ -64,6 +103,7 @@ export const multiAgentServerTool: ServerToolRegistration = (invocation, gateway if (!requested && !replaying) return { type: 'inactive' }; const maxConcurrent = invocation.payload.multi_agent?.max_concurrent_subagents; + // The guide states no fixed upper bound, so only the shape is checked. if (maxConcurrent !== undefined && maxConcurrent !== null && (!Number.isInteger(maxConcurrent) || maxConcurrent < 1)) { return { type: 'invalid-request', @@ -72,6 +112,15 @@ export const multiAgentServerTool: ServerToolRegistration = (invocation, gateway }; } + // Combinations the hosted feature documents as unsupported. Both reference + // SDKs refuse them client-side; refusing them here too means a client that + // does not gets the same answer rather than silently different behaviour. + // https://developers.openai.com/api/docs/guides/tools-multi-agent + if (requested) { + const unsupported = unsupportedCombination(invocation); + if (unsupported !== null) return unsupported; + } + const { checkpoints, input: withoutCheckpoints } = extractCheckpoints(invocation.payload.input); // The client's own tools; the capability's injected functions are appended @@ -136,12 +185,29 @@ const prepareActive = (args: { return built; }; + // The capability is hosted here, so the upstream must not see its config — + // and `BetaResponse` does not echo `multi_agent` back either. + invocation.payload = { ...invocation.payload, multi_agent: null }; + delete invocation.payload.multi_agent; + return { type: 'active', baseToolNames: MULTI_AGENT_ACTIONS, transformItems: (items, toolNames) => { - ensureSession(toolNames); - return rewriteMultiAgentHistory(items, toolNames, routedCallIds); + const session = ensureSession(toolNames); + return [ + multiAgentInstructionsItem(ROOT_AGENT_PATH, session.maxConcurrentSubagents), + ...rewriteMultiAgentHistory(items, toolNames, routedCallIds), + ]; + }, + stampRootOutput: { + // `phase` is the model's own channel output. A Responses upstream on a + // channel-emitting model already sets it, and forwarding it verbatim is + // the rule for any upstream-owned field; a translated target has no + // channel at all, so the phase is reconstructed from the turn instead — + // which is only knowable once the turn's stream has been consumed. + deferMessages: invocation.targetApi !== 'responses', + stamp: (item, finalTurn) => stampRootItem(item, finalTurn, invocation.targetApi !== 'responses'), }, injected: { buildFunctionTools: toolNames => buildMultiAgentFunctionTools(toolNames), @@ -166,7 +232,7 @@ const prepareActive = (args: { const result = yield* current.executeAction(action, intercepted.arguments, ROOT_AGENT_PATH); yield { progress: 'item', - id: createRandomResponsesItemId('multi_agent_call'), + id: createRandomResponsesItemId('multi_agent_call_output'), item: { type: 'multi_agent_call_output', action, @@ -226,7 +292,7 @@ export const rewriteMultiAgentHistory = ( } // A subagent's own orchestration record lives in that subagent's // transcript, restored from the checkpoint. - if (agentAttributionOf(item) !== undefined) continue; + if (isSubagentItem(item)) continue; out.push({ type: 'function_call', call_id: item.call_id, name: toolName, arguments: item.arguments, status: 'completed' }); continue; } @@ -236,7 +302,7 @@ export const rewriteMultiAgentHistory = ( out.push(item); continue; } - if (agentAttributionOf(item) !== undefined) continue; + if (isSubagentItem(item)) continue; out.push({ type: 'function_call_output', call_id: item.call_id, @@ -245,7 +311,7 @@ export const rewriteMultiAgentHistory = ( continue; } - if (item.type === 'function_call' && agentAttributionOf(item) !== undefined) { + if (item.type === 'function_call' && isSubagentItem(item)) { subagentCallIds.add(item.call_id); continue; } @@ -256,3 +322,23 @@ export const rewriteMultiAgentHistory = ( return out; }; + +// The hosted service appends its instructions to the root agent as a developer +// message, the same way it does for every subagent. +const multiAgentInstructionsItem = (agentPath: string, maxConcurrentSubagents: number): ResponsesInputItem => ({ + type: 'message', + role: 'developer', + content: [{ type: 'input_text', text: multiAgentInstructions(agentPath, maxConcurrentSubagents) }], +}); + +// Both reference SDKs extract the run's answer as the item that is a `message` +// with `agent.agent_name === '/root'` AND `phase === 'final_answer'`, dropping +// every other message silently. Root items therefore have to carry the +// attribution the hosted service would have stamped. +const stampRootItem = (item: ResponsesOutputItem, finalTurn: boolean, synthesizePhase: boolean): ResponsesOutputItem => { + const attributed = (item as { agent?: unknown }).agent === undefined + ? { ...item, agent: { agent_name: ROOT_AGENT_PATH } } as ResponsesOutputItem + : item; + if (attributed.type !== 'message' || !synthesizePhase || attributed.phase != null) return attributed; + return { ...attributed, phase: finalTurn ? 'final_answer' : 'commentary' }; +}; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts index 7a08b34bd..0439f242d 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/session.ts @@ -18,8 +18,8 @@ import type { MultiAgentCheckpoint, CheckpointAgent } from './checkpoint.ts'; import { agentErrorPayloadText, agentMessageEnvelopeText, + multiAgentInstructions, ROOT_AGENT_PATH, - WAIT_AGENT_MINIMUM_TIMEOUT_MS, type AgentStatus, type AgentStatusReport, type AgentMessageKind, @@ -59,6 +59,9 @@ interface SessionAgent { status: AgentStatus; history: ResponsesInputItem[]; pendingClientCallIds: string[]; + // Reported by `list_agents`, which the guide describes as returning the tree, + // statuses, and each agent's last task message. + lastTaskMessage: string | null; } const isSettled = (status: AgentStatus): boolean => typeof status === 'object'; @@ -106,7 +109,8 @@ export class MultiAgentSession { path: agent.path, status: agent.status, history: agent.history, - pendingClientCallIds: agent.pendingClientCallId === undefined ? [] : [agent.pendingClientCallId], + pendingClientCallIds: [...agent.pendingClientCallIds], + lastTaskMessage: agent.lastTaskMessage, }); } } @@ -116,7 +120,8 @@ export class MultiAgentSession { path: agent.path, status: agent.status, history: agent.history, - ...(agent.pendingClientCallIds[0] === undefined ? {} : { pendingClientCallId: agent.pendingClientCallIds[0] }), + pendingClientCallIds: [...agent.pendingClientCallIds], + lastTaskMessage: agent.lastTaskMessage, })); return { type: 'floway.multi_agent_checkpoint', @@ -126,6 +131,10 @@ export class MultiAgentSession { }; } + get maxConcurrentSubagents(): number { + return this.options.maxConcurrentSubagents; + } + hasAgents(): boolean { return this.agents.size > 0; } @@ -139,11 +148,6 @@ export class MultiAgentSession { if (index === -1) continue; agent.pendingClientCallIds.splice(index, 1); agent.history.push(item); - if (agent.pendingClientCallIds.length === 0 && agent.status === 'running') { - // Nothing else is outstanding, so the agent is runnable again; the - // next `wait_agent` picks it up. - agent.status = 'running'; - } return true; } return false; @@ -171,9 +175,9 @@ export class MultiAgentSession { case 'spawn_agent': return yield* this.spawnAgent(args, callerPath); case 'send_message': - return yield* this.deliver(args, callerPath, 'MESSAGE'); + return this.sendMessage(args, callerPath); case 'followup_task': - return yield* this.deliver(args, callerPath, 'FOLLOWUP_TASK'); + return yield* this.followupTask(args, callerPath); case 'wait_agent': return yield* this.waitAgent(args); case 'interrupt_agent': @@ -212,6 +216,7 @@ export class MultiAgentSession { agentMessageItem({ kind: 'NEW_TASK', author: callerPath, recipient: path, payload: message }), ], pendingClientCallIds: [], + lastTaskMessage: message, }; this.agents.set(path, agent); @@ -219,37 +224,46 @@ export class MultiAgentSession { return { output: JSON.stringify({ task_name: path }), endResponse }; } - private async *deliver( - args: Record, - callerPath: string, - kind: Extract, - ): AsyncGenerator { + // Queues a message without triggering a turn, per the guide's description of + // `send_message`. The recipient acts on it at the next `wait_agent`. + private sendMessage(args: Record, callerPath: string): ActionResult { + const target = typeof args.target === 'string' ? args.target : ''; + const message = typeof args.message === 'string' ? args.message : ''; + const agent = this.agents.get(target); + if (agent === undefined) { + return { output: JSON.stringify({ error: `No agent at ${target}.` }), endResponse: false }; + } + agent.history.push(agentMessageItem({ kind: 'MESSAGE', author: callerPath, recipient: target, payload: message })); + if (isSettled(agent.status)) agent.status = 'running'; + return { output: '', endResponse: false }; + } + + // Assigns a new task to an existing agent and triggers its turn. + private async *followupTask(args: Record, callerPath: string): AsyncGenerator { const target = typeof args.target === 'string' ? args.target : ''; const message = typeof args.message === 'string' ? args.message : ''; const agent = this.agents.get(target); if (agent === undefined) { return { output: JSON.stringify({ error: `No agent at ${target}.` }), endResponse: false }; } - agent.history.push(agentMessageItem({ kind, author: callerPath, recipient: target, payload: message })); + if (target === ROOT_AGENT_PATH) { + return { output: JSON.stringify({ error: 'followup_task targets an existing subagent, not the root agent.' }), endResponse: false }; + } + agent.history.push(agentMessageItem({ kind: 'NEW_TASK', author: callerPath, recipient: target, payload: message })); + agent.lastTaskMessage = message; agent.status = 'running'; const endResponse = yield* this.runAgent(agent, parentOf(agent.path)); - // The captured wire returns an empty string for a plain delivery. return { output: '', endResponse }; } - private async *waitAgent(args: Record): AsyncGenerator { - const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined; - if (timeoutMs !== undefined && timeoutMs < WAIT_AGENT_MINIMUM_TIMEOUT_MS) { - return { output: `timeout_ms must be at least ${WAIT_AGENT_MINIMUM_TIMEOUT_MS}`, endResponse: false }; - } - + private async *waitAgent(_args: Record): AsyncGenerator { let endResponse = false; for (const agent of [...this.agents.values()]) { if (isSettled(agent.status) || agent.pendingClientCallIds.length > 0) continue; endResponse = (yield* this.runAgent(agent, parentOf(agent.path))) || endResponse; if (endResponse) break; } - return { output: JSON.stringify({ agents: this.statusReports(), timed_out: false }), endResponse }; + return { output: JSON.stringify({ agents: this.statusReports() }), endResponse }; } private interruptAgent(args: Record): ActionResult { @@ -271,7 +285,11 @@ export class MultiAgentSession { } private statusReports(): AgentStatusReport[] { - return [...this.agents.values()].map(agent => ({ agent_name: agent.path, agent_status: agent.status })); + return [...this.agents.values()].map(agent => ({ + agent_name: agent.path, + agent_status: agent.status, + last_task_message: agent.lastTaskMessage, + })); } // ── Agent loop ───────────────────────────────────────────────────────── @@ -289,7 +307,10 @@ export class MultiAgentSession { const result = await this.options.runtime.runTurn({ ...this.options.basePayload, - input: agent.history, + input: [ + multiAgentInstructionsItem(agent.path, this.options.maxConcurrentSubagents), + ...agent.history, + ], tools: [...this.options.clientTools, ...this.options.agentTools], // A subagent transcript is Floway-internal; it must never enter the // upstream's own stored conversation history. @@ -321,14 +342,22 @@ export class MultiAgentSession { }; } agent.status = 'running'; - const injected = await this.options.clientToolBridge?.awaitOutputs(clientCalls.map(call => call.call_id)) ?? null; - if (injected === null) return true; - // The client answered on the still-open connection, so this agent - // picks straight back up and the response never has to end. - for (const item of injected) { - const callId = item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.call_id : undefined; - if (callId === undefined) continue; - this.routeClientOutput(callId, item); + const bridge = this.options.clientToolBridge; + // Without a bridge the response ends here with every parked call + // outstanding, and the client answers all of them on its next request — + // which is exactly what the guide describes for the HTTP boundary. + if (bridge === undefined) return true; + // With one, await the calls one at a time rather than as a batch: a + // client that pauses at the first `function_call` it reads has not yet + // seen the second, so it cannot answer both at once. The items are + // already on the wire, so it picks each up as it resumes. + for (const call of clientCalls) { + const injected = await bridge.awaitOutputs([call.call_id]); + if (injected === null) return true; + for (const item of injected) { + const callId = item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.call_id : undefined; + if (callId !== undefined) this.routeClientOutput(callId, item); + } } continue; } @@ -386,7 +415,7 @@ export class MultiAgentSession { }; yield { progress: 'item', - id: createRandomResponsesItemId('multi_agent_call'), + id: createRandomResponsesItemId('multi_agent_call_output'), item: { type: 'multi_agent_call_output', action, @@ -411,8 +440,11 @@ export class MultiAgentSession { if (caller !== undefined) caller.history.push(item); yield { progress: 'item', - id: createRandomResponsesItemId('agent_message'), - item: { ...item, agent: { agent_name: args.author } } as ResponsesOutputAgentMessageItem, + // On an `agent_message` the `agent` tag names the RECIPIENT, not the + // producer; `author` and `recipient` carry the direction. + // https://developers.openai.com/api/docs/guides/tools-multi-agent + id: item.id!, + item: { ...item, agent: { agent_name: args.recipient } } as ResponsesOutputAgentMessageItem, }; } @@ -447,8 +479,8 @@ const agentMessageItem = (args: { type: 'input_text', text: agentMessageEnvelopeText({ kind: args.kind, - taskName: args.recipient, - sender: args.author, + recipient: args.recipient, + author: args.author, payload: args.payload, }), }], @@ -462,3 +494,11 @@ const parseArguments = (argumentsJson: string): Record | null = return null; } }; + +// The hosted service appends its instructions as a developer message; the same +// shape keeps them out of the user turn on every target protocol. +const multiAgentInstructionsItem = (agentPath: string, maxConcurrentSubagents: number): ResponsesInputItem => ({ + type: 'message', + role: 'developer', + content: [{ type: 'input_text', text: multiAgentInstructions(agentPath, maxConcurrentSubagents) }], +}); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts index 1613c03d8..b61ab1b73 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/multi-agent/wire.ts @@ -1,39 +1,29 @@ // Wire contract for the Responses beta `multi_agent` capability. // -// The upstream publishes the item shapes but deliberately keeps each action's -// `arguments` an opaque JSON string, and offers no per-action JSON Schema: -// https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L7911-L8008 +// The upstream publishes the item shapes and the six action names, but keeps +// each action's `arguments` an opaque JSON string and documents a per-action +// argument schema for exactly one of them. What is and is not grounded: // -// The lowering below therefore has to define the schemas the orchestrator -// model actually sees. They are transcribed from live OpenAI collaboration -// traffic captured in Codex CLI rollouts, which exercises the same six -// actions against the same agent-path namespace: +// - `spawn_agent`'s `task_name` / `fork_turns` / `message` come from the +// guide's own worked example, and `fork_turns` is named again in the +// injected prompt below. +// - The other five actions' argument names are Floway's, transcribed from +// live OpenAI collaboration traffic captured in Codex CLI rollouts, which +// drives six identically-named actions over the same `/root/...` namespace. +// That is a different feature reusing the same vocabulary, so treat these as +// a reasonable convention rather than an upstream contract. +// - Result shapes are likewise Floway's outside `spawn_agent`, whose result +// the guide shows echoing the fully-qualified path. // -// spawn_agent {"task_name":"review_apps_web","message":"…", -// "model":"…","reasoning_effort":"…","fork_turns":"none"} -// -> {"task_name":"/root/review_apps_web"} -// send_message {"target":"/root/prior_art","message":"…"} -> "" -// interrupt_agent {"target":"/root/prior_art"} -// -> {"previous_status":{"completed":"…"}} -// list_agents {"path_prefix":"/root/r2"} -// -> {"agents":[{"agent_name":"/root","agent_status":"running"}]} -// wait_agent {"timeout_ms":10000} -// -> {"message":"Wait timed out.","timed_out":true} -// (and `timeout_ms must be at least 10000` below that floor) -// -// `followup_task` never appeared in the captured traffic; it takes the same -// target/message pair as `send_message` and differs only in that it re-tasks -// an agent that already produced a final answer. +// https://developers.openai.com/api/docs/guides/tools-multi-agent import type { ResponsesFunctionTool, ResponsesMultiAgentAction } from '@floway-dev/protocols/responses'; export const ROOT_AGENT_PATH = '/root'; -export const WAIT_AGENT_MINIMUM_TIMEOUT_MS = 10_000; - // Base names of the six functions the capability lowers to. They are hidden // implementation detail: the shim resolves each against the client's own tool -// names and strips the resulting calls out of the response it echoes back. +// names and replaces the resulting calls with `multi_agent_call` records. export const MULTI_AGENT_ACTIONS: readonly ResponsesMultiAgentAction[] = [ 'spawn_agent', 'send_message', @@ -52,25 +42,87 @@ export type AgentStatus = export interface AgentStatusReport { agent_name: string; agent_status: AgentStatus; + // The guide describes `list_agents` as returning the tree, statuses, and + // each agent's last task message. + last_task_message: string | null; } -// Message kinds carried by an `agent_message` item's leading text part. The -// captured envelope is a four-line header followed by the payload body. -export type AgentMessageKind = 'NEW_TASK' | 'FOLLOWUP_TASK' | 'MESSAGE' | 'FINAL_ANSWER'; +// The three message kinds the injected prompts tell agents to expect. The root +// prompt lists two; a subagent additionally receives `NEW_TASK`. +export type AgentMessageKind = 'NEW_TASK' | 'MESSAGE' | 'FINAL_ANSWER'; export const agentMessageEnvelopeText = (args: { kind: AgentMessageKind; - taskName: string; - sender: string; + recipient: string; + author: string; payload: string; }): string => - `Message Type: ${args.kind}\nTask name: ${args.taskName}\nSender: ${args.sender}\nPayload:\n${args.payload}`; + `Message Type: ${args.kind}\nTask name: ${args.recipient}\nSender: ${args.author}\nPayload:\n${args.payload}`; -// Emitted in place of a final answer when an agent's own turn failed. Mirrors -// the captured wording so an orchestrator prompted on the upstream's behaviour -// reads the same recovery hint. +// Emitted in place of a final answer when an agent's own turn failed. The +// upstream documents no failure semantics for a subagent at all, so this is +// Floway's: report it to the parent as a delivered answer rather than failing +// the whole tree, and say plainly that the agent can be re-tasked. export const agentErrorPayloadText = (error: string): string => - `Agent errored: ${error}\n\nThis agent's turn failed. If you still need this agent, use the available collaboration tools to give it another task.`; + `Agent errored: ${error}\n\nThis agent's turn failed. If you still need this agent, give it another task.`; + +// ── Injected instructions ──────────────────────────────────────────────── +// +// When multi-agent is enabled the hosted service appends its own developer +// message to the root agent and to every subagent — "You cannot edit or remove +// these instructions" — and the guide publishes both texts verbatim. A hosted +// implementation that omits them leaves the model holding six tools with no +// contract for using them, so they are vendored here exactly as published, +// with `{max_concurrent_subagents + 1}` resolved per request. +// +// Vendored from the hosted multi-agent guide, §Prompt guidance: +// https://developers.openai.com/api/docs/guides/tools-multi-agent + +const rootInstructions = (concurrencySlots: number): string => + 'You are `/root`, the primary agent in a team of agents collaborating to fulfill the user\'s goals.\n\n' + + 'At the start of your turn, you are the active agent.\n' + + 'You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents.\n' + + 'All agents in the team, including the agents that you can assign tasks to, are equally intelligent and capable, and have access to the same set of tools.\n\n' + + 'You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task and trigger a turn, and `send_message` to pass a message to a running agent without triggering a turn.\n' + + 'Child agents can also spawn their own sub-agents.\n' + + 'You can decide how much context you want to propagate to your sub-agents with the `fork_turns` parameter.\n\n' + + 'You will receive messages in the form:\n' + + '```\n' + + 'Message Type: MESSAGE | FINAL_ANSWER\n' + + 'Task name: \n' + + 'Sender: \n' + + 'Payload:\n' + + '\n' + + '```\n' + + 'They may be addressed as to=/root\n\n' + + `There are ${concurrencySlots} available concurrency slots, meaning that up to ${concurrencySlots} agents can be active at once, including you.`; + +const subagentInstructions = (concurrencySlots: number): string => + 'You are an agent in a team of agents collaborating to complete a task.\n\n' + + 'You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents. All agents in the team, including the agents that you can assign tasks to, are equally intelligent and capable, and have access to the same set of tools.\n\n' + + 'You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task and trigger a turn, and `send_message` to pass a message to a running agent.\n' + + 'Child agents can also spawn their own sub-agents.\n\n' + + 'When you provide a response in the final channel, that content is immediately delivered back to your parent agent.\n\n' + + 'You will receive messages in the form:\n' + + '```\n' + + 'Message Type: NEW_TASK | MESSAGE | FINAL_ANSWER\n' + + 'Task name: \n' + + 'Sender: \n' + + 'Payload:\n' + + '\n' + + '```\n' + + 'You may also see them addressed as to=/root/..., which indicates your identity is /root/...\n\n' + + `There are ${concurrencySlots} available concurrency slots, meaning that up to ${concurrencySlots} agents can be active at once, including you.`; + +// The guide's texts are templated on `{max_concurrent_subagents + 1}` — the +// config counts the tree excluding the root, while the prompt counts the slot +// the reader itself occupies. +export const multiAgentInstructions = (agentPath: string, maxConcurrentSubagents: number): string => + agentPath === ROOT_AGENT_PATH + ? rootInstructions(maxConcurrentSubagents + 1) + : subagentInstructions(maxConcurrentSubagents + 1); + +// ── Lowering ───────────────────────────────────────────────────────────── const functionTool = (name: string, description: string, properties: Record, required: readonly string[]): ResponsesFunctionTool => ({ type: 'function', @@ -97,19 +149,19 @@ export const buildMultiAgentFunctionTools = (toolNames: ReadonlyMap { if (!parsed || typeof parsed !== 'object') return null; const candidate = parsed as Partial; if (candidate.type !== 'response.inject') return null; - return { - type: 'response.inject', - response_id: typeof candidate.response_id === 'string' ? candidate.response_id : '', - input: Array.isArray(candidate.input) ? candidate.input : [], - }; + // A malformed injection is a protocol violation rather than a routing + // failure: the two documented `response.inject.failed` codes both describe a + // response that could not accept the input, which is not what happened here. + if (typeof candidate.response_id !== 'string' || candidate.response_id.length === 0 || !Array.isArray(candidate.input)) { + return { type: 'response.inject', response_id: '', input: [] }; + } + return { type: 'response.inject', response_id: candidate.response_id, input: candidate.input }; }; const handleInjectMessage = async ( @@ -218,6 +220,14 @@ const handleInjectMessage = async ( sendError(socket, 401, { type: 'authentication_error', code: 'invalid_api_key', message: 'Invalid API key.' }); return; } + if (injection.response_id.length === 0) { + sendError(socket, 400, { + type: 'invalid_request_error', + code: 'invalid_request_error', + message: 'response.inject requires a response_id string and an input array.', + }); + return; + } const committed = bridges.commit(injection.response_id, injection.input); if (committed.ok) { @@ -459,8 +469,9 @@ const respondResponsesWebSocket = async (input: { // pull is where the runtime blocks on an injection, and the client can // only address one by response id. The id is only authoritative here, // after every egress transform has run. - if (frame.type === 'event' && 'response' in frame.event) { - bridges.register(bridge, frame.event.response.id); + if (frame.type === 'event') { + if ('response' in frame.event) bridges.register(bridge, frame.event.response.id); + if (typeof frame.event.sequence_number === 'number') bridges.observeSequenceNumber(frame.event.sequence_number); } pendingNext = pendingWsFrameResult(iterator.next()); if (frame.type !== 'event') continue; diff --git a/packages/protocols/__tests__/responses/item-id_test.ts b/packages/protocols/__tests__/responses/item-id_test.ts index d4ca170b7..037c7f71f 100644 --- a/packages/protocols/__tests__/responses/item-id_test.ts +++ b/packages/protocols/__tests__/responses/item-id_test.ts @@ -12,6 +12,7 @@ const expectedPrefixes = { image_generation_call: 'ig', agent_message: 'amsg', multi_agent_call: 'mac', + multi_agent_call_output: 'maco', } as const satisfies Record; test.each(Object.entries(expectedPrefixes))('creates unique %s ids with the canonical prefix', (type, prefix) => { diff --git a/packages/protocols/src/responses/index.ts b/packages/protocols/src/responses/index.ts index 6bd5f37f6..3cc66b26d 100644 --- a/packages/protocols/src/responses/index.ts +++ b/packages/protocols/src/responses/index.ts @@ -119,6 +119,7 @@ export interface ResponsesInputMessage { role: 'user' | 'assistant' | 'system' | 'developer'; content: string | ResponsesInputContent[]; phase?: ResponsesMessagePhase; + agent?: ResponsesAgentAttribution | null; } // The Responses request schema's EasyInputMessage makes the constant @@ -203,10 +204,12 @@ export type ResponsesToolCaller = | { type: 'direct' } | { type: 'program'; caller_id: string }; -// Items a subagent produced carry the agent that produced them; the beta's own -// hosted multi-agent example shows a `function_call` attributed to -// `/root/researcher` while the root is suspended on it. -// https://github.com/openai/openai-agents-python/blob/a2d82707d94bfcf2ffbcc62ea9746c5fb183804f/tests/extensions/experimental/hosted_multi_agent/test_model.py#L334-L415 +// "The canonical name of the agent that produced this item." The beta puts +// this on every item variant, so it is declared once and attached wherever +// Floway itself produces or reads it. One exception is worth knowing: on an +// `agent_message` the tag names the RECIPIENT, and `author` / `recipient` +// carry the direction. +// https://developers.openai.com/api/docs/guides/tools-multi-agent export interface ResponsesAgentAttribution { agent_name: string; } @@ -934,6 +937,7 @@ export interface ResponsesOutputMessage { role: 'assistant'; content: ResponsesOutputContentBlock[]; phase?: ResponsesMessagePhase; + agent?: ResponsesAgentAttribution | null; } export type ResponsesOutputContentBlock = ResponsesOutputText | ResponsesOutputRefusal; diff --git a/packages/protocols/src/responses/item-id.ts b/packages/protocols/src/responses/item-id.ts index 514709a30..0d6c69866 100644 --- a/packages/protocols/src/responses/item-id.ts +++ b/packages/protocols/src/responses/item-id.ts @@ -13,11 +13,13 @@ const generatedItemPrefixes = { compaction: 'cmp', // https://github.com/openai/codex/blob/8c41ed33ce3e39460e7b13b14c35e0c39bb5980d/codex-rs/protocol/src/models.rs#L1076-L1094 image_generation_call: 'ig', - // Beta multi-agent items. OpenAI's own examples show `amsg_` on agent - // messages and `mac_` on the orchestration call/output pair. - // https://github.com/openai/openai-node/blob/228c224393ef4bf3bda2a9d7eb40f387499299b5/src/resources/beta/responses/responses.ts#L7911-L8008 + // Beta multi-agent items. The hosted multi-agent guide's worked example + // shows `amsg_`, `mac_`, and `maco_` on these three; no prose states the + // prefixes normatively. + // https://developers.openai.com/api/docs/guides/tools-multi-agent agent_message: 'amsg', multi_agent_call: 'mac', + multi_agent_call_output: 'maco', } as const; export type GeneratedResponsesItemType = keyof typeof generatedItemPrefixes; From f42e5e8a619606ecada1b4b0a3342aceecc99d89 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 14:34:26 +0800 Subject: [PATCH 10/11] docs: correct the scheduling summary for the queue-only send_message --- docs/MULTI-AGENT.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/MULTI-AGENT.md b/docs/MULTI-AGENT.md index 94b3a2daf..9a8ce48db 100644 --- a/docs/MULTI-AGENT.md +++ b/docs/MULTI-AGENT.md @@ -89,10 +89,11 @@ The upstream runs subagents asynchronously behind its own coordinator. Floway has none: there is no Durable Object and no execution that survives the response. Scheduling is therefore eager and serial. -- `spawn_agent`, `send_message`, and `followup_task` each drive their target's - loop until it settles or parks on a client tool, then return. -- `wait_agent` drives whatever is still runnable — the agents a freshly - supplied tool output just woke — and reports statuses. +- `spawn_agent` and `followup_task` each drive their target's loop until it + settles or parks on a client tool, then return. +- `send_message` only queues, per the guide; `wait_agent` is what drives + whatever is runnable — an agent a queued message or a freshly supplied tool + output just woke — and reports statuses. - `list_agents` and `interrupt_agent` are pure reads and writes over the tree. Nothing runs between actions, so every action's `multi_agent_call_output` From a29a914c55668c21a506fd78df03278e496a6464 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 28 Jul 2026 15:33:31 +0800 Subject: [PATCH 11/11] fix(gateway): merge shim usage structurally instead of by field list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-turn merge enumerated five fields and rebuilt the result from them, so anything not on that list was dropped. Two things were: `cache_write_tokens`, which the current upstream usage schema requires, and — because it hangs off a symbol key that Object.entries never sees — the billing metadata Floway threads alongside the native fields, which mispriced every multi-turn shim response. Usage is a bag of counters the upstream owns, so it now merges by shape: numbers add, nested breakdowns merge the same way, and anything that cannot be summed takes the later turn's value. The wire projection still fills the three required totals and forwards everything else as it arrived. --- .../interceptors/server-tool-shim_test.ts | 24 +++++- .../interceptors/server-tool-shim.ts | 78 ++++++++----------- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts index 3ef9f44e4..043c53964 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tool-shim_test.ts @@ -7,6 +7,7 @@ import { materializeAccumulatedOutput, parseServerToolArguments, sumUsage, + type MergeUsage, type InterceptedFunctionCall, type ServerToolResultSlot, type TurnSummary, @@ -31,7 +32,7 @@ import type { import { getRepo, initRepo } from '../../../../../src/repo/index.ts'; import { InMemoryRepo } from '../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; +import { eventFrame, USAGE_BILLING } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalResponsesPayload, @@ -5874,6 +5875,27 @@ test('sumUsage omits detail subfields neither side reported (sparse)', () => { }); }); +test('sumUsage keeps detail fields it was never told about', () => { + // The old field-list merge silently dropped `cache_write_tokens`, which the + // current upstream schema requires. + const a: MergeUsage = { input_tokens_details: { cached_tokens: 1, cache_write_tokens: 4 } }; + const b: MergeUsage = { input_tokens_details: { cached_tokens: 2, cache_write_tokens: 5 } }; + assertEquals(sumUsage(a, b), { input_tokens_details: { cached_tokens: 3, cache_write_tokens: 9 } }); +}); + +test('sumUsage carries the symbol-keyed billing metadata across turns', () => { + // Symbol keys are invisible to Object.entries, so a field-list merge dropped + // billing entirely and mispriced every multi-turn shim response. + const a: MergeUsage = { [USAGE_BILLING]: { cacheWriteTokenCount: 10, serviceTier: 'standard' } }; + const b: MergeUsage = { [USAGE_BILLING]: { cacheWriteTokenCount: 5, serviceTier: 'priority' } }; + + const merged = sumUsage(a, b); + + assertEquals(merged[USAGE_BILLING]?.cacheWriteTokenCount, 15); + // A tier cannot be summed, so the later turn's value stands. + assertEquals(merged[USAGE_BILLING]?.serviceTier, 'priority'); +}); + test('sumUsage of two empty operands returns an empty object (no fabricated zeros)', () => { assertEquals(sumUsage({}, {}), {}); }); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts index 789b73748..56705acae 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tool-shim.ts @@ -21,13 +21,12 @@ import { } from '@floway-dev/protocols/responses'; import type { EventResultMetadata, ExecuteResult } from '@floway-dev/provider'; -export interface MergeUsage { - input_tokens?: number; - output_tokens?: number; - total_tokens?: number; - input_tokens_details?: { cached_tokens: number }; - output_tokens_details?: { reasoning_tokens: number }; -} +// Usage is a bag of counters the upstream owns, so the shim merges it +// structurally instead of by field list. Enumerating fields silently dropped +// everything not listed: `cache_write_tokens`, and — because it is symbol-keyed +// — the billing metadata Floway threads alongside the native fields, which +// mispriced every multi-turn shim response. +export type MergeUsage = Partial>; export interface MergeState { sequenceNumber: number; @@ -251,57 +250,44 @@ export const materializeAccumulatedOutput = (state: MergeState): ResponsesOutput return sorted.map(k => state.accumulatedOutput.get(k)!); }; -export const sumUsage = (a: MergeUsage, b: MergeUsage): MergeUsage => { - const out: MergeUsage = {}; - const sumScalar = (key: 'input_tokens' | 'output_tokens' | 'total_tokens') => { - if (a[key] !== undefined || b[key] !== undefined) out[key] = (a[key] ?? 0) + (b[key] ?? 0); - }; - sumScalar('input_tokens'); - sumScalar('output_tokens'); - sumScalar('total_tokens'); - if (a.input_tokens_details !== undefined || b.input_tokens_details !== undefined) { - out.input_tokens_details = { - cached_tokens: (a.input_tokens_details?.cached_tokens ?? 0) + (b.input_tokens_details?.cached_tokens ?? 0), - }; - } - if (a.output_tokens_details !== undefined || b.output_tokens_details !== undefined) { - out.output_tokens_details = { - reasoning_tokens: (a.output_tokens_details?.reasoning_tokens ?? 0) + (b.output_tokens_details?.reasoning_tokens ?? 0), - }; - } +const isMergeableUsageObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +// Counters add; nested breakdowns merge the same way. Anything else cannot be +// summed — `serviceTier` is the live example — so the later turn wins. +const mergeUsageValue = (a: unknown, b: unknown): unknown => { + if (a === undefined) return b; + if (b === undefined) return a; + if (typeof a === 'number' && typeof b === 'number') return a + b; + if (isMergeableUsageObject(a) && isMergeableUsageObject(b)) return mergeUsageObjects(a, b); + return b; +}; + +// `Reflect.ownKeys` rather than `Object.entries`: the billing metadata hangs +// off a symbol key and would otherwise be invisible to the merge. +const mergeUsageObjects = (a: Record, b: Record): Record => { + const out: Record = { ...a }; + for (const key of Reflect.ownKeys(b)) out[key] = mergeUsageValue(a[key], b[key]); return out; }; +export const sumUsage = (a: MergeUsage, b: MergeUsage): MergeUsage => + mergeUsageObjects(a as Record, b as Record) as MergeUsage; + const usageForWire = (state: MergeState): ResponsesResult['usage'] => { const u = state.accumulatedUsage; - if ( - u.input_tokens === undefined - && u.output_tokens === undefined - && u.total_tokens === undefined - && u.input_tokens_details === undefined - && u.output_tokens_details === undefined - ) { - return undefined; - } + if (Reflect.ownKeys(u).length === 0) return undefined; + // The three totals are required on the wire; everything else the upstream + // reported rides along exactly as it arrived. return { + ...u, input_tokens: u.input_tokens ?? 0, output_tokens: u.output_tokens ?? 0, total_tokens: u.total_tokens ?? 0, - ...(u.input_tokens_details !== undefined ? { input_tokens_details: u.input_tokens_details } : {}), - ...(u.output_tokens_details !== undefined ? { output_tokens_details: u.output_tokens_details } : {}), }; }; -const usageOf = (usage: ResponsesResult['usage']): MergeUsage => { - if (usage === undefined) return {}; - const out: MergeUsage = {}; - if (usage.input_tokens !== undefined) out.input_tokens = usage.input_tokens; - if (usage.output_tokens !== undefined) out.output_tokens = usage.output_tokens; - if (usage.total_tokens !== undefined) out.total_tokens = usage.total_tokens; - if (usage.input_tokens_details !== undefined) out.input_tokens_details = usage.input_tokens_details; - if (usage.output_tokens_details !== undefined) out.output_tokens_details = usage.output_tokens_details; - return out; -}; +const usageOf = (usage: ResponsesResult['usage']): MergeUsage => usage ?? {}; const rewriteHostedToolChoice = ( toolChoice: ResponsesToolChoice | null | undefined,