From 2ce0a4c20ff9dab39b191a83fab5b46b8a069a1b Mon Sep 17 00:00:00 2001 From: zack-dev-cm <112114097+zack-dev-cm@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:04:28 +0400 Subject: [PATCH 1/3] fix: preserve optional vision through provider wrappers Signed-off-by: zack-dev-cm <112114097+zack-dev-cm@users.noreply.github.com> --- src/providers/fallback-chain.ts | 26 ++- src/providers/resilient.ts | 6 + test/provider-vision.test.ts | 303 ++++++++++++++++++++++++++++++++ 3 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 test/provider-vision.test.ts diff --git a/src/providers/fallback-chain.ts b/src/providers/fallback-chain.ts index e2c979f929..74686db28a 100644 --- a/src/providers/fallback-chain.ts +++ b/src/providers/fallback-chain.ts @@ -1,25 +1,41 @@ import type { MemoryProvider } from "../types.js"; +type ImageProvider = MemoryProvider & + Required>; + export class FallbackChainProvider implements MemoryProvider { name: string; + declare readonly describeImage?: MemoryProvider["describeImage"]; constructor(private providers: MemoryProvider[]) { this.name = `fallback(${providers.map((p) => p.name).join(" -> ")})`; + const imageProviders = providers.filter( + (provider): provider is ImageProvider => + typeof provider.describeImage === "function", + ); + if (imageProviders.length > 0) { + this.describeImage = (imageData, mimeType, prompt) => + this.tryAll( + (provider) => provider.describeImage(imageData, mimeType, prompt), + imageProviders, + ); + } } async compress(systemPrompt: string, userPrompt: string): Promise { - return this.tryAll((p) => p.compress(systemPrompt, userPrompt)); + return this.tryAll((p) => p.compress(systemPrompt, userPrompt), this.providers); } async summarize(systemPrompt: string, userPrompt: string): Promise { - return this.tryAll((p) => p.summarize(systemPrompt, userPrompt)); + return this.tryAll((p) => p.summarize(systemPrompt, userPrompt), this.providers); } - private async tryAll( - fn: (p: MemoryProvider) => Promise, + private async tryAll( + fn: (p: T) => Promise, + providers: T[], ): Promise { let lastError: Error | null = null; - for (const provider of this.providers) { + for (const provider of providers) { try { return await fn(provider); } catch (err) { diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index f82e578d72..e407398d56 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -12,9 +12,15 @@ export function isPayloadRejection(err: unknown): boolean { export class ResilientProvider implements MemoryProvider { private breaker = new CircuitBreaker(); name: string; + declare readonly describeImage?: MemoryProvider["describeImage"]; constructor(private inner: MemoryProvider) { this.name = `resilient(${inner.name})`; + const describeImage = inner.describeImage; + if (typeof describeImage === "function") { + this.describeImage = (imageData, mimeType, prompt) => + this.call(() => describeImage.call(inner, imageData, mimeType, prompt)); + } } private async call(fn: () => Promise): Promise { diff --git a/test/provider-vision.test.ts b/test/provider-vision.test.ts new file mode 100644 index 0000000000..0d294d4594 --- /dev/null +++ b/test/provider-vision.test.ts @@ -0,0 +1,303 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { IIIClient } from "iii-sdk"; +import type { StateKV } from "../src/state/kv.js"; +import type { MemoryProvider, RawObservation, CompressedObservation } from "../src/types.js"; +import { mockKV, mockSdk } from "./helpers/mocks.js"; + +const sdkDouble = vi.hoisted(() => ({ create: vi.fn() })); +vi.mock("@anthropic-ai/sdk", () => ({ default: class { messages = { create: sdkDouble.create }; } })); +vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); +vi.mock("../src/functions/search.js", () => ({ + getSearchIndex: () => ({ add: vi.fn() }), getVectorIndex: () => null, + scheduleIndexSave: vi.fn(), vectorIndexAddGuarded: vi.fn().mockResolvedValue(false), +})); + +import { AnthropicProvider } from "../src/providers/anthropic.js"; +import { createProvider, createFallbackProvider } from "../src/providers/index.js"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import { FallbackChainProvider } from "../src/providers/fallback-chain.js"; +import { registerCompressFunction, type CompressResult } from "../src/functions/compress.js"; +import { VISION_DESCRIPTION_PROMPT } from "../src/prompts/vision.js"; +import { KV } from "../src/state/schema.js"; + +const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="; +const CAPTION = "TEST_VISION_RESULT: I see a red dot"; +const XML = `imageScreenshot of Red Dot +Test image observationImage shows a single red pixel +A vision model described a screenshot showing a red dot on a white background +testingscreenshot +5`; +type Request = { messages: Array<{ role?: string; content: string | Array<{ type: string; text?: string; source?: { data: string; media_type: string } }> }> }; +type VisionCall = { name: string; receiverMatches: boolean; args: string[] }; +let visionCalls: VisionCall[] = []; +let prompts: string[] = []; + +function plain(name: string): MemoryProvider { + return { name, compress: async (_system, user) => { prompts.push(user); return XML; }, summarize: async () => "summary" }; +} + +function capable(name: string, fails = false): MemoryProvider { + const inner: MemoryProvider = { ...plain(name), async describeImage(image, mime, prompt) { + visionCalls.push({ name, receiverMatches: this === inner, args: [image, mime, prompt] }); + if (fails) throw new Error(name + " vision unavailable"); + return CAPTION; + } }; + return inner; +} + +async function compress(provider: MemoryProvider, modality: "image" | "text" = "image") { + const sdk = mockSdk({ looseTrigger: true }); + const kv = mockKV(); + const raw: RawObservation = { id: "obs_vision-boundary", sessionId: "session_vision-boundary", + timestamp: "2026-06-01T00:00:00.000Z", hookType: "post_tool_use", toolName: "screenshot", + toolOutput: "A screenshot was captured", raw: {}, modality, + ...(modality === "image" ? { imageData: PNG } : {}) }; + await kv.set(KV.observations(raw.sessionId), raw.id, raw); + registerCompressFunction(sdk as unknown as IIIClient, kv as unknown as StateKV, provider); + const result = await sdk.trigger({ function_id: "mem::compress", payload: { + observationId: raw.id, sessionId: raw.sessionId, + } }) as CompressResult; + const stored = await kv.get(KV.observations(raw.sessionId), raw.id); + return { result, stored, raw }; +} + +async function expectCaption(provider: MemoryProvider) { + const { result, stored, raw } = await compress(provider); + expect(result.success).toBe(true); + expect(stored?.imageDescription).toBe(CAPTION); + expect(result.compressed?.imageDescription).toBe(CAPTION); + expect(stored?.id).toBe(raw.id); + expect(stored?.sessionId).toBe(raw.sessionId); + expect(stored?.imageRef).toBe(PNG); + expect(prompts).toHaveLength(1); + expect(prompts[0]).toContain(CAPTION); +} + +describe("provider vision boundary", () => { + beforeEach(() => { + visionCalls = []; prompts = []; + vi.stubEnv("ANTHROPIC_API_KEY", "fixture-not-a-real-key"); + vi.stubEnv("OPENAI_API_KEY", "fixture-not-a-real-key"); + vi.stubEnv("ANTHROPIC_MODEL", "fixture-vision-model"); + vi.stubEnv("OPENAI_MODEL", "fixture-text-model"); + vi.stubEnv("OPENAI_BASE_URL", "http://127.0.0.1:1/v1"); + sdkDouble.create.mockReset(); + sdkDouble.create.mockImplementation(async (request: Request) => { + const content = request.messages[0]!.content; + if (Array.isArray(content)) { + const image = content.find((part) => part.type === "image")!; + const text = content.find((part) => part.type === "text")!; + visionCalls.push({ name: "Anthropic SDK", receiverMatches: true, + args: [image.source!.data, image.source!.media_type, text.text!] }); + return { content: [{ type: "text", text: CAPTION }] }; + } + prompts.push(content); + return { content: [{ type: "text", text: XML }] }; + }); + vi.stubGlobal("fetch", vi.fn(async (_url: string, init: RequestInit) => { + const request = JSON.parse(String(init.body)) as Request; + const content = request.messages.find((message) => message.role === "user")!.content; + if (typeof content !== "string") throw new Error("Unexpected text-provider payload"); + prompts.push(content); + return new Response(JSON.stringify({ choices: [{ message: { content: XML }, finish_reason: "stop" }] }), + { status: 200, headers: { "content-type": "application/json" } }); + })); + }); + afterEach(() => { + vi.unstubAllEnvs(); vi.unstubAllGlobals(); + }); + + it("bare real Anthropic provider retains the existing image description", async () => { + await expectCaption(new AnthropicProvider("fixture-key", "fixture-model", 512)); + expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + }); + + it("createProvider Anthropic preserves caption through the resilience wrapper", async () => { + await expectCaption(createProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 })); + expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + }); + + it("createFallbackProvider preserves a capable primary through both wrappers", async () => { + await expectCaption(createFallbackProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 }, + { providers: ["openai"] })); + expect(visionCalls).toHaveLength(1); + }); + + it("createFallbackProvider preserves a capable fallback after a text-only primary", async () => { + await expectCaption(createFallbackProvider({ provider: "openai", model: "fixture-model", maxTokens: 512 }, + { providers: ["anthropic"] })); + expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + }); + + it("supported fallback preserves receiver arguments and order after the first vision failure", async () => { + const p = new ResilientProvider(new FallbackChainProvider([plain("text"), capable("first", true), capable("second")])); + await expectCaption(p); + expect(visionCalls).toEqual(["first", "second"].map((name) => ({ name, receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }))); + }); + + it("all capable failures are attempted in order and retain existing text fallback", async () => { + const p = new ResilientProvider(new FallbackChainProvider([plain("text"), capable("first", true), capable("second", true)])); + const { result, stored } = await compress(p); + expect(result.success).toBe(true); + expect(stored?.imageDescription).toBeUndefined(); + expect(stored?.imageRef).toBe(PNG); + expect(visionCalls).toEqual(["first", "second"].map((name) => ({ name, receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }))); + }); + + it("unsupported wrappers and chains keep capability absent and the text path works", async () => { + const p = new ResilientProvider(new FallbackChainProvider([plain("text"), plain("other")])); + expect(p).not.toHaveProperty("describeImage"); + expect(createProvider({ provider: "noop", model: "noop", maxTokens: 512 })).not.toHaveProperty("describeImage"); + const { result, stored } = await compress(p); + expect(result.success).toBe(true); + expect(stored?.imageDescription).toBeUndefined(); + expect(visionCalls).toEqual([]); + }); + + it("ordinary text through the real factory does not invoke image description", async () => { + const p = createProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 }); + const { result, stored } = await compress(p, "text"); + expect(result.success).toBe(true); + expect(stored?.imageDescription).toBeUndefined(); + expect(visionCalls).toEqual([]); + expect(prompts).toHaveLength(1); + }); + + it("empty and text-only chains leave image capability absent", async () => { + const empty = new FallbackChainProvider([]); + expect(empty).not.toHaveProperty("describeImage"); + expect(new ResilientProvider(empty)).not.toHaveProperty("describeImage"); + expect(new ResilientProvider(plain("text"))).not.toHaveProperty("describeImage"); + expect(new FallbackChainProvider([plain("text")])).not.toHaveProperty("describeImage"); + await expect(empty.compress("system", "user")).rejects.toThrow("No providers available"); + await expect(empty.summarize("system", "user")).rejects.toThrow("No providers available"); + }); + + it("stops at the first successful image provider with its receiver and arguments", async () => { + const chain = new FallbackChainProvider([ + plain("text"), capable("first"), capable("unused"), + ]); + await expect(chain.describeImage?.("image", "image/jpeg", "describe")).resolves.toBe(CAPTION); + expect(visionCalls).toEqual([ + { name: "first", receiverMatches: true, args: ["image", "image/jpeg", "describe"] }, + ]); + }); + + it("propagates the last capable provider error after trying each in order", async () => { + const lastError = new Error("last image provider unavailable"); + const first = capable("first", true); + const last: MemoryProvider = { + ...plain("last"), + async describeImage(image, mime, prompt) { + visionCalls.push({ name: "last", receiverMatches: this === last, args: [image, mime, prompt] }); + throw lastError; + }, + }; + const chain = new FallbackChainProvider([first, plain("text"), last]); + await expect(chain.describeImage?.("image", "image/webp", "describe")).rejects.toBe(lastError); + expect(visionCalls).toEqual(["first", "last"].map((name) => ({ + name, receiverMatches: true, args: ["image", "image/webp", "describe"], + }))); + }); + + it("normalizes a non-Error rejection from the last image provider", async () => { + const inner: MemoryProvider = { + ...plain("text"), + describeImage: async () => { throw "image request rejected"; }, + }; + const chain = new FallbackChainProvider([inner]); + await expect(chain.describeImage?.("image", "image/png", "describe")) + .rejects.toEqual(new Error("image request rejected")); + }); + + it("preserves the direct resilience delegate receiver and all image arguments", async () => { + const provider = new ResilientProvider(capable("inner")); + await expect(provider.describeImage?.("image", "image/gif", "describe")).resolves.toBe(CAPTION); + expect(visionCalls).toEqual([ + { name: "inner", receiverMatches: true, args: ["image", "image/gif", "describe"] }, + ]); + }); + + it("counts image gateway failures and blocks all operations with the same breaker", async () => { + const error = new Error('OpenAI API error (502): {"error":{"message":"content_filter","code":"bad_gateway"}}'); + const image = vi.fn().mockRejectedValue(error); + const inner = { ...plain("inner"), describeImage: image }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) { + await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toBe(error); + } + expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); + await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toThrow("circuit_breaker_open"); + await expect(provider.compress("system", "user")).rejects.toThrow("circuit_breaker_open"); + await expect(provider.summarize("system", "user")).rejects.toThrow("circuit_breaker_open"); + expect(image).toHaveBeenCalledTimes(3); + expect(prompts).toEqual([]); + }); + + it("blocks image requests after text failures have opened the shared breaker", async () => { + const error = new Error("provider unavailable"); + const image = vi.fn().mockResolvedValue(CAPTION); + const inner: MemoryProvider = { + ...plain("inner"), describeImage: image, + compress: async () => { throw error; }, + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 3; i++) { + await expect(provider.compress("system", "user")).rejects.toBe(error); + } + await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toThrow("circuit_breaker_open"); + expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); + expect(image).not.toHaveBeenCalled(); + }); + + it("resets prior text failures when an image request succeeds", async () => { + const error = new Error("text request failed"); + const inner: MemoryProvider = { + ...capable("inner"), compress: async () => { throw error; }, + }; + const provider = new ResilientProvider(inner); + for (let i = 0; i < 2; i++) { + await expect(provider.compress("system", "user")).rejects.toBe(error); + } + expect(provider.circuitState.failures).toBe(2); + await expect(provider.describeImage?.("image", "image/png", "describe")).resolves.toBe(CAPTION); + expect(provider.circuitState).toEqual({ state: "closed", failures: 0, lastFailureAt: null, openedAt: null }); + expect(visionCalls).toHaveLength(1); + }); + + it("does not count image content-filter rejections as service failures", async () => { + const error = new Error('OpenAI API error (400): {"error":{"code":"content_filter"}}'); + const image = vi.fn().mockRejectedValue(error); + const provider = new ResilientProvider({ ...plain("inner"), describeImage: image }); + for (let i = 0; i < 5; i++) { + await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toBe(error); + } + expect(provider.circuitState).toEqual({ state: "closed", failures: 0, lastFailureAt: null, openedAt: null }); + await expect(provider.compress("system", "user")).resolves.toBe(XML); + expect(image).toHaveBeenCalledTimes(5); + }); + + it("sanitizes factory-provided captions before returning and persisting them", async () => { + const secret = ["fixture", "password"].join("-"); + const caption = `terminal shows postgres://deploy:${secret}@db.example/app`; + const sanitized = "terminal shows postgres://[REDACTED_SECRET]@db.example/app"; + sdkDouble.create.mockImplementation(async (request: Request) => ({ + content: [{ type: "text", text: Array.isArray(request.messages[0]!.content) ? caption : XML }], + })); + const { result, stored, raw } = await compress(createProvider({ + provider: "anthropic", model: "fixture-model", maxTokens: 512, + })); + expect(result.success).toBe(true); + expect(result.compressed?.imageDescription).toBe(sanitized); + expect(stored?.imageDescription).toBe(sanitized); + expect(stored?.id).toBe(raw.id); + expect(stored?.imageRef).toBe(PNG); + expect(JSON.stringify({ result, stored })).not.toContain(secret); + expect(sdkDouble.create).toHaveBeenCalledTimes(2); + }); +}); From dc193dfda309fdbd01d0759d6b6596c61bc2a1b2 Mon Sep 17 00:00:00 2001 From: zack-dev-cm <112114097+zack-dev-cm@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:24:59 +0400 Subject: [PATCH 2/3] style: format provider vision regressions Signed-off-by: zack-dev-cm <112114097+zack-dev-cm@users.noreply.github.com> --- test/provider-vision.test.ts | 413 +++++++++++++++++++++++++++-------- 1 file changed, 318 insertions(+), 95 deletions(-) diff --git a/test/provider-vision.test.ts b/test/provider-vision.test.ts index 0d294d4594..f293eadaa0 100644 --- a/test/provider-vision.test.ts +++ b/test/provider-vision.test.ts @@ -1,63 +1,128 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { IIIClient } from "iii-sdk"; import type { StateKV } from "../src/state/kv.js"; -import type { MemoryProvider, RawObservation, CompressedObservation } from "../src/types.js"; +import type { + MemoryProvider, + RawObservation, + CompressedObservation, +} from "../src/types.js"; import { mockKV, mockSdk } from "./helpers/mocks.js"; const sdkDouble = vi.hoisted(() => ({ create: vi.fn() })); -vi.mock("@anthropic-ai/sdk", () => ({ default: class { messages = { create: sdkDouble.create }; } })); -vi.mock("../src/logger.js", () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); +vi.mock("@anthropic-ai/sdk", () => ({ + default: class { + messages = { create: sdkDouble.create }; + }, +})); +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); vi.mock("../src/functions/search.js", () => ({ - getSearchIndex: () => ({ add: vi.fn() }), getVectorIndex: () => null, - scheduleIndexSave: vi.fn(), vectorIndexAddGuarded: vi.fn().mockResolvedValue(false), + getSearchIndex: () => ({ add: vi.fn() }), + getVectorIndex: () => null, + scheduleIndexSave: vi.fn(), + vectorIndexAddGuarded: vi.fn().mockResolvedValue(false), })); import { AnthropicProvider } from "../src/providers/anthropic.js"; -import { createProvider, createFallbackProvider } from "../src/providers/index.js"; +import { + createProvider, + createFallbackProvider, +} from "../src/providers/index.js"; import { ResilientProvider } from "../src/providers/resilient.js"; import { FallbackChainProvider } from "../src/providers/fallback-chain.js"; -import { registerCompressFunction, type CompressResult } from "../src/functions/compress.js"; +import { + registerCompressFunction, + type CompressResult, +} from "../src/functions/compress.js"; import { VISION_DESCRIPTION_PROMPT } from "../src/prompts/vision.js"; import { KV } from "../src/state/schema.js"; -const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="; +const PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="; const CAPTION = "TEST_VISION_RESULT: I see a red dot"; const XML = `imageScreenshot of Red Dot Test image observationImage shows a single red pixel A vision model described a screenshot showing a red dot on a white background testingscreenshot 5`; -type Request = { messages: Array<{ role?: string; content: string | Array<{ type: string; text?: string; source?: { data: string; media_type: string } }> }> }; +type Request = { + messages: Array<{ + role?: string; + content: + | string + | Array<{ + type: string; + text?: string; + source?: { data: string; media_type: string }; + }>; + }>; +}; type VisionCall = { name: string; receiverMatches: boolean; args: string[] }; let visionCalls: VisionCall[] = []; let prompts: string[] = []; function plain(name: string): MemoryProvider { - return { name, compress: async (_system, user) => { prompts.push(user); return XML; }, summarize: async () => "summary" }; + return { + name, + compress: async (_system, user) => { + prompts.push(user); + return XML; + }, + summarize: async () => "summary", + }; } function capable(name: string, fails = false): MemoryProvider { - const inner: MemoryProvider = { ...plain(name), async describeImage(image, mime, prompt) { - visionCalls.push({ name, receiverMatches: this === inner, args: [image, mime, prompt] }); - if (fails) throw new Error(name + " vision unavailable"); - return CAPTION; - } }; + const inner: MemoryProvider = { + ...plain(name), + async describeImage(image, mime, prompt) { + visionCalls.push({ + name, + receiverMatches: this === inner, + args: [image, mime, prompt], + }); + if (fails) throw new Error(name + " vision unavailable"); + return CAPTION; + }, + }; return inner; } -async function compress(provider: MemoryProvider, modality: "image" | "text" = "image") { +async function compress( + provider: MemoryProvider, + modality: "image" | "text" = "image", +) { const sdk = mockSdk({ looseTrigger: true }); const kv = mockKV(); - const raw: RawObservation = { id: "obs_vision-boundary", sessionId: "session_vision-boundary", - timestamp: "2026-06-01T00:00:00.000Z", hookType: "post_tool_use", toolName: "screenshot", - toolOutput: "A screenshot was captured", raw: {}, modality, - ...(modality === "image" ? { imageData: PNG } : {}) }; + const raw: RawObservation = { + id: "obs_vision-boundary", + sessionId: "session_vision-boundary", + timestamp: "2026-06-01T00:00:00.000Z", + hookType: "post_tool_use", + toolName: "screenshot", + toolOutput: "A screenshot was captured", + raw: {}, + modality, + ...(modality === "image" ? { imageData: PNG } : {}), + }; await kv.set(KV.observations(raw.sessionId), raw.id, raw); - registerCompressFunction(sdk as unknown as IIIClient, kv as unknown as StateKV, provider); - const result = await sdk.trigger({ function_id: "mem::compress", payload: { - observationId: raw.id, sessionId: raw.sessionId, - } }) as CompressResult; - const stored = await kv.get(KV.observations(raw.sessionId), raw.id); + registerCompressFunction( + sdk as unknown as IIIClient, + kv as unknown as StateKV, + provider, + ); + const result = (await sdk.trigger({ + function_id: "mem::compress", + payload: { + observationId: raw.id, + sessionId: raw.sessionId, + }, + })) as CompressResult; + const stored = await kv.get( + KV.observations(raw.sessionId), + raw.id, + ); return { result, stored, raw }; } @@ -75,7 +140,8 @@ async function expectCaption(provider: MemoryProvider) { describe("provider vision boundary", () => { beforeEach(() => { - visionCalls = []; prompts = []; + visionCalls = []; + prompts = []; vi.stubEnv("ANTHROPIC_API_KEY", "fixture-not-a-real-key"); vi.stubEnv("OPENAI_API_KEY", "fixture-not-a-real-key"); vi.stubEnv("ANTHROPIC_MODEL", "fixture-vision-model"); @@ -87,72 +153,143 @@ describe("provider vision boundary", () => { if (Array.isArray(content)) { const image = content.find((part) => part.type === "image")!; const text = content.find((part) => part.type === "text")!; - visionCalls.push({ name: "Anthropic SDK", receiverMatches: true, - args: [image.source!.data, image.source!.media_type, text.text!] }); + visionCalls.push({ + name: "Anthropic SDK", + receiverMatches: true, + args: [image.source!.data, image.source!.media_type, text.text!], + }); return { content: [{ type: "text", text: CAPTION }] }; } prompts.push(content); return { content: [{ type: "text", text: XML }] }; }); - vi.stubGlobal("fetch", vi.fn(async (_url: string, init: RequestInit) => { - const request = JSON.parse(String(init.body)) as Request; - const content = request.messages.find((message) => message.role === "user")!.content; - if (typeof content !== "string") throw new Error("Unexpected text-provider payload"); - prompts.push(content); - return new Response(JSON.stringify({ choices: [{ message: { content: XML }, finish_reason: "stop" }] }), - { status: 200, headers: { "content-type": "application/json" } }); - })); + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + const request = JSON.parse(String(init.body)) as Request; + const content = request.messages.find( + (message) => message.role === "user", + )!.content; + if (typeof content !== "string") + throw new Error("Unexpected text-provider payload"); + prompts.push(content); + return new Response( + JSON.stringify({ + choices: [{ message: { content: XML }, finish_reason: "stop" }], + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + }), + ); }); afterEach(() => { - vi.unstubAllEnvs(); vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); }); it("bare real Anthropic provider retains the existing image description", async () => { - await expectCaption(new AnthropicProvider("fixture-key", "fixture-model", 512)); - expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + await expectCaption( + new AnthropicProvider("fixture-key", "fixture-model", 512), + ); + expect(visionCalls).toEqual([ + { + name: "Anthropic SDK", + receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], + }, + ]); }); it("createProvider Anthropic preserves caption through the resilience wrapper", async () => { - await expectCaption(createProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 })); - expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + await expectCaption( + createProvider({ + provider: "anthropic", + model: "fixture-model", + maxTokens: 512, + }), + ); + expect(visionCalls).toEqual([ + { + name: "Anthropic SDK", + receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], + }, + ]); }); it("createFallbackProvider preserves a capable primary through both wrappers", async () => { - await expectCaption(createFallbackProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 }, - { providers: ["openai"] })); + await expectCaption( + createFallbackProvider( + { provider: "anthropic", model: "fixture-model", maxTokens: 512 }, + { providers: ["openai"] }, + ), + ); expect(visionCalls).toHaveLength(1); }); it("createFallbackProvider preserves a capable fallback after a text-only primary", async () => { - await expectCaption(createFallbackProvider({ provider: "openai", model: "fixture-model", maxTokens: 512 }, - { providers: ["anthropic"] })); - expect(visionCalls).toEqual([{ name: "Anthropic SDK", receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }]); + await expectCaption( + createFallbackProvider( + { provider: "openai", model: "fixture-model", maxTokens: 512 }, + { providers: ["anthropic"] }, + ), + ); + expect(visionCalls).toEqual([ + { + name: "Anthropic SDK", + receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], + }, + ]); }); it("supported fallback preserves receiver arguments and order after the first vision failure", async () => { - const p = new ResilientProvider(new FallbackChainProvider([plain("text"), capable("first", true), capable("second")])); + const p = new ResilientProvider( + new FallbackChainProvider([ + plain("text"), + capable("first", true), + capable("second"), + ]), + ); await expectCaption(p); - expect(visionCalls).toEqual(["first", "second"].map((name) => ({ name, receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }))); + expect(visionCalls).toEqual( + ["first", "second"].map((name) => ({ + name, + receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], + })), + ); }); it("all capable failures are attempted in order and retain existing text fallback", async () => { - const p = new ResilientProvider(new FallbackChainProvider([plain("text"), capable("first", true), capable("second", true)])); + const p = new ResilientProvider( + new FallbackChainProvider([ + plain("text"), + capable("first", true), + capable("second", true), + ]), + ); const { result, stored } = await compress(p); expect(result.success).toBe(true); expect(stored?.imageDescription).toBeUndefined(); expect(stored?.imageRef).toBe(PNG); - expect(visionCalls).toEqual(["first", "second"].map((name) => ({ name, receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT] }))); + expect(visionCalls).toEqual( + ["first", "second"].map((name) => ({ + name, + receiverMatches: true, + args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], + })), + ); }); it("unsupported wrappers and chains keep capability absent and the text path works", async () => { - const p = new ResilientProvider(new FallbackChainProvider([plain("text"), plain("other")])); + const p = new ResilientProvider( + new FallbackChainProvider([plain("text"), plain("other")]), + ); expect(p).not.toHaveProperty("describeImage"); - expect(createProvider({ provider: "noop", model: "noop", maxTokens: 512 })).not.toHaveProperty("describeImage"); + expect( + createProvider({ provider: "noop", model: "noop", maxTokens: 512 }), + ).not.toHaveProperty("describeImage"); const { result, stored } = await compress(p); expect(result.success).toBe(true); expect(stored?.imageDescription).toBeUndefined(); @@ -160,7 +297,11 @@ describe("provider vision boundary", () => { }); it("ordinary text through the real factory does not invoke image description", async () => { - const p = createProvider({ provider: "anthropic", model: "fixture-model", maxTokens: 512 }); + const p = createProvider({ + provider: "anthropic", + model: "fixture-model", + maxTokens: 512, + }); const { result, stored } = await compress(p, "text"); expect(result.success).toBe(true); expect(stored?.imageDescription).toBeUndefined(); @@ -172,19 +313,35 @@ describe("provider vision boundary", () => { const empty = new FallbackChainProvider([]); expect(empty).not.toHaveProperty("describeImage"); expect(new ResilientProvider(empty)).not.toHaveProperty("describeImage"); - expect(new ResilientProvider(plain("text"))).not.toHaveProperty("describeImage"); - expect(new FallbackChainProvider([plain("text")])).not.toHaveProperty("describeImage"); - await expect(empty.compress("system", "user")).rejects.toThrow("No providers available"); - await expect(empty.summarize("system", "user")).rejects.toThrow("No providers available"); + expect(new ResilientProvider(plain("text"))).not.toHaveProperty( + "describeImage", + ); + expect(new FallbackChainProvider([plain("text")])).not.toHaveProperty( + "describeImage", + ); + await expect(empty.compress("system", "user")).rejects.toThrow( + "No providers available", + ); + await expect(empty.summarize("system", "user")).rejects.toThrow( + "No providers available", + ); }); it("stops at the first successful image provider with its receiver and arguments", async () => { const chain = new FallbackChainProvider([ - plain("text"), capable("first"), capable("unused"), + plain("text"), + capable("first"), + capable("unused"), ]); - await expect(chain.describeImage?.("image", "image/jpeg", "describe")).resolves.toBe(CAPTION); + await expect( + chain.describeImage?.("image", "image/jpeg", "describe"), + ).resolves.toBe(CAPTION); expect(visionCalls).toEqual([ - { name: "first", receiverMatches: true, args: ["image", "image/jpeg", "describe"] }, + { + name: "first", + receiverMatches: true, + args: ["image", "image/jpeg", "describe"], + }, ]); }); @@ -194,47 +351,76 @@ describe("provider vision boundary", () => { const last: MemoryProvider = { ...plain("last"), async describeImage(image, mime, prompt) { - visionCalls.push({ name: "last", receiverMatches: this === last, args: [image, mime, prompt] }); + visionCalls.push({ + name: "last", + receiverMatches: this === last, + args: [image, mime, prompt], + }); throw lastError; }, }; const chain = new FallbackChainProvider([first, plain("text"), last]); - await expect(chain.describeImage?.("image", "image/webp", "describe")).rejects.toBe(lastError); - expect(visionCalls).toEqual(["first", "last"].map((name) => ({ - name, receiverMatches: true, args: ["image", "image/webp", "describe"], - }))); + await expect( + chain.describeImage?.("image", "image/webp", "describe"), + ).rejects.toBe(lastError); + expect(visionCalls).toEqual( + ["first", "last"].map((name) => ({ + name, + receiverMatches: true, + args: ["image", "image/webp", "describe"], + })), + ); }); it("normalizes a non-Error rejection from the last image provider", async () => { const inner: MemoryProvider = { ...plain("text"), - describeImage: async () => { throw "image request rejected"; }, + describeImage: async () => { + throw "image request rejected"; + }, }; const chain = new FallbackChainProvider([inner]); - await expect(chain.describeImage?.("image", "image/png", "describe")) - .rejects.toEqual(new Error("image request rejected")); + await expect( + chain.describeImage?.("image", "image/png", "describe"), + ).rejects.toEqual(new Error("image request rejected")); }); it("preserves the direct resilience delegate receiver and all image arguments", async () => { const provider = new ResilientProvider(capable("inner")); - await expect(provider.describeImage?.("image", "image/gif", "describe")).resolves.toBe(CAPTION); + await expect( + provider.describeImage?.("image", "image/gif", "describe"), + ).resolves.toBe(CAPTION); expect(visionCalls).toEqual([ - { name: "inner", receiverMatches: true, args: ["image", "image/gif", "describe"] }, + { + name: "inner", + receiverMatches: true, + args: ["image", "image/gif", "describe"], + }, ]); }); it("counts image gateway failures and blocks all operations with the same breaker", async () => { - const error = new Error('OpenAI API error (502): {"error":{"message":"content_filter","code":"bad_gateway"}}'); + const error = new Error( + 'OpenAI API error (502): {"error":{"message":"content_filter","code":"bad_gateway"}}', + ); const image = vi.fn().mockRejectedValue(error); const inner = { ...plain("inner"), describeImage: image }; const provider = new ResilientProvider(inner); for (let i = 0; i < 3; i++) { - await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toBe(error); + await expect( + provider.describeImage?.("image", "image/png", "describe"), + ).rejects.toBe(error); } expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); - await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toThrow("circuit_breaker_open"); - await expect(provider.compress("system", "user")).rejects.toThrow("circuit_breaker_open"); - await expect(provider.summarize("system", "user")).rejects.toThrow("circuit_breaker_open"); + await expect( + provider.describeImage?.("image", "image/png", "describe"), + ).rejects.toThrow("circuit_breaker_open"); + await expect(provider.compress("system", "user")).rejects.toThrow( + "circuit_breaker_open", + ); + await expect(provider.summarize("system", "user")).rejects.toThrow( + "circuit_breaker_open", + ); expect(image).toHaveBeenCalledTimes(3); expect(prompts).toEqual([]); }); @@ -243,14 +429,19 @@ describe("provider vision boundary", () => { const error = new Error("provider unavailable"); const image = vi.fn().mockResolvedValue(CAPTION); const inner: MemoryProvider = { - ...plain("inner"), describeImage: image, - compress: async () => { throw error; }, + ...plain("inner"), + describeImage: image, + compress: async () => { + throw error; + }, }; const provider = new ResilientProvider(inner); for (let i = 0; i < 3; i++) { await expect(provider.compress("system", "user")).rejects.toBe(error); } - await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toThrow("circuit_breaker_open"); + await expect( + provider.describeImage?.("image", "image/png", "describe"), + ).rejects.toThrow("circuit_breaker_open"); expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); expect(image).not.toHaveBeenCalled(); }); @@ -258,26 +449,48 @@ describe("provider vision boundary", () => { it("resets prior text failures when an image request succeeds", async () => { const error = new Error("text request failed"); const inner: MemoryProvider = { - ...capable("inner"), compress: async () => { throw error; }, + ...capable("inner"), + compress: async () => { + throw error; + }, }; const provider = new ResilientProvider(inner); for (let i = 0; i < 2; i++) { await expect(provider.compress("system", "user")).rejects.toBe(error); } expect(provider.circuitState.failures).toBe(2); - await expect(provider.describeImage?.("image", "image/png", "describe")).resolves.toBe(CAPTION); - expect(provider.circuitState).toEqual({ state: "closed", failures: 0, lastFailureAt: null, openedAt: null }); + await expect( + provider.describeImage?.("image", "image/png", "describe"), + ).resolves.toBe(CAPTION); + expect(provider.circuitState).toEqual({ + state: "closed", + failures: 0, + lastFailureAt: null, + openedAt: null, + }); expect(visionCalls).toHaveLength(1); }); it("does not count image content-filter rejections as service failures", async () => { - const error = new Error('OpenAI API error (400): {"error":{"code":"content_filter"}}'); + const error = new Error( + 'OpenAI API error (400): {"error":{"code":"content_filter"}}', + ); const image = vi.fn().mockRejectedValue(error); - const provider = new ResilientProvider({ ...plain("inner"), describeImage: image }); + const provider = new ResilientProvider({ + ...plain("inner"), + describeImage: image, + }); for (let i = 0; i < 5; i++) { - await expect(provider.describeImage?.("image", "image/png", "describe")).rejects.toBe(error); + await expect( + provider.describeImage?.("image", "image/png", "describe"), + ).rejects.toBe(error); } - expect(provider.circuitState).toEqual({ state: "closed", failures: 0, lastFailureAt: null, openedAt: null }); + expect(provider.circuitState).toEqual({ + state: "closed", + failures: 0, + lastFailureAt: null, + openedAt: null, + }); await expect(provider.compress("system", "user")).resolves.toBe(XML); expect(image).toHaveBeenCalledTimes(5); }); @@ -285,13 +498,23 @@ describe("provider vision boundary", () => { it("sanitizes factory-provided captions before returning and persisting them", async () => { const secret = ["fixture", "password"].join("-"); const caption = `terminal shows postgres://deploy:${secret}@db.example/app`; - const sanitized = "terminal shows postgres://[REDACTED_SECRET]@db.example/app"; + const sanitized = + "terminal shows postgres://[REDACTED_SECRET]@db.example/app"; sdkDouble.create.mockImplementation(async (request: Request) => ({ - content: [{ type: "text", text: Array.isArray(request.messages[0]!.content) ? caption : XML }], - })); - const { result, stored, raw } = await compress(createProvider({ - provider: "anthropic", model: "fixture-model", maxTokens: 512, + content: [ + { + type: "text", + text: Array.isArray(request.messages[0]!.content) ? caption : XML, + }, + ], })); + const { result, stored, raw } = await compress( + createProvider({ + provider: "anthropic", + model: "fixture-model", + maxTokens: 512, + }), + ); expect(result.success).toBe(true); expect(result.compressed?.imageDescription).toBe(sanitized); expect(stored?.imageDescription).toBe(sanitized); From fe078739e72f88c38f5bdd5b84795107cbad2db8 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 16 Nov 2026 16:09:28 +0530 Subject: [PATCH 3/3] test(providers): replace vision boundary test with focused wrapper tests The previous test mocked the Anthropic SDK, KV, search and the compress pipeline across 526 lines to check a 27 line change. Test the two wrappers directly: ResilientProvider forwards describeImage and counts its failures in the shared breaker, FallbackChainProvider skips text-only providers and falls through in order, and both leave describeImage absent when nothing supports it. Signed-off-by: Rohit Ghumare --- test/provider-vision.test.ts | 541 +++-------------------------------- 1 file changed, 43 insertions(+), 498 deletions(-) diff --git a/test/provider-vision.test.ts b/test/provider-vision.test.ts index f293eadaa0..3d63bb0148 100644 --- a/test/provider-vision.test.ts +++ b/test/provider-vision.test.ts @@ -1,526 +1,71 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { IIIClient } from "iii-sdk"; -import type { StateKV } from "../src/state/kv.js"; -import type { - MemoryProvider, - RawObservation, - CompressedObservation, -} from "../src/types.js"; -import { mockKV, mockSdk } from "./helpers/mocks.js"; - -const sdkDouble = vi.hoisted(() => ({ create: vi.fn() })); -vi.mock("@anthropic-ai/sdk", () => ({ - default: class { - messages = { create: sdkDouble.create }; - }, -})); -vi.mock("../src/logger.js", () => ({ - logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, -})); -vi.mock("../src/functions/search.js", () => ({ - getSearchIndex: () => ({ add: vi.fn() }), - getVectorIndex: () => null, - scheduleIndexSave: vi.fn(), - vectorIndexAddGuarded: vi.fn().mockResolvedValue(false), -})); - -import { AnthropicProvider } from "../src/providers/anthropic.js"; -import { - createProvider, - createFallbackProvider, -} from "../src/providers/index.js"; -import { ResilientProvider } from "../src/providers/resilient.js"; +import { describe, expect, it, vi } from "vitest"; import { FallbackChainProvider } from "../src/providers/fallback-chain.js"; -import { - registerCompressFunction, - type CompressResult, -} from "../src/functions/compress.js"; -import { VISION_DESCRIPTION_PROMPT } from "../src/prompts/vision.js"; -import { KV } from "../src/state/schema.js"; - -const PNG = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg=="; -const CAPTION = "TEST_VISION_RESULT: I see a red dot"; -const XML = `imageScreenshot of Red Dot -Test image observationImage shows a single red pixel -A vision model described a screenshot showing a red dot on a white background -testingscreenshot -5`; -type Request = { - messages: Array<{ - role?: string; - content: - | string - | Array<{ - type: string; - text?: string; - source?: { data: string; media_type: string }; - }>; - }>; -}; -type VisionCall = { name: string; receiverMatches: boolean; args: string[] }; -let visionCalls: VisionCall[] = []; -let prompts: string[] = []; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; -function plain(name: string): MemoryProvider { +function textOnly(name: string): MemoryProvider { return { name, - compress: async (_system, user) => { - prompts.push(user); - return XML; - }, - summarize: async () => "summary", + compress: async () => `compressed by ${name}`, + summarize: async () => `summarized by ${name}`, }; } -function capable(name: string, fails = false): MemoryProvider { - const inner: MemoryProvider = { - ...plain(name), - async describeImage(image, mime, prompt) { - visionCalls.push({ - name, - receiverMatches: this === inner, - args: [image, mime, prompt], - }); - if (fails) throw new Error(name + " vision unavailable"); - return CAPTION; - }, - }; - return inner; +function withVision( + name: string, + describeImage: MemoryProvider["describeImage"], +): MemoryProvider { + return { ...textOnly(name), describeImage }; } -async function compress( - provider: MemoryProvider, - modality: "image" | "text" = "image", -) { - const sdk = mockSdk({ looseTrigger: true }); - const kv = mockKV(); - const raw: RawObservation = { - id: "obs_vision-boundary", - sessionId: "session_vision-boundary", - timestamp: "2026-06-01T00:00:00.000Z", - hookType: "post_tool_use", - toolName: "screenshot", - toolOutput: "A screenshot was captured", - raw: {}, - modality, - ...(modality === "image" ? { imageData: PNG } : {}), - }; - await kv.set(KV.observations(raw.sessionId), raw.id, raw); - registerCompressFunction( - sdk as unknown as IIIClient, - kv as unknown as StateKV, - provider, - ); - const result = (await sdk.trigger({ - function_id: "mem::compress", - payload: { - observationId: raw.id, - sessionId: raw.sessionId, - }, - })) as CompressResult; - const stored = await kv.get( - KV.observations(raw.sessionId), - raw.id, - ); - return { result, stored, raw }; -} - -async function expectCaption(provider: MemoryProvider) { - const { result, stored, raw } = await compress(provider); - expect(result.success).toBe(true); - expect(stored?.imageDescription).toBe(CAPTION); - expect(result.compressed?.imageDescription).toBe(CAPTION); - expect(stored?.id).toBe(raw.id); - expect(stored?.sessionId).toBe(raw.sessionId); - expect(stored?.imageRef).toBe(PNG); - expect(prompts).toHaveLength(1); - expect(prompts[0]).toContain(CAPTION); -} +describe("provider wrappers keep image description", () => { + it("ResilientProvider forwards describeImage with its arguments", async () => { + const describe = vi.fn(async () => "a red dot"); + const provider = new ResilientProvider(withVision("vision", describe)); -describe("provider vision boundary", () => { - beforeEach(() => { - visionCalls = []; - prompts = []; - vi.stubEnv("ANTHROPIC_API_KEY", "fixture-not-a-real-key"); - vi.stubEnv("OPENAI_API_KEY", "fixture-not-a-real-key"); - vi.stubEnv("ANTHROPIC_MODEL", "fixture-vision-model"); - vi.stubEnv("OPENAI_MODEL", "fixture-text-model"); - vi.stubEnv("OPENAI_BASE_URL", "http://127.0.0.1:1/v1"); - sdkDouble.create.mockReset(); - sdkDouble.create.mockImplementation(async (request: Request) => { - const content = request.messages[0]!.content; - if (Array.isArray(content)) { - const image = content.find((part) => part.type === "image")!; - const text = content.find((part) => part.type === "text")!; - visionCalls.push({ - name: "Anthropic SDK", - receiverMatches: true, - args: [image.source!.data, image.source!.media_type, text.text!], - }); - return { content: [{ type: "text", text: CAPTION }] }; - } - prompts.push(content); - return { content: [{ type: "text", text: XML }] }; - }); - vi.stubGlobal( - "fetch", - vi.fn(async (_url: string, init: RequestInit) => { - const request = JSON.parse(String(init.body)) as Request; - const content = request.messages.find( - (message) => message.role === "user", - )!.content; - if (typeof content !== "string") - throw new Error("Unexpected text-provider payload"); - prompts.push(content); - return new Response( - JSON.stringify({ - choices: [{ message: { content: XML }, finish_reason: "stop" }], - }), - { status: 200, headers: { "content-type": "application/json" } }, - ); - }), + await expect(provider.describeImage?.("data", "image/png", "prompt")).resolves.toBe( + "a red dot", ); - }); - afterEach(() => { - vi.unstubAllEnvs(); - vi.unstubAllGlobals(); + expect(describe).toHaveBeenCalledWith("data", "image/png", "prompt"); }); - it("bare real Anthropic provider retains the existing image description", async () => { - await expectCaption( - new AnthropicProvider("fixture-key", "fixture-model", 512), - ); - expect(visionCalls).toEqual([ - { - name: "Anthropic SDK", - receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], - }, - ]); + it("ResilientProvider leaves describeImage absent for a text-only provider", () => { + expect(new ResilientProvider(textOnly("text")).describeImage).toBeUndefined(); }); - it("createProvider Anthropic preserves caption through the resilience wrapper", async () => { - await expectCaption( - createProvider({ - provider: "anthropic", - model: "fixture-model", - maxTokens: 512, + it("ResilientProvider counts image failures in the shared circuit breaker", async () => { + const provider = new ResilientProvider( + withVision("vision", async () => { + throw new Error("gateway down"); }), ); - expect(visionCalls).toEqual([ - { - name: "Anthropic SDK", - receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], - }, - ]); - }); - - it("createFallbackProvider preserves a capable primary through both wrappers", async () => { - await expectCaption( - createFallbackProvider( - { provider: "anthropic", model: "fixture-model", maxTokens: 512 }, - { providers: ["openai"] }, - ), - ); - expect(visionCalls).toHaveLength(1); - }); - - it("createFallbackProvider preserves a capable fallback after a text-only primary", async () => { - await expectCaption( - createFallbackProvider( - { provider: "openai", model: "fixture-model", maxTokens: 512 }, - { providers: ["anthropic"] }, - ), - ); - expect(visionCalls).toEqual([ - { - name: "Anthropic SDK", - receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], - }, - ]); - }); - - it("supported fallback preserves receiver arguments and order after the first vision failure", async () => { - const p = new ResilientProvider( - new FallbackChainProvider([ - plain("text"), - capable("first", true), - capable("second"), - ]), - ); - await expectCaption(p); - expect(visionCalls).toEqual( - ["first", "second"].map((name) => ({ - name, - receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], - })), - ); - }); - - it("all capable failures are attempted in order and retain existing text fallback", async () => { - const p = new ResilientProvider( - new FallbackChainProvider([ - plain("text"), - capable("first", true), - capable("second", true), - ]), - ); - const { result, stored } = await compress(p); - expect(result.success).toBe(true); - expect(stored?.imageDescription).toBeUndefined(); - expect(stored?.imageRef).toBe(PNG); - expect(visionCalls).toEqual( - ["first", "second"].map((name) => ({ - name, - receiverMatches: true, - args: [PNG, "image/png", VISION_DESCRIPTION_PROMPT], - })), - ); - }); - - it("unsupported wrappers and chains keep capability absent and the text path works", async () => { - const p = new ResilientProvider( - new FallbackChainProvider([plain("text"), plain("other")]), - ); - expect(p).not.toHaveProperty("describeImage"); - expect( - createProvider({ provider: "noop", model: "noop", maxTokens: 512 }), - ).not.toHaveProperty("describeImage"); - const { result, stored } = await compress(p); - expect(result.success).toBe(true); - expect(stored?.imageDescription).toBeUndefined(); - expect(visionCalls).toEqual([]); + for (let i = 0; i < 5; i++) { + await provider.describeImage?.("data", "image/png", "prompt").catch(() => {}); + } + await expect(provider.compress("sys", "user")).rejects.toThrow(/circuit/i); }); - it("ordinary text through the real factory does not invoke image description", async () => { - const p = createProvider({ - provider: "anthropic", - model: "fixture-model", - maxTokens: 512, + it("FallbackChainProvider skips text-only providers and falls through in order", async () => { + const failing = vi.fn(async () => { + throw new Error("first down"); }); - const { result, stored } = await compress(p, "text"); - expect(result.success).toBe(true); - expect(stored?.imageDescription).toBeUndefined(); - expect(visionCalls).toEqual([]); - expect(prompts).toHaveLength(1); - }); - - it("empty and text-only chains leave image capability absent", async () => { - const empty = new FallbackChainProvider([]); - expect(empty).not.toHaveProperty("describeImage"); - expect(new ResilientProvider(empty)).not.toHaveProperty("describeImage"); - expect(new ResilientProvider(plain("text"))).not.toHaveProperty( - "describeImage", - ); - expect(new FallbackChainProvider([plain("text")])).not.toHaveProperty( - "describeImage", - ); - await expect(empty.compress("system", "user")).rejects.toThrow( - "No providers available", - ); - await expect(empty.summarize("system", "user")).rejects.toThrow( - "No providers available", - ); - }); - - it("stops at the first successful image provider with its receiver and arguments", async () => { + const working = vi.fn(async () => "from second"); const chain = new FallbackChainProvider([ - plain("text"), - capable("first"), - capable("unused"), - ]); - await expect( - chain.describeImage?.("image", "image/jpeg", "describe"), - ).resolves.toBe(CAPTION); - expect(visionCalls).toEqual([ - { - name: "first", - receiverMatches: true, - args: ["image", "image/jpeg", "describe"], - }, + textOnly("text"), + withVision("first", failing), + withVision("second", working), ]); - }); - it("propagates the last capable provider error after trying each in order", async () => { - const lastError = new Error("last image provider unavailable"); - const first = capable("first", true); - const last: MemoryProvider = { - ...plain("last"), - async describeImage(image, mime, prompt) { - visionCalls.push({ - name: "last", - receiverMatches: this === last, - args: [image, mime, prompt], - }); - throw lastError; - }, - }; - const chain = new FallbackChainProvider([first, plain("text"), last]); - await expect( - chain.describeImage?.("image", "image/webp", "describe"), - ).rejects.toBe(lastError); - expect(visionCalls).toEqual( - ["first", "last"].map((name) => ({ - name, - receiverMatches: true, - args: ["image", "image/webp", "describe"], - })), + await expect(chain.describeImage?.("data", "image/png", "prompt")).resolves.toBe( + "from second", ); + expect(failing).toHaveBeenCalledOnce(); + expect(working).toHaveBeenCalledWith("data", "image/png", "prompt"); }); - it("normalizes a non-Error rejection from the last image provider", async () => { - const inner: MemoryProvider = { - ...plain("text"), - describeImage: async () => { - throw "image request rejected"; - }, - }; - const chain = new FallbackChainProvider([inner]); - await expect( - chain.describeImage?.("image", "image/png", "describe"), - ).rejects.toEqual(new Error("image request rejected")); - }); - - it("preserves the direct resilience delegate receiver and all image arguments", async () => { - const provider = new ResilientProvider(capable("inner")); - await expect( - provider.describeImage?.("image", "image/gif", "describe"), - ).resolves.toBe(CAPTION); - expect(visionCalls).toEqual([ - { - name: "inner", - receiverMatches: true, - args: ["image", "image/gif", "describe"], - }, - ]); - }); - - it("counts image gateway failures and blocks all operations with the same breaker", async () => { - const error = new Error( - 'OpenAI API error (502): {"error":{"message":"content_filter","code":"bad_gateway"}}', - ); - const image = vi.fn().mockRejectedValue(error); - const inner = { ...plain("inner"), describeImage: image }; - const provider = new ResilientProvider(inner); - for (let i = 0; i < 3; i++) { - await expect( - provider.describeImage?.("image", "image/png", "describe"), - ).rejects.toBe(error); - } - expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); - await expect( - provider.describeImage?.("image", "image/png", "describe"), - ).rejects.toThrow("circuit_breaker_open"); - await expect(provider.compress("system", "user")).rejects.toThrow( - "circuit_breaker_open", - ); - await expect(provider.summarize("system", "user")).rejects.toThrow( - "circuit_breaker_open", - ); - expect(image).toHaveBeenCalledTimes(3); - expect(prompts).toEqual([]); - }); - - it("blocks image requests after text failures have opened the shared breaker", async () => { - const error = new Error("provider unavailable"); - const image = vi.fn().mockResolvedValue(CAPTION); - const inner: MemoryProvider = { - ...plain("inner"), - describeImage: image, - compress: async () => { - throw error; - }, - }; - const provider = new ResilientProvider(inner); - for (let i = 0; i < 3; i++) { - await expect(provider.compress("system", "user")).rejects.toBe(error); - } - await expect( - provider.describeImage?.("image", "image/png", "describe"), - ).rejects.toThrow("circuit_breaker_open"); - expect(provider.circuitState).toMatchObject({ state: "open", failures: 3 }); - expect(image).not.toHaveBeenCalled(); - }); - - it("resets prior text failures when an image request succeeds", async () => { - const error = new Error("text request failed"); - const inner: MemoryProvider = { - ...capable("inner"), - compress: async () => { - throw error; - }, - }; - const provider = new ResilientProvider(inner); - for (let i = 0; i < 2; i++) { - await expect(provider.compress("system", "user")).rejects.toBe(error); - } - expect(provider.circuitState.failures).toBe(2); - await expect( - provider.describeImage?.("image", "image/png", "describe"), - ).resolves.toBe(CAPTION); - expect(provider.circuitState).toEqual({ - state: "closed", - failures: 0, - lastFailureAt: null, - openedAt: null, - }); - expect(visionCalls).toHaveLength(1); - }); - - it("does not count image content-filter rejections as service failures", async () => { - const error = new Error( - 'OpenAI API error (400): {"error":{"code":"content_filter"}}', - ); - const image = vi.fn().mockRejectedValue(error); - const provider = new ResilientProvider({ - ...plain("inner"), - describeImage: image, - }); - for (let i = 0; i < 5; i++) { - await expect( - provider.describeImage?.("image", "image/png", "describe"), - ).rejects.toBe(error); - } - expect(provider.circuitState).toEqual({ - state: "closed", - failures: 0, - lastFailureAt: null, - openedAt: null, - }); - await expect(provider.compress("system", "user")).resolves.toBe(XML); - expect(image).toHaveBeenCalledTimes(5); - }); - - it("sanitizes factory-provided captions before returning and persisting them", async () => { - const secret = ["fixture", "password"].join("-"); - const caption = `terminal shows postgres://deploy:${secret}@db.example/app`; - const sanitized = - "terminal shows postgres://[REDACTED_SECRET]@db.example/app"; - sdkDouble.create.mockImplementation(async (request: Request) => ({ - content: [ - { - type: "text", - text: Array.isArray(request.messages[0]!.content) ? caption : XML, - }, - ], - })); - const { result, stored, raw } = await compress( - createProvider({ - provider: "anthropic", - model: "fixture-model", - maxTokens: 512, - }), - ); - expect(result.success).toBe(true); - expect(result.compressed?.imageDescription).toBe(sanitized); - expect(stored?.imageDescription).toBe(sanitized); - expect(stored?.id).toBe(raw.id); - expect(stored?.imageRef).toBe(PNG); - expect(JSON.stringify({ result, stored })).not.toContain(secret); - expect(sdkDouble.create).toHaveBeenCalledTimes(2); + it("FallbackChainProvider leaves describeImage absent when no provider supports it", () => { + expect( + new FallbackChainProvider([textOnly("a"), textOnly("b")]).describeImage, + ).toBeUndefined(); }); });