diff --git a/src/providers/fallback-chain.ts b/src/providers/fallback-chain.ts index e2c979f929..74686db28a 100644 --- a/src/providers/fallback-chain.ts +++ b/src/providers/fallback-chain.ts @@ -1,25 +1,41 @@ import type { MemoryProvider } from "../types.js"; +type ImageProvider = MemoryProvider & + Required>; + export class FallbackChainProvider implements MemoryProvider { name: string; + declare readonly describeImage?: MemoryProvider["describeImage"]; constructor(private providers: MemoryProvider[]) { this.name = `fallback(${providers.map((p) => p.name).join(" -> ")})`; + const imageProviders = providers.filter( + (provider): provider is ImageProvider => + typeof provider.describeImage === "function", + ); + if (imageProviders.length > 0) { + this.describeImage = (imageData, mimeType, prompt) => + this.tryAll( + (provider) => provider.describeImage(imageData, mimeType, prompt), + imageProviders, + ); + } } async compress(systemPrompt: string, userPrompt: string): Promise { - return this.tryAll((p) => p.compress(systemPrompt, userPrompt)); + return this.tryAll((p) => p.compress(systemPrompt, userPrompt), this.providers); } async summarize(systemPrompt: string, userPrompt: string): Promise { - return this.tryAll((p) => p.summarize(systemPrompt, userPrompt)); + return this.tryAll((p) => p.summarize(systemPrompt, userPrompt), this.providers); } - private async tryAll( - fn: (p: MemoryProvider) => Promise, + private async tryAll( + fn: (p: T) => Promise, + providers: T[], ): Promise { let lastError: Error | null = null; - for (const provider of this.providers) { + for (const provider of providers) { try { return await fn(provider); } catch (err) { diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts index f82e578d72..e407398d56 100644 --- a/src/providers/resilient.ts +++ b/src/providers/resilient.ts @@ -12,9 +12,15 @@ export function isPayloadRejection(err: unknown): boolean { export class ResilientProvider implements MemoryProvider { private breaker = new CircuitBreaker(); name: string; + declare readonly describeImage?: MemoryProvider["describeImage"]; constructor(private inner: MemoryProvider) { this.name = `resilient(${inner.name})`; + const describeImage = inner.describeImage; + if (typeof describeImage === "function") { + this.describeImage = (imageData, mimeType, prompt) => + this.call(() => describeImage.call(inner, imageData, mimeType, prompt)); + } } private async call(fn: () => Promise): Promise { diff --git a/test/provider-vision.test.ts b/test/provider-vision.test.ts new file mode 100644 index 0000000000..3d63bb0148 --- /dev/null +++ b/test/provider-vision.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from "vitest"; +import { FallbackChainProvider } from "../src/providers/fallback-chain.js"; +import { ResilientProvider } from "../src/providers/resilient.js"; +import type { MemoryProvider } from "../src/types.js"; + +function textOnly(name: string): MemoryProvider { + return { + name, + compress: async () => `compressed by ${name}`, + summarize: async () => `summarized by ${name}`, + }; +} + +function withVision( + name: string, + describeImage: MemoryProvider["describeImage"], +): MemoryProvider { + return { ...textOnly(name), describeImage }; +} + +describe("provider wrappers keep image description", () => { + it("ResilientProvider forwards describeImage with its arguments", async () => { + const describe = vi.fn(async () => "a red dot"); + const provider = new ResilientProvider(withVision("vision", describe)); + + await expect(provider.describeImage?.("data", "image/png", "prompt")).resolves.toBe( + "a red dot", + ); + expect(describe).toHaveBeenCalledWith("data", "image/png", "prompt"); + }); + + it("ResilientProvider leaves describeImage absent for a text-only provider", () => { + expect(new ResilientProvider(textOnly("text")).describeImage).toBeUndefined(); + }); + + it("ResilientProvider counts image failures in the shared circuit breaker", async () => { + const provider = new ResilientProvider( + withVision("vision", async () => { + throw new Error("gateway down"); + }), + ); + for (let i = 0; i < 5; i++) { + await provider.describeImage?.("data", "image/png", "prompt").catch(() => {}); + } + await expect(provider.compress("sys", "user")).rejects.toThrow(/circuit/i); + }); + + it("FallbackChainProvider skips text-only providers and falls through in order", async () => { + const failing = vi.fn(async () => { + throw new Error("first down"); + }); + const working = vi.fn(async () => "from second"); + const chain = new FallbackChainProvider([ + textOnly("text"), + withVision("first", failing), + withVision("second", working), + ]); + + await expect(chain.describeImage?.("data", "image/png", "prompt")).resolves.toBe( + "from second", + ); + expect(failing).toHaveBeenCalledOnce(); + expect(working).toHaveBeenCalledWith("data", "image/png", "prompt"); + }); + + it("FallbackChainProvider leaves describeImage absent when no provider supports it", () => { + expect( + new FallbackChainProvider([textOnly("a"), textOnly("b")]).describeImage, + ).toBeUndefined(); + }); +});