diff --git a/.changeset/quiet-adapters-images.md b/.changeset/quiet-adapters-images.md new file mode 100644 index 000000000..e85c6c2dc --- /dev/null +++ b/.changeset/quiet-adapters-images.md @@ -0,0 +1,5 @@ +--- +"@agentskit/adapters": minor +--- + +Serialize supported image/file parts in OpenAI-compatible, Anthropic, Gemini/Vertex and Ollama requests. Preserve text-only model behavior and older multimodal histories by falling back to text parts and short attachment markers, omitting binary data and data URLs, with at most one warning per request when multimodal capability is disabled; OpenAI and Anthropic assistant binary parts also use the text fallback. Unsupported modalities and sources still raise CAPABILITY_UNSUPPORTED when multimodal input is enabled. Explicit capabilities override the expanded model-name heuristic. Add configurable path, headers and fetch, optional Bearer authentication for compatible endpoints, OpenAI-compatible and Cloudflare REST presets, an OpenRouter alias, and explicit catalog URL variables. Existing text-only and default endpoint behavior is preserved. diff --git a/.doc-bridge/capabilities.json b/.doc-bridge/capabilities.json index af9370989..d04f65c4a 100644 --- a/.doc-bridge/capabilities.json +++ b/.doc-bridge/capabilities.json @@ -7,7 +7,7 @@ "name": "AgentsKit", "root": "." }, - "contentHash": "e62e317cdcb1a17046123607ae239ca0be9045c7bf9e9c0804ab66c7a6ffb01b", + "contentHash": "de95fec2af4608ff2a437882c6c08c0cdc82c86637b2a49cc29ce29de2a43cca", "artifacts": { "index": ".doc-bridge/index.json", "llmsTxt": ".doc-bridge/llms.txt" diff --git a/.doc-bridge/index.json b/.doc-bridge/index.json index ba330ffed..487787324 100644 --- a/.doc-bridge/index.json +++ b/.doc-bridge/index.json @@ -1,8 +1,8 @@ { "schemaVersion": 1, - "contentHash": "e62e317cdcb1a17046123607ae239ca0be9045c7bf9e9c0804ab66c7a6ffb01b", + "contentHash": "de95fec2af4608ff2a437882c6c08c0cdc82c86637b2a49cc29ce29de2a43cca", "contentHashAlgo": "sha256-normalized-v1", - "generatedAt": "2026-10-07T18:43:39.882Z", + "generatedAt": "2026-10-08T01:36:58.952Z", "project": { "name": "AgentsKit", "root": "." @@ -2308,7 +2308,7 @@ "document", "human" ], - "contentHash": "8649f719440a37aca548f897790f465b8d79504d41e535e291d7346485917757" + "contentHash": "aeb25b23b2eca677189218974c32d3f042e317c690b4e7c464187c83034601e9" }, { "id": "document:apps/docs-next/content/docs/data/providers/openrouter.mdx", @@ -4426,12 +4426,12 @@ "type": "document", "title": "multi-modal.mdx", "path": "apps/docs-next/content/docs/reference/recipes/multi-modal.mdx", - "description": "Every provider has its own multi-modal shape. OpenAI wants { type: 'image_url', image_url: {...} }, Anthropic wants { type: 'image', source: {...} }, Gemini wants parts-with-inline- data. @agentskit/core provides a provider-neutral ContentPart model — adapters that understand a modality read the parts, the rest fall back to a text projection.", + "description": "Every provider has its own multi-modal shape. OpenAI wants { type: 'image_url', image_url: {...} }, Anthropic wants { type: 'image', source: {...} }, Gemini wants parts-with-inline- data. @agentskit/core provides a provider-neutral ContentPart model.", "tags": [ "document", "human" ], - "contentHash": "cb2013f2d179b7f36db952432cea91745e92160c23db10325ec078b5ebf805b1" + "contentHash": "9eb91d2547f891f6a179bac361fa6a3d23902a822103480f355100cf19758376" }, { "id": "document:apps/docs-next/content/docs/reference/recipes/open-specs.mdx", @@ -5597,6 +5597,17 @@ ], "contentHash": "3077743f11a0f1247886eba780727275a818b99d1cb4abca3febaa84dfb031ca" }, + { + "id": "document:docs/adapter-modalities.md", + "type": "document", + "title": "adapter-modalities.md", + "path": "docs/adapter-modalities.md", + "tags": [ + "document", + "human" + ], + "contentHash": "cecf0b22615d11742cb07d40791edd49ff4d4a4aab9b0789cecb72c64b14e2b9" + }, { "id": "document:docs/architecture/adrs/0001-adapter-contract.md", "type": "document", @@ -6017,6 +6028,18 @@ ], "contentHash": "697a4c044d5d149a6215b4fcd99733612db5add00ad76c415c7f2b3ff95a918d" }, + { + "id": "document:docs/architecture/adrs/0041-adapter-content-serialization.md", + "type": "document", + "title": "0041 — Serialize attachments at the provider boundary", + "path": "docs/architecture/adrs/0041-adapter-content-serialization.md", + "description": "Status: proposed; implementation ready for maintainer review", + "tags": [ + "document", + "human" + ], + "contentHash": "ee7624765cefcbd803ffa7f0666307f5cf732aa75b517e25620f4b9d2d6096ac" + }, { "id": "document:docs/architecture/adrs/0042-durable-tool-decisions.md", "type": "document", @@ -6039,7 +6062,7 @@ "document", "human" ], - "contentHash": "9c62e4e61d8e6e91046e39aa4bfa30f72ec792b5f5b2653dd9e1989acc02b612" + "contentHash": "5c07710d50c06de02f9f2f29b5bdc395d6a645d96fd9d819624690599872948a" }, { "id": "document:docs/architecture/ecosystem-manifest.md", @@ -6485,6 +6508,18 @@ ], "contentHash": "77706fd8391e1e6cc4c31c835ee0e1685796559ef284d16675b983fb0cbacbdb" }, + { + "id": "document:docs/evidence/adapters-multimodal-transport.md", + "type": "document", + "title": "Adapter multimodal transport — local implementation report", + "path": "docs/evidence/adapters-multimodal-transport.md", + "description": "Status: AWAITING_HUMAN_APPROVAL: all CH-A2 checks passed; coordinator evidence review remains required. Earlier BLOCKED results below are historical evidence. No release, publication, deployment, push or PR was performed.", + "tags": [ + "document", + "human" + ], + "contentHash": "4821916edffb5279caafdb46f646423762ea03e5b5a00faf6d6ae246247f6a38" + }, { "id": "document:docs/evidence/ecosystem-documentation-quality/README.md", "type": "document", @@ -6867,7 +6902,7 @@ "document", "project" ], - "contentHash": "81357860a2849bf61ee3e04ae289e76d5f2000fd3a8fdb401295013053ce7f31" + "contentHash": "337241356418b3cfc59e3fb28afa15beb2a157449e8b1b1257a1a7881fd07557" }, { "id": "document:packages/angular/CONVENTIONS.md", @@ -14115,7 +14150,7 @@ "ts", "catalog" ], - "contentHash": "9f9e345ff56621ebb4522b8bb0eabde563fe9bbfa9c48a8486cc7f44f45d4fe9" + "contentHash": "5e3a07cad66d3db3e61cd5fe939e5e176bbd4da3902fd4fe98563a396077dece" }, { "id": "module:packages/adapters/src/catalog/drift.ts", @@ -14471,6 +14506,24 @@ ], "contentHash": "6a98e2a39cddfbded09b62bdc3d53a3c1b38f36b797c31b5d032187e2376e0ea" }, + { + "id": "module:packages/adapters/src/cloudflare.ts", + "type": "module", + "title": "cloudflare.ts", + "path": "packages/adapters/src/cloudflare.ts", + "symbols": [ + "CloudflareAiGatewayConfig", + "WorkersAiConfig", + "cloudflareAiGateway", + "workersAi" + ], + "tags": [ + "module", + "ts", + "src" + ], + "contentHash": "79620d21cd40807b05e76f231a76b9b24bd6e87186508e9569decbfbc7e8b73b" + }, { "id": "module:packages/adapters/src/cohere.ts", "type": "module", @@ -14488,6 +14541,22 @@ ], "contentHash": "65c009fb57c33513f74adea37dd218e8eba5735ebac0ab511f3e7a3d3e2c731a" }, + { + "id": "module:packages/adapters/src/content-parts.ts", + "type": "module", + "title": "content-parts.ts", + "path": "packages/adapters/src/content-parts.ts", + "symbols": [ + "CapabilityUnsupportedError", + "providerParts" + ], + "tags": [ + "module", + "ts", + "src" + ], + "contentHash": "4541c6764430fb8e9e7546cfe457fa26b7ceb8e46debbbddc4c9e02b58d179f2" + }, { "id": "module:packages/adapters/src/createAdapter.ts", "type": "module", @@ -14843,8 +14912,10 @@ "BailConfig", "BedrockConfig", "BedrockRuntimeClientLike", + "CapabilityUnsupportedError", "CarbonTable", "CerebrasConfig", + "CloudflareAiGatewayConfig", "CohereConfig", "CreateAdapterConfig", "CredentialRefreshable", @@ -14881,7 +14952,9 @@ "ModelDeprecation", "OllamaConfig", "OllamaEmbedderConfig", + "OpenAICompatibleConfig", "OpenAICompatibleEmbedderConfig", + "OpenAICompatibleTransportConfig", "OpenAIConfig", "OpenAIEmbedderConfig", "OpenRouterConfig", @@ -14903,6 +14976,7 @@ "VertexConfig", "WebLlmConfig", "WebLlmEngineLike", + "WorkersAiConfig", "anthropic", "applyCarbonTable", "azureOpenAI", @@ -14914,6 +14988,7 @@ "cerebras", "cerebrasAdapter", "chunkText", + "cloudflareAiGateway", "cohere", "cohereAdapter", "createAdapter", @@ -14946,7 +15021,9 @@ "mockAdapter", "ollama", "ollamaEmbedder", + "openRouter", "openai", + "openaiCompatible", "openaiEmbedder", "openrouter", "qwen", @@ -14964,14 +15041,15 @@ "vllm", "webllm", "webllmAdapter", - "withDeprecationPolicy" + "withDeprecationPolicy", + "workersAi" ], "tags": [ "module", "ts", "src" ], - "contentHash": "219fd72d02c77632ca34d8c9247d0b3427a9b9cdcaf3837d7d400e386aafec1d" + "contentHash": "fc0b76056685ae8b01ec4c33e2d89fddebf702733de3993b9477b5af0d6f920a" }, { "id": "module:packages/adapters/src/kimi.ts", @@ -15110,7 +15188,7 @@ "ts", "src" ], - "contentHash": "3335d32093c3c2e58df7d5ad0c42288a52f9d84d2545ca6352142694c752c6d2" + "contentHash": "9f1a2e4365a8baf6b3c07d817b0b6fa574eb1d6e3c2c0adbe56ca4c580b4596f" }, { "id": "module:packages/adapters/src/openai-compatible.ts", @@ -15119,14 +15197,16 @@ "path": "packages/adapters/src/openai-compatible.ts", "symbols": [ "OpenAICompatibleConfig", - "createOpenAICompatibleAdapter" + "OpenAICompatibleTransportConfig", + "createOpenAICompatibleAdapter", + "openaiCompatible" ], "tags": [ "module", "ts", "src" ], - "contentHash": "6810d8d6c52d0f3d659207dd7a186d82c78a0df9bbbf7e608dacc6b2169b7e64" + "contentHash": "a334373f594863a9392bb5d463db6c51e06c1ce78f3b57ce18aaae7ee3123cd7" }, { "id": "module:packages/adapters/src/openai-stream.ts", @@ -15157,7 +15237,7 @@ "ts", "src" ], - "contentHash": "483f7dc8e5e72fd3c6fb9823acc4f563144387927d40cfe9ece59d70d124d3a4" + "contentHash": "131ca615f9ae12c76a9850f04752d20bcf6a23109c6bcbbbcb334291c0f50c2b" }, { "id": "module:packages/adapters/src/openrouter.ts", @@ -15166,6 +15246,7 @@ "path": "packages/adapters/src/openrouter.ts", "symbols": [ "OpenRouterConfig", + "openRouter", "openrouter" ], "tags": [ @@ -15173,7 +15254,7 @@ "ts", "src" ], - "contentHash": "f7ca8a907096b80892536d2e4af51c2d810082e6076f1f04a03ade7924ebc15f" + "contentHash": "50f46dccd29e8ed0ef860d1466b69f2f40865c421fa0d8c6ed923f8ef9015b2f" }, { "id": "module:packages/adapters/src/replicate.ts", @@ -15322,7 +15403,7 @@ "ts", "src" ], - "contentHash": "6fbcd004ffe0aa203d2e1338b87b7e4566ee8bddcaed5fd5a7a4fd5ea79e0ac1" + "contentHash": "16417cad54eee7d552ed45b70cfb870a2b90b6a45ca67b300bda51b5d89b5f28" }, { "id": "module:packages/adapters/src/types.ts", @@ -15364,7 +15445,7 @@ "ts", "src" ], - "contentHash": "a116b36e8da34bd4cf227428863e807c1eccd9999bd5ef183f8cbdcd1fee9883" + "contentHash": "6bd9abaf3091be58f9c3de4cceb496b21931daa6d0bfce6eb0dd743f0e75869f" }, { "id": "module:packages/adapters/src/vercel-ai.ts", @@ -15484,7 +15565,7 @@ "test", "tests" ], - "contentHash": "9e3804a4d7229c9e3bba9931675f8ec34995671fa12df068b624fde000f5f2e6" + "contentHash": "966aabd64ada7923c6bb1de3857e776f256bae73e3ab45e480414bda2b1ce39d" }, { "id": "module:packages/adapters/tests/carbon.test.ts", @@ -15649,6 +15730,19 @@ ], "contentHash": "0ebc486d8b6527079a6611a145036e9580c1fa402f1efc127e5c022ef5ca368b" }, + { + "id": "module:packages/adapters/tests/content-parts.test.ts", + "type": "module", + "title": "content-parts.test.ts", + "path": "packages/adapters/tests/content-parts.test.ts", + "tags": [ + "module", + "ts", + "test", + "tests" + ], + "contentHash": "63bcc1875bcc679bbd824d2bbc6d84873d2e54ea37fc0dd41addf1f715b73e68" + }, { "id": "module:packages/adapters/tests/contract.test.ts", "type": "module", @@ -15818,6 +15912,19 @@ ], "contentHash": "45587e0795c55b9f5beb24614fcaf92d2844591d7d052ef3d5ad2ade2c59e7b2" }, + { + "id": "module:packages/adapters/tests/gateway-types.test-d.ts", + "type": "module", + "title": "gateway-types.test-d.ts", + "path": "packages/adapters/tests/gateway-types.test-d.ts", + "tags": [ + "module", + "ts", + "test", + "tests" + ], + "contentHash": "cb7ac059b1f0aa928142ea6359d62c57c7355221b4c97656ee51589a15b3dc6c" + }, { "id": "module:packages/adapters/tests/gemini.test.ts", "type": "module", @@ -15896,6 +16003,22 @@ ], "contentHash": "13a968c5c0c06720f03fe2730f9898950bf1c20d917340d4141f34ee6141c280" }, + { + "id": "module:packages/adapters/tests/multimodal-http.test.ts", + "type": "module", + "title": "multimodal-http.test.ts", + "path": "packages/adapters/tests/multimodal-http.test.ts", + "symbols": [ + "multimodalCases" + ], + "tags": [ + "module", + "ts", + "test", + "tests" + ], + "contentHash": "e83d348b812952d2a5fe8d3bf30e6894a5191a11af07cbb3b9f12c7200f521d4" + }, { "id": "module:packages/adapters/tests/net-migration-http.test.ts", "type": "module", @@ -35564,7 +35687,7 @@ "test", "scripts" ], - "contentHash": "887101ed2c2622cb50bbaece632d06aa2e60f74dd8d6ede5f3caec0cec355291" + "contentHash": "ee5a0098d2323e8078ee64fad92efabd47bef684621a2342f1aa0ad36642e6e6" }, { "id": "module:scripts/doc-bridge-benchmark.mjs", @@ -38027,8 +38150,8 @@ "changes": {} }, "inputs": { - "hash": "fa78e13c81cd38831e338a46af2db9f8eb05344ad308e1182c06b4b38ce7798e", - "fileCount": 2528, + "hash": "24f8bb213b1054a6b0b61251b4ce7f91cb854760740820a7f028a72434cf0c24", + "fileCount": 2536, "projectionVersion": 2 }, "retrieval": { @@ -38050,9 +38173,9 @@ "projection": { "type": "retrieval-index", "schemaVersion": 1, - "contentHash": "568aa991dcff6321ebc2af4233efa9ffb8a35bced470c97187037484bc9009b5", + "contentHash": "a3281eb0ac9d555bb67038a03f9124810b70d157a35a8577ab54a68a5eeab1b4", "contentHashAlgo": "sha256-normalized-v1", - "snapshotHash": "066ddf71be9a8a9c494a064711680bb58eb6d616da854e45fabcc62fbdfb0bdc", + "snapshotHash": "b7f4c0990a396b5c64bf74375a2406e9ed9de4ef1e20bc36611d78e976d37d2b", "overlayHash": "f99666ff9119084dd25c2974ae64eb6935ec2aa785ff5ad478fb9d541f7a5e94", "configurationHash": "9662cc07ea4ab203ff2ddf457d2db48b5375a7ab4a1f8fbc79dfebcaa8f9a00c", "lexiconVersion": 1, @@ -38072,7 +38195,7 @@ }, "lexical": { "version": 1, - "documentCount": 2483, + "documentCount": 2491, "fieldNames": [ "aliases", "body", @@ -38083,13 +38206,13 @@ "title" ], "averageFieldLength": { - "aliases": 3.0144985904148207, - "body": 60.88078936770036, - "headings": 4.00724929520741, - "path": 7.614579138139348, - "summary": 4.558598469593234, - "symbols": 7.443817962142569, - "title": 3.349979863068868 + "aliases": 3.0144520272982738, + "body": 61.262545162585305, + "headings": 4.014853472501003, + "path": 7.613809714973906, + "summary": 4.553191489361702, + "symbols": 7.450020072260137, + "title": 3.3540746688077077 } }, "entries": [ @@ -38508,7 +38631,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/examples/discord-bot.mdx" @@ -38547,7 +38670,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/examples/dspy.mdx" @@ -38625,7 +38748,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/examples/embedded.mdx" @@ -38946,7 +39069,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/examples/slack-bot.mdx" @@ -38985,7 +39108,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/examples/teams-bot.mdx" @@ -39341,7 +39464,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/data/providers/embedders.mdx" @@ -39456,7 +39579,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/cli/index.mdx" @@ -39619,7 +39742,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/security/index.mdx" @@ -39782,7 +39905,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/reference/packages/eval-braintrust.mdx" @@ -39866,7 +39989,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/evals/snapshots.mdx" @@ -39905,7 +40028,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/evals/replay.mdx" @@ -39944,7 +40067,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/evals/snapshots.mdx" @@ -39983,7 +40106,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.00239, + "pagerank": 0.002371, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/ui/index.mdx" @@ -40053,7 +40176,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/agents/tools/integrations-catalog.mdx" @@ -40123,7 +40246,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/data/memory/index.mdx" @@ -40162,7 +40285,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/data/rag/loaders.mdx" @@ -40201,7 +40324,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/data/rag/rerank.mdx" @@ -40240,7 +40363,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.00239, + "pagerank": 0.002371, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/ui/index.mdx" @@ -40279,7 +40402,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/security/sandbox.mdx" @@ -40318,7 +40441,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.00239, + "pagerank": 0.002371, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/ui/index.mdx" @@ -40357,7 +40480,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.00239, + "pagerank": 0.002371, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/ui/index.mdx" @@ -40427,7 +40550,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.002808, + "pagerank": 0.002786, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/agents/tools/integrations-catalog.mdx" @@ -40517,7 +40640,7 @@ "aliases": "area" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [ "document:apps/docs-next/content/docs/production/observability/devtools.mdx" @@ -41179,6 +41302,11 @@ "kind": "mentions-symbol", "id": "module:packages/adapters/src/ollama.ts", "confidence": "observed" + }, + { + "kind": "mentions-symbol", + "id": "module:packages/adapters/src/openai-compatible.ts", + "confidence": "observed" } ] }, @@ -41338,7 +41466,7 @@ "aliases": "document unclassified" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -41424,7 +41552,7 @@ "aliases": "document unclassified" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -41477,7 +41605,7 @@ "aliases": "document unclassified" }, "graph": { - "pagerank": 0.00588, + "pagerank": 0.005834, "inboundLinks": 10, "coveredBy": [], "mentionedBy": [ @@ -41673,7 +41801,7 @@ "aliases": "document project" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -44034,7 +44162,7 @@ "aliases": "document human" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -46073,7 +46201,7 @@ "aliases": "document human" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -46215,7 +46343,7 @@ "aliases": "document human" }, "graph": { - "pagerank": 0.003646, + "pagerank": 0.003617, "inboundLinks": 1, "coveredBy": [], "mentionedBy": [], @@ -46894,7 +47022,7 @@ "aliases": "document human" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -48286,7 +48414,7 @@ "aliases": "document human" }, "graph": { - "pagerank": 0.001971, + "pagerank": 0.001955, "inboundLinks": 0, "coveredBy": [], "mentionedBy": [], @@ -49499,19 +49627,25 @@ ], "fields": { "title": "openai.mdx", - "headings": "Options Model examples Env Notes Related", + "headings": "Options Model examples Env Notes Related Gateway presets", "path": "apps/docs-next/content/docs/data/providers/openai.mdx", "symbols": "", "summary": "gpt-4o · gpt-4o-mini · gpt-5 · o1 · o3-mini · gpt-4.1.", - "body": "Options Option Type Default --- --- --- apiKey string required model string required baseUrl string https://api.openai.com retry RetryOptions package default includeUsage boolean true for the canonical OpenAI host Model examples gpt-4o · gpt-4o-mini · gpt-5 · o1 · o3-mini · gpt-4.1 . Env Var Purpose --- --- OPENAI API KEY API key (standard or session key) Notes - Streaming via SSE; emits text + tool-call + done chunks. - Parallel tool calling enabled by default. - Multimodal: pass { type: 'image', url } content parts. - Azure OpenAI users should use — different routing. Related - · · — openaiEmbedder - Recipe:", + "body": "Options Option Type Default --- --- --- apiKey string required for openai ; optional for openaiCompatible and Cloudflare presets model string required baseUrl string https://api.openai.com path string /v1/chat/completions ; custom path appended to baseUrl headers HeadersInit additional headers; override defaults case-insensitively fetch typeof fetch global fetch capabilities Partialmajor-package", - "body": "@agentskit/adapters Profile: ` | model-name defaults; override for compatible models | | `retry` | `RetryOptions` | package default | | `includeUsage` | `boolean` | `true` for the canonical OpenAI host | @@ -36,10 +40,55 @@ const adapter = openai({ - Streaming via SSE; emits `text` + `tool-call` + `done` chunks. - Parallel tool calling enabled by default. -- Multimodal: pass `{ type: 'image', url }` content parts. +- Images: pass `message.parts` entries `{ type: 'image', source }` (HTTPS/data URL), with optional `detail`. Files accept data URLs or uploaded `file-*` IDs. See the [tested modality matrix](/docs/reference/recipes/multi-modal). +- Unsupported modalities/sources raise `CAPABILITY_UNSUPPORTED` when multimodal input is enabled. Text-only models and assistant messages with binary parts use text parts and short attachment markers, omitting binary data and data URLs, with at most one warning per request, preserving older conversations. Explicit `capabilities.multiModal` overrides model-name heuristics (GPT-4/5, o-series, vision, Gemini and Claude 3+, including provider prefixes); set it for custom models or differing provider support. +- `openaiCompatible()` uses the same transport, with optional `apiKey`; omitting it sends no automatic Bearer header. Existing `openrouter()` remains available, with `openRouter()` as an alias. +- With the default path, trailing `/v1` on `baseUrl` is normalized. With an explicit `path`, the base URL is preserved (apart from the trailing slash). - Azure OpenAI users should use [azureOpenAIAdapter](https://github.com/AgentsKit-io/agentskit/issues/428) — different routing. ## Related - [Providers overview](./) · [Hosted](./hosted) · [Embedders](./embedders) — `openaiEmbedder` - Recipe: [custom-adapter](/docs/reference/recipes/custom-adapter) + +## Gateway presets + +```ts +import { cloudflareAiGateway, workersAi, openRouter } from '@agentskit/adapters' + +const gateway = cloudflareAiGateway({ + accountId: 'your-account', gatewayId: 'your-gateway', + gatewayToken: process.env.CF_AIG_TOKEN, + model: 'openrouter/google/gemini-2.5-flash-lite', + capabilities: { multiModal: true }, +}) +const workers = workersAi({ + accountId: 'your-account', apiKey: process.env.CLOUDFLARE_API_TOKEN!, + model: '@cf/your-model', +}) +const router = openRouter({ + apiKey: process.env.OPENROUTER_API_KEY!, + model: 'google/gemini-2.5-flash-lite', + capabilities: { multiModal: true }, + headers: { 'HTTP-Referer': 'https://example.com', 'X-Title': 'Example' }, +}) +``` + +`cloudflareAiGateway()` targets +`https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/compat/chat/completions` +and adds `cf-aig-authorization: Bearer {gatewayToken}` when supplied. +A provider `apiKey` is optional for gateway-managed keys. `workersAi()` targets +`https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/v1/chat/completions` +with the supplied API token as Bearer, using REST without runtime bindings. +`openRouter()` targets `https://openrouter.ai/api/v1/chat/completions`. +All presets support `baseUrl`, `path`, `headers`, `fetch` and capability overrides. + +For catalog dispatch, pass URL variables explicitly: +`env: { CLOUDFLARE_ACCOUNT_ID: 'your-account' }`. Missing placeholders raise +`MISSING_URL_VARIABLE`; no environment variables are read implicitly. + +URL/header contracts were tested through local HTTP capture. Authenticated live +OpenRouter, AI Gateway and Workers AI calls remain pending and require separate authorization. +See [Cloudflare Unified API](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), +[Workers AI REST](https://developers.cloudflare.com/ai-gateway/get-started/), and +[OpenRouter API](https://openrouter.ai/docs/api_reference/overview). diff --git a/apps/docs-next/content/docs/for-agents/adapters.mdx b/apps/docs-next/content/docs/for-agents/adapters.mdx index bb49d362d..990117e50 100644 --- a/apps/docs-next/content/docs/for-agents/adapters.mdx +++ b/apps/docs-next/content/docs/for-agents/adapters.mdx @@ -151,6 +151,16 @@ npm install @agentskit/adapters `createOpenAICompatibleAdapter` base; each exposes a default `baseUrl` and accepts an override. +`openaiCompatible` accepts an explicit endpoint, path, headers and fetch +transport with optional Bearer authentication. `openRouter` aliases +`openrouter`; `cloudflareAiGateway` and `workersAi` provide Cloudflare REST +presets without runtime bindings. Ordered image/file parts are serialized at +the provider boundary; unsupported inputs throw `CapabilityUnsupportedError` +before transport I/O when multimodal input is enabled. Text-only models fall +back to `message.content` with a warning; OpenAI assistant binary parts do the +same. Explicit `capabilities.multiModal` overrides model-name heuristics. +See the [modality limits](/docs/reference/recipes/provider-swap). + ### Composition - `createRouter({ candidates, policy, classify, onRoute })` — pick diff --git a/apps/docs-next/content/docs/reference/recipes/multi-modal.mdx b/apps/docs-next/content/docs/reference/recipes/multi-modal.mdx index 40d49f4a8..73f838d87 100644 --- a/apps/docs-next/content/docs/reference/recipes/multi-modal.mdx +++ b/apps/docs-next/content/docs/reference/recipes/multi-modal.mdx @@ -1,14 +1,20 @@ --- title: Unified multi-modal -description: One API for text, image, audio, video, and file inputs — regardless of provider. +description: Provider-neutral parts with tested image and file serialization and explicit unsupported-input errors. --- Every provider has its own multi-modal shape. OpenAI wants `{ type: 'image_url', image_url: {...} }`, Anthropic wants `{ type: 'image', source: {...} }`, Gemini wants parts-with-inline- data. `@agentskit/core` provides a provider-neutral `ContentPart` -model — adapters that understand a modality read the parts, the rest -fall back to a text projection. +model. The adapters listed below serialize supported image/file parts and reject +unsupported inputs with `CapabilityUnsupportedError` (`code: CAPABILITY_UNSUPPORTED`) +from `createSource`, before network I/O. Audio/video builders exist in core; +these adapters do not yet send them. Text-only models fall back to +text parts and short attachment markers, omitting binary data and data URLs, +with at most one warning per request, so existing image/file histories remain usable. +OpenAI and Anthropic assistant binary parts also use this text fallback. Explicit +`capabilities.multiModal` overrides OpenAI-compatible model-name heuristics. ## Install @@ -49,32 +55,44 @@ const message: Message = { | `imagePart(src, { mimeType?, detail? })` | `'image'` | Data URL, http(s), or provider-hosted id | | `audioPart(src, { durationSec? })` | `'audio'` | | | `videoPart(src, { durationSec? })` | `'video'` | | -| `filePart(src, { filename? })` | `'file'` | PDF, CSV, arbitrary binary | - -## In an adapter - -A vision-aware adapter reads `msg.parts` and maps each entry to its -provider's shape. A text-only adapter keeps reading `msg.content` and -sees a safe projection like `"caption\n[image: pic.png]"`. - -```ts -import { normalizeContent, filterParts } from '@agentskit/core' - -function toOpenAIMessage(m: Message) { - const { parts } = normalizeContent(m.content, m.parts) - return { - role: m.role, - content: parts.map(p => { - if (p.type === 'text') return { type: 'text', text: p.text } - if (p.type === 'image') return { type: 'image_url', image_url: { url: p.source, detail: p.detail } } - return { type: 'text', text: `[${p.type}]` } - }), - } -} - -// Quickly grab every attached image: -const images = filterParts(parts, 'image') -``` +| `filePart(src, { filename?, mimeType? })` | `'file'` | Support depends on the adapter and source format below | + +## Tested adapter capabilities + +This table is generated from `packages/adapters/tests/multimodal-http.test.ts` +and checked against its HTTP payload cases. It proves serialization to a local +server, not acceptance or inference quality on a live provider. + +| Adapter | Image wire format | File wire format | Tested sources / limits | +| --- | --- | --- | --- | +| openai | image_url | file | image URL/data URL; file data URL/id | +| openai-compatible | image_url | file | same wire format; opt into model capability | +| anthropic | image | document | image URL/data URL; PDF URL/data URL; text data URL | +| gemini | inlineData/fileData | inlineData/fileData | data URL or provider URI with MIME type | +| vertex | inlineData/fileData | inlineData/fileData | data URL or provider URI with MIME type | +| ollama | images | image data URL only | image data URL only; vision model required; PDF rejected | + +For user/assistant messages, parts replace the legacy `content` projection when present; their order is preserved. +Messages without parts keep the previous wire format. OpenAI-compatible custom models +must opt into `capabilities: { multiModal: true }`; transport compatibility alone +says nothing about the upstream model. Ollama accepts `multiModal: true` for +vision models whose names do not match the default `llava`/`vision` heuristic. +An Ollama file part containing an image data URL is sent in `images`; PDFs and +other files are rejected. Remote image URLs are not fetched by the adapter. + +Gemini/Vertex remote sources require `mimeType` and a provider-accessible URI +(`https://` or `gs://`); data URLs supply their own MIME type. Anthropic accepts +PDFs and inline UTF-8 text documents; other file MIME types are rejected. +OpenAI Chat Completions accepts file data URLs or uploaded `file-*` identifiers, +not remote file URLs. Uploading and resolving storage references is the caller's responsibility. +Provider-hosted image identifiers, audio and video are currently rejected. +Model-specific MIME/size/access limits still apply upstream. + +See the provider wire contracts: [OpenAI file inputs](https://developers.openai.com/api/docs/guides/file-inputs), +[Anthropic images](https://platform.claude.com/docs/en/build-with-claude/vision), +[Anthropic PDFs](https://platform.claude.com/docs/en/build-with-claude/pdf-support), +[Gemini parts](https://ai.google.dev/api/generate-content), and +[Ollama chat](https://docs.ollama.com/api/chat). ## See also diff --git a/apps/docs-next/lib/ecosystem-claims.snapshot.json b/apps/docs-next/lib/ecosystem-claims.snapshot.json index b215f019d..4e98c6dd1 100644 --- a/apps/docs-next/lib/ecosystem-claims.snapshot.json +++ b/apps/docs-next/lib/ecosystem-claims.snapshot.json @@ -35,7 +35,7 @@ }, { "id": "native-adapters", - "value": 25, + "value": 28, "noun": "native adapters", "evidence": { "type": "repository-derivation", @@ -43,7 +43,7 @@ "path": "scripts/compute-stats.mjs", "summary": "Provider exports in the adapters package excluding composition helpers." }, - "conservativeFloor": 25 + "conservativeFloor": 28 }, { "id": "integrations", diff --git a/apps/landing/lib/ecosystem-claims.snapshot.json b/apps/landing/lib/ecosystem-claims.snapshot.json index b215f019d..4e98c6dd1 100644 --- a/apps/landing/lib/ecosystem-claims.snapshot.json +++ b/apps/landing/lib/ecosystem-claims.snapshot.json @@ -35,7 +35,7 @@ }, { "id": "native-adapters", - "value": 25, + "value": 28, "noun": "native adapters", "evidence": { "type": "repository-derivation", @@ -43,7 +43,7 @@ "path": "scripts/compute-stats.mjs", "summary": "Provider exports in the adapters package excluding composition helpers." }, - "conservativeFloor": 25 + "conservativeFloor": 28 }, { "id": "integrations", diff --git a/docs/adapter-modalities.md b/docs/adapter-modalities.md new file mode 100644 index 000000000..ffa7bbd81 --- /dev/null +++ b/docs/adapter-modalities.md @@ -0,0 +1,8 @@ +| Adapter | Image wire format | File wire format | Tested sources / limits | +| --- | --- | --- | --- | +| openai | image_url | file | image URL/data URL; file data URL/id | +| openai-compatible | image_url | file | same wire format; opt into model capability | +| anthropic | image | document | image URL/data URL; PDF URL/data URL; text data URL | +| gemini | inlineData/fileData | inlineData/fileData | data URL or provider URI with MIME type | +| vertex | inlineData/fileData | inlineData/fileData | data URL or provider URI with MIME type | +| ollama | images | image data URL only | image data URL only; vision model required; PDF rejected | diff --git a/docs/architecture/adrs/0041-adapter-content-serialization.md b/docs/architecture/adrs/0041-adapter-content-serialization.md new file mode 100644 index 000000000..d85f3b951 --- /dev/null +++ b/docs/architecture/adrs/0041-adapter-content-serialization.md @@ -0,0 +1,35 @@ +# 0041 — Serialize attachments at the provider boundary + +Status: proposed; implementation ready for maintainer review + +## Context + +The existing Message contract contains ordered ContentPart arrays alongside a +legacy string projection. Native provider serializers read only the projection, +so advertising image capability does not ensure that an image reaches the wire. +OpenAI-compatible gateways also need configurable authentication and endpoint paths. + +## Decision + +Reuse the existing adapter and message contracts. Serialize parts in the shared +provider history encoders, preserving order and tool-call correlation. Parts are +authoritative when present. Unsupported modalities, source formats or disabled +image capability raise an adapter-local CAPABILITY_UNSUPPORTED error before I/O. +No core types or contracts change; text-only wire behavior remains unchanged. + +Use the existing OpenAI stream transport for configurable path, headers and fetch +and for gateway presets. Authentication is optional and supplied headers override +defaults case-insensitively. Resolve catalog placeholders from explicit caller +values; never read environment credentials inside the library. No new dependency. + +## Consequences and evidence + +Model-level capability must be supplied explicitly for custom compatible models; +transport support is not a claim about provider/model inference. Ollama supports +inline image data only; unsupported PDFs/URLs fail explicitly. Provider MIME and +access restrictions still apply. Shared encoders also serve Azure and Bedrock; +existing tool-history suites must pass alongside the new HTTP payload tests. + +`packages/adapters/tests/multimodal-http.test.ts` exercises local HTTP capture, +stream/usage preservation, presets, source errors and tool history. Live provider +calls are a separate, currently pending evidence gate. No publication is authorized. diff --git a/docs/architecture/adrs/README.md b/docs/architecture/adrs/README.md index 563147c13..3a70b59c7 100644 --- a/docs/architecture/adrs/README.md +++ b/docs/architecture/adrs/README.md @@ -75,6 +75,7 @@ Do NOT write an ADR for: routine features, bug fixes, internal refactors, or any | [0038](./0038-canonical-json-hash.md) | Canonical JSON and SHA-256 helpers | Accepted | | [0039](./0039-cryptographic-ids.md) | Cryptographic IDs for generated identifiers | Accepted | | [0040](./0040-eval-replay-fingerprint-compatibility.md) | Preserve eval replay fingerprint compatibility | Proposed | +| [0041](./0041-adapter-content-serialization.md) | Serialize attachments at the provider boundary | Proposed | | [0042](./0042-durable-tool-decisions.md) | Durable tool decisions in core 1.x | Proposed | diff --git a/docs/evidence/adapters-multimodal-transport.json b/docs/evidence/adapters-multimodal-transport.json new file mode 100644 index 000000000..2d284b2cd --- /dev/null +++ b/docs/evidence/adapters-multimodal-transport.json @@ -0,0 +1,139 @@ +{ + "sourceRevision": "f78b492b0345684c920c1ce2245248cccb21cc46", + "adapterTree": "e7eb02cdeaef97f042f7c7b81e288431d89c9855", + "scope": "local transport serialization; no live provider inference", + "http": { + "command": "pnpm --filter @agentskit/adapters exec vitest run tests/multimodal-http.test.ts --maxWorkers=2 --reporter=json", + "total": 16, + "passed": 16, + "failed": 0, + "tests": [ + { + "name": "multimodal HTTP contract openai preserves ordered parts, detail and files", + "status": "passed" + }, + { + "name": "multimodal HTTP contract openai-compatible preserves ordered parts, detail and files", + "status": "passed" + }, + { + "name": "multimodal HTTP contract Anthropic sends image and PDF blocks", + "status": "passed" + }, + { + "name": "multimodal HTTP contract gemini sends inlineData and fileData", + "status": "passed" + }, + { + "name": "multimodal HTTP contract vertex sends inlineData and fileData", + "status": "passed" + }, + { + "name": "multimodal HTTP contract Ollama sends images without the data URL prefix", + "status": "passed" + }, + { + "name": "multimodal HTTP contract rejects unsupported modalities before network I/O", + "status": "passed" + }, + { + "name": "multimodal HTTP contract custom gateway path/headers/fetch preserves streaming and usage without Bearer", + "status": "passed" + }, + { + "name": "OpenAI-compatible presets routes OpenRouter, AI Gateway and Workers AI with documented headers", + "status": "passed" + }, + { + "name": "OpenAI-compatible presets catalog interpolates explicit Cloudflare account id and rejects missing variables", + "status": "passed" + }, + { + "name": "multimodal boundaries and history sends URL image sources and provider file identifiers", + "status": "passed" + }, + { + "name": "multimodal boundaries and history rejects unknown sources, missing MIME types, and unsupported audio/video without sending", + "status": "passed" + }, + { + "name": "multimodal boundaries and history keeps parts on assistant tool calls and merged user turns", + "status": "passed" + }, + { + "name": "keeps the generated capability table and recipe aligned with HTTP cases", + "status": "passed" + }, + { + "name": "encodes inline text documents and Ollama image file parts", + "status": "passed" + }, + { + "name": "honors case-insensitive caller authentication overrides", + "status": "passed" + } + ] + }, + "runs": [ + { + "name": "regressions-original", + "command": "pnpm --filter @agentskit/adapters exec vitest run tests/multimodal-http.test.ts", + "result": "expected failure: 8/8 on original source", + "logSha256": "349d391a5c38a754122a82a038c72018f05d7dbd4b403f7f562797a785b1ea67" + }, + { + "name": "adapters-current", + "command": "pnpm --filter @agentskit/adapters exec vitest run --maxWorkers=2", + "result": "passed: 53 files, 584 tests", + "logSha256": "776211ae74ef31ebb4d87cb1a0d4a17aab88666c19f3e9050b7a482e8b0367f6" + }, + { + "name": "global-test-1", + "command": "pnpm test", + "result": "failed: integrations HTTP deadline assertion", + "logSha256": "a4270411fdb8d3af7bf772fa061d78ec9521ce1a835ecae600613d6a1628dc1a" + }, + { + "name": "global-test-2", + "command": "pnpm test", + "result": "failed: sandbox process exit assertion", + "logSha256": "67dc185aa3baa398a1c1a85b345a88bc3e494c016121879408be9cffa6224d13" + }, + { + "name": "global-test-3", + "command": "pnpm exec turbo run test --concurrency=1", + "result": "failed: eval bundler interop timeout 90000ms", + "logSha256": "8c49160fe54d4b9c75d1eafa081da4d7ef7e602187a133aeb07bfca14bafbd4e" + }, + { + "name": "mdx", + "command": "pnpm --filter @agentskit/docs-next lint:mdx", + "result": "passed: 417 files, 87 warnings", + "logSha256": "2e821e8dfa885f4b2b27148bcbab27ad3661e08e0ab56862206b95bc44671f42" + }, + { + "name": "handoff", + "command": "pnpm exec ak-docs query package adapters --agent", + "result": "blocked: index stale", + "logSha256": "05dfff1e17a1cc2f318c8a71eae16f3b00e8bf72aedcc37f5cf7114a67d899e2" + }, + { + "name": "package-build", + "command": "pnpm --filter @agentskit/adapters build", + "result": "passed: ESM, CJS and DTS", + "logSha256": "5a5caf33ec4cd92fa25e16d542a69d48770a1ae055c2c24bde70c048b9061161" + }, + { + "name": "package-lint", + "command": "pnpm --filter @agentskit/adapters lint", + "result": "passed: source and test typecheck, including required credential types", + "logSha256": "9fafd74610b7358c7a0059fe6127a94c3c761fd15f15b95ca7d7811832217fcd" + }, + { + "name": "built-api", + "command": "node built ESM/CJS export and deferred-I/O assertions", + "result": "passed", + "logSha256": "9277aa1c312e66758234daa2522a69fead1268b28ccd63e104a6084c7d9ed7ee" + } + ] +} diff --git a/docs/evidence/adapters-multimodal-transport.md b/docs/evidence/adapters-multimodal-transport.md new file mode 100644 index 000000000..dbbc2247e --- /dev/null +++ b/docs/evidence/adapters-multimodal-transport.md @@ -0,0 +1,129 @@ +# Adapter multimodal transport — local implementation report + +Status: **AWAITING_HUMAN_APPROVAL**: all CH-A2 checks passed; coordinator evidence review remains required. Earlier BLOCKED results below are historical evidence. No release, publication, deployment, push or PR was performed. + +## Contract and authorization + +Intent: implement configurable OpenAI-compatible transport and serialize image/file inputs in the named native adapter families, with synthetic HTTP regression tests, local changeset and commits. The coordinator confirmed RF-01, RF-02, RF-04, RF-05 and RF-06 as the worker criteria; RF-03 belongs to a separate starter and is outside this contract. No core contract changes, new dependencies or real provider credentials are authorized. + +Scope: `packages/adapters`, its catalog dispatch, corresponding provider/recipe documentation, an ADR, local changeset and this evidence. Other repositories, starter/protocol/storage/cost work, live accounts and UI implementation are outside scope. Intermediate implementation and local verification were authorized together. The dispatch task specification is the approved contract; no separate parent contract hash was supplied and no subagents were dispatched. + +Validation budget: local and cached dependencies only; no CI or real inference calls; three repository-wide test attempts, one repository-wide build attempt, scoped checks for changed revisions, and documented doc-bridge operations. Redundant/stale task-owned jobs were stopped after source changes and repeated global failures, rather than weakening their gates. Temporary logs are reduced to allowlisted durable metadata below and removed at settlement; ambiguous/pre-existing files are retained. + +## Source and implementation + +The source revision and adapter tree are recorded in [the evidence metadata](./adapters-multimodal-transport.json). Subsequent report-only commits do not change that adapter tree. + +Local implementation commits: + +- `434ba87a`: attachment serializers, configurable transport, Cloudflare presets, OpenRouter alias, catalog variables, regression suite and changeset. +- `93a02265`: media validation and documented source limits. +- `e0cab7bc`: explicit case-insensitive gateway authentication overrides. +- `f78b492b`: retain required credential types for existing providers; add static API compatibility checks. + +The existing OpenAI transport supplies `path`, `headers` and injected `fetch`. The new `openaiCompatible` and Cloudflare presets allow omitted Bearer authentication. Existing OpenAI and provider configuration types retain required `apiKey`; existing default `/v1` routing and text-only requests without parts remain compatible. + +Shared history serializers send OpenAI `image_url`/`file`, Anthropic `image`/`document`, Gemini/Vertex `inlineData`/`fileData`, and Ollama `images`. Unsupported sources/modalities fail with `CapabilityUnsupportedError.code === 'CAPABILITY_UNSUPPORTED'` before I/O. Ollama accepts inline images, including image file parts, and rejects PDFs and remote URLs. Catalog placeholders are resolved only from explicitly supplied `env` values. No Cloudflare runtime binding is imported. + +The [modality table](../adapter-modalities.md) is generated by a Vitest file snapshot from the HTTP test cases and checked against the recipe. Refreshing it uses the existing suite's `-u` option. [ADR 0041](../architecture/adrs/0041-adapter-content-serialization.md) records the implementation decision for maintainer review; it does not revise a core contract. + +## Criterion reconciliation + +| Criterion | Status | Current evidence and limit | +| --- | --- | --- | +| RF-01: configurable endpoint/path/headers/fetch, no implicit Bearer, streaming/usage | validated locally | HTTP capture asserts full configured path, custom headers, injected fetch invocation, absent Authorization, text stream and token usage; explicit caller authentication overrides also pass | +| RF-02: OpenRouter, AI Gateway, Workers AI presets and catalog account URL | validated locally | Captured requests assert all three official URLs and headers; missing catalog variables produce MISSING_URL_VARIABLE; library source contains no `cloudflare:` import | +| RF-03: starter default/fallback model and ledger | not applicable | Separate starter contract, explicitly excluded by coordinator | +| RF-04: image/file provider payloads | validated locally | Real loopback HTTP capture checks ordered data URL/URL/hosted-ID parts, Gemini/Vertex payloads, Anthropic PDF/text and Ollama images; shared tool-history tests also pass | +| RF-05: capability contract and explicit unsupported errors | validated locally | Named families preserve their asserted image payloads; disabled model capability, unsupported audio/video, invalid data URLs/MIME/source references, PDF on Ollama and invalid UTF-8 documents fail before network I/O | +| RF-06: truthful provider/recipe docs and generated capability table | validated for content | File-snapshot table and recipe equality check passes; semantic review matches documented limits to payload tests; MDX lint passed; rendered docs/browser behavior was not analyzed | +| Stage 0: green build/typecheck/test across repository | blocked | All three global test attempts failed in other packages; global build/typecheck runs were stopped when stale, so no current repository-wide green claim is made | +| Changeset, local commits, report and cleanup | implemented | Local minor changeset exists; four implementation commits recorded; no package versioning/publication; final cleanup/status recorded below | + +## Validated + +The current adapter suite passed **53 files / 584 tests** using `--maxWorkers=2`, without increasing test timeouts. It includes **16 HTTP/documentation checks**, existing stream contracts, shared tool-history encoders, Azure/Bedrock tests, provider routing and compatibility tests. The HTTP fixtures contain synthetic base64 strings and synthetic credentials; they validate payload structure rather than real PNG/PDF inference. + +Provider source formats were checked against public primary documentation: [Cloudflare Unified API](https://developers.cloudflare.com/ai-gateway/usage/chat-completion/), [Workers AI REST](https://developers.cloudflare.com/ai-gateway/get-started/), [OpenRouter](https://openrouter.ai/docs/api_reference/overview), [OpenAI files](https://developers.openai.com/api/docs/guides/file-inputs), [Anthropic vision](https://platform.claude.com/docs/en/build-with-claude/vision), [Anthropic PDFs](https://platform.claude.com/docs/en/build-with-claude/pdf-support), [Gemini parts](https://ai.google.dev/api/generate-content) and [Ollama chat](https://docs.ollama.com/api/chat). + +The original implementation failed all eight initial loopback regressions: six provider image/file payload cases, capability rejection and custom gateway transport. These were actual assertion failures after dependencies built, rather than missing-import failures. The first attempted run before dependency builds failed to load `@agentskit/net` and is not regression evidence. + +## Partially validated + +Global lint passed earlier (60 tasks) but predates the final type signatures. The repository build produced package outputs and progressed through docs generation; it was interrupted after 17m23s when its source became stale. Its 43 successful tasks are supporting evidence, not a green build. A later global lint was also stopped as stale; an interrupted command's zero wrapper exit status is not interpreted as success. + +The initially used pre-install doc-bridge tool rejected `analysis.areas`. After `pnpm install --frozen-lockfile`, the repository-local tool accepted the configuration and reported a stale index instead. The documented gate/index operations were attempted but stopped as stale during subsequent edits; freshness is not validated and no harness configuration was changed. `ORCA_PLAYBOOK.md` was absent from this worktree; no workers or terminal tasks were dispatched. + +## Blocked + +Repository-wide test attempts returned explicit failures: + +1. `pnpm test`: `packages/integrations/tests/http.test.ts:630`, deadline retry count was 0 instead of 1. The same HTTP file subsequently passed its 34 tests in isolation. +2. `pnpm test`: `packages/sandbox/tests/local-spawner.test.ts:27`, process exit assertion was false. +3. `pnpm exec turbo run test --concurrency=1`: `packages/eval/tests/bundler-interop.test.ts:5`, test exceeded its existing 90000ms timeout. + +A current adapter rerun under unrestricted workers also timed out in CLI deadline and LangChain bridge tests; the full current suite passed with two workers. These observations support resource contention as a possible cause, but do not prove the global failures are resolved. No unrelated package was changed and no timeout or harness gate was weakened. + +## Not analyzed + +Authenticated live OpenRouter, AI Gateway, Workers AI, OpenAI, Anthropic, Gemini/Vertex and Ollama inference; real image/document interpretation; provider MIME/size/quota/access acceptance; performance/latency claims; browser rendering and interaction of the docs; other adapters outside the named families beyond their existing regression tests. + +## Not applicable + +UI implementation, screenshots, visual approval, database persistence, upload/storage references, business-domain flows, starter model fallback, billing, release/versioning, external tracking systems, new dependencies and core size checks: none is part of this implementation contract. + +## Final gate and next action + +The coordinator must review the implementation and proposed ADR, restore/complete current doc-bridge evidence, and obtain passing current repository-wide checks before moving this work beyond BLOCKED. Live provider validation remains a separately authorized follow-up; this report does not claim enterprise readiness or parent-task completion. + +Current package checks passed on source `f78b492b0345684c920c1ce2245248cccb21cc46`: + +- `pnpm --filter @agentskit/adapters build`: ESM, CJS and declarations generated successfully. +- `pnpm --filter @agentskit/adapters lint`: both source and test typechecks passed, including required-key types for existing providers and optional-key construction for the new APIs. +- `pnpm --filter @agentskit/adapters exec vitest run --maxWorkers=2`: 53 files, 584 tests passed. +- Current JSON HTTP rerun: all 16 checks passed, with assertion-level statuses in the evidence metadata. +- Built ESM/CJS API smoke checks passed for all new exports and deferred I/O. + +Cleanup: prebuild-generated ecosystem/knowledge/API-route changes outside the implementation diff were restored to their initially clean state; the task-created generated knowledge blob was removed. Stale task-owned build/lint/doc-bridge processes were stopped. Compiler outputs and installed dependencies remain ignored and available locally; no environment data or credentials were created. Raw task-owned temporary logs are removed after extracting allowlisted metadata. The final commit is report-only; the adapter tree hash remains the one recorded in the evidence metadata. + +The final tracked diff contains only adapters, matching docs, proposed ADR, generated modality table, local changeset and the evidence report. The coordinator's next action is to review these artifacts and resolve the documented global/doc-bridge gates; this dispatch reports failure because Stage 0 is not green. + +## CH-A2 — ratchet regression and branch gate reconciliation + +Intent: fix the CH-A regression reported by GATES, preserving data-URI validation and the existing guardrail rules/allowlists. The coordinator approved native URL protocol classification because `net-sse-data-prefix` targets SSE, while this serializer handles media data URIs; `parseSSE` is not applicable. `new URL` with `try/catch` avoids depending on `URL.canParse` availability in workerd. MIME matching, base64 syntax and length checks remain strict. + +The coordinator subsequently authorized fixing every quality-gate failure introduced by CH-A and comparing failures with clean main `0e1c1aa9`. Scope expanded to the direct serializer regression test, synthetic HTTP redirects, provider-recipe assertion, for-agents documentation, ADR index, public API snapshot and generated claims/README hashes. No gate rule, suppression, baseline allowance, dependency, core contract or UI implementation was changed. The public API snapshot records nine already-implemented exports; JSDoc coverage remains shrink-only and README review dates are preserved. + +Acceptance and evidence map: + +| ID | Criterion | Evidence method | +| --- | --- | --- | +| A2-01 | Ratchet passes without weakened rules or allowlists | Real `node scripts/check-net-rules.mjs`; net `tests/rules.test.ts` (14 tests); diff review of rules and baselines | +| A2-02 | Media serialization and strict rejection remain correct | Adapters build, full package tests, source/test typechecks; real loopback HTTP payload checks plus direct URI regression | +| A2-03 | Repository suite passes, or remaining failures are reproduced on main | `VITEST_MAX_WORKERS=1 pnpm test --env-mode=loose --force --continue=always` (Turbo concurrency 2), with adapters built first; matching main acceptance flow for each remaining failure | +| A2-04 | CH-A quality regressions are fixed and remaining failures classified | Full `check:quality-gates` on branch and a task-owned clean main worktree; no relaxed gates | +| A2-05 | Locked doc-bridge tool accepts a current index | `pnpm docs:bridge:index` only after observed staleness, then `pnpm docs:bridge:gate` | +| A2-06 | Local commits, truthful evidence and cleanup | Commit/diff/status review, this report, removal of task-owned main worktree and temporary logs; coordinator evidence review remains required | + +Budget: local cached dependencies only, no downloads/CI/provider inference; sequential build/test/gate commands within this worktree. A temporary main worktree installed the frozen lockfile offline with zero downloads. Other agents' processes and worktrees are untouched. The first overlapping global attempt failed on missing adapters declarations and is invalid due to competing builds; the next attempt was interrupted after discovering Turbo's strict environment could omit `VITEST_MAX_WORKERS`, and its wrapper exit code is not passing evidence. The final run explicitly uses loose environment forwarding and bypasses cache for whole-repository scanners. + +Implementation commits: `84bf9e90` (native URI classifier and direct regression), `46a0fa0e` (HTTP fixture redirects and supported OpenAI fetch assertion), `de0a94da` (documented exports, ADR link and generated evidence). The current implementation source is `de0a94da3bbf7f07b0405a87f881da6e8e587d5a`; later evidence/index-only changes do not change it. + +### Final current-source results + +Implementation source: `de0a94da3bbf7f07b0405a87f881da6e8e587d5a`; adapters tree: `71f24b88ef19ecf0d757c7f3d68377bf9c385539`. The commands below ran against that source, with the evidence note/index updated afterward. No runtime or test source changed after these results. + +| Criterion | Status | Current evidence | +| --- | --- | --- | +| A2-01 | validated | Ratchet CLI passed with 111 baselined findings; isolated net rules 14/14; global net package 8 files / 99 tests passed; no rule, allowance or suppression changed | +| A2-02 | validated locally | ESM/CJS/declarations build passed; source and test typechecks passed; adapters 54 files / 585 tests passed under one worker, including direct strict URI rejection and real loopback payload tests; global rerun also passed 585/585 | +| A2-03 | validated | `VITEST_MAX_WORKERS=1 pnpm test --env-mode=loose --force --continue=always`: 77/77 tasks, zero cache, 6m07s, exit 0; CLI 57 files / 526 tests, WebLLM build and docs-next production build passed | +| A2-04 | validated | `VITEST_MAX_WORKERS=1 pnpm check:quality-gates`: all 52 gates passed, exit 0; JSDoc 1957/1963 documented with the unchanged shrink-only baseline; all three recipe acceptance flows passed | +| A2-05 | validated | Locked `@agentskit/doc-bridge` 1.12.0 regenerated the observed stale index; gate passed freshness, 27 human-guide links and all seven required documentation rules; final evidence/index changes are followed by another gate before settlement | +| A2-06 | implemented; awaiting coordinator review | Three scoped implementation commits above, report and regenerated index; task-owned main worktree removed; generated build artifacts outside this follow-up restored; temporary run logs removed at settlement; final diff/status inspected | + +Baseline comparison: task-owned clean main `0e1c1aa92e69f29f41241e49753ecab8ae279119` installed the frozen lockfile offline with zero downloads, then ran the same quality-gate command. Main passed 50/52 gates, including JSDoc, test-reference parity, for-agents coverage, ADR index, claims, README, content-pipeline tests and cross-platform/net ratchets. Its two failures were `verified recipe factory` (stale doc-bridge index) and `ecosystem documentation quality contract` (live doctor certification failure). Both pass on the final branch; no pre-existing-failure exception is needed for the final result. + +**Validated:** A2-01 through A2-05 as scoped above. **Partially validated:** synthetic HTTP proves serialization and transport behavior, not live provider inference. **Not analyzed:** live accounts, media interpretation, quotas, latency and browser rendering. **Blocked:** none in CH-A2 verification; the earlier report's failed attempts remain historical. **Not applicable:** UI interaction/visual approval, database/storage, release, push, PR, CI and changes to core contracts. + +Cleanup and next action: the clean baseline worktree was removed after inspection; other agents' processes/worktrees were untouched. Task-owned generated API routes, deterministic knowledge/site blobs and statistics snapshots were restored to their starting tracked contents; generated claims snapshots remain because their documented freshness gate required them. Task-owned raw logs are deleted after reconciliation. The coordinator must review the current commits and this evidence before approving the parent delivery; this worker reports verified assigned work, not parent completion or enterprise readiness. diff --git a/docs/stability/public-api-v1.json b/docs/stability/public-api-v1.json index cae82b460..c3932e28c 100644 --- a/docs/stability/public-api-v1.json +++ b/docs/stability/public-api-v1.json @@ -94,6 +94,13 @@ "type" ] }, + { + "name": "CapabilityUnsupportedError", + "kinds": [ + "type", + "value" + ] + }, { "name": "CarbonTable", "kinds": [ @@ -124,6 +131,18 @@ "value" ] }, + { + "name": "cloudflareAiGateway", + "kinds": [ + "value" + ] + }, + { + "name": "CloudflareAiGatewayConfig", + "kinds": [ + "type" + ] + }, { "name": "cohere", "kinds": [ @@ -538,12 +557,30 @@ "value" ] }, + { + "name": "openaiCompatible", + "kinds": [ + "value" + ] + }, + { + "name": "OpenAICompatibleConfig", + "kinds": [ + "type" + ] + }, { "name": "OpenAICompatibleEmbedderConfig", "kinds": [ "type" ] }, + { + "name": "OpenAICompatibleTransportConfig", + "kinds": [ + "type" + ] + }, { "name": "OpenAIConfig", "kinds": [ @@ -568,6 +605,12 @@ "value" ] }, + { + "name": "openRouter", + "kinds": [ + "value" + ] + }, { "name": "OpenRouterConfig", "kinds": [ @@ -777,6 +820,18 @@ "kinds": [ "value" ] + }, + { + "name": "workersAi", + "kinds": [ + "value" + ] + }, + { + "name": "WorkersAiConfig", + "kinds": [ + "type" + ] } ] }, diff --git a/ecosystem-claims.json b/ecosystem-claims.json index b215f019d..4e98c6dd1 100644 --- a/ecosystem-claims.json +++ b/ecosystem-claims.json @@ -35,7 +35,7 @@ }, { "id": "native-adapters", - "value": 25, + "value": 28, "noun": "native adapters", "evidence": { "type": "repository-derivation", @@ -43,7 +43,7 @@ "path": "scripts/compute-stats.mjs", "summary": "Provider exports in the adapters package excluding composition helpers." }, - "conservativeFloor": 25 + "conservativeFloor": 28 }, { "id": "integrations", diff --git a/packages/adapters/README.md b/packages/adapters/README.md index d1e0cb750..f196eafd6 100644 --- a/packages/adapters/README.md +++ b/packages/adapters/README.md @@ -40,6 +40,27 @@ Docs: [package guide](https://www.agentskit.io/docs/reference/packages/adapters) - **CLI-backed agents** — `@agentskit/adapters/cli` normalizes text, JSON, and ACP-based local LLM CLIs - **LangChain bridge** — `@agentskit/adapters/langchain-bridge` turns any adapter into a LangChain `BaseChatModel` for `createAgent` +## Gateway and multimodal transport + +`openaiCompatible({ model, baseUrl, path, headers, fetch })` supports custom +OpenAI-compatible endpoints; `apiKey` is optional and headers override defaults. +`openRouter` aliases `openrouter`; `cloudflareAiGateway` and `workersAi` supply +Cloudflare REST URLs without runtime bindings. Catalog URL variables are supplied +explicitly through `env`, including `CLOUDFLARE_ACCOUNT_ID`. + +OpenAI-compatible, Anthropic, Gemini/Vertex and Ollama serializers read +`message.parts`. Unsupported sources/modalities raise `CapabilityUnsupportedError` +with `code: CAPABILITY_UNSUPPORTED` when multimodal input is enabled. Models +without multimodal capability fall back to text parts and short attachment markers, +omitting binary data and data URLs, and log at most one warning per request. +OpenAI and Anthropic assistant binary parts also use this text fallback. +Explicit `capabilities.multiModal` takes precedence over model-name heuristics +(GPT-4/5, o-series, vision, Gemini and Claude 3+ names, including provider prefixes). +Set it explicitly for custom models or providers whose capabilities differ. +See the [tested modality matrix](../../docs/adapter-modalities.md) and +[configuration guide](../../apps/docs-next/content/docs/data/providers/openai.mdx). +These are local wire-format tests; live provider acceptance remains unverified. + ## Install diff --git a/packages/adapters/src/catalog/dispatch.ts b/packages/adapters/src/catalog/dispatch.ts index c895d7b57..86f2f4160 100644 --- a/packages/adapters/src/catalog/dispatch.ts +++ b/packages/adapters/src/catalog/dispatch.ts @@ -1,19 +1,19 @@ import type { AdapterFactory } from '@agentskit/core' -import { openai } from '../openai' -import type { RetryOptions } from '../utils' +import { openai, type OpenAIConfig } from '../openai' import { getProvider } from './loader' import type { CatalogProvider } from './types' /** * Provider, model, credentials, and optional transport settings for catalog-based dispatch. */ -export interface CatalogDispatchConfig { +export interface CatalogDispatchConfig extends OpenAIConfig { provider: string model: string apiKey: string /** Override the catalog base URL (e.g. proxy / self-hosted gateway). */ baseUrl?: string - retry?: RetryOptions + /** Explicit values for catalog URL placeholders; never reads process.env. */ + env?: Record } /** @@ -22,14 +22,14 @@ export interface CatalogDispatchConfig { export class CatalogDispatchError extends Error { constructor( message: string, - readonly code: 'UNKNOWN_PROVIDER' | 'NOT_OPENAI_COMPATIBLE' | 'NO_BASE_URL', + readonly code: 'UNKNOWN_PROVIDER' | 'NOT_OPENAI_COMPATIBLE' | 'NO_BASE_URL' | 'MISSING_URL_VARIABLE', ) { super(message) this.name = 'CatalogDispatchError' } } -function resolveBaseUrl(provider: CatalogProvider, override?: string): string { +function resolveBaseUrl(provider: CatalogProvider, override?: string, env: Record = {}): string { const baseUrl = override ?? provider.baseUrl if (!baseUrl) { throw new CatalogDispatchError( @@ -37,7 +37,11 @@ function resolveBaseUrl(provider: CatalogProvider, override?: string): string { 'NO_BASE_URL', ) } - return baseUrl + return baseUrl.replace(/\$\{([A-Z_][A-Z_0-9]*)\}/g, (_match, name: string) => { + const value = env[name] + if (!value) throw new CatalogDispatchError(`missing catalog URL variable ${name}`, 'MISSING_URL_VARIABLE') + return encodeURIComponent(value) + }) } /** @@ -63,9 +67,9 @@ export function dispatchFromCatalog(config: CatalogDispatchConfig): AdapterFacto ) } return openai({ - apiKey: config.apiKey, + ...config, model: config.model, - baseUrl: resolveBaseUrl(provider, config.baseUrl), + baseUrl: resolveBaseUrl(provider, config.baseUrl, config.env), retry: config.retry, }) } diff --git a/packages/adapters/src/cloudflare.ts b/packages/adapters/src/cloudflare.ts new file mode 100644 index 000000000..da8c6520c --- /dev/null +++ b/packages/adapters/src/cloudflare.ts @@ -0,0 +1,35 @@ +import type { AdapterFactory } from '@agentskit/core' +import { openaiCompatible, type OpenAICompatibleTransportConfig } from './openai-compatible' + +/** Cloudflare account and optional AI Gateway authentication. */ +export interface CloudflareAiGatewayConfig extends OpenAICompatibleTransportConfig { + accountId: string + gatewayId: string + gatewayToken?: string +} + +/** OpenAI-compatible AI Gateway endpoint; supports gateway-only authentication. */ +export function cloudflareAiGateway(config: CloudflareAiGatewayConfig): AdapterFactory { + const headers = new Headers(config.headers) + if (config.gatewayToken && !headers.has('cf-aig-authorization')) headers.set('cf-aig-authorization', `Bearer ${config.gatewayToken}`) + return openaiCompatible({ + ...config, + baseUrl: config.baseUrl ?? `https://gateway.ai.cloudflare.com/v1/${encodeURIComponent(config.accountId)}/${encodeURIComponent(config.gatewayId)}/compat`, + path: config.path ?? '/chat/completions', + headers, + }) +} + +/** Workers AI REST configuration; no Cloudflare runtime binding is required. */ +export interface WorkersAiConfig extends OpenAICompatibleTransportConfig { + accountId: string +} + +/** Workers AI's OpenAI-compatible REST endpoint. */ +export function workersAi(config: WorkersAiConfig): AdapterFactory { + return openaiCompatible({ + ...config, + baseUrl: config.baseUrl ?? `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(config.accountId)}/ai/v1`, + path: config.path ?? '/chat/completions', + }) +} diff --git a/packages/adapters/src/content-parts.ts b/packages/adapters/src/content-parts.ts new file mode 100644 index 000000000..ea70af004 --- /dev/null +++ b/packages/adapters/src/content-parts.ts @@ -0,0 +1,88 @@ +import type { ContentPart, Message } from '@agentskit/core' + +type Provider = 'openai' | 'anthropic' | 'gemini' | 'ollama' + +/** A modality or source representation that the selected adapter cannot send. */ +export class CapabilityUnsupportedError extends Error { + readonly code = 'CAPABILITY_UNSUPPORTED' + constructor(readonly provider: string, readonly partType: ContentPart['type']) { + super(`${provider} cannot send this ${partType} part`) + this.name = 'CapabilityUnsupportedError' + } +} + +function unsupported(provider: Provider, part: ContentPart): never { + throw new CapabilityUnsupportedError(provider, part.type) +} + +function dataSource(part: Exclude, provider: Provider) { + try { + if (new URL(part.source).protocol !== 'data:') return undefined + } catch { + return undefined + } + const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]+={0,2})$/.exec(part.source) + if (!match || match[2]!.length % 4 !== 0 || (part.mimeType && part.mimeType !== match[1])) unsupported(provider, part) + return { mimeType: match[1]!, data: match[2]! } +} + +/** Serialize ordered parts, using short text markers when binary parts cannot be sent. */ +export function providerParts(message: Message, provider: Provider, multiModal = true, warning?: { emitted: boolean }): Array> | undefined { + if (!message.parts?.length) return undefined + if ((!multiModal || ((provider === 'openai' || provider === 'anthropic') && message.role === 'assistant')) && message.parts.some(part => part.type !== 'text')) { + if (!warning?.emitted) { + console.warn(`${provider}: binary content parts omitted; using text fallback`) + if (warning) warning.emitted = true + } + const text = message.parts.map(part => { + if (part.type === 'text') return part.text + if (part.type === 'file') { + const name = part.filename && !/^\s*data:/i.test(part.filename) ? part.filename : 'omitted' + return `[file: ${name}]` + } + return `[${part.type} omitted]` + }).join('\n') + return provider === 'gemini' ? [{ text }] : [{ type: 'text', text }] + } + return message.parts.map(part => { + if (part.type === 'text') return provider === 'gemini' ? { text: part.text } : { type: 'text', text: part.text } + if (!multiModal || (message.role !== 'user' && message.role !== 'assistant') || (part.type !== 'image' && part.type !== 'file')) unsupported(provider, part) + const inline = dataSource(part, provider) + const isUrl = /^https?:\/\//.test(part.source) + if (part.type === 'image' && (inline?.mimeType ?? part.mimeType) && !(inline?.mimeType ?? part.mimeType)!.startsWith('image/')) unsupported(provider, part) + if (provider === 'openai') { + if (part.type === 'image') { + if ((!inline && !isUrl) || (inline && !inline.mimeType.startsWith('image/'))) unsupported(provider, part) + return { type: 'image_url', image_url: { url: part.source, ...(part.detail ? { detail: part.detail } : {}) } } + } + if (inline) return { type: 'file', file: { file_data: part.source, filename: part.filename ?? 'attachment' } } + if (/^file-[\w-]+$/.test(part.source)) return { type: 'file', file: { file_id: part.source } } + unsupported(provider, part) + } + if (provider === 'anthropic') { + if (part.type === 'image' && inline && !['image/png', 'image/jpeg', 'image/gif', 'image/webp'].includes(inline.mimeType)) unsupported(provider, part) + const mimeType = inline?.mimeType ?? part.mimeType + if (part.type === 'file' && mimeType !== 'application/pdf' && mimeType !== 'text/plain') unsupported(provider, part) + if (part.type === 'file' && mimeType === 'text/plain' && !inline) unsupported(provider, part) + if (inline?.mimeType === 'text/plain' && part.type === 'file') { + const bytes = Uint8Array.from(atob(inline.data), c => c.charCodeAt(0)) + try { + return { type: 'document', source: { type: 'text', media_type: 'text/plain', data: new TextDecoder('utf-8', { fatal: true }).decode(bytes) } } + } catch { + unsupported(provider, part) + } + } + if (!inline && !isUrl) unsupported(provider, part) + return { type: part.type === 'image' ? 'image' : 'document', source: inline + ? { type: 'base64', media_type: inline.mimeType, data: inline.data } + : { type: 'url', url: part.source } } + } + if (provider === 'gemini') { + if (inline) return { inlineData: inline } + if (!part.mimeType || (!isUrl && !part.source.startsWith('gs://'))) unsupported(provider, part) + return { fileData: { mimeType: part.mimeType, fileUri: part.source } } + } + if (!inline || !inline.mimeType.startsWith('image/')) unsupported(provider, part) + return { image: inline.data } + }) +} diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 59c9f12d2..207901d64 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -124,3 +124,10 @@ export type { CredentialRefreshable, RotatingCredentials, } from './credential-rotation' + +export { openaiCompatible } from './openai-compatible' +export type { OpenAICompatibleConfig, OpenAICompatibleTransportConfig } from './openai-compatible' +export { openRouter } from './openrouter' +export { cloudflareAiGateway, workersAi } from './cloudflare' +export type { CloudflareAiGatewayConfig, WorkersAiConfig } from './cloudflare' +export { CapabilityUnsupportedError } from './content-parts' diff --git a/packages/adapters/src/ollama.ts b/packages/adapters/src/ollama.ts index 5d2d2af2e..f05251f7c 100644 --- a/packages/adapters/src/ollama.ts +++ b/packages/adapters/src/ollama.ts @@ -1,5 +1,6 @@ import type { AdapterFactory, AdapterRequest, StreamSource } from '@agentskit/core' import { parseOllamaStream, type RetryOptions } from './utils' +import { providerParts } from './content-parts' import { createStreamSource } from './stream-source' /** @@ -8,6 +9,7 @@ import { createStreamSource } from './stream-source' export interface OllamaConfig { model: string baseUrl?: string + multiModal?: boolean retry?: RetryOptions } @@ -26,16 +28,22 @@ export function ollama(config: OllamaConfig): AdapterFactory { capabilities: { streaming: true, tools: false, // varies by model; default 'false' — enable via capabilities.extensions if your model supports it - multiModal: model.includes('llava') || model.includes('vision'), + multiModal: config.multiModal ?? (model.includes('llava') || model.includes('vision')), }, createSource: (request: AdapterRequest): StreamSource => { + const warning = { emitted: false } const body = { model, stream: true, - messages: request.messages.map(message => ({ - role: message.role, - content: message.content, - })), + messages: request.messages.map(message => { + const parts = providerParts(message, 'ollama', config.multiModal ?? (model.includes('llava') || model.includes('vision')), warning) + const images = parts?.filter(part => part.image !== undefined).map(part => part.image) + return { + role: message.role, + content: parts ? parts.filter(part => part.type === 'text').map(part => part.text).join('\n') : message.content, + ...(images?.length ? { images } : {}), + } + }), } return createStreamSource( diff --git a/packages/adapters/src/openai-compatible.ts b/packages/adapters/src/openai-compatible.ts index 7d406abba..d009dc230 100644 --- a/packages/adapters/src/openai-compatible.ts +++ b/packages/adapters/src/openai-compatible.ts @@ -1,8 +1,14 @@ import type { AdapterFactory } from '@agentskit/core' import { openai, type OpenAIConfig } from './openai' +/** Configuration for provider presets that require Bearer authentication. */ export interface OpenAICompatibleConfig extends OpenAIConfig {} +/** Transport configuration for endpoints with optional Bearer authentication. */ +export interface OpenAICompatibleTransportConfig extends Omit { + apiKey?: string +} + export function createOpenAICompatibleAdapter(defaultBaseUrl: string) { return function openAICompatibleAdapter(config: OpenAICompatibleConfig): AdapterFactory { return openai({ @@ -11,3 +17,8 @@ export function createOpenAICompatibleAdapter(defaultBaseUrl: string) { }) } } + +/** An OpenAI-compatible endpoint with explicit transport configuration. */ +export function openaiCompatible(config: OpenAICompatibleTransportConfig): AdapterFactory { + return openai({ ...config, apiKey: config.apiKey ?? '' }) +} diff --git a/packages/adapters/src/openai.ts b/packages/adapters/src/openai.ts index 92103ec0f..ddd5e12f9 100644 --- a/packages/adapters/src/openai.ts +++ b/packages/adapters/src/openai.ts @@ -9,6 +9,9 @@ export interface OpenAIConfig { apiKey: string model: string baseUrl?: string + path?: string + headers?: HeadersInit + fetch?: typeof globalThis.fetch retry?: RetryOptions /** * Ask the provider to include token usage in the final stream chunk via @@ -18,7 +21,7 @@ export interface OpenAIConfig { * Turn this on for vanilla `api.openai.com`. */ includeUsage?: boolean - /** Explicit capability facts for OpenAI-compatible providers with custom models. */ + /** Explicit capability facts take precedence over model-name heuristics. */ capabilities?: Partial } @@ -34,9 +37,10 @@ export function openai(config: OpenAIConfig): AdapterFactory { const { apiKey, model, baseUrl = 'https://api.openai.com', retry } = config // Normalize: many compatible endpoints are declared WITH a trailing `/v1` // (together, mistral, fireworks, openrouter `/api/v1`, …) while others are - // declared without it. We always append `/v1/chat/completions`, so strip a + // declared without it. The default path appends `/v1/chat/completions`, so strip a // trailing `/v1` first to avoid a double `/v1/v1/...` (→ 404). - const apiRoot = baseUrl.replace(/\/v1\/?$/, '') + const apiRoot = config.path === undefined ? baseUrl.replace(/\/v1\/?$/, '') : baseUrl + const url = `${apiRoot.replace(/\/$/, '')}/${(config.path ?? '/v1/chat/completions').replace(/^\//, '')}` // Auto: on for canonical OpenAI, off for every other compatible endpoint // where the param is a known source of 4xx surprises. // Match the canonical OpenAI host exactly — a substring/prefix check @@ -50,21 +54,22 @@ export function openai(config: OpenAIConfig): AdapterFactory { })() const includeUsage = config.includeUsage ?? isCanonicalOpenAI + const capabilities: AdapterCapabilities = { + streaming: true, + tools: config.capabilities?.tools ?? true, + // o1 / o3 models emit reasoning; older models don't. Accurate per-model + // detection would need a model registry; 'true' is the safer default here. + reasoning: config.capabilities?.reasoning ?? (model.startsWith('o1') || model.startsWith('o3')), + multiModal: config.capabilities?.multiModal ?? /(^|\/)(gpt-[45]|o\d|gemini-|claude-(?:[3-9]|(?:sonnet|opus|haiku)-[3-9]))|vision/i.test(model), + usage: true, + ...config.capabilities, + } return { - capabilities: { - streaming: true, - tools: config.capabilities?.tools ?? true, - // o1 / o3 models emit reasoning; older models don't. Accurate per-model - // detection would need a model registry; 'true' is the safer default here. - reasoning: config.capabilities?.reasoning ?? (model.startsWith('o1') || model.startsWith('o3')), - multiModal: config.capabilities?.multiModal ?? (model.startsWith('gpt-4') || model.startsWith('o')), - usage: true, - ...config.capabilities, - }, + capabilities, createSource: (request: AdapterRequest): StreamSource => { const body: Record = { model, - messages: toProviderMessages(request.messages), + messages: toProviderMessages(request.messages, capabilities.multiModal), tools: request.context?.tools?.map(tool => ({ type: 'function', function: { @@ -80,15 +85,17 @@ export function openai(config: OpenAIConfig): AdapterFactory { if (includeUsage) body.stream_options = { include_usage: true } return createStreamSource( - (signal) => fetch(`${apiRoot}/v1/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify(body), - signal, - }), + (signal) => { + const headers = new Headers({ 'Content-Type': 'application/json' }) + if (apiKey) headers.set('Authorization', `Bearer ${apiKey}`) + new Headers(config.headers).forEach((value, name) => headers.set(name, value)) + return (config.fetch ?? globalThis.fetch)(url, { + method: 'POST', + headers: Object.fromEntries([...headers].map(([name, value]) => [name === 'authorization' ? 'Authorization' : name, value])), + body: JSON.stringify(body), + signal, + }) + }, parseOpenAIStream, 'OpenAI API', retry, diff --git a/packages/adapters/src/openrouter.ts b/packages/adapters/src/openrouter.ts index 7664f00f3..3acacc145 100644 --- a/packages/adapters/src/openrouter.ts +++ b/packages/adapters/src/openrouter.ts @@ -1,9 +1,10 @@ -import { createOpenAICompatibleAdapter, type OpenAICompatibleConfig } from './openai-compatible' +import { createOpenAICompatibleAdapter } from './openai-compatible' +import type { OpenAIConfig } from './openai' /** * Configuration options for the OpenRouter chat adapter. */ -export interface OpenRouterConfig extends OpenAICompatibleConfig {} +export interface OpenRouterConfig extends OpenAIConfig {} /** * OpenRouter — routes to 300+ provider models behind a single @@ -11,3 +12,6 @@ export interface OpenRouterConfig extends OpenAICompatibleConfig {} * `model`, e.g. `anthropic/claude-sonnet-4-6`. */ export const openrouter = createOpenAICompatibleAdapter('https://openrouter.ai/api/v1') + +/** Camel-case alias for the existing OpenRouter factory. */ +export const openRouter = openrouter diff --git a/packages/adapters/src/tool-history.ts b/packages/adapters/src/tool-history.ts index 153f21538..14b36c487 100644 --- a/packages/adapters/src/tool-history.ts +++ b/packages/adapters/src/tool-history.ts @@ -1,4 +1,5 @@ import type { Message } from '@agentskit/core' +import { providerParts } from './content-parts' /** Normalize tool-call args for provider wire formats. */ function toolInput(args: unknown): Record { @@ -43,13 +44,15 @@ export function toAnthropicMessages( const knownNames = new Map() const output: Array<{ role: string; content: unknown }> = [] + const warning = { emitted: false } for (const message of messages) { + const parts = providerParts(message, 'anthropic', true, warning) if (message.role === 'system') continue if (message.role === 'assistant' && message.toolCalls && message.toolCalls.length > 0) { for (const tc of message.toolCalls) knownNames.set(tc.id, tc.name) - const blocks: Array> = [] - if (message.content) { + const blocks: Array> = [...(parts ?? [])] + if (!parts && message.content) { blocks.push({ type: 'text', text: message.content }) } for (const tc of message.toolCalls) { @@ -87,16 +90,16 @@ export function toAnthropicMessages( continue } - if (message.role === 'assistant' && !message.content) continue + if (message.role === 'assistant' && !message.content && !parts) continue const previous = output[output.length - 1] if (message.role === 'user' && previous?.role === 'user') { const previousBlocks = Array.isArray(previous.content) ? previous.content : [{ type: 'text', text: previous.content }] - previous.content = [...previousBlocks, { type: 'text', text: message.content }] + previous.content = [...previousBlocks, ...(parts ?? [{ type: 'text', text: message.content }])] } else { - output.push({ role: message.role, content: message.content }) + output.push({ role: message.role, content: parts ?? message.content }) } } @@ -115,12 +118,13 @@ export function toGeminiContents( const output: Array<{ role: string; parts: Array> }> = [] for (const message of messages) { + const contentParts = providerParts(message, 'gemini') if (message.role === 'system') continue if (message.role === 'assistant' && message.toolCalls && message.toolCalls.length > 0) { for (const tc of message.toolCalls) knownNames.set(tc.id, tc.name) - const parts: Array> = [] - if (message.content) { + const parts: Array> = [...(contentParts ?? [])] + if (!contentParts && message.content) { parts.push({ text: message.content }) } for (const tc of message.toolCalls) { @@ -159,14 +163,14 @@ export function toGeminiContents( continue } - if (message.role === 'assistant' && !message.content) continue + if (message.role === 'assistant' && !message.content && !contentParts) continue const role = message.role === 'assistant' ? 'model' : 'user' const previous = output[output.length - 1] if (previous?.role === role) { - previous.parts.push({ text: message.content }) + previous.parts.push(...(contentParts ?? [{ text: message.content }])) } else { - output.push({ role, parts: [{ text: message.content }] }) + output.push({ role, parts: contentParts ?? [{ text: message.content }] }) } } diff --git a/packages/adapters/src/utils.ts b/packages/adapters/src/utils.ts index 713a9c24a..f7b13444f 100644 --- a/packages/adapters/src/utils.ts +++ b/packages/adapters/src/utils.ts @@ -1,5 +1,6 @@ import { type Message, type StreamChunk, type StreamSource } from '@agentskit/core' import { parseSSE, sleep as netSleep } from '@agentskit/net' +import { providerParts } from './content-parts' import { fetchWithRetry, type RetryOptions } from './retry' import { readNDJSONLines } from './stream-lines' import { @@ -16,7 +17,7 @@ export { parseOpenAIStream } from './openai-stream' export { readNDJSONLines, readSSELines } from './stream-lines' export type { StreamParser } from './stream-types' -export function toProviderMessages(messages: Message[]) { +export function toProviderMessages(messages: Message[], multiModal = true) { // Track which tool_call ids were declared by preceding assistant turns so // we can drop orphan tool messages — the OpenAI Chat Completions API // rejects a tool message whose tool_call_id isn't bound to a previous @@ -24,14 +25,17 @@ export function toProviderMessages(messages: Message[]) { const knownToolCallIds = new Set() const output: Array> = [] + const warning = { emitted: false } for (const message of messages) { + const parts = providerParts(message, 'openai', multiModal, warning) + const content = parts ?? message.content if (message.role === 'tool') { const id = message.toolCallId // Orphan (no id, or id not declared) — skip; it would 400 the API. if (!id || !knownToolCallIds.has(id)) continue output.push({ role: 'tool' as const, - content: message.content, + content, tool_call_id: id, }) continue @@ -41,7 +45,7 @@ export function toProviderMessages(messages: Message[]) { for (const tc of message.toolCalls) knownToolCallIds.add(tc.id) output.push({ role: 'assistant' as const, - content: message.content || null, + content: content || null, tool_calls: message.toolCalls.map(tc => ({ id: tc.id, type: 'function' as const, @@ -59,9 +63,9 @@ export function toProviderMessages(messages: Message[]) { // and the placeholder assistant message never received content — // sending `{role:'assistant', content:''}` back to the provider either // 400s or confuses the model into silence on the next turn. - if (message.role === 'assistant' && !message.content) continue + if (message.role === 'assistant' && !message.content && !parts) continue - output.push({ role: message.role, content: message.content }) + output.push({ role: message.role, content }) } return output diff --git a/packages/adapters/tests/capabilities.test.ts b/packages/adapters/tests/capabilities.test.ts index a60566b4b..873ebd4f0 100644 --- a/packages/adapters/tests/capabilities.test.ts +++ b/packages/adapters/tests/capabilities.test.ts @@ -4,6 +4,11 @@ import { openai, anthropic, gemini, ollama, mockAdapter, vercelAI } from '../src import { chunkText, simulateStream } from '../src/utils' describe('adapter capabilities', () => { + it.each(['meta-llama/llama-vision', 'google/gemini-2.5-flash', 'anthropic/claude-3.5-sonnet', 'anthropic/claude-sonnet-4-6', 'openai/gpt-4o', 'openai/o3'])('detects compatible multimodal model %s and honors explicit overrides', model => { + expect(openai({ apiKey: 'x', model }).capabilities?.multiModal).toBe(true) + expect(openai({ apiKey: 'x', model, capabilities: { multiModal: false } }).capabilities?.multiModal).toBe(false) + expect(openai({ apiKey: 'x', model: 'custom', capabilities: { multiModal: true } }).capabilities?.multiModal).toBe(true) + }) it('openai advertises streaming, tools, usage', () => { const adapter = openai({ apiKey: 'x', model: 'gpt-4o' }) expect(adapter.capabilities?.streaming).toBe(true) diff --git a/packages/adapters/tests/content-parts.test.ts b/packages/adapters/tests/content-parts.test.ts new file mode 100644 index 000000000..28ac9e40a --- /dev/null +++ b/packages/adapters/tests/content-parts.test.ts @@ -0,0 +1,19 @@ +import { expect, it } from 'vitest' +import type { Message } from '@agentskit/core' +import { CapabilityUnsupportedError, providerParts } from '../src/content-parts' + +it('classifies data URIs while retaining strict media validation and non-URL sources', () => { + const message: Message = { id: 'synthetic', role: 'user', content: '', status: 'complete', createdAt: new Date(0) } + for (const source of ['data:image/png;base64,aW1hZ2U=', 'https://example.test/image.png']) { + expect(providerParts({ ...message, parts: [{ type: 'image', source }] }, 'openai')).toEqual([ + { type: 'image_url', image_url: { url: source } }, + ]) + } + expect(providerParts({ ...message, parts: [{ type: 'file', source: 'file-synthetic' }] }, 'openai')).toEqual([ + { type: 'file', file: { file_id: 'file-synthetic' } }, + ]) + for (const source of ['unknown-reference', 'data:image/png;base64,invalid!', 'DATA:image/png;base64,aW1hZ2U=', 'data:image/png;base64,YQ=']) { + expect(() => providerParts({ ...message, parts: [{ type: 'image', source }] }, 'openai')).toThrow(CapabilityUnsupportedError) + } + expect(() => providerParts({ ...message, parts: [{ type: 'image', source: 'data:image/png;base64,aW1hZ2U=', mimeType: 'image/jpeg' }] }, 'openai')).toThrow(CapabilityUnsupportedError) +}) diff --git a/packages/adapters/tests/gateway-types.test-d.ts b/packages/adapters/tests/gateway-types.test-d.ts new file mode 100644 index 000000000..55a1447ed --- /dev/null +++ b/packages/adapters/tests/gateway-types.test-d.ts @@ -0,0 +1,13 @@ +import { expectTypeOf } from 'vitest' +import { openaiCompatible, cloudflareAiGateway, workersAi } from '../src' +import type { DeepSeekConfig, OpenAIConfig, OpenRouterConfig } from '../src' +import type { OpenAICompatibleConfig } from '../src/openai-compatible' + +expectTypeOf().toEqualTypeOf() +expectTypeOf().toEqualTypeOf() +expectTypeOf().toEqualTypeOf() +expectTypeOf().toEqualTypeOf() + +openaiCompatible({ model: 'synthetic', headers: { 'X-Synthetic': 'present' }, path: '/custom' }) +cloudflareAiGateway({ model: 'synthetic', accountId: 'synthetic', gatewayId: 'synthetic' }) +workersAi({ model: 'synthetic', accountId: 'synthetic' }) diff --git a/packages/adapters/tests/multimodal-http.test.ts b/packages/adapters/tests/multimodal-http.test.ts new file mode 100644 index 000000000..d5bd2dc09 --- /dev/null +++ b/packages/adapters/tests/multimodal-http.test.ts @@ -0,0 +1,302 @@ +import { readFileSync } from 'node:fs' +import { createServer, type IncomingHttpHeaders } from 'node:http' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { partsToText, type AdapterFactory, type ContentPart, type Message } from '@agentskit/core' +import { openai } from '../src/openai' +import { openaiCompatible } from '../src/openai-compatible' +import { anthropic } from '../src/anthropic' +import { gemini } from '../src/gemini' +import { vertex } from '../src/vertex' +import { ollama } from '../src/ollama' +import { dispatchFromCatalog } from '../src/catalog/dispatch' +import { listProviders } from '../src/catalog/loader' + +export const multimodalCases = [ + { provider: 'openai', image: 'image_url', file: 'file', sources: 'image URL/data URL; file data URL/id' }, + { provider: 'openai-compatible', image: 'image_url', file: 'file', sources: 'same wire format; opt into model capability' }, + { provider: 'anthropic', image: 'image', file: 'document', sources: 'image URL/data URL; PDF URL/data URL; text data URL' }, + { provider: 'gemini', image: 'inlineData/fileData', file: 'inlineData/fileData', sources: 'data URL or provider URI with MIME type' }, + { provider: 'vertex', image: 'inlineData/fileData', file: 'inlineData/fileData', sources: 'data URL or provider URI with MIME type' }, + { provider: 'ollama', image: 'images', file: 'image data URL only', sources: 'image data URL only; vision model required; PDF rejected' }, +] as const + +const png = 'data:image/png;base64,aW1hZ2U=' +const pdf = 'data:application/pdf;base64,cGRm' +function message(parts: ContentPart[], role: Message['role'] = 'user'): Message { + return { id: 'synthetic', role, content: partsToText(parts), parts, status: 'complete', createdAt: new Date(0) } +} +let server: ReturnType +let baseUrl: string +let captured: { path: string; headers: IncomingHttpHeaders; body: Record }[] +beforeEach(async () => { + captured = [] + server = createServer(async (req, res) => { + let body = '' + for await (const chunk of req) body += chunk + captured.push({ path: req.url!, headers: req.headers, body: JSON.parse(body) }) + res.setHeader('Content-Type', 'text/event-stream') + if (req.url!.includes('/messages')) res.end('data: {"type":"message_stop"}\n\n') + else if (req.url!.includes('streamGenerateContent')) res.end('data: {"candidates":[{"finishReason":"STOP"}]}\n\n') + else if (req.url === '/api/chat') res.end('{"done":true}\n') + else res.end('data: {"choices":[{"delta":{"content":"ok"}}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\ndata: [DONE]\n\n') + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('No local address') + baseUrl = `http://127.0.0.1:${address.port}` +}) +afterEach(async () => { + vi.unstubAllGlobals() + await new Promise((resolve, reject) => server.close(err => err ? reject(err) : resolve())) +}) +async function send(adapter: AdapterFactory, parts: ContentPart[]) { + const source = adapter.createSource({ messages: [message(parts)] }) + const chunks = [] + for await (const chunk of source.stream()) chunks.push(chunk) + expect(chunks.at(-1)?.type).toBe('done') + return captured.at(-1)!.body +} + +describe('multimodal HTTP contract', () => { + it.each(['openai', 'openai-compatible'])('%s preserves ordered parts, detail and files', async provider => { + const body = await send((provider === 'openai' ? openai : openaiCompatible)({ apiKey: 'synthetic', model: provider === 'openai' ? 'gpt-4o' : 'custom', baseUrl, capabilities: { multiModal: true } }), [ + { type: 'text', text: 'inspect' }, { type: 'image', source: png, detail: 'high' }, + { type: 'file', source: pdf, filename: 'sample.pdf' }, + ]) + expect(body.messages).toEqual([{ role: 'user', content: [ + { type: 'text', text: 'inspect' }, { type: 'image_url', image_url: { url: png, detail: 'high' } }, + { type: 'file', file: { file_data: pdf, filename: 'sample.pdf' } }, + ] }]) + }) + it('Anthropic sends image and PDF blocks', async () => { + const body = await send(anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }), [ + { type: 'image', source: png }, { type: 'file', source: pdf }, + ]) + expect(body.messages).toEqual([{ role: 'user', content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'aW1hZ2U=' } }, + { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: 'cGRm' } }, + ] }]) + }) + it.each(['gemini', 'vertex'])('%s sends inlineData and fileData', async provider => { + const nativeFetch = globalThis.fetch + if (provider === 'vertex') vi.stubGlobal('fetch', (input: Parameters[0], init: Parameters[1]) => nativeFetch(`${baseUrl}/streamGenerateContent`, init)) + const adapter = provider === 'gemini' + ? gemini({ apiKey: 'synthetic', model: 'gemini', baseUrl }) + : vertex({ project: 'synthetic', region: 'synthetic', model: 'gemini', accessToken: 'synthetic' }) + const body = await send(adapter, [{ type: 'image', source: png }, { type: 'file', source: 'gs://synthetic/sample.pdf', mimeType: 'application/pdf' }]) + expect(body.contents).toEqual([{ role: 'user', parts: [ + { inlineData: { mimeType: 'image/png', data: 'aW1hZ2U=' } }, + { fileData: { mimeType: 'application/pdf', fileUri: 'gs://synthetic/sample.pdf' } }, + ] }]) + }) + it('Ollama sends images without the data URL prefix', async () => { + const body = await send(ollama({ model: 'llava', baseUrl }), [{ type: 'text', text: 'inspect' }, { type: 'image', source: png }]) + expect(body.messages).toEqual([{ role: 'user', content: 'inspect', images: ['aW1hZ2U='] }]) + }) + it('rejects unsupported modalities before network I/O', () => { + expect(() => ollama({ model: 'llava', baseUrl }).createSource({ messages: [message([{ type: 'file', source: pdf }])] })).toThrow(expect.objectContaining({ code: 'CAPABILITY_UNSUPPORTED' })) + expect(captured).toHaveLength(0) + }) + it.each([ + ['gpt-3.5', undefined], ['gpt-4o', false], + ] as const)('falls back to text with a warning for %s, including old image history', async (model, multiModal) => { + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const adapter = openai({ apiKey: 'synthetic', model, baseUrl, ...(multiModal === undefined ? {} : { capabilities: { multiModal } }) }) + const messages = [message([{ type: 'text', text: 'inspect' }, { type: 'image', source: png }, { type: 'file', source: pdf, filename: 'sample.pdf' }]), + message([{ type: 'image', source: png }]), + message([{ type: 'file', source: pdf }]), + { ...message([], 'assistant'), content: 'old answer' }, + { ...message([{ type: 'text', text: 'followup' }]), content: 'followup' }] + const before = JSON.stringify(messages) + const chunks = [] + for await (const chunk of adapter.createSource({ messages }).stream()) chunks.push(chunk) + expect(chunks.at(-1)?.type).toBe('done') + expect(captured[0]?.body.messages).toEqual([ + { role: 'user', content: [{ type: 'text', text: 'inspect\n[image omitted]\n[file: sample.pdf]' }] }, + { role: 'user', content: [{ type: 'text', text: '[image omitted]' }] }, + { role: 'user', content: [{ type: 'text', text: '[file: omitted]' }] }, + { role: 'assistant', content: 'old answer' }, + { role: 'user', content: [{ type: 'text', text: 'followup' }] }, + ]) + expect(warning).toHaveBeenCalledExactlyOnceWith(expect.stringContaining('text fallback')) + expect(JSON.stringify(captured[0]?.body)).not.toContain('data:') + expect(JSON.stringify(captured[0]?.body)).not.toContain('aW1hZ2U=') + for await (const _chunk of adapter.createSource({ messages }).stream()) { /* drain */ } + expect(warning).toHaveBeenCalledTimes(2) + expect(JSON.stringify(messages)).toBe(before) + } finally { warning.mockRestore() } + }) + it.each(['openai', 'anthropic'])('%s omits binary assistant parts while preserving text and tool calls', async provider => { + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const messages = [message([{ type: 'image', source: png }], 'assistant'), + { ...message([{ type: 'text', text: 'checking' }, { type: 'file', source: pdf, filename: 'sample.pdf' }], 'assistant'), toolCalls: [{ id: 'call', name: 'inspect', args: {}, status: 'complete' as const }] }] + const adapter = provider === 'openai' ? openai({ apiKey: 'synthetic', model: 'gpt-4o', baseUrl }) : anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }) + for await (const _chunk of adapter.createSource({ messages }).stream()) { /* drain */ } + const text = { type: 'text', text: 'checking\n[file: sample.pdf]' } + expect(captured[0]?.body.messages).toEqual([ + { role: 'assistant', content: [{ type: 'text', text: '[image omitted]' }] }, + provider === 'openai' + ? { role: 'assistant', content: [text], tool_calls: [{ id: 'call', type: 'function', function: { name: 'inspect', arguments: '{}' } }] } + : { role: 'assistant', content: [text, { type: 'tool_use', id: 'call', name: 'inspect', input: {} }] }, + ]) + expect(warning).toHaveBeenCalledOnce() + expect(JSON.stringify(captured[0]?.body)).not.toContain('data:') + } finally { warning.mockRestore() } + }) + it('Ollama text-only requests omit binary sources and warn once per request', async () => { + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const messages = [ + message([{ type: 'image', source: png }]), + message([{ type: 'text', text: 'inspect' }, { type: 'file', source: pdf, filename: 'sample.pdf' }]), + message([{ type: 'file', source: pdf }]), + ] + const before = JSON.stringify(messages) + expect(messages[0]!.content).toBe(partsToText(messages[0]!.parts!)) + expect(messages[0]!.content).toContain(png) + const adapter = ollama({ model: 'llama3.1', baseUrl }) + for await (const _chunk of adapter.createSource({ messages }).stream()) { /* drain */ } + expect(captured[0]?.body.messages).toEqual([ + { role: 'user', content: '[image omitted]' }, + { role: 'user', content: 'inspect\n[file: sample.pdf]' }, + { role: 'user', content: '[file: omitted]' }, + ]) + expect(warning).toHaveBeenCalledOnce() + expect(JSON.stringify(captured[0]?.body)).not.toContain('data:') + expect(JSON.stringify(messages)).toBe(before) + for await (const _chunk of adapter.createSource({ messages }).stream()) { /* drain */ } + expect(warning).toHaveBeenCalledTimes(2) + } finally { warning.mockRestore() } + }) + it('custom gateway path/headers/fetch preserves streaming and usage without Bearer', async () => { + const transport = vi.fn(globalThis.fetch) + const config = { model: 'custom', baseUrl: `${baseUrl}/gateway`, path: '/compat/chat/completions', headers: { 'cf-aig-authorization': 'synthetic', 'X-Custom': 'present' }, fetch: transport, includeUsage: true } + const adapter = openaiCompatible(config) + const chunks = [] + for await (const chunk of adapter.createSource({ messages: [message([{ type: 'text', text: 'hello' }])] }).stream()) chunks.push(chunk) + expect(captured[0]?.path).toBe('/gateway/compat/chat/completions') + expect(captured[0]?.headers.authorization).toBeUndefined() + expect(captured[0]?.headers['cf-aig-authorization']).toBe('synthetic') + expect(captured[0]?.headers['x-custom']).toBe('present') + expect(transport).toHaveBeenCalledOnce() + expect(chunks).toContainEqual({ type: 'text', content: 'ok' }) + expect(chunks).toContainEqual(expect.objectContaining({ type: 'usage', usage: { promptTokens: 2, completionTokens: 1, totalTokens: 3 } })) + }) +}) + +// Presets still use a real HTTP server: only the destination host is redirected. +describe('OpenAI-compatible presets', () => { + it('routes OpenRouter, AI Gateway and Workers AI with documented headers', async () => { + const { openRouter, openrouter } = await import('../src/openrouter') + const { cloudflareAiGateway, workersAi } = await import('../src/cloudflare') + expect(openRouter).toBe(openrouter) + const nativeFetch = globalThis.fetch + const destinations: string[] = [] + const transport: typeof fetch = (input, init) => { + const url = String(input) + destinations.push(url) + const redirected = new URL(url) + redirected.host = new URL(baseUrl).host + redirected.protocol = 'http:' + return nativeFetch(redirected, init) + } + await send(openRouter({ model: 'google/gemini-2.5-flash-lite', apiKey: 'synthetic', fetch: transport, headers: { 'HTTP-Referer': 'https://example.test', 'X-Title': 'Synthetic' } }), [{ type: 'text', text: 'hello' }]) + await send(cloudflareAiGateway({ accountId: 'account', gatewayId: 'gateway', gatewayToken: 'synthetic', model: 'openrouter/google/gemini-2.5-flash-lite', fetch: transport }), [{ type: 'text', text: 'hello' }]) + await send(workersAi({ accountId: 'account', apiKey: 'synthetic', model: '@cf/meta/model', fetch: transport }), [{ type: 'text', text: 'hello' }]) + expect(destinations).toEqual([ + 'https://openrouter.ai/api/v1/chat/completions', + 'https://gateway.ai.cloudflare.com/v1/account/gateway/compat/chat/completions', + 'https://api.cloudflare.com/client/v4/accounts/account/ai/v1/chat/completions', + ]) + expect(captured[0]?.headers).toMatchObject({ authorization: 'Bearer synthetic', 'http-referer': 'https://example.test', 'x-title': 'Synthetic' }) + expect(captured[1]?.headers['cf-aig-authorization']).toBe('Bearer synthetic') + expect(captured[1]?.headers.authorization).toBeUndefined() + expect(captured[2]?.headers.authorization).toBe('Bearer synthetic') + }) + it('catalog interpolates explicit Cloudflare account id and rejects missing variables', async () => { + const provider = listProviders().find(p => p.baseUrl?.includes('${CLOUDFLARE_ACCOUNT_ID}') && p.openaiCompatible)! + expect(provider).toBeDefined() + expect(() => dispatchFromCatalog({ provider: provider.id, model: 'custom', apiKey: 'synthetic' })).toThrow(expect.objectContaining({ code: 'MISSING_URL_VARIABLE' })) + const nativeFetch = globalThis.fetch + let destination = '' + const transport: typeof fetch = (input, init) => { + destination = String(input) + const redirected = new URL(destination) + redirected.host = new URL(baseUrl).host + redirected.protocol = 'http:' + return nativeFetch(redirected, init) + } + await send(dispatchFromCatalog({ provider: provider.id, model: 'custom', apiKey: 'synthetic', env: { CLOUDFLARE_ACCOUNT_ID: 'synthetic-account' }, fetch: transport }), [{ type: 'text', text: 'hello' }]) + expect(destination).toBe('https://api.cloudflare.com/client/v4/accounts/synthetic-account/ai/v1/chat/completions') + }) +}) + +describe('multimodal boundaries and history', () => { + it('sends URL image sources and provider file identifiers', async () => { + expect((await send(openai({ apiKey: 'synthetic', model: 'gpt-4o', baseUrl }), [{ type: 'image', source: 'https://example.test/image.png' }, { type: 'file', source: 'file-synthetic' }])).messages).toEqual([{ role: 'user', content: [ + { type: 'image_url', image_url: { url: 'https://example.test/image.png' } }, { type: 'file', file: { file_id: 'file-synthetic' } }, + ] }]) + expect((await send(anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }), [{ type: 'image', source: 'https://example.test/image.png' }, { type: 'file', source: 'https://example.test/sample.pdf', mimeType: 'application/pdf' }])).messages).toEqual([{ role: 'user', content: [ + { type: 'image', source: { type: 'url', url: 'https://example.test/image.png' } }, { type: 'document', source: { type: 'url', url: 'https://example.test/sample.pdf' } }, + ] }]) + expect((await send(gemini({ apiKey: 'synthetic', model: 'gemini', baseUrl }), [{ type: 'file', source: pdf }])).contents).toEqual([{ role: 'user', parts: [{ inlineData: { mimeType: 'application/pdf', data: 'cGRm' } }] }]) + }) + it('rejects unknown sources, missing MIME types, and unsupported audio/video without sending', () => { + const adapters = [openai({ apiKey: 'synthetic', model: 'gpt-4o', baseUrl }), anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }), gemini({ apiKey: 'synthetic', model: 'gemini', baseUrl }), ollama({ model: 'llava', baseUrl })] + for (const adapter of adapters) for (const part of [ + { type: 'audio', source: 'https://example.test/audio.wav' }, + { type: 'video', source: 'https://example.test/video.mp4' }, + { type: 'image', source: 'data:image/png;base64,invalid!' }, + { type: 'image', source: 'unknown-reference' }, + { type: 'image', source: pdf }, + ] as ContentPart[]) expect(() => adapter.createSource({ messages: [message([part])] })).toThrow(expect.objectContaining({ code: 'CAPABILITY_UNSUPPORTED' })) + expect(() => adapters[2]!.createSource({ messages: [message([{ type: 'file', source: 'gs://synthetic/no-mime' }])] })).toThrow(expect.objectContaining({ code: 'CAPABILITY_UNSUPPORTED' })) + expect(() => adapters[1]!.createSource({ messages: [message([{ type: 'file', source: 'data:text/plain;base64,/w==' }])] })).toThrow(expect.objectContaining({ code: 'CAPABILITY_UNSUPPORTED' })) + expect(captured).toHaveLength(0) + }) + it('keeps parts on assistant tool calls and merged user turns', async () => { + const messages: Message[] = [ + message([{ type: 'image', source: png }]), + { ...message([{ type: 'text', text: 'checking' }], 'assistant'), id: 'assistant', toolCalls: [{ id: 'call', name: 'inspect', args: {}, status: 'complete' }] }, + { ...message([], 'tool'), id: 'tool', toolCallId: 'call', content: 'result' }, + { ...message([{ type: 'file', source: pdf }]), id: 'followup' }, + ] + for (const adapter of [openai({ apiKey: 'synthetic', model: 'gpt-4o', baseUrl }), anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }), gemini({ apiKey: 'synthetic', model: 'gemini', baseUrl })]) { + for await (const _chunk of adapter.createSource({ messages }).stream()) { /* drain */ } + const wire = JSON.stringify(captured.at(-1)!.body) + expect(wire).toContain('aW1hZ2U=') + expect(wire).toContain('cGRm') + expect(wire).toContain('checking') + expect(wire).toContain('result') + expect(wire).toContain('call') + expect(wire).not.toContain('legacy projection') + } + }) +}) + +const capabilityTable = [ + '| Adapter | Image wire format | File wire format | Tested sources / limits |', + '| --- | --- | --- | --- |', + ...multimodalCases.map(row => `| ${row.provider} | ${row.image} | ${row.file} | ${row.sources} |`), +].join('\n') + '\n' + +it('keeps the generated capability table and recipe aligned with HTTP cases', async () => { + await expect(capabilityTable).toMatchFileSnapshot('../../../docs/adapter-modalities.md') + const recipe = readFileSync(new URL('../../../apps/docs-next/content/docs/reference/recipes/multi-modal.mdx', import.meta.url), 'utf8') + expect(recipe).toContain(capabilityTable.trim()) +}) + +it('encodes inline text documents and Ollama image file parts', async () => { + expect((await send(anthropic({ apiKey: 'synthetic', model: 'claude', baseUrl }), [{ type: 'file', source: 'data:text/plain;base64,aGVsbG8=' }])).messages).toEqual([{ role: 'user', content: [{ type: 'document', source: { type: 'text', media_type: 'text/plain', data: 'hello' } }] }]) + expect((await send(ollama({ model: 'custom-vision-model', multiModal: true, baseUrl }), [{ type: 'file', source: png }])).messages).toEqual([{ role: 'user', content: '', images: ['aW1hZ2U='] }]) +}) + +it('honors case-insensitive caller authentication overrides', async () => { + const { cloudflareAiGateway } = await import('../src/cloudflare') + await send(openaiCompatible({ apiKey: 'synthetic', model: 'custom', baseUrl, headers: { authorization: 'Basic synthetic' } }), [{ type: 'text', text: 'hello' }]) + expect(captured[0]?.headers.authorization).toBe('Basic synthetic') + await send(cloudflareAiGateway({ accountId: 'synthetic', gatewayId: 'synthetic', gatewayToken: 'default-synthetic', model: 'custom', baseUrl, headers: { 'CF-AIG-Authorization': 'Bearer override-synthetic' } }), [{ type: 'text', text: 'hello' }]) + expect(captured[1]?.headers['cf-aig-authorization']).toBe('Bearer override-synthetic') +}) diff --git a/readme-standard-v1.json b/readme-standard-v1.json index c5e782325..cb12d04d9 100644 --- a/readme-standard-v1.json +++ b/readme-standard-v1.json @@ -207,7 +207,7 @@ "ecosystem.json", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:a85fbd255e698f3e689d34c150c15e38be267ce58aee5c79a9d290a76f6f8881" + "sourceHash": "sha256:3a5921eaaf01df895ff645c1a6e2cb95252d3cd2ee9b1c807957619098d191c7" }, "exceptions": [] }, @@ -285,7 +285,7 @@ "docs/STABILITY.md", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:38e4ff3f70e7463ab700af5ebd9ac3b1815365e4968d47107ccd107f02fcfd64" + "sourceHash": "sha256:5b20b7ab61f667e9a5801a71d15c6e9ad689bf25a22dc8d264cee669ba13032d" }, "exceptions": [] }, @@ -2382,7 +2382,7 @@ "ecosystem-claims.json", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:0e4e7ae036016cb1258f3899acab4817d9827736f660049f9e7315567eb67277" + "sourceHash": "sha256:1e69ab8fd2fc64d2549d288bd07d1222aff23b9dd76c5a04d6e1fc705b6f4b65" }, "exceptions": [ { @@ -2466,7 +2466,7 @@ "ecosystem-claims.json", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:fa59326d7fbdd34731f081627a839dc14d38465f61a0e3e5fd5cfe749b1972dd" + "sourceHash": "sha256:d0cb36c6b27cc4a081192e7b38efb149acda80cd28c155d5ade9fc24b8c46176" }, "exceptions": [ { @@ -2550,7 +2550,7 @@ "ecosystem-claims.json", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:58bcdd16a16ef989d33a167ad23dedbacc257077fa591e213c6bc141d7470977" + "sourceHash": "sha256:34f6b0851407cf08a39c0746f9937d05258098c445e16675f0b1e26151bd4a97" }, "exceptions": [ { @@ -2634,7 +2634,7 @@ "ecosystem-claims.json", "scripts/readme-fixtures.test.mjs" ], - "sourceHash": "sha256:3e729fefb23f11b621248e557a5ae76283fd8569f751675d49a6f91981c103ee" + "sourceHash": "sha256:260d8b8e8631d8f46c5d3920206b664962a74600ae104ec21bae8be369bdb725" }, "exceptions": [ { diff --git a/scripts/content-pipeline.test.mjs b/scripts/content-pipeline.test.mjs index 62792e24e..30175f5fc 100644 --- a/scripts/content-pipeline.test.mjs +++ b/scripts/content-pipeline.test.mjs @@ -221,7 +221,7 @@ test('provider recipe and canonical docs stay aligned with adapter configuration assert.match(recipe, /adapter compatibility matrix/) const unsupportedOptions = { - openai: ['organization', 'project', 'fetch'], + openai: ['organization', 'project'], anthropic: ['version', 'fetch'], gemini: ['apiVersion', 'fetch'], openrouter: ['appUrl', 'appName', 'fetch'],