From 0e9cd85289cc7882b251d05af7cd0de011132742 Mon Sep 17 00:00:00 2001 From: Hauke Walden Date: Fri, 28 Aug 2026 00:00:20 +0200 Subject: [PATCH 1/2] fix(unsloth): detect vision and thinking for the loaded model via /api/inference/status (#35, #37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /v1/models carries no modality or thinking metadata. An earlier revision of this PR probed /api/models/check-vision/{id} for loaded models — but for a model id that is not a resolvable HF repo (e.g. a local GGUF quant name like Qwen3.8-27B-IQ4_NL), Studio's handler falls back to fetching config.json from huggingface.co and 401s in a retry loop on EVERY listModels; with Crossbar polling every 15s (Studio has no health endpoint) that became sustained HuggingFace traffic. Both capabilities are now read from ONE local call, GET /api/inference/status, which describes the loaded backend and has no HF fallback path: - vision (#35): active model with is_vision: true → input ["text", "image"] - thinking (#37): active model with supports_reasoning: true → reasoning active_model carries the same public id as the /v1/models entry (fallback: the loaded: true entry), so the flags attach to exactly one model. Capabilities are known only for the loaded model; unloaded models pick up their real flags once loaded. Best-effort by contract: any probe failure degrades to the conservative pre-fix behaviour, never throws, never blocks registration. --- CAPABILITY-MATRIX.md | 2 +- src/adapters/unsloth.ts | 144 +++++++++++++-- tests/adapters/unsloth.fixture.ts | 39 +++- tests/adapters/unsloth.test.ts | 290 +++++++++++++++++++++++++++++- 4 files changed, 453 insertions(+), 22 deletions(-) diff --git a/CAPABILITY-MATRIX.md b/CAPABILITY-MATRIX.md index e75fe5e..2f59617 100644 --- a/CAPABILITY-MATRIX.md +++ b/CAPABILITY-MATRIX.md @@ -18,7 +18,7 @@ adapter registers under (`oai` = `openai-completions`, `ant` = `anthropic-messag | **oobabooga** | 5000 | oai | ✅ `/v1/models` | ✅ `/v1/internal/model/info` | ✅ load | ✅ `/v1/internal/model/{load,unload}` | ❌ | ◐ `--api-key` | ◐ | ◐ | ✅ | `/v1/internal/*` namespace | | **Jan** | 1337 | oai | ✅ `/v1/models` | ◐ | ◐ engine | ◐ engine | ❌ | ◐ Bearer | ❌ | ◐ | ✅ | weak (log line) | | **llamafile** | 8080 | oai | ✅ `/v1/models` | ◐ `/props` | ❌ | ❌ | ❌ (always on) | ◐ `--api-key` | ✅ `/health` | ◐ via `/props` | ✅ | `/props` w/ non-`bNNNN` build_info | -| **Unsloth Studio** | 8888 | oai | ✅ `/v1/models` (keyed) | ✅ per-model `loaded` field | ❌ (UI-only) | ❌ (UI-only) | ✅ `/api/settings/openai-auto-switch` | ✅ Bearer `sk-unsloth-…` (required) | ❌ (poll via listModels) | ◐ ctx fields only while loaded | ✅ | `Server: unsloth-studio` header (every response) | +| **Unsloth Studio** | 8888 | oai | ✅ `/v1/models` (keyed) | ✅ per-model `loaded` field | ❌ (UI-only) | ❌ (UI-only) | ✅ `/api/settings/openai-auto-switch` | ✅ Bearer `sk-unsloth-…` (required) | ❌ (poll via listModels) | ◐ ctx fields only while loaded; vision + reasoning for the active model via `/api/inference/status` (#35, #37) — both best-effort | ✅ | `Server: unsloth-studio` header (every response) | | **generic OpenAI-compat** | varies | oai | ✅ `/v1/models` | ❌ | ❌ | ❌ | ❌ | ◐ optional Bearer | ◐ | ◐ | ✅ | anything serving `/v1/models` (fallback) | ## Capability-driven UX rules (derived) diff --git a/src/adapters/unsloth.ts b/src/adapters/unsloth.ts index cf4e5a3..92b3baa 100644 --- a/src/adapters/unsloth.ts +++ b/src/adapters/unsloth.ts @@ -40,6 +40,41 @@ * `owned_by: "unsloth-studio"` and a `loaded: boolean` residency flag, used below for * IntrospectLoaded. * + * # Vision detection (issue #35) + * + * The OpenAI-compatible `GET /v1/models` exposes NO modality information — entries carry only + * `id`, `owned_by`, `quant`, context-length fields and `loaded`. Registering every model as + * text-only (the old behaviour) made Pi silently replace attached images with the placeholder + * `(image omitted: model does not support images)`, so VLMs appeared to "have no vision". + * + * Vision is read from the SAME loaded-backend status call used for thinking: + * `GET /api/inference/status` reports `is_vision: boolean` for the resident model (verified + * against a live instance: `is_vision: true` for a loaded Qwen3-VL). This is local and instant + * — it never touches huggingface.co. + * + * We deliberately do NOT use `GET /api/models/check-vision/{model_name}`: for a model whose id + * is not a resolvable HF repo (e.g. a local GGUF quant name like `Qwen3.8-27B-IQ4_NL`) Studio's + * handler falls back to fetching `config.json` from huggingface.co and 401s in a retry loop on + * EVERY `listModels`. Since Unsloth has no health endpoint Crossbar polls via `listModels` + * every 15s → sustained HF noise/latency even for the loaded model. The status endpoint has no + * such fallback. + * + * Vision is therefore known ONLY for the loaded model (its `active_model`). Unloaded models + * stay text-only until they are loaded, when the next `listModels` picks up their real + * modality. Best-effort: ANY failure (older Studio → 404, refused, malformed body) degrades + * to text-only, the conservative pre-fix behaviour. Never throws, never blocks registration. + * + * # Thinking detection (issue #37) + * + * The model catalogue (`/v1/models`, `/api/models/*`) carries no thinking metadata at all. + * Studio knows it only for the LOADED backend: `GET /api/inference/status` reports + * `supports_reasoning`, `reasoning_style`, `reasoning_effort_levels` — plus `active_model`, + * which carries the same public id as the `/v1/models` entry, so the flag can be matched to + * exactly one model (verified against a live instance: `supports_reasoning: true` for a Qwen3 + * `enable_thinking` template). Unloaded models have no clean detection path and stay + * `reasoning: false`. Same best-effort contract as vision: any probe failure degrades to the + * conservative pre-fix behaviour and never throws. + * * Uses ONLY the injected Probe — never calls fetch directly. */ @@ -85,6 +120,18 @@ interface UnslothAutoSwitchSettings { enabled?: unknown; } +/** + * Shape of `GET /api/inference/status` — the loaded-backend status surface. Only the fields + * needed for vision and thinking detection are declared; everything else is ignored. + */ +interface UnslothInferenceStatus { + /** Public id of the loaded model — same namespace as `/v1/models` entries. */ + active_model?: string; + /** Whether the loaded model accepts image input. Trusted only when strictly `true`. */ + is_vision?: boolean; + supports_reasoning?: boolean; +} + /** The literal header value Unsloth Studio sets on every response. */ const SERVER_HEADER_VALUE = "unsloth-studio"; @@ -111,6 +158,9 @@ const AUTO_SWITCH_SETTINGS_PATH = "/api/settings/openai-auto-switch"; const FALLBACK_CONTEXT_WINDOW = 128_000; const FALLBACK_MAX_TOKENS = 0; +/** Loaded-backend status endpoint (thinking metadata; see the file header, issue #37). */ +const INFERENCE_STATUS_PATH = "/api/inference/status"; + function isUnslothStudioResponse(headers: Record): boolean { // Probe lowercases header names AND we compare the value case-insensitively — cheap // insurance against a future casing change upstream, no behavioural cost today. @@ -151,6 +201,41 @@ function contextWindowFor(entry: UnslothModelEntry): number | undefined { ); } +/** + * Read Studio's loaded-backend status: which model is resident (`active_model`), whether it + * accepts image input (`is_vision`), and whether it supports thinking (`supports_reasoning`). + * + * The status endpoint describes the loaded backend only: `active_model` carries the same public + * id as the `/v1/models` entry (verified against a live instance), so the caller can match it + * exactly; when absent, the caller falls back to the `loaded: true` entry. Both vision and + * thinking are known ONLY for the loaded model — unloaded models have no detection path + * (issues #35, #37). Crucially this call never touches huggingface.co, unlike the per-model + * `check-vision` probe (see file header). + * + * Best-effort by contract: any failure (older Studio versions without the endpoint → 404, + * 401, refused connection, malformed body) yields `undefined` — vision and thinking stay off + * everywhere, i.e. the conservative pre-fix behaviour. Never throws. + */ +async function loadedStatus( + probe: Probe, + headers: Record, +): Promise<{ id?: string; vision: boolean; reasoning: boolean } | undefined> { + try { + const r = await probe(INFERENCE_STATUS_PATH, { headers }); + if (!r.ok || r.status !== 200) return undefined; + const body = r.json as UnslothInferenceStatus | undefined; + if (!body) return undefined; + const result: { id?: string; vision: boolean; reasoning: boolean } = { + vision: body.is_vision === true, + reasoning: body.supports_reasoning === true, + }; + if (typeof body.active_model === "string") result.id = body.active_model; + return result; + } catch { + return undefined; + } +} + // --------------------------------------------------------------------------- // UnslothAdapter // --------------------------------------------------------------------------- @@ -212,24 +297,47 @@ class UnslothAdapter implements BackendAdapter { const body = r.json as UnslothModelsResponse | undefined; if (!Array.isArray(body?.data)) return []; - return body.data - .filter((entry): entry is UnslothModelEntry => typeof entry?.id === "string") - .map((entry): ModelDescriptor => { - const contextWindow = contextWindowFor(entry); - const descriptor: ModelDescriptor = { - id: entry.id, - name: entry.display_name ?? entry.id, - input: ["text"], - reasoning: false, - embeddings: isEmbeddingId(entry.id), - loaded: entry.loaded === true, - raw: entry, - }; - // Omitted entirely when unknown, so the cached descriptor never asserts a context - // the server did not report — and picks up the real value once the model is loaded. - if (contextWindow !== undefined) descriptor.contextWindow = contextWindow; - return descriptor; - }); + const entries = body.data.filter( + (entry): entry is UnslothModelEntry => typeof entry?.id === "string", + ); + + // /v1/models carries no modality/thinking info. Both vision and thinking come from ONE + // /api/inference/status call describing the loaded backend — which, unlike the per-model + // check-vision probe, never touches HuggingFace (see file header). Known only for the + // loaded model; unloaded models degrade to text-only / no-thinking. The helper is + // internally defensive and never rejects. + const status = await loadedStatus(probe, headers); + + // The status endpoint describes exactly one model. Match it by active_model id; fall back + // to the `loaded: true` entry when it names nothing. + const isLoadedTarget = (entry: UnslothModelEntry): boolean => + status === undefined + ? false + : status.id === undefined + ? entry.loaded === true + : entry.id === status.id; + + return entries.map((entry): ModelDescriptor => { + const contextWindow = contextWindowFor(entry); + const target = isLoadedTarget(entry); + // Vision known only for the loaded model (issue #35); unloaded → text-only. + const isVision = target && status?.vision === true; + // Thinking metadata exists only for the loaded model (issue #37). + const isReasoning = target && status?.reasoning === true; + const descriptor: ModelDescriptor = { + id: entry.id, + name: entry.display_name ?? entry.id, + input: isVision ? ["text", "image"] : ["text"], + reasoning: isReasoning, + embeddings: isEmbeddingId(entry.id), + loaded: entry.loaded === true, + raw: entry, + }; + // Omitted entirely when unknown, so the cached descriptor never asserts a context + // the server did not report — and picks up the real value once the model is loaded. + if (contextWindow !== undefined) descriptor.contextWindow = contextWindow; + return descriptor; + }); } // --- introspectLoaded ---------------------------------------------------------------------- diff --git a/tests/adapters/unsloth.fixture.ts b/tests/adapters/unsloth.fixture.ts index 93246b7..c14685e 100644 --- a/tests/adapters/unsloth.fixture.ts +++ b/tests/adapters/unsloth.fixture.ts @@ -24,6 +24,11 @@ import { unslothAdapter } from "../../src/adapters/unsloth.ts"; const LOADED_MODEL_ID = "unsloth/Qwen3.8-27B-GGUF"; const UNLOADED_MODEL_ID = "Qwen3.6-35B-A3B-UD-Q4_K_XL"; const EMBED_ID = "nomic-embed-text-v1.5"; +/** + * A second loaded entry that is NOT the status endpoint's `active_model` — must stay + * text-only / no-thinking, since both capabilities are known only for the active model. + */ +const VLM_ID = "unsloth/Qwen2.5-VL-7B-Instruct-GGUF"; /** Captured verbatim: `GET /v1/models` with no (or an invalid) Authorization header. */ const UNAUTHENTICATED_RESPONSE: ProbeResult = { @@ -72,10 +77,39 @@ const AUTHENTICATED_RESPONSE: ProbeResult = { loaded: false, display_name: EMBED_ID, }, + { + // Loaded but not active — see the VLM_ID note above. + id: VLM_ID, + object: "model", + owned_by: "unsloth-studio", + loaded: true, + display_name: VLM_ID, + }, ], }, }; +/** + * Studio's loaded-backend status (`GET /api/inference/status`) — the single source for BOTH + * vision (#35) and thinking (#37) metadata, and the only capability probe this adapter issues + * (deliberately NOT `/api/models/check-vision/{id}`, whose HF fallback makes Studio hit + * huggingface.co). Reports the fixture's active model as a reasoning-capable TEXT model; + * `active_model` carries the same public id as the `/v1/models` entry. + */ +const INFERENCE_STATUS_RESPONSE: ProbeResult = { + status: 200, + ok: true, + headers: { server: "unsloth-studio", "content-type": "application/json" }, + json: { + active_model: LOADED_MODEL_ID, + is_vision: false, + supports_reasoning: true, + reasoning_style: "enable_thinking_effort", + reasoning_effort_levels: ["low", "medium", "high", "xhigh"], + reasoning_always_on: false, + }, +}; + /** * Real Unsloth Studio behaviour: `GET /v1/models` 401s without a bearer token and 200s with * one, but sets the SAME `Server: unsloth-studio` header either way. `fingerprint()` calls the @@ -141,6 +175,7 @@ export const unslothFixture: AdapterFixture = { routes: { "/v1/models": MODELS_ROUTE, "/api/settings/openai-auto-switch": AUTO_SWITCH_OFF_RESPONSE, + "/api/inference/status": INFERENCE_STATUS_RESPONSE, }, negativeRoutes: NEGATIVE_ROUTES, @@ -153,9 +188,9 @@ export const unslothFixture: AdapterFixture = { confidenceMax: 1.0, }, models: { - includedIds: [LOADED_MODEL_ID, UNLOADED_MODEL_ID], + includedIds: [LOADED_MODEL_ID, UNLOADED_MODEL_ID, VLM_ID], excludedIds: [EMBED_ID], - minCount: 2, + minCount: 3, }, loadedState: { anyOf: [LOADED_MODEL_ID], diff --git a/tests/adapters/unsloth.test.ts b/tests/adapters/unsloth.test.ts index 9c30f29..1dac0d7 100644 --- a/tests/adapters/unsloth.test.ts +++ b/tests/adapters/unsloth.test.ts @@ -11,7 +11,7 @@ import { describe, it, expect } from "vitest"; import { runConformance } from "../conformance/run-conformance.ts"; -import { createFakeProbe } from "../conformance/fake-probe.ts"; +import { createFakeProbe, type RouteMap } from "../conformance/fake-probe.ts"; import { unslothAdapter } from "../../src/adapters/unsloth.ts"; import { unslothFixture } from "./unsloth.fixture.ts"; import { Capability } from "../../src/core/capability.ts"; @@ -198,6 +198,294 @@ describe("[unsloth] adapter-specific", () => { }); }); + // Regression (issue #35): the adapter used to hardcode input: ["text"] for every model + // because /v1/models carries no modality info — Pi then silently replaced attached images + // with "(image omitted: model does not support images)", so VLMs appeared to have no vision. + describe("vision detection (issue #35)", () => { + const server = { + kind: "unsloth" as const, + baseUrl: "http://127.0.0.1:8888", + auth: "apiKey" as const, + label: "Unsloth Studio", + confidence: 0.95, + }; + const cred = { mode: "apiKey" as const, apiKey: "sk-unsloth-test-key" }; + + // Vision is probed ONLY for loaded models (unloaded ones would make Studio hit HF), so + // every model under test here is marked loaded to exercise the probe path. + // Vision comes from /api/inference/status (the loaded backend), NOT from a per-model + // check-vision probe (which would make Studio hit huggingface.co). The status endpoint + // describes exactly one loaded model, matched by active_model. + const visionProbe = (id: string, isVision: boolean) => + createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id, owned_by: "unsloth-studio", loaded: true }] }, + }, + "/api/inference/status": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { active_model: id, is_vision: isVision }, + }, + }); + + it("registers a model as [text, image] when status reports is_vision: true", async () => { + const models = await unslothAdapter.listModels( + server, + cred, + visionProbe("unsloth/Qwen2.5-VL-7B-Instruct-GGUF", true), + ); + expect(models).toHaveLength(1); + expect(models[0]?.input).toEqual(["text", "image"]); + }); + + it("keeps text models at [text] when status reports is_vision: false", async () => { + const models = await unslothAdapter.listModels( + server, + cred, + visionProbe("unsloth/Qwen3.8-27B-GGUF", false), + ); + expect(models[0]?.input).toEqual(["text"]); + }); + + it("passes the image modality through toPiModel", () => { + const entry = unslothAdapter.toPiModel(server, { + id: "vlm", + name: "vlm", + input: ["text", "image"], + }); + expect(entry.input).toEqual(["text", "image"]); + }); + + it("does NOT probe unloaded models — they stay text-only even if the backend is a VLM", async () => { + const requestedPaths: string[] = []; + const probe: Probe = async (path) => { + requestedPaths.push(path); + if (path === "/v1/models") { + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "unloaded-vlm", owned_by: "unsloth-studio", loaded: false }] }, + }; + } + // If this were ever hit it would report a VLM — the test asserts check-vision is NOT hit + // and that status names a DIFFERENT (loaded) model, so the unloaded one stays text-only. + if (path === "/api/inference/status") { + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { active_model: "some-other-loaded-model", is_vision: true }, + }; + } + return { status: 0, ok: false, headers: {}, json: null }; + }; + const models = await unslothAdapter.listModels(server, cred, probe); + expect(models[0]?.input).toEqual(["text"]); + // The whole point of the fix: no check-vision request at all (no HF hit). + expect(requestedPaths.some((p) => p.startsWith("/api/models/check-vision/"))).toBe(false); + }); + + it("degrades to [text] when the status endpoint is missing (older Studio → status 0)", async () => { + const probe = createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "some-vlm", owned_by: "unsloth-studio", loaded: true }] }, + }, + }); // no /api/inference/status fixture → refused connection + const models = await unslothAdapter.listModels(server, cred, probe); + expect(models[0]?.input).toEqual(["text"]); + }); + + it("degrades to [text] on 401 / non-200 / malformed bodies — never throws", async () => { + for (const json of [ + { active_model: "m", is_vision: "yes" }, // not a boolean + { active_model: "m", other: true }, + undefined, + ]) { + const probe = createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "m", owned_by: "unsloth-studio", loaded: true }] }, + }, + "/api/inference/status": + json === undefined + ? { status: 401, ok: false, headers: { server: "unsloth-studio" }, json: null } + : { status: 200, ok: true, headers: { server: "unsloth-studio" }, json }, + }); + const models = await unslothAdapter.listModels(server, cred, probe); + expect(models[0]?.input).toEqual(["text"]); + } + }); + + it("reads vision from /api/inference/status and sends the bearer key — never check-vision", async () => { + const seenPaths: string[] = []; + let statusHeaders: Record | undefined; + const probe: Probe = async (path, init) => { + seenPaths.push(path); + if (path === "/v1/models") { + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { + data: [{ id: "unsloth/Qwen2.5-VL-7B-Instruct-GGUF", owned_by: "unsloth-studio", loaded: true }], + }, + }; + } + statusHeaders = init?.headers; + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { active_model: "unsloth/Qwen2.5-VL-7B-Instruct-GGUF", is_vision: true }, + }; + }; + const models = await unslothAdapter.listModels(server, cred, probe); + expect(seenPaths).toContain("/api/inference/status"); + expect(seenPaths.some((p) => p.startsWith("/api/models/check-vision/"))).toBe(false); + expect(statusHeaders?.["Authorization"]).toBe("Bearer sk-unsloth-test-key"); + expect(models[0]?.input).toEqual(["text", "image"]); + }); + }); + + // Regression (issue #37): the adapter used to hardcode reasoning: false for every model. + // Studio exposes thinking metadata only for the LOADED model, via GET /api/inference/status + // (supports_reasoning + active_model carrying the same public id as /v1/models). + describe("thinking detection (issue #37)", () => { + const server = { + kind: "unsloth" as const, + baseUrl: "http://127.0.0.1:8888", + auth: "apiKey" as const, + label: "Unsloth Studio", + confidence: 0.95, + }; + const cred = { mode: "apiKey" as const, apiKey: "sk-unsloth-test-key" }; + + const thinkingProbe = (statusJson: unknown) => + createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { + data: [ + { id: "thinker", owned_by: "unsloth-studio", loaded: true }, + { id: "plain", owned_by: "unsloth-studio", loaded: false }, + ], + }, + }, + "/api/inference/status": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: statusJson, + }, + }); + + it("registers the loaded model as reasoning-capable, matched by active_model", async () => { + const models = await unslothAdapter.listModels( + server, + cred, + thinkingProbe({ active_model: "thinker", supports_reasoning: true }), + ); + expect(models.find((m) => m.id === "thinker")?.reasoning).toBe(true); + expect(models.find((m) => m.id === "plain")?.reasoning).toBe(false); + }); + + it("falls back to the loaded entry when active_model is absent", async () => { + const models = await unslothAdapter.listModels( + server, + cred, + thinkingProbe({ supports_reasoning: true }), + ); + expect(models.find((m) => m.id === "thinker")?.reasoning).toBe(true); + expect(models.find((m) => m.id === "plain")?.reasoning).toBe(false); + }); + + it("applies nothing when supports_reasoning is false or malformed", async () => { + for (const json of [{ active_model: "thinker", supports_reasoning: false }, { supports_reasoning: "yes" }]) { + const models = await unslothAdapter.listModels(server, cred, thinkingProbe(json)); + expect(models.every((m) => m.reasoning === false)).toBe(true); + } + }); + + it("degrades to all-false when the endpoint is missing (older Studio) or fails — never throws", async () => { + // No /api/inference/status fixture → refused connection (status 0). + const probe = createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "thinker", owned_by: "unsloth-studio", loaded: true }] }, + }, + }); + const models = await unslothAdapter.listModels(server, cred, probe); + expect(models[0]?.reasoning).toBe(false); + + for (const status of [401, 404] as const) { + const failingProbe = createFakeProbe({ + "/v1/models": { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "thinker", owned_by: "unsloth-studio", loaded: true }] }, + }, + "/api/inference/status": { + status, + ok: false, + headers: { server: "unsloth-studio" }, + json: null, + }, + }); + const failingModels = await unslothAdapter.listModels(server, cred, failingProbe); + expect(failingModels[0]?.reasoning).toBe(false); + } + }); + + it("sends the bearer key with the status request", async () => { + let seenHeaders: Record | undefined; + const probe: Probe = async (path, init) => { + if (path === "/v1/models") { + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { data: [{ id: "thinker", owned_by: "unsloth-studio", loaded: true }] }, + }; + } + seenHeaders = init?.headers; + return { + status: 200, + ok: true, + headers: { server: "unsloth-studio" }, + json: { active_model: "thinker", supports_reasoning: true }, + }; + }; + const models = await unslothAdapter.listModels(server, cred, probe); + expect(seenHeaders?.["Authorization"]).toBe("Bearer sk-unsloth-test-key"); + expect(models[0]?.reasoning).toBe(true); + }); + + it("passes reasoning through toPiModel", () => { + const entry = unslothAdapter.toPiModel(server, { + id: "thinker", + name: "thinker", + input: ["text"], + reasoning: true, + }); + expect(entry.reasoning).toBe(true); + }); + }); + it("inferenceBaseUrl appends /v1 exactly once", () => { const server = { kind: "unsloth" as const, From 65878e33b0c089a7cb7c09ee72c202edd480aac4 Mon Sep 17 00:00:00 2001 From: Hauke Walden Date: Mon, 21 Sep 2026 21:06:32 +0200 Subject: [PATCH 2/2] fix(llamacpp): detect vision capability from /props modalities object map llama-server reports /props modalities as an object map of the loaded model's capabilities (e.g. { vision: true, video: true, audio: false }), but the adapter expected a string array, so Array.isArray(...) was always false and every llama.cpp model was advertised to pi as text-only - even with --mmproj loaded. The llama-server web UI reads the same field correctly, which is why image uploads worked there but not in pi. Detect vision for both the object map (current servers, verified on b10709 and b10840) and legacy string-array payloads, and align the conformance fixture with the live object-map shape. --- CAPABILITY-MATRIX.md | 2 +- src/adapters/llamacpp.ts | 32 +++++++++++++++++++-- tests/adapters/llamacpp.fixture.ts | 3 +- tests/adapters/llamacpp.test.ts | 46 ++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/CAPABILITY-MATRIX.md b/CAPABILITY-MATRIX.md index 2f59617..68299c4 100644 --- a/CAPABILITY-MATRIX.md +++ b/CAPABILITY-MATRIX.md @@ -8,7 +8,7 @@ adapter registers under (`oai` = `openai-completions`, `ant` = `anthropic-messag |---|---|---|---|---|---|---|---|---|---|---|---|---| | **Ollama** | 11434 | oai | ✅ `/api/tags`,`/v1/models` | ✅ `/api/ps` | ✅ implicit (request id) | ✅ `keep_alive:0` | ❌ (always on) | ◐ none local | ✅ `GET /` text | ✅ `/api/show` caps + ctx | ✅ | `GET /` → `Ollama is running` | | **LM Studio** | 1234 | oai | ✅ `/api/v1/models` (v0 fallback) | ✅ `state` field | ✅ JIT + `/api/v1/models/load` | ✅ load/unload + `lms` | ❌ (always on) | ◐ Bearer, none default | ◐ infer 200 | ✅ type+`max_context_length` | ✅ | `/api/v1/models` (v0 fallback) w/ `state`,`compatibility_type` | -| **llama-server** | 8080 | oai | ✅ `/v1/models` | ◐ `/props`,`/slots` (single) | ❌ (1/instance) | ❌ classic | ❌ (always on) | ◐ none / `--api-key` | ✅ `/health` | ◐ ctx via `meta.n_ctx`, router args, `/props` | ✅ | `/props` w/ `default_generation_settings`+`build_info` | +| **llama-server** | 8080 | oai | ✅ `/v1/models` | ◐ `/props`,`/slots` (single) | ❌ (1/instance) | ❌ classic | ❌ (always on) | ◐ none / `--api-key` | ✅ `/health` | ◐ ctx via `meta.n_ctx`, router args, `/props`; vision via `/props` `modalities` | ✅ | `/props` w/ `default_generation_settings`+`build_info` | | **llama-swap** | 8080 | oai/ant | ✅ `/v1/models` (all config) | ✅ `/running` | ✅ via `model` → restart upstream | ✅ `/api/models/unload`, ttl | ❌ (always on) | ◐ optional multi-scheme | ✅ `/health`→OK | ◐ ctx via `context_length`; output unknown | ✅ | `/` → `/ui/`; `/running`,`/upstream/{model}` | | **vLLM** | 8000 | oai | ✅ `/v1/models` | ◐ `/is_sleeping` (dev) | ❌ base · ◐ LoRA | ◐ sleep/wake + LoRA | ❌ | ◐ none / `--api-key` | ✅ `/health` | ◐ `max_model_len` only | ✅ | `/version` + `/metrics` `vllm:` + `owned_by:"vllm"` | | **OpenAI** | cloud | oai | ✅ `/v1/models` | ❌ | ✅ (pick id) | ❌ managed | ❌ | ✅ Bearer | ❌ (status page) | ❌ (static table needed) | ✅ | n/a (configured, not probed) | diff --git a/src/adapters/llamacpp.ts b/src/adapters/llamacpp.ts index 77a10dd..0c26f3c 100644 --- a/src/adapters/llamacpp.ts +++ b/src/adapters/llamacpp.ts @@ -34,7 +34,12 @@ interface PropsBody { } | null; build_info?: unknown; model_path?: string; - modalities?: string[]; + /** + * Modalities of the loaded model. Current llama-server reports an object + * map with boolean values (e.g. `{ vision: true, video: true, audio: false }`); + * legacy string-array payloads (`["text","image"]`) are tolerated as well. + */ + modalities?: string[] | Record; } interface V1ModelsBody { @@ -101,6 +106,28 @@ function positiveIntegerArg( return found; } +/** + * Detect vision capability from `/props` modalities. Current llama-server + * reports an object map keyed by modality with boolean values + * (e.g. `{ vision: true, video: true, audio: false }`); legacy payloads may + * use a string array (`["text","image"]`). Both shapes are accepted. + */ +function hasVisionModality(modalities: PropsBody["modalities"]): boolean { + if (!modalities) return false; + if (Array.isArray(modalities)) { + return modalities.some( + (m) => + typeof m === "string" && + (m.toLowerCase().includes("vision") || m.toLowerCase().includes("image")), + ); + } + return Object.keys(modalities).some( + (key) => + (key.toLowerCase().includes("vision") || key.toLowerCase().includes("image")) && + Boolean(modalities[key]), + ); +} + // --------------------------------------------------------------------------- // LlamacppAdapter // --------------------------------------------------------------------------- @@ -185,8 +212,7 @@ class LlamacppAdapter implements BackendAdapter { propsParams?.max_tokens, props?.default_generation_settings?.n_predict, ); - const hasVision = Array.isArray(props?.modalities) && - props.modalities.some((m) => m.toLowerCase().includes("vision") || m.toLowerCase().includes("image")); + const hasVision = hasVisionModality(props?.modalities); return data.map((entry) => { const isRouterRow = entry.status != null; diff --git a/tests/adapters/llamacpp.fixture.ts b/tests/adapters/llamacpp.fixture.ts index 07db316..c05d766 100644 --- a/tests/adapters/llamacpp.fixture.ts +++ b/tests/adapters/llamacpp.fixture.ts @@ -40,7 +40,8 @@ const PROPS_RESPONSE: ProbeResult = { compiler: "gcc-13", }, model_path: MODEL_PATH, - modalities: ["text"], + // Current llama-server shape: object map of the loaded model's modalities. + modalities: { vision: false }, }, }; diff --git a/tests/adapters/llamacpp.test.ts b/tests/adapters/llamacpp.test.ts index e80ce9d..548c79b 100644 --- a/tests/adapters/llamacpp.test.ts +++ b/tests/adapters/llamacpp.test.ts @@ -271,6 +271,52 @@ describe("llama.cpp reported model limits", () => { }); }); +describe("llama.cpp vision detection (/props modalities)", () => { + it.each([ + { + name: "object map with vision enabled (live llama-server shape)", + props: { modalities: { vision: true, video: true, audio: false } }, + expected: ["text", "image"], + }, + { + name: "object map with vision disabled", + props: { modalities: { vision: false, video: false, audio: false } }, + expected: ["text"], + }, + { + name: "object map using an image key", + props: { modalities: { image: true } }, + expected: ["text", "image"], + }, + { + name: "legacy string array with an image entry", + props: { modalities: ["text", "image"] }, + expected: ["text", "image"], + }, + { + name: "legacy string array with a vision entry", + props: { modalities: ["text", "vision"] }, + expected: ["text", "image"], + }, + { + name: "text-only string array", + props: { modalities: ["text"] }, + expected: ["text"], + }, + { + name: "missing modalities field", + props: {}, + expected: ["text"], + }, + ])("$name", async ({ props, expected }) => { + const [model] = await listModels( + [{ id: "vision-check", meta: null, status: null }], + props, + ); + expect(model?.input).toEqual(expected); + }); +}); + describe("llama.cpp Pi model limits", () => { it("uses the Pi-boundary sentinels for absent or non-positive descriptor limits", () => { const absent = llamacppAdapter.toPiModel(SERVER, {