fix(unsloth): detect vision and thinking for the loaded model via /api/inference/status

CAPABILITY-MATRIX.md

@@ -18,7 +18,7 @@ adapter registers under (`oai` = `openai-completions`, `ant` = `anthropic-messag
 | **oobabooga** | 5000 | oai | ✅ `/v1/models` | ✅ `/v1/internal/model/info` | ✅ load | ✅ `/v1/internal/model/{load,unload}` | ❌ | ◐ `--api-key` | ◐ | ◐ | ✅ | `/v1/internal/*` namespace |
 | **Jan** | 1337 | oai | ✅ `/v1/models` | ◐ | ◐ engine | ◐ engine | ❌ | ◐ Bearer | ❌ | ◐ | ✅ | weak (log line) |
 | **llamafile** | 8080 | oai | ✅ `/v1/models` | ◐ `/props` | ❌ | ❌ | ❌ (always on) | ◐ `--api-key` | ✅ `/health` | ◐ via `/props` | ✅ | `/props` w/ non-`bNNNN` build_info |
-| **Unsloth Studio** | 8888 | oai | ✅ `/v1/models` (keyed) | ✅ per-model `loaded` field | ❌ (UI-only) | ❌ (UI-only) | ✅ `/api/settings/openai-auto-switch` | ✅ Bearer `sk-unsloth-…` (required) | ❌ (poll via listModels) | ◐ ctx fields only while loaded | ✅ | `Server: unsloth-studio` header (every response) |
+| **Unsloth Studio** | 8888 | oai | ✅ `/v1/models` (keyed) | ✅ per-model `loaded` field | ❌ (UI-only) | ❌ (UI-only) | ✅ `/api/settings/openai-auto-switch` | ✅ Bearer `sk-unsloth-…` (required) | ❌ (poll via listModels) | ◐ ctx fields only while loaded; vision + reasoning for the active model via `/api/inference/status` (#35, #37) — both best-effort | ✅ | `Server: unsloth-studio` header (every response) |
 | **generic OpenAI-compat** | varies | oai | ✅ `/v1/models` | ❌ | ❌ | ❌ | ❌ | ◐ optional Bearer | ◐ | ◐ | ✅ | anything serving `/v1/models` (fallback) |
 
 ## Capability-driven UX rules (derived)

src/adapters/unsloth.ts

@@ -40,6 +40,41 @@
  * `owned_by: "unsloth-studio"` and a `loaded: boolean` residency flag, used below for
  * IntrospectLoaded.
  *
+ * # Vision detection (issue #35)
+ *
+ * The OpenAI-compatible `GET /v1/models` exposes NO modality information — entries carry only
+ * `id`, `owned_by`, `quant`, context-length fields and `loaded`. Registering every model as
+ * text-only (the old behaviour) made Pi silently replace attached images with the placeholder
+ * `(image omitted: model does not support images)`, so VLMs appeared to "have no vision".
+ *
+ * Vision is read from the SAME loaded-backend status call used for thinking:
+ * `GET /api/inference/status` reports `is_vision: boolean` for the resident model (verified
+ * against a live instance: `is_vision: true` for a loaded Qwen3-VL). This is local and instant
+ * — it never touches huggingface.co.
+ *
+ * We deliberately do NOT use `GET /api/models/check-vision/{model_name}`: for a model whose id
+ * is not a resolvable HF repo (e.g. a local GGUF quant name like `Qwen3.8-27B-IQ4_NL`) Studio's
+ * handler falls back to fetching `config.json` from huggingface.co and 401s in a retry loop on
+ * EVERY `listModels`. Since Unsloth has no health endpoint Crossbar polls via `listModels`
+ * every 15s → sustained HF noise/latency even for the loaded model. The status endpoint has no
+ * such fallback.
+ *
+ * Vision is therefore known ONLY for the loaded model (its `active_model`). Unloaded models
+ * stay text-only until they are loaded, when the next `listModels` picks up their real
+ * modality. Best-effort: ANY failure (older Studio → 404, refused, malformed body) degrades
+ * to text-only, the conservative pre-fix behaviour. Never throws, never blocks registration.
+ *
+ * # Thinking detection (issue #37)
+ *
+ * The model catalogue (`/v1/models`, `/api/models/*`) carries no thinking metadata at all.
+ * Studio knows it only for the LOADED backend: `GET /api/inference/status` reports
+ * `supports_reasoning`, `reasoning_style`, `reasoning_effort_levels` — plus `active_model`,
+ * which carries the same public id as the `/v1/models` entry, so the flag can be matched to
+ * exactly one model (verified against a live instance: `supports_reasoning: true` for a Qwen3
+ * `enable_thinking` template). Unloaded models have no clean detection path and stay
+ * `reasoning: false`. Same best-effort contract as vision: any probe failure degrades to the
+ * conservative pre-fix behaviour and never throws.
+ *
  * Uses ONLY the injected Probe — never calls fetch directly.
  */
 
@@ -85,6 +120,18 @@ interface UnslothAutoSwitchSettings {
   enabled?: unknown;
 }
 
+/**
+ * Shape of `GET /api/inference/status` — the loaded-backend status surface. Only the fields
+ * needed for vision and thinking detection are declared; everything else is ignored.
+ */
+interface UnslothInferenceStatus {
+  /** Public id of the loaded model — same namespace as `/v1/models` entries. */
+  active_model?: string;
+  /** Whether the loaded model accepts image input. Trusted only when strictly `true`. */
+  is_vision?: boolean;
+  supports_reasoning?: boolean;
+}
+
 /** The literal header value Unsloth Studio sets on every response. */
 const SERVER_HEADER_VALUE = "unsloth-studio";
 
@@ -111,6 +158,9 @@ const AUTO_SWITCH_SETTINGS_PATH = "/api/settings/openai-auto-switch";
 const FALLBACK_CONTEXT_WINDOW = 128_000;
 const FALLBACK_MAX_TOKENS = 0;
 
+/** Loaded-backend status endpoint (thinking metadata; see the file header, issue #37). */
+const INFERENCE_STATUS_PATH = "/api/inference/status";
+
 function isUnslothStudioResponse(headers: Record<string, string>): boolean {
   // Probe lowercases header names AND we compare the value case-insensitively — cheap
   // insurance against a future casing change upstream, no behavioural cost today.
@@ -151,6 +201,41 @@ function contextWindowFor(entry: UnslothModelEntry): number | undefined {
   );
 }
 
+/**
+ * Read Studio's loaded-backend status: which model is resident (`active_model`), whether it
+ * accepts image input (`is_vision`), and whether it supports thinking (`supports_reasoning`).
+ *
+ * The status endpoint describes the loaded backend only: `active_model` carries the same public
+ * id as the `/v1/models` entry (verified against a live instance), so the caller can match it
+ * exactly; when absent, the caller falls back to the `loaded: true` entry. Both vision and
+ * thinking are known ONLY for the loaded model — unloaded models have no detection path
+ * (issues #35, #37). Crucially this call never touches huggingface.co, unlike the per-model
+ * `check-vision` probe (see file header).
+ *
+ * Best-effort by contract: any failure (older Studio versions without the endpoint → 404,
+ * 401, refused connection, malformed body) yields `undefined` — vision and thinking stay off
+ * everywhere, i.e. the conservative pre-fix behaviour. Never throws.
+ */
+async function loadedStatus(
+  probe: Probe,
+  headers: Record<string, string>,
+): Promise<{ id?: string; vision: boolean; reasoning: boolean } | undefined> {
+  try {
+    const r = await probe(INFERENCE_STATUS_PATH, { headers });
+    if (!r.ok || r.status !== 200) return undefined;
+    const body = r.json as UnslothInferenceStatus | undefined;
+    if (!body) return undefined;
+    const result: { id?: string; vision: boolean; reasoning: boolean } = {
+      vision: body.is_vision === true,
+      reasoning: body.supports_reasoning === true,
+    };
+    if (typeof body.active_model === "string") result.id = body.active_model;
+    return result;
+  } catch {
+    return undefined;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // UnslothAdapter
 // ---------------------------------------------------------------------------
@@ -212,24 +297,47 @@ class UnslothAdapter implements BackendAdapter {
     const body = r.json as UnslothModelsResponse | undefined;
     if (!Array.isArray(body?.data)) return [];
 
-    return body.data
-      .filter((entry): entry is UnslothModelEntry => typeof entry?.id === "string")
-      .map((entry): ModelDescriptor => {
-        const contextWindow = contextWindowFor(entry);
-        const descriptor: ModelDescriptor = {
-          id: entry.id,
-          name: entry.display_name ?? entry.id,
-          input: ["text"],
-          reasoning: false,
-          embeddings: isEmbeddingId(entry.id),
-          loaded: entry.loaded === true,
-          raw: entry,
-        };
-        // Omitted entirely when unknown, so the cached descriptor never asserts a context
-        // the server did not report — and picks up the real value once the model is loaded.
-        if (contextWindow !== undefined) descriptor.contextWindow = contextWindow;
-        return descriptor;
-      });
+    const entries = body.data.filter(
+      (entry): entry is UnslothModelEntry => typeof entry?.id === "string",
+    );
+
+    // /v1/models carries no modality/thinking info. Both vision and thinking come from ONE
+    // /api/inference/status call describing the loaded backend — which, unlike the per-model
+    // check-vision probe, never touches HuggingFace (see file header). Known only for the
+    // loaded model; unloaded models degrade to text-only / no-thinking. The helper is
+    // internally defensive and never rejects.
+    const status = await loadedStatus(probe, headers);
+
+    // The status endpoint describes exactly one model. Match it by active_model id; fall back
+    // to the `loaded: true` entry when it names nothing.
+    const isLoadedTarget = (entry: UnslothModelEntry): boolean =>
+      status === undefined
+        ? false
+        : status.id === undefined
+          ? entry.loaded === true
+          : entry.id === status.id;
+
+    return entries.map((entry): ModelDescriptor => {
+      const contextWindow = contextWindowFor(entry);
+      const target = isLoadedTarget(entry);
+      // Vision known only for the loaded model (issue #35); unloaded → text-only.
+      const isVision = target && status?.vision === true;
+      // Thinking metadata exists only for the loaded model (issue #37).
+      const isReasoning = target && status?.reasoning === true;
+      const descriptor: ModelDescriptor = {
+        id: entry.id,
+        name: entry.display_name ?? entry.id,
+        input: isVision ? ["text", "image"] : ["text"],
+        reasoning: isReasoning,
+        embeddings: isEmbeddingId(entry.id),
+        loaded: entry.loaded === true,
+        raw: entry,
+      };
+      // Omitted entirely when unknown, so the cached descriptor never asserts a context
+      // the server did not report — and picks up the real value once the model is loaded.
+      if (contextWindow !== undefined) descriptor.contextWindow = contextWindow;
+      return descriptor;
+    });
   }
 
   // --- introspectLoaded ----------------------------------------------------------------------

tests/adapters/unsloth.fixture.ts

@@ -24,6 +24,11 @@ import { unslothAdapter } from "../../src/adapters/unsloth.ts";
 const LOADED_MODEL_ID = "unsloth/Qwen3.8-27B-GGUF";
 const UNLOADED_MODEL_ID = "Qwen3.6-35B-A3B-UD-Q4_K_XL";
 const EMBED_ID = "nomic-embed-text-v1.5";
+/**
+ * A second loaded entry that is NOT the status endpoint's `active_model` — must stay
+ * text-only / no-thinking, since both capabilities are known only for the active model.
+ */
+const VLM_ID = "unsloth/Qwen2.5-VL-7B-Instruct-GGUF";
 
 /** Captured verbatim: `GET /v1/models` with no (or an invalid) Authorization header. */
 const UNAUTHENTICATED_RESPONSE: ProbeResult = {
@@ -72,10 +77,39 @@ const AUTHENTICATED_RESPONSE: ProbeResult = {
         loaded: false,
         display_name: EMBED_ID,
       },
+      {
+        // Loaded but not active — see the VLM_ID note above.
+        id: VLM_ID,
+        object: "model",
+        owned_by: "unsloth-studio",
+        loaded: true,
+        display_name: VLM_ID,
+      },
     ],
   },
 };
 
+/**
+ * Studio's loaded-backend status (`GET /api/inference/status`) — the single source for BOTH
+ * vision (#35) and thinking (#37) metadata, and the only capability probe this adapter issues
+ * (deliberately NOT `/api/models/check-vision/{id}`, whose HF fallback makes Studio hit
+ * huggingface.co). Reports the fixture's active model as a reasoning-capable TEXT model;
+ * `active_model` carries the same public id as the `/v1/models` entry.
+ */
+const INFERENCE_STATUS_RESPONSE: ProbeResult = {
+  status: 200,
+  ok: true,
+  headers: { server: "unsloth-studio", "content-type": "application/json" },
+  json: {
+    active_model: LOADED_MODEL_ID,
+    is_vision: false,
+    supports_reasoning: true,
+    reasoning_style: "enable_thinking_effort",
+    reasoning_effort_levels: ["low", "medium", "high", "xhigh"],
+    reasoning_always_on: false,
+  },
+};
+
 /**
  * Real Unsloth Studio behaviour: `GET /v1/models` 401s without a bearer token and 200s with
  * one, but sets the SAME `Server: unsloth-studio` header either way. `fingerprint()` calls the
@@ -141,6 +175,7 @@ export const unslothFixture: AdapterFixture = {
   routes: {
     "/v1/models": MODELS_ROUTE,
     "/api/settings/openai-auto-switch": AUTO_SWITCH_OFF_RESPONSE,
+    "/api/inference/status": INFERENCE_STATUS_RESPONSE,
   },
 
   negativeRoutes: NEGATIVE_ROUTES,
@@ -153,9 +188,9 @@ export const unslothFixture: AdapterFixture = {
       confidenceMax: 1.0,
     },
     models: {
-      includedIds: [LOADED_MODEL_ID, UNLOADED_MODEL_ID],
+      includedIds: [LOADED_MODEL_ID, UNLOADED_MODEL_ID, VLM_ID],
       excludedIds: [EMBED_ID],
-      minCount: 2,
+      minCount: 3,
     },
     loadedState: {
       anyOf: [LOADED_MODEL_ID],