From 5ab0968916400cbd0972cf85e7a6dad5b1c61e51 Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Fri, 13 Nov 2026 05:44:35 +0100 Subject: [PATCH 1/2] fix(ai): prioritize Claude 5.5 and keep HF as final fallback (UB-178) --- .env.example | 27 +++-- CLAUDE.md | 12 +- src/config/env.test.ts | 12 +- src/config/env.ts | 8 +- src/services/ai/model-presets.test.ts | 14 ++- src/services/ai/model-presets.ts | 9 +- src/services/ai/provider-ids.ts | 2 +- src/services/ai/streaming.breaker.test.ts | 114 +++++++++--------- src/services/ai/streaming.chain-skips.test.ts | 12 +- src/services/ai/streaming.chains.test.ts | 65 ++++++---- .../ai/streaming.claude-backup.test.ts | 52 ++++++-- src/services/ai/streaming.fallback.test.ts | 80 ++++++------ src/services/ai/streaming.requests.test.ts | 74 +++++++++--- src/services/ai/streaming.test-support.ts | 4 +- src/services/ai/streaming.usage.test.ts | 84 ++++++------- src/services/ai/streaming/chains.ts | 28 +++-- src/services/ai/streaming/request-params.ts | 25 ++++ src/services/ai/streaming/slot.ts | 24 +++- src/services/ai/usage-report.test.ts | 2 +- 19 files changed, 401 insertions(+), 247 deletions(-) diff --git a/.env.example b/.env.example index 3ed5689..ae03ab8 100644 --- a/.env.example +++ b/.env.example @@ -31,17 +31,19 @@ NODE_ENV=development # AI providers — all OpenAI-compatible (via OpenAI SDK) # # Chains: -# SMART_CHAIN: z.ai AI_MODEL → Claude CLAUDE_MODEL → HF HF_MODEL -# FAST_CHAIN: z.ai AI_FAST_MODEL → Claude CLAUDE_FAST_MODEL → HF HF_FAST_MODEL +# SMART_CHAIN: Claude CLAUDE_MODEL → z.ai AI_MODEL → HF HF_MODEL +# FAST_CHAIN: Claude CLAUDE_FAST_MODEL → z.ai AI_FAST_MODEL → HF HF_FAST_MODEL # OCR_CHAIN: Claude CLAUDE_VISION_MODEL → HF HF_VISION_MODEL (fixed order) # # The SMART/FAST order is configuration: AI_SMART_CHAIN / AI_FAST_CHAIN, comma-separated provider ids -# (zai, claude, hf, openrouter). Unset = the default order above. Unknown ids are ignored with a warning. -#AI_SMART_CHAIN=zai,claude,hf -#AI_FAST_CHAIN=zai,claude,hf +# (claude, zai, openrouter, hf). Unset = the default order above. Unknown ids are ignored with a warning. +# Among selected providers Claude is primary and HF is last; other providers keep their order. +# Health can demote Claude behind other priority providers, while HF remains the final reserve. +#AI_SMART_CHAIN=claude,zai,hf +#AI_FAST_CHAIN=claude,zai,hf # # OpenRouter is optional and in no default order: it runs only when named in a chain order above -# (e.g. AI_SMART_CHAIN=zai,openrouter,claude,hf) AND both its key and the chain's model are set. +# (e.g. AI_SMART_CHAIN=claude,zai,openrouter,hf) AND both its key and the chain's model are set. # # z.ai and HF fields are required. Claude is optional: when its key is absent, the slot # is skipped and the rest of the chain runs unchanged (without either CLAUDE_API_TOKEN or CLAUDE_API_TOKEN2, OCR is HF-only). @@ -55,7 +57,7 @@ AI_BASE_URL=https://api.z.ai/api/coding/paas/v4 AI_MODEL=glm-5.3 AI_FAST_MODEL=glm-5.3-flash -# Anthropic Claude via its OpenAI-compatible endpoint (optional; next after z.ai GLM, OCR primary). +# Anthropic Claude via its OpenAI-compatible endpoint (optional key; primary in smart/fast/OCR). # Smart chain uses Sonnet, fast chain (validators, bank prefill, merchant agent) uses Haiku, # receipt OCR uses CLAUDE_VISION_MODEL (default Sonnet: on degraded receipt photos Haiku emitted a # wrong but sum-verified receipt, Sonnet refused instead — measured 2026-10-12). @@ -76,11 +78,12 @@ AI_FAST_MODEL=glm-5.3-flash # CLAUDE_VISION_MODEL=claude-sonnet-5-5 # HuggingFace Router (text fallback + vision fallback) +# Provider routes verified in the 2026-11-13 catalog: tools and structured output supported. HF_TOKEN=hf_XXX HF_BASE_URL=https://router.huggingface.co/v1 -HF_MODEL=Qwen/Qwen3-235B-A22B -HF_FAST_MODEL=meta-llama/Llama-3.3-70B-Instruct -HF_VISION_MODEL=Qwen/Qwen2.5-VL-72B-Instruct +HF_MODEL=Qwen/Qwen3.8-2.4T-A95B:deepinfra +HF_FAST_MODEL=meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra +HF_VISION_MODEL=Qwen/Qwen3.8-Flash-Next-FP8:nscale # Role routing (stage B3; off by default). AI_ROUTING=roles runs agent rounds on per-role chains: # executor = ordinary tool rounds, repair = after the stall guard steps in. Entries are `provider` @@ -90,8 +93,8 @@ HF_VISION_MODEL=Qwen/Qwen2.5-VL-72B-Instruct #AI_ROUTING=legacy #AI_ROUTING_USER_IDS= #AI_PLANNER_CHAIN= -#AI_EXECUTOR_CHAIN=openrouter:glm53-flash-low,openrouter:deepseek41-flash,claude -#AI_REPAIR_CHAIN=openrouter:glm53-low,claude +#AI_EXECUTOR_CHAIN=claude,openrouter:glm53-flash-low,openrouter:deepseek41-flash +#AI_REPAIR_CHAIN=claude,openrouter:glm53-low # Hybrid tool catalog (#203 step B; off by default; same names as HyperCalendarBot). `full` sends # every tool schema in every request; `hybrid` sends an always-on core (8 tools) plus a compact diff --git a/CLAUDE.md b/CLAUDE.md index 85f6697..10d1837 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -202,15 +202,17 @@ Bot has a full AI agent with tool calling, accessible by mentioning `@ExpenseSyn - Every provider **attempt** of `aiStreamRound` writes one `ai_usage_events` row (migration 053; [usage-recorder.ts](src/services/ai/usage-recorder.ts)): `at`, `provider` (`zai`/`claude`/`hf`/`openrouter`; rows written before Groq was dropped carry `groq` and still render), `model`, `chain` (`smart`/`fast`/`ocr`), `purpose` (required `purpose` option of `aiStreamRound`: `agent`, `agent_final`, `agent_batch`, `advice`, `validator`, `advice_validator`, `prefill`, `merchant`, `receipt_parse`, `receipt_summary`, `ocr`, `dev_agent`, `code_review`, `replay`, `probe`), `outcome` (`ok`/`error`/`skipped`), `error_class` + `http_status` for errors (`timeout`, `rate_limit`, `quota`, `auth`, `server`, `empty`, `aborted`, `bad_request`, `other`, derived from the SDK error's status/code/name — never its body), `duration_ms`, token counts and a per-round `request_id` that links a failed attempt to the fallback that answered. Token columns are **NULL when the provider did not report usage — never 0** (a reported 0 stays 0); every request carries `stream_options: { include_usage: true }`, and a provider that answers 400 to it is remembered and retried at once without it (one attempt, one row, tokens unknown). `skipped` rows are the slots a stall handoff excluded (`excludeProviders`) or a local Claude credential cooldown omitted without HTTP; a slot whose key is absent is not in the chain and leaves no row. The insert runs on a later event-loop turn: a storage failure is `logger.warn`-ed and never reaches the answer. No prompts, completions, user ids or chat ids are stored. A daily cron (03:30 UTC, plus once at startup) prunes rows older than 180 days. **Role, preset and cost** (migration 054, stage B4; nullable, appended — never renumber): `role` (`planner`/`executor`/`repair` of a role-routed slot, NULL on legacy-chain rows and rows from before 054), `preset` (registry preset id of a role-routed `provider:preset` slot), `cost_micro_usd` = prompt × price_in + completion × price_out of the preset price table in [model-presets.ts](src/services/ai/model-presets.ts) (`priceUsdPerMTok`, list price, dated `asOf`, valid for its `via` provider; found by provider + model so a legacy chain naming a priced OpenRouter model is priced too). Cost is NULL — never 0 — when the route has no price or a token count was not reported; the reports sum the known rows and print `n/a` / `(?N)` for the unknown ones. `/ai_stats` and `bun run scripts/report-ai-usage.ts` show cost per model and a per-role, per-provider block (p50/p95 latency, tokens, cost; legacy rows are `legacy`) that appears only once some row carries a role. - **Speed telemetry** (migration 055): ok attempts also store `first_token_ms` (request start → first content/tool-call delta) and `generation_ms` (first → last such delta; 0 for a one-delta answer), measured in the slot's stream loop with no extra provider call; error/empty/skipped rows keep both NULL. Tokens per second is NOT stored: `outputTokensPerS` in [usage-report.ts](src/services/ai/usage-report.ts) derives it as `(completion_tokens − reasoning_tokens) × 1000 / generation_ms` (reasoning is billed in completion but not streamed; NULL when a piece is unknown, the window is under 50 ms, or nothing visible is left). `/ai_stats` and `scripts/report-ai-usage.ts [--json]` show p50/p95 first token, p50/p95 tok/s and total latency per `provider:model` and per role, each with `n` = ok rows that had a value (unknown is never counted as 0). Same column and JSON key names as HyperCalendarBot (`firstToken`, `generationSpeed`, `durationRows`). -**Configuration:** All providers go through the OpenAI SDK with a fallback chain in [streaming.ts](src/services/ai/streaming.ts): smart = z.ai GLM → Claude Sonnet → HF; fast (validators, bank prefill, merchant agent) = z.ai GLM fast → Claude Haiku → HF; OCR = Claude (`CLAUDE_VISION_MODEL`, default Sonnet) → HF vision (fixed order, not configurable). The Claude slot is skipped when its key is absent — without either `CLAUDE_API_TOKEN` or `CLAUDE_API_TOKEN2` OCR runs HF-only. Claude stream creation uses both distinct credentials before changing provider; credential cooldown is shared by smart, fast, OCR and role chains. Authentication, billing/quota (HTTP 402 and the HTTP 400 credit-balance and "API usage limits" spend-cap bodies), transient failures and a share of the existing header budget can trigger one alternate-key attempt. An invalid, oversized or moderated request is returned as it is, without trying the other key or cooling any credential; a model-permission 403 tries the other key but cools neither. Retry-After is honored, caller cancellation stops rotation, and an already-open stream or financial tool execution is never replayed. Each failed credential followed by another HTTP attempt records its own usage error row; successful stream timing starts from the credential that answered. `ANTHROPIC_API_KEY` remains exclusively z.ai. +**Configuration:** All providers go through the OpenAI SDK with a fallback chain in [streaming.ts](src/services/ai/streaming.ts): smart = Claude Sonnet → z.ai GLM → HF; fast (validators, bank prefill, merchant agent) = Claude Haiku → z.ai GLM fast → HF; OCR = Claude (`CLAUDE_VISION_MODEL`, default Sonnet) → HF vision (fixed order, not configurable). The Claude slot is skipped when its key is absent — without either `CLAUDE_API_TOKEN` or `CLAUDE_API_TOKEN2` OCR runs HF-only. Claude stream creation uses both distinct credentials before changing provider; credential cooldown is shared by smart, fast, OCR and role chains. Authentication, billing/quota (HTTP 402 and the HTTP 400 credit-balance and "API usage limits" spend-cap bodies), transient failures and a share of the existing header budget can trigger one alternate-key attempt. An invalid, oversized or moderated request is returned as it is, without trying the other key or cooling any credential; a model-permission 403 tries the other key but cools neither. Retry-After is honored, caller cancellation stops rotation, and an already-open stream or financial tool execution is never replayed. Each failed credential followed by another HTTP attempt records its own usage error row; successful stream timing starts from the credential that answered. `ANTHROPIC_API_KEY` remains exclusively z.ai. Credential rotation edge cases: a token the SDK cannot send as an `Authorization` header (zero-width, non-Latin-1 or control characters) is excluded from rotation at client creation and never logged. A backup that becomes available only after the first attempt is not tried with a leftover budget below `min(timeout/4, 1s)`; the first failure is returned and the backup is not cooled. An attempt that times out on a per-key slice below 1s still rotates to the other key but does not cool the key it timed out on. Each key's cooldown keeps the failure that set its end time (a concurrent shorter failure never relabels it), and a request that finds every key cooling or being probed reports that cause. The `stream_options` compatibility retry runs inside the credential attempt that rejected the option; it never restarts rotation. -**Model ids are env values, refreshed against the live catalogs** (2026-10-24). z.ai coding endpoint `/models`: newest are `glm-5.3` (`AI_MODEL`) and `glm-5.3-flash` (`AI_FAST_MODEL`); `glm-5.3-flashx` is not in the Max plan (code 1311). Both answer 429 code 1310 while the weekly premium quota is exhausted (an unknown id would be 400 code 1211), so a 429 does not mean a bad id and the chain falls through to Claude. `glm-5.1` and `glm-4.5-flash` still exist but are superseded. Anthropic `/v1/models`: `claude-sonnet-5-5` and `claude-haiku-5-5` are the newest Sonnet and Haiku, both `max_tokens` 128000. HF router: `Qwen/Qwen3-235B-A22B`, `meta-llama/Llama-3.3-70B-Instruct` and the OCR model `Qwen/Qwen2.5-VL-72B-Instruct` are still listed live; HF has no credits (402) so a replacement cannot be probed and the ids are kept until they leave `/models`. +**Model ids are env values, refreshed against the live catalogs** (2026-10-24; HF examples refreshed 2026-11-13). z.ai coding endpoint `/models`: newest are `glm-5.3` (`AI_MODEL`) and `glm-5.3-flash` (`AI_FAST_MODEL`); `glm-5.3-flashx` is not in the Max plan (code 1311). Both answer 429 code 1310 while the weekly premium quota is exhausted (an unknown id would be 400 code 1211), so a 429 does not mean a bad id and the chain falls through to Claude. `glm-5.1` and `glm-4.5-flash` still exist but are superseded. Anthropic `/v1/models`: `claude-sonnet-5-5` and `claude-haiku-5-5` are the newest Sonnet and Haiku, both `max_tokens` 128000. HF Router examples now use `Qwen/Qwen3.8-2.4T-A95B:deepinfra` (smart), `meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra` (fast), and `Qwen/Qwen3.8-Flash-Next-FP8:nscale` (vision), each listed with tools and structured output in `/v1/models` on 2026-11-13. Credentialed inference still returns 402 requiring prepaid credits; catalog membership is not successful inference. Applying these examples to the live environment and evaluating cost, latency and receipt accuracy require separate rollout approval. **Provider order is configuration, not code**: `AI_SMART_CHAIN` and `AI_FAST_CHAIN` list provider ids (zai, claude, hf, openrouter); default for both is z.ai → Claude → HF (`openrouter` is in no default order — it runs only when the operator names it); providers without key/model are skipped. The ids come from `PROVIDER_IDS` in [provider-ids.ts](src/services/ai/provider-ids.ts) (a leaf module, shared with `config/env.ts`). The skip is logged, an unknown name is ignored with a warning, an order naming nothing configured falls back to the default order. The order is the same as in HyperCalendarBot. The circuit breaker ([provider-breaker.ts](src/services/ai/provider-breaker.ts)) only moves a *demoted* provider to the back, everything else keeps the configured order: a z.ai that answers 429 (quota/rate limit) or 402 (no credits) is demoted at once (Retry-After / quota reset time, else 5 min; 402 for an hour), so the next rounds start with Claude instead of paying z.ai's failed call again; the demoted z.ai is still tried last, and one clean answer lifts the demotion. A z.ai that merely hangs is not demoted (an abort is no evidence): each such round spends the 45 s per-provider timeout before Claude is tried. Risk: HF Router answered 402 (no credits) on 2026-10-12 and z.ai's quota ran out in HyperCalendarBot on 2026-09-24, so Claude is the de-facto primary when z.ai is depleted — keep its `max_tokens`/thinking handling intact. Sonnet is the OCR default because on degraded receipt photos Haiku produced a wrong yet sum-verified receipt while Sonnet refused (2026-10-12, clean fixtures: equal accuracy, Haiku ~25% faster). Request budget fields come from `chatRequestParams()`, which delegates to `claudeRequestParams()` for Claude slots (also used by `scripts/verify-ai-chains.ts`): never `temperature` (400 on Claude 5.x); Sonnet/Opus keep thinking on at the model's own level (the OpenAI-compat endpoint ignores `reasoning_effort`, so none is sent) and get `CLAUDE_THINKING_RESERVE_TOKENS` (8192) added to `max_tokens`, because thinking spends the same budget as the answer (without the reserve Sonnet truncated a 200-token prefill answer in 3/6 calls); Haiku runs with `thinking: {type: 'disabled'}` — at small budgets it otherwise streams zero text. The OpenAI-compat endpoint rejects explicit adaptive thinking. After changing `CLAUDE_MODEL`/`CLAUDE_FAST_MODEL`/`CLAUDE_VISION_MODEL`, run `scripts/verify-ai-chains.ts` (plus `scripts/run-receipt-snapshots.ts` for the vision model); to validate a provider change on real traffic, run `scripts/replay-ai-logs.ts` against `logs/chats` and a DB copy. -**Model presets** ([model-presets.ts](src/services/ai/model-presets.ts), stage B2; same ids and fields as HyperCalendarBot): `MODEL_PRESETS` (`deepseek41-flash`, `glm53-flash-low`, `glm53-low`, `haiku`) = per-provider model ids (`models`), measured `effort`/`temperature`/`maxOutputTokens`, list price (`priceUsdPerMTok` with `asOf` date and the `via` provider it is valid for; `null` = unknown, never 0), candidate `roles`, and `qualification` (`null` until a run passed `QUALIFICATION_GATE`: ≥95% own-role and 100% on the labelled history). `isQualifiedFor(preset, role)` is the only gate role routing may use. `EXCLUDED_MODEL_PATTERNS` (gpt-oss, Groq's qwen3.8-27b) are enforced in every text chain, legacy included: a slot whose model matches is skipped with a warning, so a retired model cannot come back through an env var. Legacy chains are otherwise untouched. +**HF Qwen request controls**: `hfRequestParams()` is applied only to HF slots, including bare role routes. The exact `Qwen/Qwen3.8-2.4T-A95B` id (with an optional provider suffix) requires thinking, so it uses `reasoning_effort: low` and 4096 extra output tokens above the caller budget. This is an engineering allowance, not a measured completion guarantee. The exact `Qwen/Qwen3.8-Flash-Next-FP8` id uses `chat_template_kwargs: {enable_thinking: false}` for bounded OCR output. All other models keep their existing request parameters. Source: the official [2.4T model card](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) and [Flash-Next model card](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8). + +**Model presets** ([model-presets.ts](src/services/ai/model-presets.ts), stage B2; same ids and fields as HyperCalendarBot): `MODEL_PRESETS` (`deepseek41-flash`, `glm53-flash-low`, `glm53-low`, `haiku`) = per-provider model ids (`models`), measured `effort`/`temperature`/`maxOutputTokens`, list price (`priceUsdPerMTok` with `asOf` date and the `via` provider it is valid for; `null` = unknown, never 0), candidate `roles`, and `qualification` (`null` until a run passed `QUALIFICATION_GATE`: ≥95% own-role and 100% on the labelled history). `isQualifiedFor(preset, role)` is the only gate role routing may use. `EXCLUDED_MODEL_PATTERNS` (gpt-oss and Groq's qwen3.8-27b) are enforced in text, role, and OCR chains: a matching slot is skipped and logged, including HF/OpenRouter model overrides. A chain with no allowed provider throws the normal aggregate error; it never returns an empty success. Claude leads configured text and role chains. Provider health may move it behind another priority provider; HF always remains the final reserve. **Role routing** (stage B3, [role-routing.ts](src/services/ai/role-routing.ts); same env names as HyperCalendarBot; inert by default): `AI_ROUTING=legacy|roles` (default `legacy`, anything else stops startup). In `roles` mode an agent round runs on the chain of its role — `executor` for ordinary tool rounds, `repair` from the stall guard's nudge/handoff on (the forced final answer included), `planner` is parsed and resolvable but ESB has no planner round yet — taken from `AI_EXECUTOR_CHAIN` / `AI_REPAIR_CHAIN` / `AI_PLANNER_CHAIN`: comma-separated `provider` (that provider's own smart model; fast model for the planner) or `provider:preset` (a [model-presets.ts](src/services/ai/model-presets.ts) preset routed over that provider, with its effort/temperature/ceiling; `reasoning_effort` only on OpenRouter). `AI_ROUTING_USER_IDS` (≤100 comma-separated positive Telegram ids) turns it into a canary: only those users. A preset loads only when `isQualifiedFor(preset, role)`; an unqualified, keyless or excluded entry is skipped with a logged error, and a role chain with nothing loadable runs the legacy smart chain. Only the smart chain takes a role (validators, prefill, merchant, OCR never do). Rollback = `AI_ROUTING=legacy` (every caller, no deploy). `AgentContext.telegramUserId` is the canary key. Deadline, breaker, usage rows and the stall-guard handoff are the same machinery as for the legacy chain. @@ -551,9 +553,9 @@ Required in `.env` (see [.env.example](.env.example)): - `ENCRYPTION_KEY` - 32-byte hex (generate: `openssl rand -hex 32`) - `NODE_ENV` - development/production - `ANTHROPIC_API_KEY` - historical name: actually the z.ai GLM key; `AI_BASE_URL`, `AI_MODEL`, `AI_FAST_MODEL` - z.ai endpoint and models -- `AI_SMART_CHAIN`, `AI_FAST_CHAIN` - optional comma-separated provider order (ids `zai`, `claude`, `hf`, `openrouter`); default `zai,claude,hf`. Groq is no longer a provider (gpt-oss answered poorly and no replacement model was chosen); `GROQ_*` variables are ignored — ESB has no other Groq use, unlike HyperCalendarBot where `GROQ_API_KEY` still means Whisper transcription +- `AI_SMART_CHAIN`, `AI_FAST_CHAIN` - optional comma-separated provider order (ids `zai`, `claude`, `hf`, `openrouter`); default `claude,zai,hf`. Groq is no longer a provider (gpt-oss answered poorly and no replacement model was chosen); `GROQ_*` variables are ignored — ESB has no other Groq use, unlike HyperCalendarBot where `GROQ_API_KEY` still means Whisper transcription - `HF_TOKEN`, `HF_BASE_URL`, `HF_MODEL`, `HF_FAST_MODEL`, `HF_VISION_MODEL` - HuggingFace Router, last text fallback and OCR vision fallback -- `CLAUDE_API_TOKEN`, `CLAUDE_API_TOKEN2` - optional primary and secondary Anthropic keys (Claude slot right after z.ai, OCR primary; both absent → Claude slots skipped, OCR HF-only); `CLAUDE_MODEL` (default `claude-sonnet-5-5`), `CLAUDE_FAST_MODEL` (default `claude-haiku-5-5`), `CLAUDE_VISION_MODEL` (receipt OCR, default `claude-sonnet-5-5`), `CLAUDE_BASE_URL` (default OpenAI-compat `https://api.anthropic.com/v1/`) +- `CLAUDE_API_TOKEN`, `CLAUDE_API_TOKEN2` - optional primary and secondary Anthropic keys (Claude primary in smart/fast/OCR; both absent → Claude slots skipped, OCR HF-only); `CLAUDE_MODEL` (default `claude-sonnet-5-5`), `CLAUDE_FAST_MODEL` (default `claude-haiku-5-5`), `CLAUDE_VISION_MODEL` (receipt OCR, default `claude-sonnet-5-5`), `CLAUDE_BASE_URL` (default OpenAI-compat `https://api.anthropic.com/v1/`) - `OPENROUTER_API_KEY`, `OPENROUTER_BASE_URL` (default `https://openrouter.ai/api/v1`), `OPENROUTER_MODEL`, `OPENROUTER_FAST_MODEL` - optional OpenRouter provider (id `openrouter`; same names as HyperCalendarBot); no default models — named in a chain order it still needs key and model, else it is skipped. Probed 2026-10-23: `deepseek/deepseek-v4.1-flash`, `z-ai/glm-5.3-flash`, `z-ai/glm-5.3`, `anthropic/claude-haiku-5.5` exist and accept an assistant tool-call turn with `content: null` (no `""` workaround needed) - `BOT_ADMIN_CHAT_ID` - admin Telegram chat ID (for merchant-agent notifications, feedback forwarding) diff --git a/src/config/env.test.ts b/src/config/env.test.ts index 0ad37ab..5a9f2c0 100644 --- a/src/config/env.test.ts +++ b/src/config/env.test.ts @@ -74,12 +74,12 @@ function loadedConfig(result: LoadResult): { } describe('env config loading', () => { - it('defaults both text chains to z.ai → Claude → HF', () => { + it('defaults both text chains to Claude → z.ai → HF', () => { const { smartChain, fastChain } = loadedConfig(loadEnv(REQUIRED_ENV)); const expected = { - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }; expect(smartChain).toEqual(expected); expect(fastChain).toEqual(expected); @@ -184,12 +184,12 @@ describe('env config loading', () => { it('a chain order naming no known provider falls back to the default order', () => { const { smartChain } = loadedConfig( - loadEnv({ ...REQUIRED_ENV, AI_SMART_CHAIN: 'groq,gemini' }), + loadEnv({ ...REQUIRED_ENV, AI_SMART_CHAIN: 'unknown,unsupported' }), ); expect(smartChain).toEqual({ - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); }); diff --git a/src/config/env.ts b/src/config/env.ts index aa3db36..1cc96fe 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -17,9 +17,9 @@ import { createLogger } from '../utils/logger'; const logger = createLogger('env'); -/** Default order of the smart chain: z.ai → Claude → HF. Overridden by AI_SMART_CHAIN. */ +/** Default order of the smart chain: Claude → z.ai → HF. Overridden by AI_SMART_CHAIN. */ export const DEFAULT_SMART_CHAIN: ProviderId[] = [...DEFAULT_PROVIDER_IDS]; -/** Default order of the fast chain: z.ai → Claude → HF. Overridden by AI_FAST_CHAIN. */ +/** Default order of the fast chain: Claude → z.ai → HF. Overridden by AI_FAST_CHAIN. */ export const DEFAULT_FAST_CHAIN: ProviderId[] = [...DEFAULT_PROVIDER_IDS]; /** @@ -82,7 +82,7 @@ interface EnvConfig { DATABASE_PATH: string; ENCRYPTION_KEY: string; // empty string when not set — encryption features deactivate gracefully - // z.ai provider (kept ANTHROPIC_API_KEY name — it's the z.ai key); head of the default chains + // z.ai provider (kept ANTHROPIC_API_KEY name — it's the z.ai key); fallback after Claude ANTHROPIC_API_KEY: string; AI_BASE_URL: string; AI_MODEL: string; @@ -92,7 +92,7 @@ interface EnvConfig { AI_SMART_CHAIN: ChainOrder; AI_FAST_CHAIN: ChainOrder; - // Anthropic Claude (OpenAI-compat endpoint; optional key — second in smart/fast, OCR primary). + // Anthropic Claude (OpenAI-compat endpoint; optional key — primary in smart/fast/OCR). // Without either key every Claude slot is skipped and OCR runs on HF vision alone. CLAUDE_API_TOKEN: string; CLAUDE_API_TOKEN2: string; diff --git a/src/services/ai/model-presets.test.ts b/src/services/ai/model-presets.test.ts index 66ef349..51df61f 100644 --- a/src/services/ai/model-presets.test.ts +++ b/src/services/ai/model-presets.test.ts @@ -50,8 +50,10 @@ describe('MODEL_PRESETS', () => { } }); - it('carries the exact OpenRouter ids probed on 2026-10-23', () => { - expect(findPreset('deepseek41-flash')?.models.openrouter).toBe('deepseek/deepseek-v4.1-flash'); + it('carries the exact OpenRouter ids from the dated model catalogs', () => { + expect(findPreset('deepseek41-flash')?.models.openrouter).toBe( + 'deepseek/deepseek-v4.1-flash-1017', + ); expect(findPreset('glm53-flash-low')?.models.openrouter).toBe('z-ai/glm-5.3-flash'); expect(findPreset('glm53-low')?.models.openrouter).toBe('z-ai/glm-5.3'); expect(findPreset('haiku')?.models.openrouter).toBe('anthropic/claude-haiku-5.5'); @@ -99,10 +101,10 @@ describe('isExcludedModel', () => { it.each([ 'glm-5.3-flash', - 'deepseek/deepseek-v4.1-flash', + 'deepseek/deepseek-v4.1-flash-1017', 'claude-sonnet-5-5', - 'Qwen/Qwen3-235B-A22B', - 'meta-llama/Llama-3.3-70B-Instruct', + 'Qwen/Qwen3.8-2.4T-A95B:deepinfra', + 'meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra', ])('keeps %s', (model) => { expect(isExcludedModel(model)).toBe(false); }); @@ -175,7 +177,7 @@ describe('presetRequestParams', () => { describe('computeCostMicroUsd', () => { // deepseek41-flash on OpenRouter: $0.30 in / $1.20 out per million tokens = 0.30 / 1.20 µUSD per token. - const MODEL = 'deepseek/deepseek-v4.1-flash'; + const MODEL = 'deepseek/deepseek-v4.1-flash-1017'; it('prices tokens at the list rate of the route, in whole micro-USD', () => { expect(computeCostMicroUsd('openrouter', MODEL, 1_000_000, 1_000_000)).toBe(1_500_000); diff --git a/src/services/ai/model-presets.ts b/src/services/ai/model-presets.ts index b670b22..bd6b977 100644 --- a/src/services/ai/model-presets.ts +++ b/src/services/ai/model-presets.ts @@ -50,8 +50,9 @@ export interface ModelPreset { } /** - * Prices are OpenRouter's /models listing of 2026-10-23 (USD per token × 1e6). OpenRouter ids were - * checked against the same listing. `claude-haiku-5-5` on the `claude` route has no price here: the + * Prices are OpenRouter's /models listing (USD per token × 1e6), dated per row. The default date + * is 2026-10-23; the DeepSeek 1017 route was refreshed on 2026-11-13. The direct `claude` route has + * no price here: the * direct Anthropic rate was not read, so its cost stays unknown. */ const PRICE_AS_OF = '2026-10-23'; @@ -59,8 +60,8 @@ const PRICE_AS_OF = '2026-10-23'; export const MODEL_PRESETS: readonly ModelPreset[] = [ { id: 'deepseek41-flash', - models: { openrouter: 'deepseek/deepseek-v4.1-flash' }, - priceUsdPerMTok: { in: 0.3, out: 1.2, asOf: PRICE_AS_OF, via: 'openrouter' }, + models: { openrouter: 'deepseek/deepseek-v4.1-flash-1017' }, + priceUsdPerMTok: { in: 0.3, out: 1.2, asOf: '2026-11-13', via: 'openrouter' }, roles: ['executor'], qualification: null, }, diff --git a/src/services/ai/provider-ids.ts b/src/services/ai/provider-ids.ts index 50d3b55..12259f9 100644 --- a/src/services/ai/provider-ids.ts +++ b/src/services/ai/provider-ids.ts @@ -14,4 +14,4 @@ export const PROVIDER_IDS = ['zai', 'claude', 'hf', 'openrouter'] as const; export type ProviderId = (typeof PROVIDER_IDS)[number]; /** The providers of the default chain orders; `openrouter` is optional and in none of them. */ -export const DEFAULT_PROVIDER_IDS: readonly ProviderId[] = ['zai', 'claude', 'hf']; +export const DEFAULT_PROVIDER_IDS: readonly ProviderId[] = ['claude', 'zai', 'hf']; diff --git a/src/services/ai/streaming.breaker.test.ts b/src/services/ai/streaming.breaker.test.ts index 4c01830..0fe70fe 100644 --- a/src/services/ai/streaming.breaker.test.ts +++ b/src/services/ai/streaming.breaker.test.ts @@ -23,8 +23,8 @@ describe('aiStreamRound fallback chain', () => { const order: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { order.push(params.model); - if (params.model === 'test-model') { - // z.ai unreachable (connection-type failure the breaker counts). + if (params.model === 'claude-smart') { + // Claude unreachable (connection-type failure the breaker counts). throw Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' }); } return { @@ -33,7 +33,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -45,18 +45,18 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); - // Three rounds where z.ai connection-fails first each time (Claude answers). + // Three rounds where Claude connection-fails first each time (z.ai answers). for (let i = 0; i < 3; i++) { order.length = 0; await round(); - expect(order[0]).toBe('test-model'); // still tried first while it has < 3 failures + expect(order[0]).toBe('claude-smart'); // still tried first while it has < 3 failures } - // 4th round: z.ai is now demoted → Claude is tried first, z.ai is not the head of the chain. + // 4th round: Claude is now demoted → z.ai is tried first, Claude is not the head of the chain. order.length = 0; const result = await round(); - expect(order[0]).toBe('claude-smart'); - expect(order).not.toContain('test-model'); // Claude answered, so the demoted z.ai isn't reached + expect(order[0]).toBe('test-model'); + expect(order).not.toContain('claude-smart'); // z.ai answered, so the demoted Claude isn't reached expect(result.text).toBe('ok'); }); @@ -65,7 +65,7 @@ describe('aiStreamRound fallback chain', () => { const order: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { order.push(params.model); - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { // A 4xx is a response, not a connection failure — the breaker must NOT count it. throw new OpenAI.APIError(400, { message: 'bad' }, 'bad', new Headers()); } @@ -75,7 +75,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -89,7 +89,7 @@ describe('aiStreamRound fallback chain', () => { for (let i = 0; i < 4; i++) { order.length = 0; await round(); - expect(order[0]).toBe('test-model'); // never demoted for 4xx — always tried first + expect(order[0]).toBe('claude-smart'); // never demoted for 4xx — always tried first } }); @@ -106,14 +106,14 @@ describe('aiStreamRound fallback chain', () => { const order: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { order.push(params.model); - if (params.model === 'test-model') throw noCredits(); + if (params.model === 'claude-smart') throw noCredits(); return { [Symbol.asyncIterator]: async function* () { yield { choices: [{ delta: { content: 'ok' }, finish_reason: 'stop' }] }; }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -126,13 +126,13 @@ describe('aiStreamRound fallback chain', () => { }); await round(); - expect(order).toEqual(['test-model', 'claude-smart']); + expect(order).toEqual(['claude-smart', 'test-model']); order.length = 0; await round(); - expect(order).toEqual(['claude-smart']); // the out-of-credits z.ai is not paid for again - expect(isProviderDemoted('zai', Date.now() + 59 * 60_000)).toBe(true); - expect(isProviderDemoted('zai', Date.now() + 61 * 60_000)).toBe(false); + expect(order).toEqual(['test-model']); // the out-of-credits Claude is not paid for again + expect(isProviderDemoted('claude', Date.now() + 59 * 60_000)).toBe(true); + expect(isProviderDemoted('claude', Date.now() + 61 * 60_000)).toBe(false); }); it('a 402 from the HF fallback slot demotes HF for an hour', async () => { @@ -141,7 +141,7 @@ describe('aiStreamRound fallback chain', () => { if (params.model === 'hf-test') throw noCredits(); throw new OpenAI.APIError(503, { message: 'down' }, 'down', new Headers()); }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -162,7 +162,7 @@ describe('aiStreamRound fallback chain', () => { const order: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { order.push(params.model); - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { // A real response (status 400) whose text mentions "connection" — provider WAS reachable, // so the breaker must not count it (status defined → not an unreachable failure). throw new OpenAI.APIError( @@ -178,7 +178,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -192,7 +192,7 @@ describe('aiStreamRound fallback chain', () => { for (let i = 0; i < 4; i++) { order.length = 0; await round(); - expect(order[0]).toBe('test-model'); // 400 response never demotes — always tried first + expect(order[0]).toBe('claude-smart'); // 400 response never demotes — always tried first } }); @@ -206,7 +206,7 @@ describe('aiStreamRound fallback chain', () => { throw Object.assign(new Error('refused'), { code: 'ECONNREFUSED' }); }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); for (let i = 0; i < 4; i++) { @@ -217,7 +217,7 @@ describe('aiStreamRound fallback chain', () => { ), ).rejects.toThrow(); } - expect(isProviderDemoted('zai', Date.now())).toBe(false); + expect(isProviderDemoted('claude', Date.now())).toBe(false); }); it('does not demote a provider that streamed a tool-call before dropping mid-stream', async () => { @@ -225,7 +225,7 @@ describe('aiStreamRound fallback chain', () => { // late drop must NOT count toward the breaker even though no TEXT was emitted. const clientsMod = await import('./clients'); const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { return { [Symbol.asyncIterator]: async function* () { yield { @@ -250,7 +250,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -262,7 +262,7 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(false); + expect(isProviderDemoted('claude', Date.now())).toBe(false); }); it('does not demote a provider that streamed a nameless tool-call chunk before dropping', async () => { @@ -271,7 +271,7 @@ describe('aiStreamRound fallback chain', () => { // onToolCallStart which only fires once a name is present). const clientsMod = await import('./clients'); const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { return { [Symbol.asyncIterator]: async function* () { yield { @@ -294,7 +294,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -306,7 +306,7 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(false); + expect(isProviderDemoted('claude', Date.now())).toBe(false); }); it('an abort does not mutate breaker state (does not reset accrued connection failures)', async () => { @@ -315,7 +315,7 @@ describe('aiStreamRound fallback chain', () => { const clientsMod = await import('./clients'); let round = 0; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { if (round === 2) { const abort = new Error('The operation was aborted'); abort.name = 'AbortError'; @@ -329,7 +329,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -342,7 +342,7 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(true); + expect(isProviderDemoted('claude', Date.now())).toBe(true); }); it('text-then-drop resets the accrued connection-failure cluster', async () => { @@ -351,7 +351,7 @@ describe('aiStreamRound fallback chain', () => { const clientsMod = await import('./clients'); let round = 0; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { if (round === 2) { return { [Symbol.asyncIterator]: async function* () { @@ -368,7 +368,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -386,7 +386,7 @@ describe('aiStreamRound fallback chain', () => { await pending; } } - expect(isProviderDemoted('zai', Date.now())).toBe(false); + expect(isProviderDemoted('claude', Date.now())).toBe(false); }); it('an all-provider connection outage yields a retryable aggregate', async () => { @@ -396,7 +396,7 @@ describe('aiStreamRound fallback chain', () => { const create = mock(async () => { throw new OpenAI.APIConnectionError({ message: 'Connection error.' }); }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -419,7 +419,7 @@ describe('aiStreamRound fallback chain', () => { // must NOT be mistaken for an abort, which would skip the breaker). const clientsMod = await import('./clients'); const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { throw new OpenAI.APIConnectionError({ message: 'Connection error.' }); } return { @@ -428,7 +428,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -440,7 +440,7 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(true); + expect(isProviderDemoted('claude', Date.now())).toBe(true); }); it('an APIUserAbortError does not mutate breaker state', async () => { @@ -449,7 +449,7 @@ describe('aiStreamRound fallback chain', () => { const clientsMod = await import('./clients'); let round = 0; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { if (round === 2) throw new OpenAI.APIUserAbortError(); throw Object.assign(new Error('refused'), { code: 'ECONNREFUSED' }); } @@ -459,7 +459,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -472,7 +472,7 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(true); + expect(isProviderDemoted('claude', Date.now())).toBe(true); }); it('a mixed transient+4xx chain intentionally retries and reads as provider_down', async () => { @@ -481,11 +481,11 @@ describe('aiStreamRound fallback chain', () => { // than reading as a (non-retryable) bad request — failing safe toward "try later". const clientsMod = await import('./clients'); const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') + if (params.model === 'claude-smart') throw Object.assign(new Error('refused'), { code: 'ECONNREFUSED' }); throw new OpenAI.APIError(400, { message: 'bad' }, 'bad', new Headers()); }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -515,16 +515,16 @@ describe('aiStreamRound fallback chain', () => { let round = 0; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { if (round < 3) { - if (params.model === 'test-model') throw conn(); // z.ai unreachable → accrues toward demotion - return ok(); // Claude answers + if (params.model === 'claude-smart') throw conn(); // Claude unreachable → accrues toward demotion + return ok(); // z.ai answers } - // Final round: the whole chain fails and the demoted z.ai is reached, returning a 503. - if (params.model === 'test-model') { + // Final round: the whole chain fails and the demoted Claude is reached, returning a 503. + if (params.model === 'claude-smart') { throw new OpenAI.APIError(503, { message: 'down' }, 'down', new Headers()); } throw conn(); }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -536,10 +536,10 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); for (round = 0; round < 3; round++) await runRound(); - expect(isProviderDemoted('zai', Date.now())).toBe(true); // demoted after 3 connection failures - await expect(runRound()).rejects.toThrow(); // round 3: z.ai returns a 503 (reachable) + expect(isProviderDemoted('claude', Date.now())).toBe(true); // demoted after 3 connection failures + await expect(runRound()).rejects.toThrow(); // round 3: Claude returns a 503 (reachable) // The 503 only reset the connection counter — it did NOT rehabilitate the demoted provider. - expect(isProviderDemoted('zai', Date.now())).toBe(true); + expect(isProviderDemoted('claude', Date.now())).toBe(true); }); it('a reachable empty-response between connection failures resets the breaker streak', async () => { @@ -548,7 +548,7 @@ describe('aiStreamRound fallback chain', () => { const clientsMod = await import('./clients'); let round = 0; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { if (round === 2) { // Reachable but empty → streamingSlot throws the plain "empty response" Error. return { @@ -565,7 +565,7 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -578,18 +578,18 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); } - expect(isProviderDemoted('zai', Date.now())).toBe(false); + expect(isProviderDemoted('claude', Date.now())).toBe(false); }); it('aggregate error prefers a transient (5xx) signal over a co-occurring 4xx (keeps retry)', async () => { const clientsMod = await import('./clients'); const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { throw new OpenAI.APIError(400, { message: 'bad' }, 'bad', new Headers()); // one provider 4xx } throw new OpenAI.APIError(503, { message: 'down' }, 'down', new Headers()); // the rest 5xx }); - for (const fn of ['zaiClient', 'claudeClient', 'hfClient'] as const) { + for (const fn of ['claudeClient', 'zaiClient', 'hfClient'] as const) { spyOn(clientsMod, fn).mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); diff --git a/src/services/ai/streaming.chain-skips.test.ts b/src/services/ai/streaming.chain-skips.test.ts index 09e2802..e2943e1 100644 --- a/src/services/ai/streaming.chain-skips.test.ts +++ b/src/services/ai/streaming.chain-skips.test.ts @@ -16,9 +16,9 @@ mock.module('../../utils/logger', () => ({ })); const defaultOrder = (): ChainOrder => ({ - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); const mockEnv = { @@ -69,10 +69,14 @@ beforeEach(() => { mockEnv.AI_SMART_CHAIN = defaultOrder(); mockEnv.AI_FAST_CHAIN = defaultOrder(); mockEnv.CLAUDE_API_TOKEN = 'test-claude'; + mockEnv.CLAUDE_VISION_MODEL = 'claude-vision'; mockEnv.HF_TOKEN = 'test-hf'; mockEnv.HF_MODEL = 'hf-test'; mockEnv.HF_FAST_MODEL = 'hf-fast'; + mockEnv.HF_VISION_MODEL = 'hf-vision'; mockEnv.OPENROUTER_API_KEY = ''; + mockEnv.OPENROUTER_MODEL = 'or-smart'; + mockEnv.OPENROUTER_FAST_MODEL = 'or-fast'; mockEnv.AI_ROUTING = 'legacy'; mockEnv.AI_ROLE_CHAINS = { planner: [], executor: [], repair: [] }; }); @@ -184,8 +188,8 @@ describe('role chain entry skips', () => { ], }; - expect(nextProvider('smart', [], 'executor')).toBe('z.ai (test-model)'); - expect(nextProvider('smart', [], 'executor')).toBe('z.ai (test-model)'); + expect(nextProvider('smart', [], 'executor')).toBe('Claude (claude-smart)'); + expect(nextProvider('smart', [], 'executor')).toBe('Claude (claude-smart)'); expect(logged('error', 'Role chain')).toEqual([ [ diff --git a/src/services/ai/streaming.chains.test.ts b/src/services/ai/streaming.chains.test.ts index 307fc61..6632cf6 100644 --- a/src/services/ai/streaming.chains.test.ts +++ b/src/services/ai/streaming.chains.test.ts @@ -21,8 +21,8 @@ describe('aiStreamRound fallback chain', () => { }); it.each([ - ['smart' as const, ['test-model', 'claude-smart', 'hf-test']], - ['fast' as const, ['test-fast', 'claude-fast', 'hf-fast']], + ['smart' as const, ['claude-smart', 'test-model', 'hf-test']], + ['fast' as const, ['claude-fast', 'test-fast', 'hf-fast']], ['ocr' as const, ['claude-vision', 'hf-vision']], ])('the %s chain with every key set is exactly %p', async (chain, expected) => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; @@ -39,8 +39,8 @@ describe('aiStreamRound fallback chain', () => { }); it.each([ - ['smart' as const, ['test-model', 'claude-smart', 'hf-test']], - ['fast' as const, ['test-fast', 'claude-fast', 'hf-fast']], + ['smart' as const, ['claude-smart', 'test-model', 'hf-test']], + ['fast' as const, ['claude-fast', 'test-fast', 'hf-fast']], ['ocr' as const, ['claude-vision', 'hf-vision']], ])('secondary alone keeps Claude available in the %s chain', async (chain, expected) => { mockEnv.CLAUDE_API_TOKEN = ''; @@ -48,11 +48,11 @@ describe('aiStreamRound fallback chain', () => { expect(await chainModels(chain)).toEqual(expected); }); - it('AI_SMART_CHAIN / AI_FAST_CHAIN override the order; the other chain keeps its default', async () => { + it('configured chains choose providers while Claude stays first and HF stays last', async () => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; mockEnv.AI_SMART_CHAIN = { ...DEFAULT_ORDER, order: ['hf', 'claude'], fromEnv: true }; - expect(await chainModels('smart')).toEqual(['hf-test', 'claude-smart']); - expect(await chainModels('fast')).toEqual(['test-fast', 'claude-fast', 'hf-fast']); + expect(await chainModels('smart')).toEqual(['claude-smart', 'hf-test']); + expect(await chainModels('fast')).toEqual(['claude-fast', 'test-fast', 'hf-fast']); }); it('a configured order naming nothing usable falls back to the default order', async () => { @@ -72,15 +72,15 @@ describe('aiStreamRound fallback chain', () => { it('openrouter is part of a chain only when the operator names it and its key and model are set', async () => { mockEnv.OPENROUTER_API_KEY = 'or-key'; // Not in the default order: a key alone changes nothing. - expect(await chainModels('smart')).toEqual(['test-model', 'claude-smart', 'hf-test']); - expect(await chainModels('fast')).toEqual(['test-fast', 'claude-fast', 'hf-fast']); + expect(await chainModels('smart')).toEqual(['claude-smart', 'test-model', 'hf-test']); + expect(await chainModels('fast')).toEqual(['claude-fast', 'test-fast', 'hf-fast']); mockEnv.AI_SMART_CHAIN = { ...DEFAULT_ORDER, order: ['openrouter', 'claude', 'hf'], fromEnv: true, }; mockEnv.AI_FAST_CHAIN = { ...DEFAULT_ORDER, order: ['zai', 'openrouter'], fromEnv: true }; - expect(await chainModels('smart')).toEqual(['or-smart', 'claude-smart', 'hf-test']); + expect(await chainModels('smart')).toEqual(['claude-smart', 'or-smart', 'hf-test']); expect(await chainModels('fast')).toEqual(['test-fast', 'or-fast']); expect(await chainModels('ocr')).toEqual(['claude-vision', 'hf-vision']); }); @@ -103,8 +103,8 @@ describe('aiStreamRound fallback chain', () => { mockEnv.HF_MODEL = 'openai/gpt-oss-120b'; mockEnv.HF_FAST_MODEL = 'qwen/qwen3.8-27b'; try { - expect(await chainModels('smart')).toEqual(['test-model', 'claude-smart']); - expect(await chainModels('fast')).toEqual(['test-fast', 'claude-fast']); + expect(await chainModels('smart')).toEqual(['claude-smart', 'test-model']); + expect(await chainModels('fast')).toEqual(['claude-fast', 'test-fast']); } finally { mockEnv.HF_MODEL = 'hf-test'; mockEnv.HF_FAST_MODEL = 'hf-fast'; @@ -145,24 +145,24 @@ describe('aiStreamRound fallback chain', () => { executor: [entry('openrouter', flash)], }; expect(await chainModels('smart', undefined, 'executor')).toEqual([ - 'test-model', 'claude-smart', + 'test-model', 'hf-test', ]); }); - it('roles mode runs the role chain in its order: preset routes and bare providers', async () => { + it('roles mode prioritizes Claude among selected preset routes and bare providers', async () => { mockEnv.AI_ROUTING = 'roles'; mockEnv.AI_ROLE_CHAINS = { ...mockEnv.AI_ROLE_CHAINS, executor: [entry('openrouter', flash), entry('claude')], }; expect(await chainModels('smart', undefined, 'executor')).toEqual([ - 'vendor/flash', 'claude-smart', + 'vendor/flash', ]); // A round without a role is still the legacy smart chain. - expect(await chainModels('smart')).toEqual(['test-model', 'claude-smart', 'hf-test']); + expect(await chainModels('smart')).toEqual(['claude-smart', 'test-model', 'hf-test']); }); it('sends the preset settings: reasoning_effort and temperature on OpenRouter, ceiling on max_tokens', async () => { @@ -226,14 +226,14 @@ describe('aiStreamRound fallback chain', () => { executor: [entry('openrouter', { ...flash, qualification: null })], }; expect(await chainModels('smart', undefined, 'executor')).toEqual([ - 'test-model', 'claude-smart', + 'test-model', 'hf-test', ]); // An unset role chain behaves the same. expect(await chainModels('smart', undefined, 'repair')).toEqual([ - 'test-model', 'claude-smart', + 'test-model', 'hf-test', ]); }); @@ -264,8 +264,8 @@ describe('aiStreamRound fallback chain', () => { 'test-fast', ]); expect(await chainModels('fast', undefined, 'planner')).toEqual([ - 'test-fast', 'claude-fast', + 'test-fast', 'hf-fast', ]); expect(await chainModels('ocr', undefined, 'executor')).toEqual([ @@ -284,12 +284,12 @@ describe('aiStreamRound fallback chain', () => { expect(streamingModule.nextProvider('smart', ['Claude (claude-smart)'], 'repair')).toBe( 'HF (hf-test)', ); - expect(streamingModule.nextProvider('smart', [])).toBe('z.ai (test-model)'); + expect(streamingModule.nextProvider('smart', [])).toBe('Claude (claude-smart)'); }); }); it('nextProvider names the slot the chain would try first without the excluded ones', () => { - expect(streamingModule.nextProvider('smart', [])).toBe('z.ai (test-model)'); + expect(streamingModule.nextProvider('smart', [])).toBe('Claude (claude-smart)'); expect(streamingModule.nextProvider('smart', ['z.ai (test-model)'])).toBe( 'Claude (claude-smart)', ); @@ -302,8 +302,27 @@ describe('aiStreamRound fallback chain', () => { ).toBeNull(); }); - it('nextProvider follows the breaker order: a demoted z.ai is not next, Claude is', () => { + it('a demoted Claude yields to z.ai while HF stays after every priority provider', async () => { + recordProviderRateLimit('claude', Date.now(), 60_000); + expect(streamingModule.nextProvider('smart', [])).toBe('z.ai (test-model)'); + expect(await chainModels('smart')).toEqual(['test-model', 'claude-smart', 'hf-test']); recordProviderRateLimit('zai', Date.now(), 60_000); - expect(streamingModule.nextProvider('smart', [])).toBe('Claude (claude-smart)'); + expect(await chainModels('smart')).toEqual(['claude-smart', 'test-model', 'hf-test']); + }); + + it('keeps HF last in a configured role chain with OpenRouter and a demoted Claude', async () => { + mockEnv.AI_ROUTING = 'roles'; + mockEnv.OPENROUTER_API_KEY = 'or-key'; + mockEnv.AI_ROLE_CHAINS.executor = [ + { provider: 'hf', preset: null }, + { provider: 'openrouter', preset: null }, + { provider: 'claude', preset: null }, + ]; + recordProviderRateLimit('claude', Date.now(), 60_000); + expect(await chainModels('smart', undefined, 'executor')).toEqual([ + 'or-smart', + 'claude-smart', + 'hf-test', + ]); }); }); diff --git a/src/services/ai/streaming.claude-backup.test.ts b/src/services/ai/streaming.claude-backup.test.ts index 38b8de2..44a7810 100644 --- a/src/services/ai/streaming.claude-backup.test.ts +++ b/src/services/ai/streaming.claude-backup.test.ts @@ -14,9 +14,9 @@ const log = createMockLogger(); mock.module('../../utils/logger', () => ({ createLogger: () => log, logger: log })); const defaultOrder = () => ({ - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); const env = { ANTHROPIC_API_KEY: 'zai-key', @@ -133,15 +133,15 @@ const claudeThenHf = { order: ['claude', 'hf'], fromEnv: true, fallback: ['claud const bareClaude: RoleChainEntry[] = [{ provider: 'claude', preset: null }]; describe('the secondary Claude credential alone', () => { - // z.ai answers 429 first wherever it is in the chain, so Claude is reached only through fallback. + // A working secondary Claude key serves the primary route without contacting another provider. test.each([ - ['smart', 'smart' as const, undefined, 'claude-smart', 'zai-smart'], - ['fast', 'fast' as const, undefined, 'claude-fast', 'zai-fast'], - ['ocr', 'ocr' as const, undefined, 'claude-vision', undefined], - ['planner role', 'smart' as const, 'planner' as const, 'claude-fast', undefined], - ['executor role', 'smart' as const, 'executor' as const, 'claude-smart', undefined], - ['repair role', 'smart' as const, 'repair' as const, 'claude-smart', undefined], - ])('serves the %s chain and keeps the z.ai key out of Claude requests', async (_name, chain, role, claudeModel, zaiModel) => { + ['smart', 'smart' as const, undefined, 'claude-smart'], + ['fast', 'fast' as const, undefined, 'claude-fast'], + ['ocr', 'ocr' as const, undefined, 'claude-vision'], + ['planner role', 'smart' as const, 'planner' as const, 'claude-fast'], + ['executor role', 'smart' as const, 'executor' as const, 'claude-smart'], + ['repair role', 'smart' as const, 'repair' as const, 'claude-smart'], + ])('serves the %s chain and keeps the z.ai key out of Claude requests', async (_name, chain, role, claudeModel) => { if (role) { env.AI_ROUTING = 'roles'; env.AI_ROLE_CHAINS = { planner: bareClaude, executor: bareClaude, repair: bareClaude }; @@ -154,9 +154,7 @@ describe('the secondary Claude credential alone', () => { expect(claudeCalls()).toEqual([ { host: 'claude.test', authorization: 'Bearer secondary-key', model: claudeModel }, ]); - expect(calls.filter((call) => call.host === 'zai.test')).toEqual( - zaiModel ? [{ host: 'zai.test', authorization: 'Bearer zai-key', model: zaiModel }] : [], - ); + expect(calls.filter((call) => call.host === 'zai.test')).toEqual([]); }); test('a rejected secondary is not retried and never replaced by the z.ai key', async () => { @@ -191,6 +189,34 @@ describe('request-specific rejections', () => { }); }); +describe('retired Claude model failover', () => { + test('records HTTP 404, keeps the credential usable, and serves HF only after priority routes fail', async () => { + env.CLAUDE_API_TOKEN = 'primary-key'; + env.AI_SMART_CHAIN = claudeThenHf; + fakeNetwork(async (call) => + call.host === 'claude.test' + ? errorResponse(404, 'model: claude-smart not found') + : okResponse(), + ); + expect((await aiStreamRound(request)).providerUsed).toBe('HF (hf-smart)'); + expect(claudeCalls().map((call) => call.authorization)).toEqual(['Bearer primary-key']); + expect(rows).toContainEqual( + expect.objectContaining({ + provider: 'claude', + outcome: 'error', + httpStatus: 404, + }), + ); + expect(log.warn).toHaveBeenCalled(); + fakeNetwork(async () => okResponse()); + expect((await aiStreamRound(request)).providerUsed).toBe('Claude (claude-smart)'); + expect(claudeCalls().map((call) => call.authorization)).toEqual([ + 'Bearer primary-key', + 'Bearer primary-key', + ]); + }); +}); + describe('one shared deadline', () => { /** A request that stays open until the SDK aborts it. */ const hang = (init: RequestInit | undefined) => diff --git a/src/services/ai/streaming.fallback.test.ts b/src/services/ai/streaming.fallback.test.ts index 99c962b..97d7156 100644 --- a/src/services/ai/streaming.fallback.test.ts +++ b/src/services/ai/streaming.fallback.test.ts @@ -12,16 +12,16 @@ describe('aiStreamRound fallback chain', () => { }); it('falls back to next provider on 400 BadRequest', async () => { - // Mock: first provider (z.ai) → 400, second (Claude) should be tried + // Mock: first provider (Claude) → 400, second (z.ai) should be tried // We test this via the full aiStreamRound by mocking fetch const calls: string[] = []; const createMock = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { calls.push(params.model); - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { throw new OpenAI.APIError(400, { message: 'bad request' }, 'bad request', new Headers()); } - if (params.model === 'claude-smart') { + if (params.model === 'test-model') { throw new OpenAI.APIError( 400, { message: 'bad request too' }, @@ -44,10 +44,10 @@ describe('aiStreamRound fallback chain', () => { // Mock all three clients to use our createMock const clientsMod = await import('./clients'); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); spyOn(clientsMod, 'hfClient').mockReturnValue({ @@ -63,7 +63,7 @@ describe('aiStreamRound fallback chain', () => { expect(result.text).toBe('hello from HF'); expect(result.providerUsed).toContain('HF'); // All three providers were tried - expect(calls).toEqual(['test-model', 'claude-smart', 'hf-test']); + expect(calls).toEqual(['claude-smart', 'test-model', 'hf-test']); }); it('propagates error when text was already emitted', async () => { @@ -75,7 +75,7 @@ describe('aiStreamRound fallback chain', () => { }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -138,7 +138,7 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -162,8 +162,8 @@ describe('aiStreamRound fallback chain', () => { arguments: '{"limit":5}', }); expect(result.finishReason).toBe('tool_calls'); - expect(result.providerUsed).toContain('z.ai'); - // Claude/HF should NOT be tried + expect(result.providerUsed).toContain('Claude'); + // z.ai/HF should NOT be tried expect(createMock).toHaveBeenCalledTimes(1); }); @@ -189,7 +189,7 @@ describe('aiStreamRound fallback chain', () => { }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -223,7 +223,7 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: { content: 'plain answer' }, finish_reason: 'stop' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -245,18 +245,18 @@ describe('aiStreamRound fallback chain', () => { [Symbol.asyncIterator]: async function* () { callNum++; if (callNum === 1) { - // z.ai: empty response (common z.ai quirk) + // Claude: empty response yield { choices: [{ delta: { content: '' }, finish_reason: 'stop' }] }; return; } - // Claude: actual text + // z.ai: actual text yield { choices: [{ delta: { content: 'recovered' }, finish_reason: 'stop' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -267,7 +267,7 @@ describe('aiStreamRound fallback chain', () => { }); expect(result.text).toBe('recovered'); - expect(result.providerUsed).toContain('Claude'); + expect(result.providerUsed).toContain('z.ai'); }); it('stops the chain immediately when the overall signal is pre-aborted', async () => { @@ -280,10 +280,10 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: { content: 'nope' }, finish_reason: 'stop' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); spyOn(clientsMod, 'hfClient').mockReturnValue({ @@ -323,7 +323,7 @@ describe('aiStreamRound fallback chain', () => { opts?: { signal?: AbortSignal }, ) => { calls.push(params.model); - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { // Hang until our injected per-provider timeout aborts this provider's signal. await new Promise((_resolve, reject) => { opts?.signal?.addEventListener('abort', () => { @@ -343,10 +343,10 @@ describe('aiStreamRound fallback chain', () => { }; }, ); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -358,8 +358,8 @@ describe('aiStreamRound fallback chain', () => { }); expect(result.text).toBe('fresh fallback'); - expect(result.providerUsed).toContain('Claude'); - expect(calls).toEqual(['test-model', 'claude-smart']); + expect(result.providerUsed).toContain('z.ai'); + expect(calls).toEqual(['claude-smart', 'test-model']); // The second provider received a fresh, non-aborted signal. expect(secondSignalStates).toEqual([false]); }); @@ -379,10 +379,10 @@ describe('aiStreamRound fallback chain', () => { err.name = 'AbortError'; throw err; }); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -398,7 +398,7 @@ describe('aiStreamRound fallback chain', () => { expect((err as Error).name).toBe('AbortError'); } // Only the first provider was tried — overall deadline stops the chain. - expect(calls).toEqual(['test-model']); + expect(calls).toEqual(['claude-smart']); }); it('tool-call resolution: handles missing tc.index (HF Router quirk)', async () => { @@ -439,7 +439,7 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -475,7 +475,7 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: { content: 'text anyway' }, finish_reason: 'stop' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -499,7 +499,7 @@ describe('aiStreamRound fallback chain', () => { yield { choices: [{ delta: { content: 'd' }, finish_reason: 'stop' }] }; }, })); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -518,10 +518,10 @@ describe('aiStreamRound fallback chain', () => { const createMock = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { throw new Error(`boom ${params.model}`); }); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); spyOn(clientsMod, 'hfClient').mockReturnValue({ @@ -538,8 +538,8 @@ describe('aiStreamRound fallback chain', () => { } catch (err) { const msg = (err as Error).message; expect(msg).toContain('All 3 providers'); - expect(msg).toContain('z.ai'); expect(msg).toContain('Claude'); + expect(msg).toContain('z.ai'); expect(msg).toContain('HF'); expect(msg).toContain('smart'); } @@ -550,10 +550,10 @@ describe('aiStreamRound fallback chain', () => { const createMock = mock(async () => { throw Object.assign(new Error('last-fail'), { status: 503 }); }); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); spyOn(clientsMod, 'hfClient').mockReturnValue({ @@ -618,7 +618,7 @@ describe('aiStreamRound fallback chain', () => { const modelsCalled: string[] = []; const createMock = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { modelsCalled.push(params.model); - if (params.model === 'test-model') { + if (params.model === 'claude-smart') { throw new Error('first down'); } return { @@ -627,10 +627,10 @@ describe('aiStreamRound fallback chain', () => { }, }; }); - spyOn(clientsMod, 'zaiClient').mockReturnValue({ + spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); - spyOn(clientsMod, 'claudeClient').mockReturnValue({ + spyOn(clientsMod, 'zaiClient').mockReturnValue({ chat: { completions: { create: createMock } }, } as unknown as OpenAI); @@ -640,9 +640,9 @@ describe('aiStreamRound fallback chain', () => { chain: 'smart', }); - // First provider (test-model) was tried and failed, second (claude-smart) succeeded - expect(modelsCalled).toContain('test-model'); + // First provider (claude-smart) was tried and failed, second (test-model) succeeded expect(modelsCalled).toContain('claude-smart'); + expect(modelsCalled).toContain('test-model'); expect(result.text).toBe('ok'); }); }); diff --git a/src/services/ai/streaming.requests.test.ts b/src/services/ai/streaming.requests.test.ts index 9e8312e..c999e92 100644 --- a/src/services/ai/streaming.requests.test.ts +++ b/src/services/ai/streaming.requests.test.ts @@ -1,5 +1,5 @@ -// Tests for the request each provider receives: Claude right after z.ai, the per-model Claude -// thinking params, tool_choice, and a quota-exhausted z.ai being skipped on the next round. +// Tests for Claude-first requests, model-specific thinking params, tool_choice, and failover +// after a priority provider exhausts its quota. import { beforeEach, describe, expect, it, mock, spyOn } from 'bun:test'; import OpenAI from 'openai'; @@ -18,9 +18,9 @@ describe('aiStreamRound fallback chain', () => { }); it.each([ - ['smart' as const, ['test-model', 'claude-smart']], - ['fast' as const, ['test-fast', 'claude-fast']], - ])('tries Claude right after z.ai in the %s chain', async (chain, expected) => { + ['smart' as const, ['claude-smart']], + ['fast' as const, ['claude-fast']], + ])('serves the %s chain through Claude without contacting fallbacks', async (chain, expected) => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; const calls: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { @@ -143,7 +143,7 @@ describe('aiStreamRound fallback chain', () => { }), ).rejects.toThrow(); - expect(sent.map((p) => p.model)).toEqual(['test-model', 'claude-smart', 'hf-test']); + expect(sent.map((p) => p.model)).toEqual(['claude-smart', 'test-model', 'hf-test']); for (const params of sent) { expect(params.tools).toEqual(calculateTool); expect(params.tool_choice).toBe('none'); @@ -164,7 +164,7 @@ describe('aiStreamRound fallback chain', () => { }; }); const clientsMod = await import('./clients'); - const spy = spyOn(clientsMod, 'zaiClient').mockReturnValue({ + const spy = spyOn(clientsMod, 'claudeClient').mockReturnValue({ chat: { completions: { create } }, } as unknown as OpenAI); @@ -184,16 +184,16 @@ describe('aiStreamRound fallback chain', () => { } }); - // Owner-ordered z.ai head: a depleted z.ai (429 quota) must cost one fast failure, not one per request. + // A depleted primary should fail once, then yield to the next priority provider. it.each([ ['smart' as const, 'test-model', 'claude-smart'], ['fast' as const, 'test-fast', 'claude-fast'], - ])('a quota-exhausted z.ai (429) falls through to Claude and is skipped on the next %s round', async (chain, zaiModel, claudeModel) => { + ])('a quota-exhausted Claude (429) falls through to z.ai and is skipped on the next %s round', async (chain, zaiModel, claudeModel) => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; const calls: string[] = []; const create = mock(async (params: OpenAI.ChatCompletionCreateParamsStreaming) => { calls.push(params.model); - if (params.model === zaiModel) { + if (params.model === claudeModel) { throw new OpenAI.APIError( 429, { message: 'Insufficient balance or quota' }, @@ -203,7 +203,7 @@ describe('aiStreamRound fallback chain', () => { } return { [Symbol.asyncIterator]: async function* () { - yield { choices: [{ delta: { content: 'from Claude' }, finish_reason: 'stop' }] }; + yield { choices: [{ delta: { content: 'from z.ai' }, finish_reason: 'stop' }] }; }, }; }); @@ -219,11 +219,57 @@ describe('aiStreamRound fallback chain', () => { maxTokens: 100, chain, }; - expect((await aiStreamRound(options)).text).toBe('from Claude'); - expect((await aiStreamRound(options)).text).toBe('from Claude'); - expect(calls).toEqual([zaiModel, claudeModel, claudeModel]); + expect((await aiStreamRound(options)).text).toBe('from z.ai'); + expect((await aiStreamRound(options)).text).toBe('from z.ai'); + expect(calls).toEqual([claudeModel, zaiModel, zaiModel]); } finally { for (const spy of spies) spy.mockRestore(); } }); }); + +describe('HF model request compatibility', () => { + it.each([ + ['hf', 'Qwen/Qwen3.8-2.4T-A95B', 100, 4196, 'low', undefined], + ['hf', 'Qwen/Qwen3.8-2.4T-A95B:deepinfra', 100, 4196, 'low', undefined], + ['hf', 'Qwen/Qwen3.8-2.4T-A95B:deepinfra', 8192, 12288, 'low', undefined], + ['hf', 'Qwen/Qwen3.8-Flash-Next-FP8', 100, 100, undefined, false], + ['hf', 'Qwen/Qwen3.8-Flash-Next-FP8:nscale', 100, 100, undefined, false], + ['hf', 'vendor/unrelated-model:deepinfra', 100, 100, undefined, undefined], + ['hf', 'meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra', 100, 100, undefined, undefined], + ['openrouter', 'Qwen/Qwen3.8-2.4T-A95B', 100, 100, undefined, undefined], + ['openrouter', 'Qwen/Qwen3.8-Flash-Next-FP8', 100, 100, undefined, undefined], + ] as const)('%s %s preserves the answer budget and sends only supported reasoning controls', async (provider, model, maxTokens, expectedMaxTokens, expectedEffort, expectedThinking) => { + type SentParams = OpenAI.ChatCompletionCreateParamsStreaming & { + chat_template_kwargs?: { enable_thinking: boolean }; + }; + const sent: SentParams[] = []; + const create = mock(async (params: SentParams) => { + sent.push(params); + return { + [Symbol.asyncIterator]: async function* () { + yield { choices: [{ delta: { content: 'visible answer' }, finish_reason: 'stop' }] }; + }, + }; + }); + const client = { chat: { completions: { create } } } as unknown as OpenAI; + const { streamingSlot } = await import('./streaming/slot'); + const slot = streamingSlot('request probe', provider, () => client, model); + const output = await slot.stream( + { + messages: [{ role: 'user', content: 'hi' }], + purpose: 'probe', + maxTokens, + temperature: 0.3, + }, + {}, + ); + expect(output.result.text).toBe('visible answer'); + expect(sent).toHaveLength(1); + expect(sent[0]?.max_tokens).toBe(expectedMaxTokens); + expect(sent[0]?.reasoning_effort).toBe(expectedEffort); + expect(sent[0]?.chat_template_kwargs?.enable_thinking).toBe(expectedThinking); + expect(sent[0]).not.toHaveProperty('thinking'); + expect(sent[0]?.temperature).toBe(0.3); + }); +}); diff --git a/src/services/ai/streaming.test-support.ts b/src/services/ai/streaming.test-support.ts index 697e4d0..eb05f25 100644 --- a/src/services/ai/streaming.test-support.ts +++ b/src/services/ai/streaming.test-support.ts @@ -21,9 +21,9 @@ mock.module('../../utils/logger', () => ({ logger: logMock, })); export const defaultOrder = (): ChainOrder => ({ - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); export const DEFAULT_ORDER = defaultOrder(); diff --git a/src/services/ai/streaming.usage.test.ts b/src/services/ai/streaming.usage.test.ts index 7157e4a..e79786d 100644 --- a/src/services/ai/streaming.usage.test.ts +++ b/src/services/ai/streaming.usage.test.ts @@ -16,15 +16,15 @@ mock.module('../../utils/logger', () => ({ })); const defaultOrder = (): ChainOrder => ({ - order: ['zai', 'claude', 'hf'], + order: ['claude', 'zai', 'hf'], fromEnv: false, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); -/** Puts HF at the head of both chains, to provoke a failing head provider (the order is an env setting). */ -const hfHead = (): ChainOrder => ({ - order: ['hf', 'zai', 'claude'], +/** Selects only HF when measuring the reserve provider in isolation. */ +const hfOnly = (): ChainOrder => ({ + order: ['hf'], fromEnv: true, - fallback: ['zai', 'claude', 'hf'], + fallback: ['claude', 'zai', 'hf'], }); const mockEnv = { @@ -319,8 +319,8 @@ describe('classifyAttemptFailure', () => { describe('aiStreamRound usage events', () => { it('records an ok attempt with the tokens from the final usage chunk', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); + mockEnv.AI_FAST_CHAIN = hfOnly(); const restore = await useCreate(async () => streamOf(textChunks('hello', FULL_USAGE))); try { const result = await run(); @@ -363,8 +363,8 @@ describe('aiStreamRound usage events', () => { }); it('keeps tokens NULL when the provider reports no usage', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); + mockEnv.AI_FAST_CHAIN = hfOnly(); const restore = await useCreate(async () => streamOf(textChunks('hello'))); try { await run(); @@ -383,8 +383,8 @@ describe('aiStreamRound usage events', () => { }); it('stores no prompt, completion or identifying data', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); + mockEnv.AI_FAST_CHAIN = hfOnly(); const restore = await useCreate(async () => streamOf(textChunks('the model answer', FULL_USAGE)), ); @@ -422,11 +422,11 @@ describe('aiStreamRound usage events', () => { }); it('records one event per attempt and stops at the first success', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = defaultOrder(); + mockEnv.AI_FAST_CHAIN = defaultOrder(); mockEnv.CLAUDE_API_TOKEN = 'test-claude'; const restore = await useCreate(async (params) => { - if (params.model === 'hf-test') throw apiError(429, 'Rate limit reached for model'); + if (params.model === 'claude-smart') throw apiError(429, 'Rate limit reached for model'); return streamOf(textChunks('from z.ai', FULL_USAGE)); }); try { @@ -439,10 +439,10 @@ describe('aiStreamRound usage events', () => { expect( recorded.map((e) => [e.provider, e.model, e.outcome, e.errorClass, e.httpStatus]), ).toEqual([ - ['hf', 'hf-test', 'error', 'rate_limit', 429], + ['claude', 'claude-smart', 'error', 'rate_limit', 429], ['zai', 'test-model', 'ok', null, null], ]); - // claude and hf were never reached: no row for them. + // HF was never reached: no row is recorded for it. expect(recorded[0]?.promptTokens).toBeNull(); expect(recorded[0]?.requestId).toBe(recorded[1]?.requestId ?? 'missing'); expect(recorded[0]?.requestId).toEqual(expect.any(String)); @@ -474,8 +474,8 @@ describe('aiStreamRound usage events', () => { } expect(recorded.map((e) => [e.provider, e.outcome, e.errorClass, e.httpStatus])).toEqual([ - ['zai', 'error', 'server', 500], ['claude', 'error', 'bad_request', 400], + ['zai', 'error', 'server', 500], ['hf', 'error', 'quota', 402], ]); }); @@ -543,8 +543,8 @@ describe('aiStreamRound usage events', () => { }); it('records a caller deadline as aborted and stops the chain', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); + mockEnv.AI_FAST_CHAIN = hfOnly(); const controller = new AbortController(); const restore = await useCreate((_params, opts) => { const hang = Promise.withResolvers(); @@ -566,30 +566,30 @@ describe('aiStreamRound usage events', () => { }); it('records a per-provider timeout as timeout and falls through to the next provider', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = defaultOrder(); + mockEnv.AI_FAST_CHAIN = defaultOrder(); const restore = await useCreate((params, opts) => { - if (params.model !== 'hf-test') return Promise.resolve(streamOf(textChunks('from z.ai'))); + if (params.model !== 'test-model') return Promise.resolve(streamOf(textChunks('from HF'))); const hang = Promise.withResolvers(); opts?.signal?.addEventListener('abort', () => hang.reject(new OpenAI.APIUserAbortError())); return hang.promise; }); try { const result = await run({ perProviderTimeoutMs: 1 }); - expect(result.providerUsed).toBe('z.ai (test-model)'); + expect(result.providerUsed).toBe('HF (hf-test)'); } finally { restore(); } expect(recorded.map((e) => [e.provider, e.outcome, e.errorClass])).toEqual([ - ['hf', 'error', 'timeout'], - ['zai', 'ok', null], + ['zai', 'error', 'timeout'], + ['hf', 'ok', null], ]); }); it('records slots excluded by a handoff as skipped, without requesting them', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = defaultOrder(); + mockEnv.AI_FAST_CHAIN = defaultOrder(); const requested: string[] = []; const restore = await useCreate(async (params) => { requested.push(params.model); @@ -619,8 +619,8 @@ describe('aiStreamRound usage events', () => { }); it.each([ - ['smart', 'agent', 'zai', 'test-model'], - ['fast', 'validator', 'zai', 'test-fast'], + ['smart', 'agent', 'claude', 'claude-smart'], + ['fast', 'validator', 'claude', 'claude-fast'], ['ocr', 'ocr', 'claude', 'claude-vision'], ] as const)('labels the %s chain event with its chain, purpose, provider and model', async (chain, purpose, provider, model) => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; @@ -637,7 +637,7 @@ describe('aiStreamRound usage events', () => { }); describe('aiStreamRound usage events: role, preset and cost (stage B4)', () => { - const DEEPSEEK = 'deepseek/deepseek-v4.1-flash'; + const DEEPSEEK = 'deepseek/deepseek-v4.1-flash-1017'; // 11 prompt + 5 completion tokens at $0.30 / $1.20 per million = 9.3 micro-USD → 9. const DEEPSEEK_COST = 9; const qualifiedPreset: ModelPreset = { @@ -738,7 +738,7 @@ describe('aiStreamRound streaming usage request', () => { restore(); } - expect(params.map((p) => p.model)).toEqual(['test-model', 'claude-smart', 'hf-test']); + expect(params.map((p) => p.model)).toEqual(['claude-smart', 'test-model', 'hf-test']); for (const p of params) expect(p.stream_options).toEqual({ include_usage: true }); }); @@ -818,8 +818,8 @@ describe('aiStreamRound streaming usage request', () => { }); it('remembers a provider that rejected stream_options and does not ask it again', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); - mockEnv.AI_FAST_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); + mockEnv.AI_FAST_CHAIN = hfOnly(); const calls: CreateParams[] = []; const restore = await useCreate(async (p) => { calls.push(p); @@ -897,7 +897,7 @@ describe('aiStreamRound prompt size estimate', () => { it('records the estimated request components on every attempt that sent a request, not on skipped ones', async () => { mockEnv.CLAUDE_API_TOKEN = 'test-claude'; const restore = await useCreate(async (params) => { - if (params.model === 'test-model') throw apiError(500, 'down'); + if (params.model === 'claude-smart') throw apiError(500, 'down'); return streamOf(textChunks('ok')); }); try { @@ -925,13 +925,13 @@ describe('aiStreamRound prompt size estimate', () => { }; expect(recorded.map((e) => [e.provider, e.outcome, e.promptSize])).toEqual([ ['hf', 'skipped', null], - ['zai', 'error', size], - ['claude', 'ok', size], + ['claude', 'error', size], + ['zai', 'ok', size], ]); }); it('stores only counts: no prompt text reaches the event', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); const restore = await useCreate(async () => streamOf(textChunks('ok'))); try { await run(); @@ -960,7 +960,7 @@ describe('aiStreamRound timing: first token and generation window', () => { const delta = (content: string) => ({ choices: [{ delta: { content }, finish_reason: null }] }); it('measures the first content delta from the request start and the window to the last delta', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); const restore = await useCreate(async () => timedStream([ { waitMs: 120, chunk: delta('he') }, @@ -987,7 +987,7 @@ describe('aiStreamRound timing: first token and generation window', () => { }); it('counts a tool-call delta as the first token', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); const toolDelta = { choices: [ { @@ -1016,7 +1016,7 @@ describe('aiStreamRound timing: first token and generation window', () => { }); it('a one-chunk answer has a zero generation window, not an unknown one', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); const restore = await useCreate(async () => streamOf(textChunks('hello', FULL_USAGE))); try { await run(); @@ -1049,7 +1049,7 @@ describe('aiStreamRound timing: first token and generation window', () => { }); it('a mid-stream failure after text keeps both timings NULL', async () => { - mockEnv.AI_SMART_CHAIN = hfHead(); + mockEnv.AI_SMART_CHAIN = hfOnly(); const restore = await useCreate(async () => ({ [Symbol.asyncIterator]: async function* () { yield delta('partial'); diff --git a/src/services/ai/streaming/chains.ts b/src/services/ai/streaming/chains.ts index 60611ad..235db87 100644 --- a/src/services/ai/streaming/chains.ts +++ b/src/services/ai/streaming/chains.ts @@ -97,7 +97,7 @@ function logUnconfiguredProvider(detail: object, named: boolean): void { /** * Builds one text chain in the configured order (AI_SMART_CHAIN / AI_FAST_CHAIN, default - * z.ai → Claude → HF), skipping — and logging once — any provider whose key or model is + * Claude → z.ai → HF), skipping — and logging once — any provider whose key or model is * missing. A named-but-unconfigured provider warns; one only in the default order is debug. * When nothing in an operator-configured order is usable the default order is tried instead. */ @@ -225,7 +225,16 @@ function buildOcrChain(): ProviderSlot[] { ); } chain.push(streamingSlot(`HF (${env.HF_VISION_MODEL})`, 'hf', hfClient, env.HF_VISION_MODEL)); - return chain; + return chain.filter((slot) => { + if (!isExcludedModel(slot.model)) return true; + logOnce(`ocr:${slot.key}:excluded:${slot.model}`, () => + logger.warn( + { chain: 'ocr', provider: slot.key, model: slot.model }, + '[AI_STREAM] Model is on the excluded list — skipping it', + ), + ); + return false; + }); } /** The smart chain: the role's own chain when one loads, else the legacy order. */ @@ -259,17 +268,22 @@ function buildChain(chain: ChainName, role?: Role): ProviderSlot[] { } } -/** The chain's slots minus the excluded names, in breaker order (demoted slots last). */ +/** + * Claude leads the selected providers; health can demote it behind another priority provider. + * HF stays last even when a priority provider is demoted, so reserve credits are used last. + * Explicit chains still choose which providers participate and preserve the other providers' order. + */ export function availableSlots( chain: ChainName, exclude: readonly string[], now: number, role?: Role, ): ProviderSlot[] { - return orderByHealth( - buildChain(chain, role).filter((slot) => !exclude.includes(slot.name)), - now, - ); + const slots = buildChain(chain, role).filter((slot) => !exclude.includes(slot.name)); + const primary = slots.filter((slot) => slot.key === 'claude'); + const fallback = slots.filter((slot) => slot.key !== 'claude' && slot.key !== 'hf'); + const reserve = slots.filter((slot) => slot.key === 'hf'); + return [...orderByHealth([...primary, ...fallback], now), ...reserve]; } /** The chain's slots named in `exclude` — a round skips them without requesting. */ diff --git a/src/services/ai/streaming/request-params.ts b/src/services/ai/streaming/request-params.ts index 9415fa2..b4cd579 100644 --- a/src/services/ai/streaming/request-params.ts +++ b/src/services/ai/streaming/request-params.ts @@ -55,6 +55,31 @@ export function chatRequestParams( return anthropic ? claudeRequestParams(model, maxTokens) : { max_tokens: maxTokens, temperature }; } +/** HF Router fields forwarded as-is by the OpenAI SDK. */ +export interface HfParams { + max_tokens: number; + reasoning_effort?: 'low'; + chat_template_kwargs?: { enable_thinking: false }; +} + +/** + * Exact Qwen 3.8 routes have different thinking contracts (official model cards, 2026-11-13). + * The 2.4T model cannot disable thinking: choose low effort and allow 4096 additional output + * tokens so a small caller budget is not also its whole reasoning budget. This allowance is an + * engineering choice, not a measured guarantee of answer completion. Flash-Next supports disabling + * thinking for bounded OCR output. Provider suffixes select a backend without changing the model. + */ +export function hfRequestParams(model: string, maxTokens: number): HfParams { + const modelId = model.split(':', 1)[0]; + if (modelId === 'Qwen/Qwen3.8-2.4T-A95B') { + return { max_tokens: maxTokens + 4096, reasoning_effort: 'low' }; + } + if (modelId === 'Qwen/Qwen3.8-Flash-Next-FP8') { + return { max_tokens: maxTokens, chat_template_kwargs: { enable_thinking: false } }; + } + return { max_tokens: maxTokens }; +} + /** * Endpoint+model pairs (`key:model`) that answered 400/422 to `stream_options`. They are asked * without it from then on; their usage rows carry NULL tokens. Per model, not per endpoint: HF diff --git a/src/services/ai/streaming/slot.ts b/src/services/ai/streaming/slot.ts index 02c72a1..bc4d964 100644 --- a/src/services/ai/streaming/slot.ts +++ b/src/services/ai/streaming/slot.ts @@ -6,7 +6,13 @@ import type OpenAI from 'openai'; import { type ModelPreset, presetRequestParams } from '../model-presets'; import type { ProviderId } from '../provider-ids'; -import { type ClaudeParams, chatRequestParams, openChatStream } from './request-params'; +import { + type ClaudeParams, + chatRequestParams, + type HfParams, + hfRequestParams, + openChatStream, +} from './request-params'; import { StreamCollector } from './stream-collector'; import type { ProviderSlot, StreamRoundOptions } from './types'; @@ -28,22 +34,28 @@ export interface SlotOptions { provider?: ProviderId; } -type StreamParams = OpenAI.ChatCompletionCreateParamsStreaming & Omit; +type StreamParams = OpenAI.ChatCompletionCreateParamsStreaming & + Omit & + Pick; /** The streaming chat request of one round: the slot's model, the caller's messages, tools and budget. */ function buildStreamParams( + key: string, model: string, opts: StreamRoundOptions, slotOptions: SlotOptions, ): StreamParams { const temperature = opts.temperature ?? DEFAULT_TEMPERATURE; + const budget = + slotOptions.preset && slotOptions.provider && !slotOptions.anthropic + ? presetRequestParams(slotOptions.preset, slotOptions.provider, temperature, opts.maxTokens) + : chatRequestParams(model, opts.maxTokens, temperature, slotOptions.anthropic ?? false); const params: StreamParams = { model, messages: opts.messages, stream: true, - ...(slotOptions.preset && slotOptions.provider && !slotOptions.anthropic - ? presetRequestParams(slotOptions.preset, slotOptions.provider, temperature, opts.maxTokens) - : chatRequestParams(model, opts.maxTokens, temperature, slotOptions.anthropic ?? false)), + ...budget, + ...(key === 'hf' ? hfRequestParams(model, budget.max_tokens) : {}), }; if (opts.tools && opts.tools.length > 0) { params.tools = opts.tools; @@ -69,7 +81,7 @@ export function streamingSlot( model, ...(slotOptions.preset ? { presetId: slotOptions.preset.id } : {}), stream: async (opts, cbs) => { - const params = buildStreamParams(model, opts, slotOptions); + const params = buildStreamParams(key, model, opts, slotOptions); let requestStartedPerf = performance.now(); const client = getClient(); const stream = await openChatStream(client, params, opts.signal, key, model, { diff --git a/src/services/ai/usage-report.test.ts b/src/services/ai/usage-report.test.ts index bafb291..fb75cb3 100644 --- a/src/services/ai/usage-report.test.ts +++ b/src/services/ai/usage-report.test.ts @@ -299,7 +299,7 @@ describe('formatUsageTelegram', () => { }); describe('cost and roles (stage B4)', () => { - const OR = { provider: 'openrouter', model: 'deepseek/deepseek-v4.1-flash' }; + const OR = { provider: 'openrouter', model: 'deepseek/deepseek-v4.1-flash-1017' }; test('model cost sums the known rows and counts the unknown ones, like tokens', () => { const r = aggregateUsage( From 97bcc3c70fa41c2cd7a611ea641887ac94c05f78 Mon Sep 17 00:00:00 2001 From: alex-mextner <111590326+alex-mextner@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:51:45 +0100 Subject: [PATCH 2/2] fix(UB-178): align diagnostic HF requests with runtime --- .env.example | 2 +- CLAUDE.md | 8 ++-- scripts/test-vision.ts | 10 ++-- scripts/verify-ai-chains.ts | 15 ++++-- src/config/env.ts | 2 +- src/services/ai/streaming.ts | 10 ++-- src/tooling/claude-probes.test.ts | 76 ++++++++++++++++++++++++++++--- 7 files changed, 100 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index ae03ab8..6ff621c 100644 --- a/.env.example +++ b/.env.example @@ -105,7 +105,7 @@ HF_VISION_MODEL=Qwen/Qwen3.8-Flash-Next-FP8:nscale #AI_TOOL_SCHEMA_USER_IDS=123456789,987654321 # OpenRouter (optional, OpenAI-compatible; skipped when key or model is unset). Model ids are OpenRouter's, -# e.g. deepseek/deepseek-v4.1-flash, z-ai/glm-5.3-flash, anthropic/claude-haiku-5.5. +# e.g. deepseek/deepseek-v4.1-flash-1017, z-ai/glm-5.3-flash, anthropic/claude-haiku-5.5. #OPENROUTER_API_KEY=sk-or-v1-XXX #OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 #OPENROUTER_MODEL= diff --git a/CLAUDE.md b/CLAUDE.md index 10d1837..b9177d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,11 +206,11 @@ Bot has a full AI agent with tool calling, accessible by mentioning `@ExpenseSyn Credential rotation edge cases: a token the SDK cannot send as an `Authorization` header (zero-width, non-Latin-1 or control characters) is excluded from rotation at client creation and never logged. A backup that becomes available only after the first attempt is not tried with a leftover budget below `min(timeout/4, 1s)`; the first failure is returned and the backup is not cooled. An attempt that times out on a per-key slice below 1s still rotates to the other key but does not cool the key it timed out on. Each key's cooldown keeps the failure that set its end time (a concurrent shorter failure never relabels it), and a request that finds every key cooling or being probed reports that cause. The `stream_options` compatibility retry runs inside the credential attempt that rejected the option; it never restarts rotation. -**Model ids are env values, refreshed against the live catalogs** (2026-10-24; HF examples refreshed 2026-11-13). z.ai coding endpoint `/models`: newest are `glm-5.3` (`AI_MODEL`) and `glm-5.3-flash` (`AI_FAST_MODEL`); `glm-5.3-flashx` is not in the Max plan (code 1311). Both answer 429 code 1310 while the weekly premium quota is exhausted (an unknown id would be 400 code 1211), so a 429 does not mean a bad id and the chain falls through to Claude. `glm-5.1` and `glm-4.5-flash` still exist but are superseded. Anthropic `/v1/models`: `claude-sonnet-5-5` and `claude-haiku-5-5` are the newest Sonnet and Haiku, both `max_tokens` 128000. HF Router examples now use `Qwen/Qwen3.8-2.4T-A95B:deepinfra` (smart), `meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra` (fast), and `Qwen/Qwen3.8-Flash-Next-FP8:nscale` (vision), each listed with tools and structured output in `/v1/models` on 2026-11-13. Credentialed inference still returns 402 requiring prepaid credits; catalog membership is not successful inference. Applying these examples to the live environment and evaluating cost, latency and receipt accuracy require separate rollout approval. +**Model ids are env values, refreshed against the live catalogs** (2026-10-24; HF examples refreshed 2026-11-13). z.ai coding endpoint `/models`: newest are `glm-5.3` (`AI_MODEL`) and `glm-5.3-flash` (`AI_FAST_MODEL`); `glm-5.3-flashx` is not in the Max plan (code 1311). Both answer 429 code 1310 while the weekly premium quota is exhausted (an unknown id would be 400 code 1211), so a 429 does not mean a bad id and the chain continues to the next available provider. `glm-5.1` and `glm-4.5-flash` still exist but are superseded. Anthropic `/v1/models`: `claude-sonnet-5-5` and `claude-haiku-5-5` are the newest Sonnet and Haiku, both `max_tokens` 128000. HF Router examples now use `Qwen/Qwen3.8-2.4T-A95B:deepinfra` (smart), `meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra` (fast), and `Qwen/Qwen3.8-Flash-Next-FP8:nscale` (vision), each listed with tools and structured output in `/v1/models` on 2026-11-13. Credentialed inference still returns 402 requiring prepaid credits; catalog membership is not successful inference. Applying these examples to the live environment and evaluating cost, latency and receipt accuracy require separate rollout approval. -**Provider order is configuration, not code**: `AI_SMART_CHAIN` and `AI_FAST_CHAIN` list provider ids (zai, claude, hf, openrouter); default for both is z.ai → Claude → HF (`openrouter` is in no default order — it runs only when the operator names it); providers without key/model are skipped. The ids come from `PROVIDER_IDS` in [provider-ids.ts](src/services/ai/provider-ids.ts) (a leaf module, shared with `config/env.ts`). The skip is logged, an unknown name is ignored with a warning, an order naming nothing configured falls back to the default order. The order is the same as in HyperCalendarBot. The circuit breaker ([provider-breaker.ts](src/services/ai/provider-breaker.ts)) only moves a *demoted* provider to the back, everything else keeps the configured order: a z.ai that answers 429 (quota/rate limit) or 402 (no credits) is demoted at once (Retry-After / quota reset time, else 5 min; 402 for an hour), so the next rounds start with Claude instead of paying z.ai's failed call again; the demoted z.ai is still tried last, and one clean answer lifts the demotion. A z.ai that merely hangs is not demoted (an abort is no evidence): each such round spends the 45 s per-provider timeout before Claude is tried. Risk: HF Router answered 402 (no credits) on 2026-10-12 and z.ai's quota ran out in HyperCalendarBot on 2026-09-24, so Claude is the de-facto primary when z.ai is depleted — keep its `max_tokens`/thinking handling intact. Sonnet is the OCR default because on degraded receipt photos Haiku produced a wrong yet sum-verified receipt while Sonnet refused (2026-10-12, clean fixtures: equal accuracy, Haiku ~25% faster). Request budget fields come from `chatRequestParams()`, which delegates to `claudeRequestParams()` for Claude slots (also used by `scripts/verify-ai-chains.ts`): never `temperature` (400 on Claude 5.x); Sonnet/Opus keep thinking on at the model's own level (the OpenAI-compat endpoint ignores `reasoning_effort`, so none is sent) and get `CLAUDE_THINKING_RESERVE_TOKENS` (8192) added to `max_tokens`, because thinking spends the same budget as the answer (without the reserve Sonnet truncated a 200-token prefill answer in 3/6 calls); Haiku runs with `thinking: {type: 'disabled'}` — at small budgets it otherwise streams zero text. The OpenAI-compat endpoint rejects explicit adaptive thinking. After changing `CLAUDE_MODEL`/`CLAUDE_FAST_MODEL`/`CLAUDE_VISION_MODEL`, run `scripts/verify-ai-chains.ts` (plus `scripts/run-receipt-snapshots.ts` for the vision model); to validate a provider change on real traffic, run `scripts/replay-ai-logs.ts` against `logs/chats` and a DB copy. +**Provider selection and order**: `AI_SMART_CHAIN` and `AI_FAST_CHAIN` select provider ids (zai, claude, hf, openrouter); the default is Claude → z.ai → HF. Claude leads the configured priority providers and HF remains the final reserve; other priority providers keep their relative configured order. OpenRouter runs only when explicitly selected. Providers without key/model are skipped and logged. The ids come from `PROVIDER_IDS` in [provider-ids.ts](src/services/ai/provider-ids.ts), shared with `config/env.ts`; an unknown name is ignored with a warning, and an order naming nothing configured falls back to the default. The circuit breaker ([provider-breaker.ts](src/services/ai/provider-breaker.ts)) demotes an unhealthy provider within its priority group, so Claude may move behind another priority provider while HF still stays last. HTTP 429 demotes a provider for Retry-After / quota reset time, else 5 min; HTTP 402 demotes it for an hour. A demoted provider remains eligible at the end of its group, and a clean answer lifts the demotion. Repeated connection-type failures also demote a provider; an abort alone does not. Claude is primary whenever configured and healthy; preserve its `max_tokens`/thinking handling. Sonnet is the OCR default because on degraded receipt photos Haiku produced a wrong yet sum-verified receipt while Sonnet refused (2026-10-12, clean fixtures: equal accuracy, Haiku ~25% faster). Request budget fields come from `chatRequestParams()`, which delegates to `claudeRequestParams()` for Claude slots (also used by `scripts/verify-ai-chains.ts`): never `temperature` (400 on Claude 5.x); Sonnet/Opus keep thinking on at the model's own level (the OpenAI-compat endpoint ignores `reasoning_effort`, so none is sent) and get `CLAUDE_THINKING_RESERVE_TOKENS` (8192) added to `max_tokens`, because thinking spends the same budget as the answer (without the reserve Sonnet truncated a 200-token prefill answer in 3/6 calls); Haiku runs with `thinking: {type: 'disabled'}` — at small budgets it otherwise streams zero text. The OpenAI-compat endpoint rejects explicit adaptive thinking. After changing `CLAUDE_MODEL`/`CLAUDE_FAST_MODEL`/`CLAUDE_VISION_MODEL`, run `scripts/verify-ai-chains.ts` (plus `scripts/run-receipt-snapshots.ts` for the vision model); to validate a provider change on real traffic, run `scripts/replay-ai-logs.ts` against `logs/chats` and a DB copy. -**HF Qwen request controls**: `hfRequestParams()` is applied only to HF slots, including bare role routes. The exact `Qwen/Qwen3.8-2.4T-A95B` id (with an optional provider suffix) requires thinking, so it uses `reasoning_effort: low` and 4096 extra output tokens above the caller budget. This is an engineering allowance, not a measured completion guarantee. The exact `Qwen/Qwen3.8-Flash-Next-FP8` id uses `chat_template_kwargs: {enable_thinking: false}` for bounded OCR output. All other models keep their existing request parameters. Source: the official [2.4T model card](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) and [Flash-Next model card](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8). +**HF Qwen request controls**: `hfRequestParams()` is applied only to HF slots, including bare role routes and the text/vision diagnostic probes. The exact `Qwen/Qwen3.8-2.4T-A95B` id (with an optional provider suffix) requires thinking, so it uses `reasoning_effort: low` and 4096 extra output tokens above the caller budget. This is an engineering allowance, not a measured completion guarantee. The exact `Qwen/Qwen3.8-Flash-Next-FP8` id uses `chat_template_kwargs: {enable_thinking: false}` for bounded OCR output. All other models keep their existing request parameters. Source: the official [2.4T model card](https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B) and [Flash-Next model card](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8). **Model presets** ([model-presets.ts](src/services/ai/model-presets.ts), stage B2; same ids and fields as HyperCalendarBot): `MODEL_PRESETS` (`deepseek41-flash`, `glm53-flash-low`, `glm53-low`, `haiku`) = per-provider model ids (`models`), measured `effort`/`temperature`/`maxOutputTokens`, list price (`priceUsdPerMTok` with `asOf` date and the `via` provider it is valid for; `null` = unknown, never 0), candidate `roles`, and `qualification` (`null` until a run passed `QUALIFICATION_GATE`: ≥95% own-role and 100% on the labelled history). `isQualifiedFor(preset, role)` is the only gate role routing may use. `EXCLUDED_MODEL_PATTERNS` (gpt-oss and Groq's qwen3.8-27b) are enforced in text, role, and OCR chains: a matching slot is skipped and logged, including HF/OpenRouter model overrides. A chain with no allowed provider throws the normal aggregate error; it never returns an empty success. Claude leads configured text and role chains. Provider health may move it behind another priority provider; HF always remains the final reserve. @@ -556,7 +556,7 @@ Required in `.env` (see [.env.example](.env.example)): - `AI_SMART_CHAIN`, `AI_FAST_CHAIN` - optional comma-separated provider order (ids `zai`, `claude`, `hf`, `openrouter`); default `claude,zai,hf`. Groq is no longer a provider (gpt-oss answered poorly and no replacement model was chosen); `GROQ_*` variables are ignored — ESB has no other Groq use, unlike HyperCalendarBot where `GROQ_API_KEY` still means Whisper transcription - `HF_TOKEN`, `HF_BASE_URL`, `HF_MODEL`, `HF_FAST_MODEL`, `HF_VISION_MODEL` - HuggingFace Router, last text fallback and OCR vision fallback - `CLAUDE_API_TOKEN`, `CLAUDE_API_TOKEN2` - optional primary and secondary Anthropic keys (Claude primary in smart/fast/OCR; both absent → Claude slots skipped, OCR HF-only); `CLAUDE_MODEL` (default `claude-sonnet-5-5`), `CLAUDE_FAST_MODEL` (default `claude-haiku-5-5`), `CLAUDE_VISION_MODEL` (receipt OCR, default `claude-sonnet-5-5`), `CLAUDE_BASE_URL` (default OpenAI-compat `https://api.anthropic.com/v1/`) -- `OPENROUTER_API_KEY`, `OPENROUTER_BASE_URL` (default `https://openrouter.ai/api/v1`), `OPENROUTER_MODEL`, `OPENROUTER_FAST_MODEL` - optional OpenRouter provider (id `openrouter`; same names as HyperCalendarBot); no default models — named in a chain order it still needs key and model, else it is skipped. Probed 2026-10-23: `deepseek/deepseek-v4.1-flash`, `z-ai/glm-5.3-flash`, `z-ai/glm-5.3`, `anthropic/claude-haiku-5.5` exist and accept an assistant tool-call turn with `content: null` (no `""` workaround needed) +- `OPENROUTER_API_KEY`, `OPENROUTER_BASE_URL` (default `https://openrouter.ai/api/v1`), `OPENROUTER_MODEL`, `OPENROUTER_FAST_MODEL` - optional OpenRouter provider (id `openrouter`; same names as HyperCalendarBot); no default models — named in a chain order it still needs key and model, else it is skipped. Current example: `deepseek/deepseek-v4.1-flash-1017`, refreshed from the 2026-11-13 catalog. That catalog check does not establish successful inference for the refreshed id. Earlier provider probes accepted an assistant tool-call turn with `content: null` (no `""` workaround needed) - `BOT_ADMIN_CHAT_ID` - admin Telegram chat ID (for merchant-agent notifications, feedback forwarding) ## Production Deployment diff --git a/scripts/test-vision.ts b/scripts/test-vision.ts index 58c62fd..4ddf717 100644 --- a/scripts/test-vision.ts +++ b/scripts/test-vision.ts @@ -1,7 +1,7 @@ /** * Quick probe: send one receipt image to a single vision provider and print the raw answer. - * Uses the bot's own clients, base URLs, models and request shape (chatRequestParams), so it - * tests exactly what production sends. Override a model inline, e.g. + * Uses the bot's own clients, base URLs, models and request helpers (chatRequestParams + * and hfRequestParams), so it tests the production request shape. Override a model inline, e.g. * CLAUDE_VISION_MODEL=claude-haiku-5-5 bun scripts/test-vision.ts receipt.jpg claude */ import { readFileSync } from 'node:fs'; @@ -9,7 +9,7 @@ import OpenAI from 'openai'; import { env } from '../src/config/env'; import { claudeClient, hfClient, zaiClient } from '../src/services/ai/clients'; import { withClaudeCredentials } from '../src/services/ai/claude-client-pool'; -import { chatRequestParams } from '../src/services/ai/streaming'; +import { chatRequestParams, hfRequestParams } from '../src/services/ai/streaming'; const PROVIDERS: Record OpenAI; model: string; anthropic: boolean }> = { claude: { client: claudeClient, model: env.CLAUDE_VISION_MODEL, anthropic: true }, @@ -43,6 +43,7 @@ say('---'); try { const t0 = Date.now(); const client = provider.client(); + const budget = chatRequestParams(provider.model, 2000, 0.1, provider.anthropic); const params: OpenAI.ChatCompletionCreateParamsNonStreaming = { model: provider.model, messages: [ @@ -57,7 +58,8 @@ try { ], }, ], - ...chatRequestParams(provider.model, 2000, 0.1, provider.anthropic), + ...budget, + ...(providerName === 'hf' ? hfRequestParams(provider.model, budget.max_tokens) : {}), }; const response = provider.anthropic ? await withClaudeCredentials( diff --git a/scripts/verify-ai-chains.ts b/scripts/verify-ai-chains.ts index 784e2bb..5840333 100644 --- a/scripts/verify-ai-chains.ts +++ b/scripts/verify-ai-chains.ts @@ -18,7 +18,7 @@ import OpenAI from 'openai'; import { env } from '../src/config/env'; import { claudeClient, hfClient, openrouterClient, zaiClient } from '../src/services/ai/clients'; import { withClaudeCredentials } from '../src/services/ai/claude-client-pool'; -import { chatRequestParams } from '../src/services/ai/streaming'; +import { chatRequestParams, hfRequestParams } from '../src/services/ai/streaming'; interface SlotSpec { chain: 'smart' | 'fast' | 'ocr'; @@ -28,6 +28,8 @@ interface SlotSpec { vision: boolean; /** Anthropic request shape, same as the Claude slots in streaming.ts: no `temperature` (400), per-model thinking. */ anthropic?: boolean; + /** HF-only thinking controls, matching the runtime slots. */ + hf?: boolean; } const SLOTS: SlotSpec[] = [ @@ -57,6 +59,7 @@ const SLOTS: SlotSpec[] = [ client: hfClient, model: env.HF_MODEL, vision: false, + hf: true, }, ...(env.OPENROUTER_API_KEY && env.OPENROUTER_MODEL && @@ -97,6 +100,7 @@ const SLOTS: SlotSpec[] = [ client: hfClient, model: env.HF_FAST_MODEL, vision: false, + hf: true, }, ...(env.OPENROUTER_API_KEY && env.OPENROUTER_FAST_MODEL && @@ -130,6 +134,7 @@ const SLOTS: SlotSpec[] = [ client: hfClient, model: env.HF_VISION_MODEL, vision: true, + hf: true, }, ]; @@ -200,11 +205,13 @@ async function testTextSlot(spec: SlotSpec): Promise { // before producing visible output. Give them headroom. const isReasoning = /qwen3|deepseek-r1/i.test(spec.model); const maxTokens = isReasoning ? 2048 : 256; + const budget = chatRequestParams(spec.model, maxTokens, 0.1, spec.anthropic ?? false); try { const response = await createCompletion(spec, { model: spec.model, messages: [{ role: 'user', content: TEXT_PROMPT }], - ...chatRequestParams(spec.model, maxTokens, 0.1, spec.anthropic ?? false), + ...budget, + ...(spec.hf ? hfRequestParams(spec.model, budget.max_tokens) : {}), }); const text = response.choices[0]?.message?.content?.trim() ?? ''; const ms = Date.now() - start; @@ -238,6 +245,7 @@ async function testVisionSlot(spec: SlotSpec): Promise { error: 'no test image available (place a .jpg/.png in debug-images/)', }; } + const budget = chatRequestParams(spec.model, 128, 0.1, spec.anthropic ?? false); try { const response = await createCompletion(spec, { model: spec.model, @@ -253,7 +261,8 @@ async function testVisionSlot(spec: SlotSpec): Promise { ], }, ], - ...chatRequestParams(spec.model, 128, 0.1, spec.anthropic ?? false), + ...budget, + ...(spec.hf ? hfRequestParams(spec.model, budget.max_tokens) : {}), }); const text = response.choices[0]?.message?.content?.trim() ?? ''; const ms = Date.now() - start; diff --git a/src/config/env.ts b/src/config/env.ts index 1cc96fe..79424e9 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -121,7 +121,7 @@ interface EnvConfig { AI_TOOL_SCHEMA_USER_IDS: number[] | undefined; // OpenRouter (optional; named in a chain order to be used, skipped without key and model). - // One key fronts DeepSeek / GLM / Claude; the model id is the OpenRouter one, e.g. `deepseek/deepseek-v4.1-flash`. + // One key fronts DeepSeek / GLM / Claude; the model id is the OpenRouter one, e.g. `deepseek/deepseek-v4.1-flash-1017`. OPENROUTER_API_KEY: string; OPENROUTER_BASE_URL: string; OPENROUTER_MODEL: string; diff --git a/src/services/ai/streaming.ts b/src/services/ai/streaming.ts index 1bbe1f7..2626228 100644 --- a/src/services/ai/streaming.ts +++ b/src/services/ai/streaming.ts @@ -2,8 +2,8 @@ * Unified AI streaming round with automatic provider fallback. * * Three chains, selected via options.chain: - * SMART: configured AI_SMART_CHAIN, default z.ai ${AI_MODEL} → Claude ${CLAUDE_MODEL} → HF ${HF_MODEL} - * FAST: configured AI_FAST_CHAIN, default z.ai ${AI_FAST_MODEL} → Claude ${CLAUDE_FAST_MODEL} → HF ${HF_FAST_MODEL} + * SMART: configured AI_SMART_CHAIN, default Claude ${CLAUDE_MODEL} → z.ai ${AI_MODEL} → HF ${HF_MODEL} + * FAST: configured AI_FAST_CHAIN, default Claude ${CLAUDE_FAST_MODEL} → z.ai ${AI_FAST_MODEL} → HF ${HF_FAST_MODEL} * OCR: Claude ${CLAUDE_VISION_MODEL} → HF ${HF_VISION_MODEL} (vision-only) * The Claude slot is skipped when its key is absent (OCR then runs HF-only); the OCR order is fixed. * @@ -16,8 +16,9 @@ * - All providers exhausted → throw an aggregated error (status/code chosen * order-independently, see pickRepresentativeError) * - * A circuit breaker (provider-breaker.ts) demotes a provider that hits repeated connection-type - * failures to the back of the chain for a cooldown, so a flapping endpoint stops being tried first. + * Configured chains put Claude first and HF last. The circuit breaker (provider-breaker.ts) + * demotes an unhealthy provider within its priority group during cooldown; HF stays the final reserve. + * Repeated connection-type failures also demote a provider; aborts alone do not. * The agent's stall handoff (stall-guard.ts) passes `excludeProviders` to keep a stalled model out * of the rest of a run; `nextProvider` names the slot such a round would try first. * @@ -43,6 +44,7 @@ export { CLAUDE_THINKING_RESERVE_TOKENS, chatRequestParams, claudeRequestParams, + hfRequestParams, resetStreamingUsageCompatibility, } from './streaming/request-params'; export { aiStreamRound } from './streaming/round'; diff --git a/src/tooling/claude-probes.test.ts b/src/tooling/claude-probes.test.ts index 3577752..0ac41d7 100644 --- a/src/tooling/claude-probes.test.ts +++ b/src/tooling/claude-probes.test.ts @@ -16,6 +16,7 @@ function runProbe( script: string, vision = false, secondaryOnly = false, + options: { hfRoutes?: boolean; provider?: 'claude' | 'hf' } = {}, ): { code: number; stdout: string; stderr: string; attempts: Attempt[] } { const dir = mkdtempSync(join(tmpdir(), 'esb-claude-probe-')); try { @@ -40,21 +41,30 @@ const logger = { info() {}, warn() {}, error() {}, debug() {} }; mock.module(${JSON.stringify(join(ROOT, 'src/utils/logger'))}, () => ({ createLogger: () => logger, logger })); const env = { ANTHROPIC_API_KEY: 'zai-key', AI_BASE_URL: 'https://zai.test/v1', AI_MODEL: 'zai-fixture', AI_FAST_MODEL: 'zai-fast-fixture', - HF_TOKEN: 'hf-key', HF_BASE_URL: 'https://hf.test/v1', HF_MODEL: 'hf-fixture', HF_FAST_MODEL: 'hf-fast-fixture', HF_VISION_MODEL: 'hf-vision-fixture', + HF_TOKEN: 'hf-key', HF_BASE_URL: 'https://hf.test/v1', + HF_MODEL: ${JSON.stringify(options.hfRoutes ? 'Qwen/Qwen3.8-2.4T-A95B:deepinfra' : 'hf-fixture')}, + HF_FAST_MODEL: 'hf-fast-fixture', + HF_VISION_MODEL: ${JSON.stringify(options.hfRoutes ? 'Qwen/Qwen3.8-Flash-Next-FP8:nscale' : 'hf-vision-fixture')}, + OPENROUTER_API_KEY: ${JSON.stringify(options.hfRoutes ? 'openrouter-key' : '')}, + OPENROUTER_BASE_URL: 'https://openrouter.test/v1', + OPENROUTER_MODEL: 'Qwen/Qwen3.8-2.4T-A95B:deepinfra', + OPENROUTER_FAST_MODEL: 'Qwen/Qwen3.8-Flash-Next-FP8:nscale', CLAUDE_API_TOKEN: ${JSON.stringify(secondaryOnly ? '' : 'primary-key')}, CLAUDE_API_TOKEN2: 'secondary-key', CLAUDE_BASE_URL: 'https://claude.test/v1', CLAUDE_MODEL: 'claude-sonnet-fixture', CLAUDE_FAST_MODEL: 'claude-haiku-fixture', CLAUDE_VISION_MODEL: 'claude-sonnet-fixture', - AI_SMART_CHAIN: { order: ['zai', 'claude', 'hf'] }, AI_FAST_CHAIN: { order: ['zai', 'claude', 'hf'] }, + AI_SMART_CHAIN: { order: ${JSON.stringify(options.hfRoutes ? ['zai', 'claude', 'openrouter', 'hf'] : ['zai', 'claude', 'hf'])} }, + AI_FAST_CHAIN: { order: ${JSON.stringify(options.hfRoutes ? ['zai', 'claude', 'openrouter', 'hf'] : ['zai', 'claude', 'hf'])} }, }; mock.module(${JSON.stringify(join(ROOT, 'src/config/env'))}, () => ({ env })); -const { chatRequestParams } = await import(${JSON.stringify(join(ROOT, 'src/services/ai/streaming/request-params.ts'))}); -mock.module(${JSON.stringify(join(ROOT, 'src/services/ai/streaming'))}, () => ({ chatRequestParams })); +const { chatRequestParams, hfRequestParams } = await import(${JSON.stringify(join(ROOT, 'src/services/ai/streaming/request-params.ts'))}); +mock.module(${JSON.stringify(join(ROOT, 'src/services/ai/streaming'))}, () => ({ chatRequestParams, hfRequestParams })); const attempts = []; globalThis.fetch = async (url, init) => { const authorization = new Headers(init?.headers).get('authorization'); const claude = new URL(String(url)).host === 'claude.test'; attempts.push({ authorization, claude, body: init?.body }); originalFs.writeFileSync(${JSON.stringify(attemptsPath)}, JSON.stringify(attempts)); - if (!claude) return Response.json({ error: { message: 'synthetic unavailable credit' } }, { status: 402 }); + const hf = new URL(String(url)).host === 'hf.test'; + if (!claude && !(hf && ${JSON.stringify(options.hfRoutes ?? false)})) return Response.json({ error: { message: 'synthetic unavailable credit' } }, { status: 402 }); if (authorization === 'Bearer primary-key') return Response.json({ error: { message: 'synthetic unauthorized' } }, { status: 401 }); return Response.json({ id: 'synthetic', object: 'chat.completion', created: 0, model: JSON.parse(init.body).model, choices: [{ index: 0, message: { role: 'assistant', content: '{"status":"ok","test":true}' }, finish_reason: 'stop' }], @@ -69,7 +79,7 @@ globalThis.fetch = async (url, init) => { '--preload', preload, join(ROOT, script), - ...(vision ? [image, 'claude'] : []), + ...(vision ? [image, options.provider ?? 'claude'] : []), ], cwd: ROOT, env: { PATH: process.env['PATH'] ?? '' }, @@ -130,3 +140,57 @@ test('a secondary-only vision probe works before rotation is needed', () => { expect({ code: result.code, stderr: result.stderr }).toEqual({ code: 0, stderr: '' }); expect(result.attempts.map((attempt) => attempt.authorization)).toEqual(['Bearer secondary-key']); }); + +interface ProbeRequest { + model: string; + max_tokens: number; + reasoning_effort?: string; + chat_template_kwargs?: { enable_thinking: boolean }; +} + +function bodiesFor(attempts: Attempt[], authorization: string): ProbeRequest[] { + return attempts + .filter((attempt) => attempt.authorization === authorization) + .map((attempt) => JSON.parse(attempt.body) as ProbeRequest); +} + +test('the chain verification script uses HF thinking controls only for HF routes', () => { + const result = runProbe('scripts/verify-ai-chains.ts', false, false, { hfRoutes: true }); + expect({ code: result.code, stderr: result.stderr }).toEqual({ code: 0, stderr: '' }); + const hf = bodiesFor(result.attempts, 'Bearer hf-key'); + expect(hf).toHaveLength(3); + expect(hf.find((body) => body.model === 'Qwen/Qwen3.8-2.4T-A95B:deepinfra')).toMatchObject({ + max_tokens: 6144, + reasoning_effort: 'low', + }); + expect(hf.find((body) => body.model === 'Qwen/Qwen3.8-Flash-Next-FP8:nscale')).toMatchObject({ + max_tokens: 128, + chat_template_kwargs: { enable_thinking: false }, + }); + const fast = hf.find((body) => body.model === 'hf-fast-fixture'); + expect(fast).toMatchObject({ max_tokens: 256 }); + expect(fast?.reasoning_effort).toBeUndefined(); + expect(fast?.chat_template_kwargs).toBeUndefined(); + const openrouter = bodiesFor(result.attempts, 'Bearer openrouter-key'); + expect(openrouter).toHaveLength(2); + for (const body of openrouter) { + expect(body.max_tokens).toBe(2048); + expect(body.reasoning_effort).toBeUndefined(); + expect(body.chat_template_kwargs).toBeUndefined(); + } +}); + +test('the actual HF vision probe disables thinking for the Flash-Next route', () => { + const result = runProbe('scripts/test-vision.ts', true, false, { + hfRoutes: true, + provider: 'hf', + }); + expect({ code: result.code, stderr: result.stderr }).toEqual({ code: 0, stderr: '' }); + const hf = bodiesFor(result.attempts, 'Bearer hf-key'); + expect(hf).toHaveLength(1); + expect(hf[0]).toMatchObject({ + model: 'Qwen/Qwen3.8-Flash-Next-FP8:nscale', + max_tokens: 2000, + chat_template_kwargs: { enable_thinking: false }, + }); +});