diff --git a/.env.example b/.env.example index bdd57c9a..a4d5ebed 100644 --- a/.env.example +++ b/.env.example @@ -3,19 +3,19 @@ BOT_TOKEN=your_telegram_bot_token # ─── AI providers (all REQUIRED — no hardcoded defaults) ──────────────────── -# AI primary — z.ai coding endpoint (GLM 5.3 via OpenAI-compat API) +# AI fallback — z.ai coding endpoint (GLM 5.3 via OpenAI-compat API) ZAI_API_KEY=your_zai_api_key ZAI_BASE_URL=https://api.z.ai/api/coding/paas/v4 ZAI_MODEL=glm-5.3 ZAI_FAST_MODEL=glm-5.3-flash -# HuggingFace Router (fallback, tool calling capable) +# HuggingFace Router (last fallback, tool calling capable) HF_TOKEN=your_hf_token HF_BASE_URL=https://router.huggingface.co/v1 -HF_MODEL=Qwen/Qwen3-235B-A22B -HF_FAST_MODEL=meta-llama/Llama-3.3-70B-Instruct +HF_MODEL=meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra +HF_FAST_MODEL=meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra -# Anthropic Claude (optional, paid reserve): tried right after z.ai in both default chains, +# Anthropic Claude (primary when configured): tried first in both default chains, # skipped when both tokens or the chain's model are unset. OpenAI-compatible endpoint. # CLAUDE_API_TOKEN=sk-ant-... # Optional backup: prefer an independent organization/workspace for independent quota. @@ -25,20 +25,25 @@ HF_FAST_MODEL=meta-llama/Llama-3.3-70B-Instruct # deadlines remain in force; only opening the AI request and catalog discovery are retried. # CLAUDE_MODEL=claude-sonnet-5-5 # CLAUDE_FAST_MODEL=claude-haiku-5-5 +# AI_SMART_CHAIN / AI_FAST_CHAIN select providers: selected Claude always leads and HF stays last. +# Other providers retain their relative order; omit Claude to select a chain without it. +# AI_SMART_CHAIN=claude,zai,hf +# AI_FAST_CHAIN=claude,zai,hf # OpenRouter (optional): DeepSeek and GLM Flash behind one key. In NO default chain: it runs only when # AI_SMART_CHAIN / AI_FAST_CHAIN name `openrouter` and both the key and that chain's model below are set. # OPENROUTER_API_KEY=sk-or-... # OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 -# OPENROUTER_MODEL=deepseek/deepseek-v4.1-flash +# OPENROUTER_MODEL=deepseek/deepseek-v4.1-flash-1017 # OPENROUTER_FAST_MODEL=z-ai/glm-5.3-flash # Role routing (default legacy = off). In `roles` mode agent rounds use AI_EXECUTOR_CHAIN and validator retries # AI_REPAIR_CHAIN; entries are `provider` or `provider:preset`, and a preset must be qualified for the role. +# Claude leads the selected role providers. The role precedes legacy fallback; HF always stays last. # AI_ROUTING=legacy # AI_ROUTING_USER_IDS=123456789 # canary: roles for these Telegram users only -# AI_EXECUTOR_CHAIN=openrouter:glm53-flash-low,claude -# AI_REPAIR_CHAIN=openrouter:glm53-low,claude +# AI_EXECUTOR_CHAIN=claude,openrouter:glm53-flash-low +# AI_REPAIR_CHAIN=claude,openrouter:glm53-low # AI_PLANNER_CHAIN= # Groq: voice transcription (Whisper) only — not an AI chat provider diff --git a/CLAUDE.md b/CLAUDE.md index 9c23af14..dc03909b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -242,13 +242,12 @@ All `process.env.*` reads go through `src/config/env.ts` → `loadConfig()` and When adding a new env var: (1) add it to `EnvConfig` interface in `env.ts`, (2) read and validate in `loadConfig()`, (3) use via `config.VAR_NAME` everywhere else. -**Model ids are env values, refreshed against the live catalogs** (2026-10-24; `ZAI_*`/`HF_*` are required env, `CLAUDE_*` optional). z.ai coding endpoint `/models`: newest are `glm-5.3` (smart) and `glm-5.3-flash` (fast; `glm-5.3-flashx` is not in the Max plan, code 1311). Both answer 429 code 1310 while the weekly premium quota is exhausted (an unknown id would be 400 code 1211), so a 429 does not mean a bad id; the chain falls through to Claude. `glm-5.1` and `glm-4.7-flash` still exist but are superseded. Anthropic `/v1/models`: `claude-sonnet-5-5` and `claude-haiku-5-5` are the newest Sonnet and Haiku (nothing above 5.5 in those tiers). HF router: `meta-llama/Llama-3.3-70B-Instruct` and `Qwen/Qwen3-235B-A22B` are still listed live; HF has no credits (402), so a replacement cannot be probed and the ids are kept until they disappear from `/models`. +**Model ids are env values, checked against current catalogs** (2026-11-13; `ZAI_*`/`HF_*` are required env, `CLAUDE_*` optional). Claude uses `claude-sonnet-5-5` and `claude-haiku-5-5`; z.ai uses `glm-5.3` and `glm-5.3-flash`. HF examples use `meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra`. A catalog entry proves availability in the catalog, not successful inference for an account. HF remains the last reserve and its account funding is handled independently of Claude activation. -**Provider order is configuration, not code**: `AI_SMART_CHAIN` and `AI_FAST_CHAIN` list provider ids (`zai`, `claude`, `hf`, optional `openrouter`); the default for both is z.ai → Claude → HF (the owner's order, identical in ExpenseSyncBot); providers without key/model are skipped. The ids come from `PROVIDER_IDS` in +**Provider selection is configuration**: `AI_SMART_CHAIN` and `AI_FAST_CHAIN` list provider ids (`zai`, `claude`, `hf`, optional `openrouter`); the default for both is Claude → z.ai → HF; selected Claude stays first, selected HF last, and other selected providers keep their relative order; providers without key/model are skipped. The ids come from `PROVIDER_IDS` in `src/services/ai/provider-ids.ts` — a leaf module that imports nothing, so both the config layer and the AI layer can derive from it without a cycle. Add a provider there and it is -nameable at once. Gemini was removed in full (2026-10): no id, no `GEMINI_*` variable is read, and a -leftover one in `.env` is ignored. Groq as a chat provider was removed in 2026-10 because gpt-oss tested badly +nameable at once. Groq as a chat provider was removed in 2026-10 because gpt-oss tested badly and there is no replacement Groq model: no `groq` id, no `GROQ_MODEL`/`GROQ_FAST_MODEL`/`GROQ_TPM_LIMITS` (leftovers in `.env` are ignored, and `groq` in a chain variable is dropped as an unknown id). `GROQ_API_KEY` remains ONLY for Whisper voice transcription (`src/services/voice/transcription-service.ts`, initialised in @@ -265,12 +264,12 @@ a new order in `streaming.ts`. `OPENROUTER_API_KEY`, `OPENROUTER_BASE_URL` (default `https://openrouter.ai/api/v1`), `OPENROUTER_MODEL` (smart chain) and `OPENROUTER_FAST_MODEL` (fast chain); no model default. It is in neither default chain: it runs only when `AI_SMART_CHAIN` / `AI_FAST_CHAIN` name it AND both the key and that chain's model are set, otherwise its slot -is skipped like any unconfigured provider. Live-verified ids (2026-10-23, `/models`): `deepseek/deepseek-v4.1-flash` +is skipped like any unconfigured provider. Live-verified ids (2026-10-23, `/models`): `deepseek/deepseek-v4.1-flash-1017` and `z-ai/glm-5.3-flash`. Claude-specific request rules (no `temperature`, the 128k `max_tokens`, the visible-token cap) apply only to the `claude` provider, so do not route Claude models through OpenRouter in a chain without measuring. Live-model replacement is off for it (it lists hundreds of differently priced models), and OpenRouter accepts the `content: null` assistant turn that carries tool calls, -so the transcript is sent unchanged. Never reintroduce gpt-oss, Groq chat or Gemini through it. Its circuit is +so the transcript is sent unchanged. Never reintroduce gpt-oss or Groq chat through it. Its circuit is scoped to the model (`openrouterCircuitScope`) for rate limits and outages, because OpenRouter routes each model to a different upstream: a limit on one model must not bench the other. A rejected or spent key (401, 402, a non-moderation 403) opens one circuit for the whole key instead (`CircuitContext.accountKey`, one notice), and any answer closes both. @@ -300,7 +299,7 @@ legacy `smart`/`fast` bench, and its ledger `chain` reads `role:`; the gat reads `preset.qualifications[provider]`, so a preset measured through OpenRouter does not load on z.ai); a preset slot never runs a live-model replacement (neither a cached nor a fresh heuristic one) because that would serve the role unmeasured; and a role chain is followed by the legacy chain (minus models the role chain already asked), so a role chain whose slots all fail, are benched or sit behind an -open circuit at runtime hands the round to the legacy chain instead of answering nobody. +open circuit at runtime hands the round to the legacy chain instead of answering nobody. Claude leads the selected providers within each role or legacy chain; legacy Claude never moves ahead of a non-HF role override. HF slots remain last across the combined chain. **A depleted or rejected provider account is skipped, not retried**: `src/services/ai/provider-circuit.ts` keeps one durable circuit per provider account for every provider in the chain (key = fingerprint @@ -341,24 +340,16 @@ is never written as 0 and a failed attempt that was billed carries its cost. Rea provider:model counts, errors by class, token sums with the answers that reported none counted apart, cost (`—` when unknown, with a count of answers without a cost next to a partial sum), p50/p95, a Roles section of cost and latency per role and provider once any row has a role (role-less rows are `legacy`), and one verdict per provider that says `no traffic in window` for a provider that served nothing; it refuses a database that has not run 067) and the -admin-only Telegram command `/ai_stats [days]` (same report, `t(lang).aiStats`, fail closed). Claude is second in the -default order and gets no traffic while z.ai answers, so prove it alive with the real code path: +admin-only Telegram command `/ai_stats [days]` (same report, `t(lang).aiStats`, fail closed). Claude is first in the +default order. Prove it alive with the real code path: `AI_SMART_CHAIN=claude AI_FAST_CHAIN=claude bun scripts/probe-ai-chain.ts --usage-database --expect-provider claude [--smart]` prints the probe result and the `ai_usage_events` row it wrote (never point it at the live database). -Rows (and the report) of retired providers such as `gemini` and `groq` stay readable: the report lists any provider id -found in the rows after the configured ones. The retired `gemini_usage` table lives only in the sidecar -`.gemini-quota.sqlite`, which no code opens any more; it is left on disk untouched (delete the file by hand when convenient). +Historical provider rows remain readable: the report lists any provider id found in the rows after the configured ones. Existing database files remain untouched. -**Cost risk of the order**: while z.ai is out of quota (its circuit is open, skipped silently) Claude is the de-facto -primary and every request is paid for. Claude keeps its `CLAUDE_MAX_OUTPUT_TOKENS` (128000) `max_tokens` and its thinking +**Cost of the order**: Claude is the primary and every successful primary request is billed to its API account. Claude keeps its `CLAUDE_MAX_OUTPUT_TOKENS` (128000) `max_tokens` and its thinking behaviour unchanged. Watch `bun scripts/report-ai-usage.ts` and set `AI_SMART_CHAIN`/`AI_FAST_CHAIN` (e.g. `zai,hf`, or `hf` alone) if the cost -is not acceptable; the ExpenseSyncBot OCR chain is separate and stays Claude vision → HF vision. -Both fallbacks have run dry before: z.ai's quota ran out on 2026-09-24 (its circuit has been open since), and Hugging Face -answered 402 "no credits" in ExpenseSyncBot on 2026-10-12. With z.ai depleted and HF the only provider behind Claude, -Claude carries the traffic; keep an eye on its spend and on HF credit. -z.ai is first in both chains, so a slow-but-not-failing z.ai delays every call before Claude is tried: up to 8 s per provider on the -fast chain (validators, city resolver), up to 15 s on the smart chain; an open circuit costs nothing. The chain order is the owner's choice — shorten -`ZAI_TIMEOUT_MS` in `clients.ts` or reorder via `AI_FAST_CHAIN` if that latency shows up in `/ai_stats`. +is not acceptable. These variables select providers: Claude stays first when selected and HF stays last, regardless of where they are written; the other providers keep their relative order. The effective smart/fast order is logged once per explicit selection. The ExpenseSyncBot OCR chain is separate and stays Claude vision → HF vision. +The fallback accounts have had quota and credit failures before. Those failures stay visible in usage/error telemetry and do not block the primary. A configured OpenRouter route runs before HF. Circuit-open providers are skipped; the remaining selected priority providers keep their relative order. Fast and smart requests retain their existing per-provider deadlines. Optional features that depend on an env var must deactivate gracefully when the var is absent — never throw at startup. Validate at the point of use, not at startup. diff --git a/docs/plans/2026-11-13-model-routing.md b/docs/plans/2026-11-13-model-routing.md new file mode 100644 index 00000000..5e5cc227 --- /dev/null +++ b/docs/plans/2026-11-13-model-routing.md @@ -0,0 +1,23 @@ +# Claude Primary and Reserve Routing + +Task: UB-178. Base revision: `128846b7e4c31667ee41b2fc1dae274e0bef3fcd`. + +## Changes + +Claude is first among selected providers in smart, fast and role chains. Other priority providers keep their relative order; Hugging Face stays last, including the combined role and legacy fallback chain. An explicit role chain precedes the legacy fallback; legacy Claude cannot displace a qualified role override. Provider selection, credential availability, preset qualification, circuits, deadlines and output validation stay active. + +Remove obsolete provider references from active source, scripts, fixtures and current configuration guidance. Historical usage rows remain readable through the existing generic provider handling; no database content is changed. + +Keep Claude Sonnet 5.5 and Haiku 5.5. Update the optional DeepSeek preset to `deepseek/deepseek-v4.1-flash-1017` and the Hugging Face examples to `meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra`. A provider-pinned Hugging Face route cannot silently change upstream through model discovery. No preset receives a new qualification. + +## Verification + +Run focused Bun tests against the source overlay with network disabled. Cover configured priority, actual provider fallback order, failure visibility, role fallback, existing output validation, preset pricing and pinned model behavior. Full suite and whole-repository typecheck are omitted under the owner's requested hatch. Live Claude account checks are recorded separately; synthetic tests do not prove production delivery. + +## Production Plan + +Publish to a source branch first. After separate rollout approval, deploy the approved revision and set both smart and fast chains to `claude,zai,hf` (include configured OpenRouter before HF when desired). Keep both current Claude credentials and 5.5 model IDs. Update the two HF model values above and remove obsolete bot-local provider environment entries without revoking shared keys. HF billing repair is outside this change and does not block Claude activation. Verify a bounded Claude text/tool request and fallback evidence after activation; retain the preceding image and environment snapshot for rollback. + +## Incident Timeline + +Claude 5.5 integration was committed on 2026-10-10 (`c7daca394994e40b65165f5842bded3daa5b4f16`), with 5.5-only model discovery and thinking-budget compatibility on 2026-10-11 (`481b6356d8d0bf2f668ce9a168cb5cc091fd1c84`). Both precede the November 8 requests reported by Anthropic. Current source and retained logs do not identify this bot as that caller. diff --git a/pyproject.toml b/pyproject.toml index 3d3c9759..2ea0ba28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hypercalendarbot" -version = "0.3.7" +version = "0.3.8" requires-python = ">=3.12" dependencies = [ "pyrofork>=2.3", diff --git a/scripts/real-history/pools.py b/scripts/real-history/pools.py index d00ee82a..9b6e67e6 100644 --- a/scripts/real-history/pools.py +++ b/scripts/real-history/pools.py @@ -711,7 +711,6 @@ "GPT", "ChatGPT", "Claude", - "Gemini", "ИИ", "Бот", "Календарь", diff --git a/src/config/env.ts b/src/config/env.ts index ed9955fe..f69bf0d3 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -1,7 +1,7 @@ // src/config/env.ts import { MODEL_ROLES } from '../services/ai/model-presets.ts'; -import { PROVIDER_IDS, type ProviderId } from '../services/ai/provider-ids.ts'; +import { PROVIDER_IDS, type ProviderId, prioritizeProviders } from '../services/ai/provider-ids.ts'; import { AI_ROUTING_MODES, type AiRoutingMode, @@ -13,15 +13,12 @@ import { TOOL_SCHEMA_MODES, type ToolSchemaMode } from '../services/ai/tool-sche import { logger, logOnce } from '../utils/logger.ts'; import { GOOGLE_OAUTH_CALLBACK_PATH, publicUrl } from './public-url.ts'; -/** Owner-chosen order for main requests: z.ai, then Claude, then Hugging Face. - * Explicit operator order (AI_SMART_CHAIN) wins; token-fit guards, circuits and fallback stay intact. - * A provider without its key or the chain's model is skipped, so a depleted z.ai falls through to Claude. - */ -export const DEFAULT_SMART_CHAIN: ProviderId[] = ['zai', 'claude', 'hf']; +/** Claude is primary; Hugging Face is the last reserve. Selection stays configurable. */ +export const DEFAULT_SMART_CHAIN: ProviderId[] = ['claude', 'zai', 'hf']; /** Short requests use the same owner-chosen order, with each provider's fast model. - * Explicit operator order (AI_FAST_CHAIN) still wins. + * Selected Claude stays first and Hugging Face stays last. */ -export const DEFAULT_FAST_CHAIN: ProviderId[] = ['zai', 'claude', 'hf']; +export const DEFAULT_FAST_CHAIN: ProviderId[] = ['claude', 'zai', 'hf']; export const DEFAULT_OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1'; @@ -64,7 +61,11 @@ function parseChain(name: string, fallback: ProviderId[]): ChainOrder { ); return { order: fallback, fromEnv: false, fallback }; } - return { order: parsed, fromEnv: true, fallback }; + const order = prioritizeProviders(parsed, (provider) => provider); + logOnce(`${name}:priority:${parsed.join(',')}`, () => + logger.info({ name, configured: parsed, effective: order }, 'Effective provider order'), + ); + return { order, fromEnv: true, fallback }; } export interface EnvConfig { @@ -72,7 +73,7 @@ export interface EnvConfig { DATABASE_PATH: string; NODE_ENV: 'development' | 'production'; - // AI primary provider (z.ai coding endpoint) + // AI fallback provider (z.ai coding endpoint) ZAI_API_KEY: string; ZAI_BASE_URL: string; ZAI_MODEL: string; @@ -97,7 +98,7 @@ export interface EnvConfig { MTPROTO_SERVICE_USER_ID?: number; // Whisper voice transcription only; Groq is not an AI chat provider any more GROQ_API_KEY?: string; - // Anthropic Claude (paid reserve after z.ai; OpenAI-compatible endpoint) + // Anthropic Claude (primary when configured; OpenAI-compatible endpoint) CLAUDE_API_TOKEN?: string; /** Independent backup credential, used with the same Claude models and request settings. */ CLAUDE_API_TOKEN2?: string; @@ -261,7 +262,7 @@ export function loadConfig(): EnvConfig { DATABASE_PATH: process.env.DATABASE_PATH || './data/calendar.db', NODE_ENV: (process.env.NODE_ENV as EnvConfig['NODE_ENV']) || 'development', - // AI primary (z.ai) + // AI fallback (z.ai) ZAI_API_KEY: requireEnv('ZAI_API_KEY'), ZAI_BASE_URL: requireEnv('ZAI_BASE_URL'), ZAI_MODEL: requireEnv('ZAI_MODEL'), diff --git a/src/database/repositories/ai-usage.repository.ts b/src/database/repositories/ai-usage.repository.ts index 37abad95..530e65f7 100644 --- a/src/database/repositories/ai-usage.repository.ts +++ b/src/database/repositories/ai-usage.repository.ts @@ -15,8 +15,8 @@ export type AiErrorClass = | 'other'; /** - * Why a provider was passed over without a request being sent. `local_budget` (Gemini's local quota - * admission, removed in #806) and `preflight_size` (the Groq token-fit preflight, removed with Groq as + * Why a provider was passed over without a request being sent. `local_budget` (retired local quota + * admission) and `preflight_size` (the Groq token-fit preflight, removed with Groq as * a chat provider) are retired and never written any more; they stay so that historical rows keep * their type. */ diff --git a/src/services/ai/model-presets.ts b/src/services/ai/model-presets.ts index 45ce811e..7cd62ece 100644 --- a/src/services/ai/model-presets.ts +++ b/src/services/ai/model-presets.ts @@ -59,18 +59,18 @@ export const HISTORY_PASS_REQUIRED = 1; const OPENROUTER_PRICE_SOURCE = 'OpenRouter /models list price; billed cost can differ, a reported usage.cost wins'; /** - * Seeded from the 2026-09-27 benchmark (PR #429) and the OpenRouter model ids verified on 2026-10-23. + * Seeded from the 2026-09-27 benchmark (PR #429); each route's price records its verification date. * No preset is qualified: the benchmark was through another host, on reconstructed scenarios. */ export const MODEL_PRESETS: readonly ModelPreset[] = [ { id: 'deepseek41-flash', - models: { openrouter: 'deepseek/deepseek-v4.1-flash' }, + models: { openrouter: 'deepseek/deepseek-v4.1-flash-1017' }, prices: { openrouter: { inputUsdPerMTok: 0.3, outputUsdPerMTok: 1.2, - asOf: '2026-10-23', + asOf: '2026-11-13', source: OPENROUTER_PRICE_SOURCE, }, }, diff --git a/src/services/ai/model-registry.ts b/src/services/ai/model-registry.ts index 9c616316..cae90119 100644 --- a/src/services/ai/model-registry.ts +++ b/src/services/ai/model-registry.ts @@ -64,7 +64,7 @@ interface ModelPreferenceTable { /** * Known-good chat model ids per provider, best first. Claude is pinned to the 5.5 models by - * owner decision (2026-10-10): no older fallbacks. It is a paid reserve and every id + * owner decision (2026-10-10): no older fallbacks. It is the primary provider and every id * shares the `claude` family, so the heuristic would pick `claude-fable-*`/`claude-opus-*`; * it is preferred-only and never switches to a pricier tier on its own. Other * providers with an empty list fall back to the heuristic in `selectReplacementModel`. @@ -445,12 +445,18 @@ async function waitForCatalog( } } -export async function resolveModelOverride(options: ResolveModelOverrideOptions): Promise { - if ( - options.signal?.aborted || +function shouldSkipModelLookup(options: ResolveModelOverrideOptions): boolean { + // The HF catalog lists base ids without provider pins. Replacing an explicit + // route from that list would silently change its provider; use normal failover. + if (options.provider === 'hf' && options.configuredModel.includes(':')) return true; + return ( + options.signal?.aborted === true || (options.providerDeadlineAt !== undefined && options.providerDeadlineAt <= performance.now()) - ) - return null; + ); +} + +export async function resolveModelOverride(options: ResolveModelOverrideOptions): Promise { + if (shouldSkipModelLookup(options)) return null; const key = cacheKey(options.provider, options.configuredModel); const pending = inFlight.get(key); const forced = options.forceRefresh === true; diff --git a/src/services/ai/provider-ids.ts b/src/services/ai/provider-ids.ts index bb1a8d15..6cc94449 100644 --- a/src/services/ai/provider-ids.ts +++ b/src/services/ai/provider-ids.ts @@ -14,3 +14,9 @@ export const PROVIDER_IDS = ['zai', 'claude', 'hf', 'openrouter'] as const; export type ProviderId = (typeof PROVIDER_IDS)[number]; + +/** Keep Claude first and Hugging Face last among the selected providers. */ +export function prioritizeProviders(entries: readonly T[], providerOf: (entry: T) => ProviderId): T[] { + const rank = (entry: T) => (providerOf(entry) === 'claude' ? 0 : providerOf(entry) === 'hf' ? 2 : 1); + return [...entries].sort((a, b) => rank(a) - rank(b)); +} diff --git a/src/services/ai/streaming/chains.ts b/src/services/ai/streaming/chains.ts index 49c76706..d7285499 100644 --- a/src/services/ai/streaming/chains.ts +++ b/src/services/ai/streaming/chains.ts @@ -10,7 +10,7 @@ import { claudeClient, hfClient, openrouterClient, zaiClient } from '../clients. import { excludedModelReason, isQualifiedFor, type ModelRole, presetById } from '../model-presets.ts'; import { providerCircuitKey } from '../provider-circuit.ts'; import type { BenchChain } from '../provider-eligibility.ts'; -import type { ProviderId } from '../provider-ids.ts'; +import { type ProviderId, prioritizeProviders } from '../provider-ids.ts'; import { aiLogger } from './logger.ts'; import { type ProviderSlot, streamingSlot } from './slot.ts'; @@ -77,7 +77,7 @@ function buildChain( const wasChosen = chain.fromEnv; const slotsFor = (ids: readonly ProviderId[]): ProviderSlot[] => { const chain: ProviderSlot[] = []; - for (const provider of ids) { + for (const provider of prioritizeProviders(ids, (provider) => provider)) { const { model, apiKey, baseUrl } = available[provider]; if (!model || !apiKey) { logOnce(`skip:${kind}:${provider}:${!!model}:${!!apiKey}`, () => { @@ -251,7 +251,7 @@ export function buildRoleChain(role: ModelRole, cfg: EnvConfig): ProviderSlot[] ...accountCircuitKeyFor(entry.provider, account.baseUrl ?? '', account.apiKey), }); } - return slots; + return prioritizeProviders(slots, (slot) => slot.provider); } /** diff --git a/src/services/ai/streaming/round-attempts.ts b/src/services/ai/streaming/round-attempts.ts index 61dc401e..5e74dd91 100644 --- a/src/services/ai/streaming/round-attempts.ts +++ b/src/services/ai/streaming/round-attempts.ts @@ -98,13 +98,15 @@ export function attemptScopeOf(state: RoundState, slot: ProviderSlot): AttemptSc * A role chain is tried first and the legacy chain follows it, minus models the role chain already asked: * roles can fail or be skipped at runtime (open circuit, bench, deadline) as well as fail to load, and a * user must not get no answer from a role chain while the legacy chain they were on yesterday is healthy. + * HF slots stay behind every selected priority provider, retaining their role and model settings. */ export function buildRoundChain(options: StreamRoundOptions): ProviderSlot[] { const cfg = loadConfig(); const role = roleForPurpose(options.purpose); const roleChain = role && rolesApplyTo(cfg, options.userId) ? buildRoleChain(role, cfg) : []; const legacyChain = options.fast ? buildFastChain(cfg) : buildSmartChain(cfg); - return roleChain.length > 0 ? [...roleChain, ...withoutModelsOf(roleChain, legacyChain)] : legacyChain; + const combined = roleChain.length > 0 ? [...roleChain, ...withoutModelsOf(roleChain, legacyChain)] : legacyChain; + return [...combined.filter((slot) => slot.provider !== 'hf'), ...combined.filter((slot) => slot.provider === 'hf')]; } function recordSkips( diff --git a/src/services/timezone/city-resolver.ts b/src/services/timezone/city-resolver.ts index 9e1d2bcd..5d6dc9b0 100644 --- a/src/services/timezone/city-resolver.ts +++ b/src/services/timezone/city-resolver.ts @@ -118,7 +118,7 @@ export async function resolveCity( return libResult; } - // 5. AI fallback via the fast provider chain (z.ai → Claude → HF fast models). + // 5. AI fallback via the fast provider chain (Claude → z.ai → HF fast models). // aiStreamRound handles provider selection, retries, and fallback internally. cmdLogger.info({ city: trimmed }, 'City resolver: falling back to AI'); diff --git a/test/config/env.test.ts b/test/config/env.test.ts index 73db0fe4..0cd75a02 100644 --- a/test/config/env.test.ts +++ b/test/config/env.test.ts @@ -63,22 +63,6 @@ describe('loadConfig', () => { expect(loadConfig().GROQ_API_KEY).toBeUndefined(); }); - test('a leftover Gemini setting in the environment never breaks loading', () => { - process.env.GEMINI_API_KEY = 'leftover-key'; - process.env.GEMINI_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/openai/'; - process.env.GEMINI_MODEL = 'gemini-2.5-pro'; - process.env.GEMINI_FAST_MODEL = 'gemini-2.5-flash'; - process.env.GEMINI_RATE_LIMITS = '{not json'; - expect(() => loadConfig()).not.toThrow(); - }); - - test('loads with only the z.ai and Hugging Face credentials', () => { - for (const key of Object.keys(process.env)) { - if (key.startsWith('GEMINI_')) delete process.env[key]; - } - expect(() => loadConfig()).not.toThrow(); - }); - describe('optional OpenRouter provider', () => { test('is unset by default and never joins the default chains', () => { for (const name of [ @@ -114,7 +98,7 @@ describe('loadConfig', () => { test('can be named in a chain order', () => { process.env.AI_SMART_CHAIN = 'openrouter,claude'; - expect(loadConfig().AI_SMART_CHAIN.order).toEqual(['openrouter', 'claude']); + expect(loadConfig().AI_SMART_CHAIN.order).toEqual(['claude', 'openrouter']); }); }); @@ -162,8 +146,8 @@ describe('loadConfig', () => { }); describe('provider chain order', () => { - const SMART_DEFAULT: ProviderId[] = ['zai', 'claude', 'hf']; - const FAST_DEFAULT: ProviderId[] = ['zai', 'claude', 'hf']; + const SMART_DEFAULT: ProviderId[] = ['claude', 'zai', 'hf']; + const FAST_DEFAULT: ProviderId[] = ['claude', 'zai', 'hf']; test('main and fast defaults prefer responsive providers while using their own configured models', () => { const config = loadConfig(); @@ -176,7 +160,7 @@ describe('loadConfig', () => { test('the fast chain reads its own variable and keeps its own fallback', () => { process.env.AI_FAST_CHAIN = 'hf,zai'; const config = loadConfig(); - expect(config.AI_FAST_CHAIN).toEqual({ order: ['hf', 'zai'], fromEnv: true, fallback: FAST_DEFAULT }); + expect(config.AI_FAST_CHAIN).toEqual({ order: ['zai', 'hf'], fromEnv: true, fallback: FAST_DEFAULT }); expect(config.AI_SMART_CHAIN.order).toEqual(SMART_DEFAULT); }); @@ -185,7 +169,7 @@ describe('loadConfig', () => { test('takes the order from the environment', () => { process.env.AI_SMART_CHAIN = 'claude, hf ,zai'; expect(loadConfig().AI_SMART_CHAIN).toEqual({ - order: ['claude', 'hf', 'zai'], + order: ['claude', 'zai', 'hf'], fromEnv: true, fallback: SMART_DEFAULT, }); @@ -193,11 +177,11 @@ describe('loadConfig', () => { test('drops a name it does not know rather than failing to start', () => { process.env.AI_SMART_CHAIN = 'hf,openai,zai'; - expect(loadConfig().AI_SMART_CHAIN).toEqual({ order: ['hf', 'zai'], fromEnv: true, fallback: SMART_DEFAULT }); + expect(loadConfig().AI_SMART_CHAIN).toEqual({ order: ['zai', 'hf'], fromEnv: true, fallback: SMART_DEFAULT }); }); test('a retired provider name left in the order is dropped like any unknown one', () => { - process.env.AI_SMART_CHAIN = 'gemini,zai'; + process.env.AI_SMART_CHAIN = 'retired,zai'; expect(loadConfig().AI_SMART_CHAIN).toEqual({ order: ['zai'], fromEnv: true, fallback: SMART_DEFAULT }); }); @@ -208,7 +192,7 @@ describe('loadConfig', () => { test('ignores a repeated provider instead of trying it twice', () => { process.env.AI_SMART_CHAIN = 'hf,zai,hf'; - expect(loadConfig().AI_SMART_CHAIN).toEqual({ order: ['hf', 'zai'], fromEnv: true, fallback: SMART_DEFAULT }); + expect(loadConfig().AI_SMART_CHAIN).toEqual({ order: ['zai', 'hf'], fromEnv: true, fallback: SMART_DEFAULT }); }); // An order naming nothing usable would leave the bot with no providers at diff --git a/test/database/repositories/ai-usage.repository.test.ts b/test/database/repositories/ai-usage.repository.test.ts index c3fa7804..f143634f 100644 --- a/test/database/repositories/ai-usage.repository.test.ts +++ b/test/database/repositories/ai-usage.repository.test.ts @@ -60,7 +60,7 @@ describe('AiUsageRepository', () => { test('rows of retired providers and retired skip reasons are read back as stored', () => { repo.insertMany([ - event({ at: 1, provider: 'gemini', model: 'gemini-3-flash' }), + event({ at: 1, provider: 'retired', model: 'historical-model' }), event({ at: 2, provider: 'groq', @@ -68,14 +68,14 @@ describe('AiUsageRepository', () => { outcome: 'skipped', skipReason: 'preflight_size', }), - event({ at: 3, provider: 'gemini', outcome: 'skipped', skipReason: 'local_budget' }), + event({ at: 3, provider: 'retired', outcome: 'skipped', skipReason: 'local_budget' }), ]); const rows = [...repo.iterateSince(0)]; expect(rows.map((row) => [row.provider, row.skipReason])).toEqual([ - ['gemini', null], + ['retired', null], ['groq', 'preflight_size'], - ['gemini', 'local_budget'], + ['retired', 'local_budget'], ]); }); diff --git a/test/scripts/provider-circuit-state.test.ts b/test/scripts/provider-circuit-state.test.ts index a4495a2f..18c7986e 100644 --- a/test/scripts/provider-circuit-state.test.ts +++ b/test/scripts/provider-circuit-state.test.ts @@ -104,7 +104,7 @@ describe('scripts/provider-circuit-state.ts', () => { ['account', null, 'open'], ['model', 'vendor/main', 'closed'], ['model', 'vendor/fast', 'closed'], - ['model', 'deepseek/deepseek-v4.1-flash', 'closed'], + ['model', 'deepseek/deepseek-v4.1-flash-1017', 'closed'], ]); expect(openrouter[0]).toMatchObject({ failureClass: 'auth_failed', status: 401 }); expect(providers.filter((row) => row.provider !== 'openrouter').every((row) => row.scope === 'account')).toBe(true); diff --git a/test/services/ai/model-excluded-chain.test.ts b/test/services/ai/model-excluded-chain.test.ts index 08b55e4e..a092221a 100644 --- a/test/services/ai/model-excluded-chain.test.ts +++ b/test/services/ai/model-excluded-chain.test.ts @@ -22,13 +22,14 @@ interface FakeProvider { requests: FakeRequest[]; } -function fakeProvider(text: string): FakeProvider { +function fakeProvider(text: string, onRequest?: () => void): FakeProvider { const requests: FakeRequest[] = []; const client = { chat: { completions: { create: async (params: FakeRequest) => { requests.push(params); + onRequest?.(); async function* chunks() { yield { choices: [{ delta: { content: text }, finish_reason: null }] }; yield { choices: [{ delta: {}, finish_reason: 'stop' }] }; @@ -94,7 +95,7 @@ describe('an excluded configured model', () => { }); test('does not exclude the same provider when its model is allowed', async () => { - process.env.AI_FAST_CHAIN = 'hf,claude'; + process.env.AI_FAST_CHAIN = 'hf'; const result = await aiStreamRound({ messages, maxTokens: 100, fast: true }); expect(result.text).toBe('from hf'); @@ -102,6 +103,39 @@ describe('an excluded configured model', () => { }); }); +describe('provider priority', () => { + test.each([false, true])('Claude answers before configured reserves, fast=%s', async (fast) => { + process.env.AI_SMART_CHAIN = 'hf,zai,claude'; + process.env.AI_FAST_CHAIN = 'hf,zai,claude'; + process.env.HF_MODEL = 'hf-main'; + const zai = fakeProvider('from zai'); + providerClients.zai = () => zai.client; + const result = await aiStreamRound({ messages, maxTokens: 100, fast }); + expect(result.text).toBe('from claude'); + expect(zai.requests).toHaveLength(0); + expect(hf.requests).toHaveLength(0); + }); + + test('HF runs only after Claude and the other selected priority providers fail', async () => { + Object.assign(process.env, { + AI_SMART_CHAIN: 'hf,openrouter,zai,claude', + HF_MODEL: 'hf-main', + OPENROUTER_API_KEY: 'or-key', + OPENROUTER_MODEL: 'vendor/model', + }); + const order: string[] = []; + for (const id of ['claude', 'openrouter', 'zai', 'hf'] as const) { + const fake = fakeProvider(`from ${id}`, () => { + order.push(id); + if (id !== 'hf') throw Object.assign(new Error('synthetic unavailable'), { status: 500 }); + }); + providerClients[id] = () => fake.client; + } + expect((await aiStreamRound({ messages, maxTokens: 100 })).text).toBe('from hf'); + expect(order).toEqual(['claude', 'openrouter', 'zai', 'hf']); + }); +}); + describe('a preset on a slot', () => { const preset: ModelPreset = { id: 'glm53-flash-low', diff --git a/test/services/ai/model-presets.test.ts b/test/services/ai/model-presets.test.ts index 8b0b7001..78c58e80 100644 --- a/test/services/ai/model-presets.test.ts +++ b/test/services/ai/model-presets.test.ts @@ -35,8 +35,9 @@ describe('seeded presets', () => { expect(presetById('nope')).toBeNull(); }); - test('carry the ids measured on 2026-09-27 and verified on OpenRouter on 2026-10-23', () => { - expect(presetById('deepseek41-flash')?.models.openrouter).toBe('deepseek/deepseek-v4.1-flash'); + test('carry verified route ids, including the latest DeepSeek snapshot', () => { + expect(presetById('deepseek41-flash')?.models.openrouter).toBe('deepseek/deepseek-v4.1-flash-1017'); + expect(presetById('deepseek41-flash')?.prices.openrouter?.asOf).toBe('2026-11-13'); expect(presetById('glm53-flash-low')?.models.openrouter).toBe('z-ai/glm-5.3-flash'); expect(presetById('glm53-flash-low')?.models.zai).toBe('glm-5.3-flash'); expect(presetById('glm53-flash-low')?.effort).toBe('low'); @@ -84,7 +85,7 @@ describe('the excluded list', () => { }); test.each([ - 'deepseek/deepseek-v4.1-flash', + 'deepseek/deepseek-v4.1-flash-1017', 'z-ai/glm-5.3', 'claude-haiku-5-5', 'glm-5.3-flash', @@ -194,7 +195,7 @@ describe('estimateCostMicroUsd (the dated price table, used only when the provid }); test('defaults to the seeded registry, which prices the OpenRouter models', () => { - expect(estimateCostMicroUsd('openrouter', 'deepseek/deepseek-v4.1-flash', 1_000_000, 0)).toBe(300_000); + expect(estimateCostMicroUsd('openrouter', 'deepseek/deepseek-v4.1-flash-1017', 1_000_000, 0)).toBe(300_000); expect(estimateCostMicroUsd('claude', 'claude-haiku-5-5', 1000, 1000)).toBeNull(); }); }); diff --git a/test/services/ai/model-registry.test.ts b/test/services/ai/model-registry.test.ts index 964f5939..1de885a1 100644 --- a/test/services/ai/model-registry.test.ts +++ b/test/services/ai/model-registry.test.ts @@ -169,6 +169,16 @@ describe('resolveModelOverride', () => { resetModelRegistry(); }); + test('never removes an explicit Hugging Face provider pin during replacement', async () => { + const { client, list } = makeListingClient(['meta-llama/Llama-4-Scout-17B-16E-Instruct']); + const configuredModel = 'meta-llama/Llama-4-Scout-17B-16E-Instruct:deepinfra'; + const resolved = await resolveModelOverride({ provider: 'hf', client, configuredModel }); + + expect(resolved).toBeNull(); + expect(list).not.toHaveBeenCalled(); + expect(getModelOverride('hf', configuredModel)).toBeNull(); + }); + test('probes /v1/models and returns a live replacement', async () => { const { client, list } = makeListingClient(['whisper-large-v3', 'vendor/model-120b']); diff --git a/test/services/ai/openrouter-provider.test.ts b/test/services/ai/openrouter-provider.test.ts index ae48d3ff..f5e86a0d 100644 --- a/test/services/ai/openrouter-provider.test.ts +++ b/test/services/ai/openrouter-provider.test.ts @@ -74,7 +74,7 @@ beforeEach(() => { CLAUDE_MODEL: 'claude-main', CLAUDE_FAST_MODEL: 'claude-fast', OPENROUTER_API_KEY: 'or-key', - OPENROUTER_MODEL: 'deepseek/deepseek-v4.1-flash', + OPENROUTER_MODEL: 'deepseek/deepseek-v4.1-flash-1017', OPENROUTER_FAST_MODEL: 'z-ai/glm-5.3-flash', }); delete process.env.AI_SMART_CHAIN; @@ -101,15 +101,15 @@ const messages: OpenAI.ChatCompletionMessageParam[] = [{ role: 'user', content: describe('optional OpenRouter provider', () => { test('answers when named first in the smart chain, with the smart model and usage', async () => { - process.env.AI_SMART_CHAIN = 'openrouter,claude'; + process.env.AI_SMART_CHAIN = 'openrouter,zai'; const result = await aiStreamRound({ messages, maxTokens: 100 }); expect(result.text).toBe('from openrouter'); - expect(result.providerUsed).toBe('OpenRouter (deepseek/deepseek-v4.1-flash)'); + expect(result.providerUsed).toBe('OpenRouter (deepseek/deepseek-v4.1-flash-1017)'); expect(result.metrics?.provider).toBe('openrouter'); expect(result.metrics?.usage?.promptTokens).toBe(7); expect(openrouter.requests[0]).toMatchObject({ - model: 'deepseek/deepseek-v4.1-flash', + model: 'deepseek/deepseek-v4.1-flash-1017', temperature: 0.3, max_tokens: 100, stream_options: { include_usage: true }, @@ -118,7 +118,7 @@ describe('optional OpenRouter provider', () => { }); test('the fast chain uses the fast model', async () => { - process.env.AI_FAST_CHAIN = 'openrouter,claude'; + process.env.AI_FAST_CHAIN = 'openrouter,zai'; const result = await aiStreamRound({ messages, maxTokens: 100, fast: true }); expect(result.providerUsed).toBe('OpenRouter (z-ai/glm-5.3-flash)'); @@ -128,44 +128,44 @@ describe('optional OpenRouter provider', () => { test('is never tried when the chains are the defaults, even if fully configured', async () => { const result = await aiStreamRound({ messages, maxTokens: 100 }); - expect(result.text).toBe('from zai'); + expect(result.text).toBe('from claude'); expect(openrouter.requests).toHaveLength(0); }); test('is skipped without a key', async () => { - process.env.AI_SMART_CHAIN = 'openrouter,claude'; + process.env.AI_SMART_CHAIN = 'openrouter,zai'; delete process.env.OPENROUTER_API_KEY; const result = await aiStreamRound({ messages, maxTokens: 100 }); - expect(result.text).toBe('from claude'); + expect(result.text).toBe('from zai'); expect(openrouter.requests).toHaveLength(0); }); test('is skipped without the chain model', async () => { - process.env.AI_FAST_CHAIN = 'openrouter,claude'; + process.env.AI_FAST_CHAIN = 'openrouter,zai'; delete process.env.OPENROUTER_FAST_MODEL; const result = await aiStreamRound({ messages, maxTokens: 100, fast: true }); - expect(result.text).toBe('from claude'); + expect(result.text).toBe('from zai'); expect(openrouter.requests).toHaveLength(0); }); test('an OpenRouter failure falls through to the next provider like any other', async () => { - process.env.AI_SMART_CHAIN = 'openrouter,claude'; + process.env.AI_SMART_CHAIN = 'openrouter,zai'; openrouter = makeProvider({ error: new OpenAI.APIError(402, { error: { message: 'no credits' } }, 'no credits', new Headers()), }); const result = await aiStreamRound({ messages, maxTokens: 100 }); - expect(result.text).toBe('from claude'); + expect(result.text).toBe('from zai'); expect(result.metrics?.failedProviders).toEqual([ - { provider: 'openrouter', model: 'deepseek/deepseek-v4.1-flash' }, + { provider: 'openrouter', model: 'deepseek/deepseek-v4.1-flash-1017' }, ]); }); test('a rate limit on the smart model does not bench the fast model behind the same key', async () => { - process.env.AI_SMART_CHAIN = 'openrouter,claude'; - process.env.AI_FAST_CHAIN = 'openrouter,claude'; + process.env.AI_SMART_CHAIN = 'openrouter,zai'; + process.env.AI_FAST_CHAIN = 'openrouter,zai'; openrouter = makeProvider({ error: new OpenAI.APIError( 429, @@ -175,7 +175,7 @@ describe('optional OpenRouter provider', () => { ), }); await aiStreamRound({ messages, maxTokens: 100 }); - expect(openrouter.requests.map((r) => r.model)).toEqual(['deepseek/deepseek-v4.1-flash']); + expect(openrouter.requests.map((r) => r.model)).toEqual(['deepseek/deepseek-v4.1-flash-1017']); openrouter = makeProvider({ text: 'fast answer' }); const fast = await aiStreamRound({ messages, maxTokens: 100, fast: true }); diff --git a/test/services/ai/role-chain-streaming.test.ts b/test/services/ai/role-chain-streaming.test.ts index a3c8f058..4c82c40d 100644 --- a/test/services/ai/role-chain-streaming.test.ts +++ b/test/services/ai/role-chain-streaming.test.ts @@ -110,7 +110,8 @@ afterEach(() => { const messages: OpenAI.ChatCompletionMessageParam[] = [{ role: 'user', content: 'hi' }]; describe('AI_ROUTING=roles', () => { - test('an agent round runs on the executor chain with the preset settings', async () => { + test('an explicit executor preset runs before the legacy Claude fallback', async () => { + process.env.AI_EXECUTOR_CHAIN = 'openrouter:qualified-test'; const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); expect(result.text).toBe('from openrouter'); @@ -120,6 +121,7 @@ describe('AI_ROUTING=roles', () => { reasoning: { effort: 'low' }, }); expect(zai.requests).toHaveLength(0); + expect(claude.requests).toHaveLength(0); }); test('a validator retry runs on the repair chain only when the preset is qualified for repair', async () => { @@ -142,22 +144,23 @@ describe('AI_ROUTING=roles', () => { process.env.AI_EXECUTOR_CHAIN = 'openrouter:no-such-preset'; const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); - expect(result.text).toBe('from zai'); + expect(result.text).toBe('from claude'); }); test('other purposes stay on the legacy chains', async () => { const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'validator', userId: 5, fast: true }); - expect(result.text).toBe('from zai'); + expect(result.text).toBe('from claude'); expect(openrouter.requests).toHaveLength(0); }); test('the canary list limits roles to the listed users', async () => { process.env.AI_ROUTING_USER_IDS = '7'; + process.env.AI_EXECUTOR_CHAIN = 'openrouter:qualified-test'; const other = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); const listed = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 7 }); - expect(other.text).toBe('from zai'); + expect(other.text).toBe('from claude'); expect(listed.text).toBe('from openrouter'); }); }); @@ -167,7 +170,7 @@ describe('AI_ROUTING=legacy (the default)', () => { delete process.env.AI_ROUTING; const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); - expect(result.text).toBe('from zai'); + expect(result.text).toBe('from claude'); expect(openrouter.requests).toHaveLength(0); }); }); diff --git a/test/services/ai/role-ledger.test.ts b/test/services/ai/role-ledger.test.ts index e6f4d50e..a9531816 100644 --- a/test/services/ai/role-ledger.test.ts +++ b/test/services/ai/role-ledger.test.ts @@ -85,7 +85,7 @@ const realPresetById = roleRoutingSeams.presetById; const savedEnv = { ...process.env }; // A route the seeded price table knows (0.3 / 1.2 USD per million tokens), so the fallback price applies to a role row. -const OR_PRESET_MODEL = 'deepseek/deepseek-v4.1-flash'; +const OR_PRESET_MODEL = 'deepseek/deepseek-v4.1-flash-1017'; const GATE = { sha: 'abc1234', measuredAt: '2026-10-23', rolePass: { executor: 1, repair: 1 }, historyPass: 1 }; const OPENROUTER_PRESET: ModelPreset = { id: 'or-test', @@ -135,10 +135,10 @@ beforeEach(() => { OPENROUTER_MODEL: 'vendor/legacy-model', OPENROUTER_FAST_MODEL: 'vendor/legacy-fast', AI_ROUTING: 'roles', - AI_EXECUTOR_CHAIN: 'openrouter:or-test,claude', - AI_REPAIR_CHAIN: 'openrouter:or-test,claude', - AI_SMART_CHAIN: 'zai,claude', - AI_FAST_CHAIN: 'zai,claude', + AI_EXECUTOR_CHAIN: 'openrouter:or-test', + AI_REPAIR_CHAIN: 'openrouter:or-test', + AI_SMART_CHAIN: 'zai', + AI_FAST_CHAIN: 'zai', }); for (const name of ['AI_ROUTING_USER_IDS', 'AI_PLANNER_CHAIN']) delete process.env[name]; resetLogOnce(); @@ -176,11 +176,13 @@ const tools: OpenAI.ChatCompletionTool[] = [ ]; describe('the ledger tells a role round from a legacy one', () => { - test('an executor round records its role, preset, chain label and the cost from the price table', async () => { + test('an executor round records its role and preset before legacy Claude can answer', async () => { + delete process.env.AI_SMART_CHAIN; await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); const [row, ...rest] = rows(); expect(rest).toEqual([]); + expect(claude.requests).toHaveLength(0); expect(row).toMatchObject({ provider: 'openrouter', model: OR_PRESET_MODEL, @@ -247,6 +249,7 @@ describe('the ledger tells a role round from a legacy one', () => { describe('a role round cannot bench a provider for legacy traffic', () => { test('a size rejection on the role slot benches that role and model only', async () => { + process.env.AI_EXECUTOR_CHAIN = 'openrouter:or-test,zai'; process.env.AI_SMART_CHAIN = 'openrouter,zai'; openrouter = fakeProvider({ error: apiError(413, 'Request too large') }); @@ -258,6 +261,7 @@ describe('a role round cannot bench a provider for legacy traffic', () => { }); test('a legacy round still reaches OpenRouter after the role slot was benched', async () => { + process.env.AI_EXECUTOR_CHAIN = 'openrouter:or-test,zai'; process.env.AI_SMART_CHAIN = 'openrouter,zai'; openrouter = fakeProvider({ error: apiError(413, 'Request too large') }, { text: 'legacy answer' }); await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5, tools }); @@ -284,18 +288,20 @@ describe('a role round cannot bench a provider for legacy traffic', () => { describe('a preset slot runs only the measured model', () => { test('a model that is gone is not swapped for a live-model guess, and the next entry serves', async () => { - process.env.AI_EXECUTOR_CHAIN = 'zai:zai-test,claude'; + process.env.AI_EXECUTOR_CHAIN = 'zai:zai-test,hf'; + process.env.AI_SMART_CHAIN = 'hf'; zai = fakeProvider({ error: apiError(404, 'model not found') }); const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); - expect(result.text).toBe('from claude'); + expect(result.text).toBe('from hf'); expect(zai.listCalls).toBe(0); expect(zai.requests.map((request) => request.model)).toEqual(['zai-preset-model']); }); test('a cached replacement from the legacy chain is not used by the preset slot either', async () => { - process.env.AI_EXECUTOR_CHAIN = 'zai:zai-test,claude'; + process.env.AI_EXECUTOR_CHAIN = 'zai:zai-test,hf'; + process.env.AI_SMART_CHAIN = 'hf'; process.env.ZAI_MODEL = 'zai-preset-model'; zai = fakeProvider({ error: apiError(404, 'model not found') }, { text: 'from replacement', usage: USAGE }); delete process.env.AI_ROUTING; @@ -304,6 +310,7 @@ describe('a preset slot runs only the measured model', () => { expect(zai.listCalls).toBe(1); process.env.AI_ROUTING = 'roles'; + process.env.AI_SMART_CHAIN = 'hf'; const requestsBefore = zai.requests.length; await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); @@ -311,7 +318,7 @@ describe('a preset slot runs only the measured model', () => { }); test('a plain provider entry keeps live-model discovery', async () => { - process.env.AI_EXECUTOR_CHAIN = 'zai,claude'; + process.env.AI_EXECUTOR_CHAIN = 'zai,hf'; zai = fakeProvider({ error: apiError(404, 'model not found') }, { text: 'from replacement', usage: USAGE }); const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); @@ -322,6 +329,10 @@ describe('a preset slot runs only the measured model', () => { }); describe('a role chain that has nothing left to try falls through to the legacy chain', () => { + beforeEach(() => { + process.env.AI_EXECUTOR_CHAIN = 'openrouter:or-test,claude'; + process.env.AI_SMART_CHAIN = 'zai,claude'; + }); test('every role slot failing at runtime hands the round to the legacy chain', async () => { openrouter = fakeProvider({ error: apiError(500, 'upstream down') }); claude = fakeProvider({ error: apiError(500, 'also down') }); @@ -331,8 +342,8 @@ describe('a role chain that has nothing left to try falls through to the legacy expect(result.text).toBe('from zai'); const written = rows(); expect(written.map((row) => [row.provider, row.outcome, row.chain, row.role])).toEqual([ - ['openrouter', 'error', 'role:executor', 'executor'], ['claude', 'error', 'role:executor', 'executor'], + ['openrouter', 'error', 'role:executor', 'executor'], ['zai', 'ok', 'smart', null], ]); }); @@ -361,8 +372,24 @@ describe('a role chain that has nothing left to try falls through to the legacy expect(openrouter.requests).toHaveLength(1); const skipped = rows().filter((row) => row.skipReason === 'circuit_open'); expect(skipped.map((row) => [row.provider, row.chain])).toEqual([ - ['openrouter', 'role:executor'], ['claude', 'role:executor'], + ['openrouter', 'role:executor'], + ]); + }); + + test('an HF role slot stays behind the legacy priority providers and keeps its role ledger', async () => { + process.env.AI_EXECUTOR_CHAIN = 'hf'; + delete process.env.AI_SMART_CHAIN; + claude = fakeProvider({ error: apiError(503, 'primary unavailable') }); + zai = fakeProvider({ error: apiError(503, 'reserve unavailable') }); + + const result = await aiStreamRound({ messages, maxTokens: 100, purpose: 'agent', userId: 5 }); + + expect(result.text).toBe('from hf'); + expect(rows().map((row) => [row.provider, row.outcome, row.chain, row.role])).toEqual([ + ['claude', 'error', 'smart', null], + ['zai', 'error', 'smart', null], + ['hf', 'ok', 'role:executor', 'executor'], ]); }); diff --git a/test/services/ai/role-routing.test.ts b/test/services/ai/role-routing.test.ts index 79ed7c2f..4e98a75f 100644 --- a/test/services/ai/role-routing.test.ts +++ b/test/services/ai/role-routing.test.ts @@ -29,7 +29,7 @@ describe('parseRoleChain', () => { test.each([ 'groq:gpt-oss', - 'gemini', + 'unknown-provider', 'openrouter:', 'openrouter:a:b', ':preset', diff --git a/test/services/ai/streaming-failover.test.ts b/test/services/ai/streaming-failover.test.ts index 84f25428..c25e1662 100644 --- a/test/services/ai/streaming-failover.test.ts +++ b/test/services/ai/streaming-failover.test.ts @@ -227,32 +227,32 @@ function ask() { describe('aiStreamRound — one broken provider never kills the chain', () => { beforeEach(() => { resetModelRegistry(); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: ZAI_QUOTA }] }); - hf = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from Claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: ZAI_QUOTA }] }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from HF' }] }); }); - test('regression: 404 model_not_found does not abort — Claude still answers', async () => { - // 2026-09-01 outage: z.ai out of quota, another provider deleted the model, and the 404 - // propagated out of aiStreamRound, so healthy Claude was never tried. + test('regression: 404 model_not_found does not abort — HF still answers', async () => { + // 2026-09-01 outage: Claude out of quota, another provider deleted the model, and the 404 + // propagated out of aiStreamRound, so healthy HF was never tried. const result = await ask(); - expect(result.text).toBe('answer from Claude'); - expect(result.providerUsed).toContain('Claude'); - expect(hf.requestedModels).toEqual(['hf-main']); + expect(result.text).toBe('answer from HF'); + expect(result.providerUsed).toContain('HF'); + expect(zai.requestedModels).toEqual(['glm-5.1']); }); test('401 and 403 fall through instead of aborting the chain', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(401, 'Invalid API key') }] }); - hf = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(403, 'Forbidden') }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(401, 'Invalid API key') }] }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(403, 'Forbidden') }] }); const result = await ask(); - expect(result.text).toBe('answer from Claude'); + expect(result.text).toBe('answer from HF'); }); test('every slot failing throws AllProvidersFailedError naming each provider and reason', async () => { - claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(500, 'claude overloaded') }] }); + hf = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(500, 'hf overloaded') }] }); const error = await ask().then( () => null, @@ -263,20 +263,20 @@ describe('aiStreamRound — one broken provider never kills the chain', () => { if (!(error instanceof AllProvidersFailedError)) throw new Error('expected AllProvidersFailedError'); expect(error.failures.map((f) => f.status)).toEqual([429, 404, 500]); - expect(error.failures.map((f) => f.provider)).toEqual(['z.ai (glm-5.1)', 'HF (hf-main)', 'Claude (claude-main)']); + expect(error.failures.map((f) => f.provider)).toEqual(['Claude (claude-main)', 'z.ai (glm-5.1)', 'HF (hf-main)']); expect(error.failures[0]?.message).toContain('Weekly/Monthly Limit Exhausted'); - expect(error.failures[2]?.message).toContain('claude overloaded'); + expect(error.failures[2]?.message).toContain('hf overloaded'); // The aggregate message alone must be enough to diagnose the outage. + expect(error.message).toContain('z.ai (glm-5.1)'); expect(error.message).toContain('HF (hf-main)'); - expect(error.message).toContain('Claude (claude-main)'); }); test('partial text from a dying provider is discarded and the next provider answers', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: new Error('connection closed') }] }); - hf = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }] }); - // z.ai emits a delta before dying: simulate by scripting a stream that throws mid-iteration. - zai.client.chat.completions.create = async (params: { model: string }) => { - zai.requestedModels.push(params.model); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: new Error('connection closed') }] }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }] }); + // Claude emits a delta before dying: simulate by scripting a stream that throws mid-iteration. + claude.client.chat.completions.create = async (params: { model: string }) => { + claude.requestedModels.push(params.model); async function* gen(): AsyncGenerator { yield { choices: [{ delta: { content: 'partial ' }, finish_reason: null }] }; throw new Error('connection closed mid-stream'); @@ -296,8 +296,8 @@ describe('aiStreamRound — one broken provider never kills the chain', () => { }, ); - expect(result.text).toBe('answer from Claude'); - expect(deltas).toEqual(['partial ', 'answer from Claude']); + expect(result.text).toBe('answer from HF'); + expect(deltas).toEqual(['partial ', 'answer from HF']); expect(switched).toBe(1); }); @@ -305,8 +305,8 @@ describe('aiStreamRound — one broken provider never kills the chain', () => { // The caller shows a "running " label as soon as onToolCallStart fires // (agent.ts: writer.setToolLabel). If that provider then dies, the label // belongs to a round that never happened and must be cleared like text is. - zai.client.chat.completions.create = async (params: { model: string }) => { - zai.requestedModels.push(params.model); + claude.client.chat.completions.create = async (params: { model: string }) => { + claude.requestedModels.push(params.model); async function* gen(): AsyncGenerator { yield { choices: [ @@ -334,20 +334,20 @@ describe('aiStreamRound — one broken provider never kills the chain', () => { ); expect(toolLabels).toEqual(['get_events']); - expect(result.text).toBe('answer from Claude'); + expect(result.text).toBe('answer from HF'); expect(switched).toBe(1); }); test('a caller-initiated abort propagates instead of burning the whole chain', async () => { const controller = new AbortController(); controller.abort(); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(500, 'irrelevant') }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(500, 'irrelevant') }] }); await expect( aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, signal: controller.signal }), ).rejects.toMatchObject({ name: 'AbortError' }); - expect(hf.requestedModels).toEqual([]); + expect(zai.requestedModels).toEqual([]); }); test('an exhausted caller deadline cannot start another provider', async () => { @@ -356,7 +356,7 @@ describe('aiStreamRound — one broken provider never kills the chain', () => { await expect(aiStreamRound({ messages: [], maxTokens: 100, signal })).rejects.toMatchObject({ name: 'TimeoutError', }); - expect(claude.requestedModels).toEqual([]); + expect(hf.requestedModels).toEqual([]); }); }); @@ -370,44 +370,44 @@ describe('benching a provider that said it is out', () => { resetProviderAlertState(); }); - // The waste this exists to remove: z.ai answered 429 to every request for a + // The waste this exists to remove: Claude answered 429 to every request for a // day, and the chain asked it again on every round of every message. test('a spent quota is not asked again on the next request', async () => { - zai = makeProvider({ + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'Weekly/Monthly Limit Exhausted, resets tomorrow') }], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); - hf = unusedProvider(); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); + zai = unusedProvider(); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}); - expect(zai.requestedModels).toHaveLength(1); - expect(claude.requestedModels).toHaveLength(2); + expect(claude.requestedModels).toHaveLength(1); + expect(hf.requestedModels).toHaveLength(2); }); // A provider's tier may not take a request carrying the tool catalog, but the short // summaries on the fast chain fit — benching it for everything would throw // away the one thing it is still good for. test.each([413, 429])('a size rejection %i benches only the requests that carry tools', async (sizeStatus) => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [ { kind: 'throw', error: apiError(sizeStatus, 'Request too large ... tokens per minute (TPM): Limit 8000') }, - { kind: 'text', text: 'from hf' }, + { kind: 'text', text: 'from zai' }, ], }); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'Rate limit reached') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); - process.env.AI_SMART_CHAIN = 'hf,claude'; + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'Rate limit reached') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); + process.env.AI_SMART_CHAIN = 'zai,hf'; const tools = [{ type: 'function' as const, function: { name: 'get_events', description: 'x', parameters: {} } }]; await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, tools }, {}); const withTools = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, tools }, {}); - expect(withTools.text).toBe('from claude'); - expect(hf.requestedModels).toHaveLength(1); + expect(withTools.text).toBe('from hf'); + expect(zai.requestedModels).toHaveLength(1); const withoutTools = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}); - expect(withoutTools.text).toBe('from hf'); + expect(withoutTools.text).toBe('from zai'); }); // A provider that reported a spent account is never forced: while every @@ -415,14 +415,14 @@ describe('benching a provider that said it is out', () => { // three guaranteed rejections (this used to be "try them anyway"). test('a chain where every account is depleted is not forced', async () => { const spent = apiError(429, 'Weekly/Monthly Limit Exhausted, resets tomorrow'); - zai = makeProvider({ + claude = makeProvider({ behaviors: [ { kind: 'throw', error: spent }, - { kind: 'text', text: 'from zai' }, + { kind: 'text', text: 'from claude' }, ], }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); hf = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); - claude = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}).catch(() => null); const second = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}).catch( @@ -434,17 +434,17 @@ describe('benching a provider that said it is out', () => { expect(second.failures).toHaveLength(3); expect(second.failures.every((f) => f.skippedBeforeRequest === true)).toBe(true); expect(second.roundMetrics.attemptCount).toBe(0); + expect(claude.requestedModels).toHaveLength(1); expect(zai.requestedModels).toHaveLength(1); - expect(hf.requestedModels).toHaveLength(1); }); // The bench suppresses calls, not observations: a chain that answers nobody // must still read as down. test('a benched chain still reports the outage', async () => { const spent = apiError(429, 'Weekly/Monthly Limit Exhausted, resets tomorrow'); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); claude = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); hf = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: spent }] }); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, {}).catch(() => null); expect(isAiChainDown()).toBe(true); @@ -482,7 +482,7 @@ describe('durable provider circuit', () => { providerCircuitClock.now = () => clock; configureProviderCircuit(statePath); startAlertLayer(); - hf = unusedProvider(); + zai = unusedProvider(); }); afterEach(() => { @@ -491,35 +491,35 @@ describe('durable provider circuit', () => { }); test('two providers with different failures are skipped afterwards, one notice each', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); - hf = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(401, 'Invalid API key') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); + zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(401, 'Invalid API key') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); await ask(); const third = await ask(); - expect(third.text).toBe('from claude'); + expect(third.text).toBe('from hf'); + expect(claude.requestedModels).toHaveLength(1); expect(zai.requestedModels).toHaveLength(1); - expect(hf.requestedModels).toHaveLength(1); - expect(claude.requestedModels).toHaveLength(3); + expect(hf.requestedModels).toHaveLength(3); expect(incidentNotices()).toHaveLength(2); }); test('a depleted account is skipped on the OTHER chain too', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, fast: true }); const smart = await ask(); - expect(smart.text).toBe('from claude'); - expect(zai.requestedModels).toEqual(['glm-5.1-air']); + expect(smart.text).toBe('from hf'); + expect(claude.requestedModels).toEqual(['claude-fast']); }); test('a restart does not repeat the notice or re-ask the provider', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); expect(incidentNotices()).toHaveLength(1); @@ -528,30 +528,30 @@ describe('durable provider circuit', () => { startAlertLayer(); await ask(); - expect(zai.requestedModels).toHaveLength(1); + expect(claude.requestedModels).toHaveLength(1); expect(incidentNotices()).toHaveLength(1); }); test('only a successful probe closes the circuit, and the next incident notifies again', async () => { - zai = makeProvider({ + claude = makeProvider({ behaviors: [ { kind: 'throw', error: apiError(402, 'Payment Required') }, - { kind: 'text', text: 'from zai' }, + { kind: 'text', text: 'from claude' }, { kind: 'throw', error: apiError(402, 'Payment Required') }, ], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); clock += 24 * 60 * MINUTE; const probe = await ask(); - expect(probe.text).toBe('from zai'); - expect(zai.requestedModels).toHaveLength(2); + expect(probe.text).toBe('from claude'); + expect(claude.requestedModels).toHaveLength(2); clock += 30 * MINUTE; - const again = await ask(); // closed now: zai is asked again and fails again - expect(again.text).toBe('from claude'); - expect(zai.requestedModels).toHaveLength(3); + const again = await ask(); // closed now: claude is asked again and fails again + expect(again.text).toBe('from hf'); + expect(claude.requestedModels).toHaveLength(3); expect(incidentNotices()).toHaveLength(2); }); @@ -560,13 +560,13 @@ describe('durable provider circuit', () => { const gate = new Promise((resolve) => { release = resolve; }); - zai = makeProvider({ + claude = makeProvider({ behaviors: [ { kind: 'throw', error: apiError(402, 'Payment Required') }, - { kind: 'hold', gate, text: 'from zai' }, + { kind: 'hold', gate, text: 'from claude' }, ], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); clock += 24 * 60 * MINUTE; @@ -575,15 +575,15 @@ describe('durable provider circuit', () => { release(); const texts = (await Promise.all([first, second])).map((r) => r.text).sort(); - expect(texts).toEqual(['from claude', 'from zai']); - expect(zai.requestedModels).toHaveLength(2); + expect(texts).toEqual(['from claude', 'from hf']); + expect(claude.requestedModels).toHaveLength(2); }); test('a provider dying mid-stream discards the partial output, opens its circuit and is not replayed', async () => { - zai = makeProvider({ + claude = makeProvider({ behaviors: [{ kind: 'partial-then-throw', text: 'half an ans', error: apiError(402, 'Payment Required') }], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); const events: string[] = []; const callbacks = { onTextDelta: (text: string) => events.push(`text:${text}`), @@ -593,78 +593,80 @@ describe('durable provider circuit', () => { const first = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, callbacks); const second = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }, callbacks); - expect(first.text).toBe('from claude'); - expect(second.text).toBe('from claude'); - expect(events).toEqual(['text:half an ans', 'switch', 'text:from claude', 'text:from claude']); - expect(zai.requestedModels).toHaveLength(1); - expect(claude.requestedModels).toHaveLength(2); + expect(first.text).toBe('from hf'); + expect(second.text).toBe('from hf'); + expect(events).toEqual(['text:half an ans', 'switch', 'text:from hf', 'text:from hf']); + expect(claude.requestedModels).toHaveLength(1); + expect(hf.requestedModels).toHaveLength(2); expect(incidentNotices()).toHaveLength(1); }); test('skipped providers stay visible in telemetry and are not counted as attempts', async () => { - process.env.AI_SMART_CHAIN = 'zai,claude'; - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + process.env.AI_SMART_CHAIN = 'claude,hf'; + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); const result = await ask(); - expect(result.metrics?.skippedProviders).toEqual([{ provider: 'zai', model: 'glm-5.1' }]); + expect(result.metrics?.skippedProviders).toEqual([{ provider: 'claude', model: 'claude-main' }]); expect(result.metrics?.failedProviders).toEqual([]); expect(result.metrics?.attemptCount).toBe(1); }); test('a caller abort carrying a quota error does not open the circuit', async () => { - zai = makeProvider({ - behaviors: [{ kind: 'text', text: 'from zai' }], + claude = makeProvider({ + behaviors: [{ kind: 'text', text: 'from claude' }], }); - claude = unusedProvider(); + hf = unusedProvider(); const controller = new AbortController(); controller.abort(apiError(402, 'Payment Required')); - expect(zai.requestedModels).toEqual([]); + expect(claude.requestedModels).toEqual([]); await expect( aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, signal: controller.signal }), ).rejects.toBeDefined(); const next = await ask(); - expect(next.text).toBe('from zai'); + expect(next.text).toBe('from claude'); expect(incidentNotices()).toHaveLength(0); }); test('a per-request rejection does not poison the provider for unrelated requests', async () => { - zai = makeProvider({ + claude = makeProvider({ behaviors: [ { kind: 'throw', error: apiError(400, 'invalid tool schema') }, { kind: 'throw', error: apiError(413, 'Request too large for this tier') }, - { kind: 'text', text: 'from zai' }, + { kind: 'text', text: 'from claude' }, ], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); const tools = [{ type: 'function' as const, function: { name: 'get_events', description: 'x', parameters: {} } }]; await ask(); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, tools }); const small = await ask(); - expect(small.text).toBe('from zai'); + expect(small.text).toBe('from claude'); expect(incidentNotices()).toHaveLength(0); }); test('a rate-limit incident notifies once and is silently skipped afterward', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'Rate limit reached') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'Rate limit reached') }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); await ask(); expect(incidentNotices()).toHaveLength(1); - expect(zai.requestedModels).toHaveLength(1); + expect(claude.requestedModels).toHaveLength(1); }); test('the incident notice never contains the provider error body', async () => { - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required SECRET-BODY-TEXT') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'from claude' }] }); + claude = makeProvider({ + behaviors: [{ kind: 'throw', error: apiError(402, 'Payment Required SECRET-BODY-TEXT') }], + }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); await ask(); @@ -674,27 +676,27 @@ describe('durable provider circuit', () => { }); describe('provider order', () => { - test('default main route starts with z.ai', async () => { + test('default main route starts with Claude', async () => { delete process.env.AI_SMART_CHAIN; - zai = makeProvider({ behaviors: [{ kind: 'text', text: 'main reply' }] }); - claude = unusedProvider(); + claude = makeProvider({ behaviors: [{ kind: 'text', text: 'main reply' }] }); + zai = unusedProvider(); hf = unusedProvider(); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'synthetic request' }], maxTokens: 256 }); expect(result.text).toBe('main reply'); expect(result.metrics?.attemptCount).toBe(1); - expect(zai.requestedModels).toEqual(['glm-5.1']); - expect(claude.requestedModels).toEqual([]); + expect(claude.requestedModels).toEqual(['claude-main']); + expect(zai.requestedModels).toEqual([]); expect(hf.requestedModels).toEqual([]); }); - test('default main failure falls directly to configured Claude main, not its fast model', async () => { + test('default main failure falls directly to configured z.ai main, not its fast model', async () => { delete process.env.AI_SMART_CHAIN; - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'unavailable') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'main fallback' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'unavailable') }] }); + zai = makeProvider({ behaviors: [{ kind: 'text', text: 'main fallback' }] }); hf = unusedProvider(); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'synthetic request' }], maxTokens: 256 }); expect(result.text).toBe('main fallback'); expect(result.metrics?.attemptCount).toBe(2); - expect(claude.requestedModels).toEqual(['claude-main']); + expect(zai.requestedModels).toEqual(['glm-5.1']); expect(hf.requestedModels).toEqual([]); }); @@ -712,7 +714,7 @@ describe('provider order', () => { // The order is the whole point of the configuration: a provider that answers // 429 all week, or whose tier rejects every request of this size, must be // routed around without a deploy. - test('tries providers in the configured order', async () => { + test('keeps Claude first and HF last among configured providers', async () => { process.env.AI_SMART_CHAIN = 'claude,hf,zai'; claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(429, 'rate limited') }] }); hf = makeProvider({ behaviors: [{ kind: 'text', text: 'from hf' }] }); @@ -723,7 +725,7 @@ describe('provider order', () => { expect(result.text).toBe('from hf'); expect(claude.requestedModels).toEqual(['claude-main']); expect(hf.requestedModels).toEqual(['hf-main']); - expect(zai.requestedModels).toEqual([]); + expect(zai.requestedModels).toEqual(['glm-5.1']); }); // An order naming only unconfigured providers would leave the bot with @@ -764,15 +766,15 @@ describe('provider order', () => { test.each([ '', ' ', - ])('empty first HF response %j falls through to the exact Claude fast model', async (emptyText) => { - process.env.AI_FAST_CHAIN = 'hf,claude'; - hf = makeProvider({ behaviors: [{ kind: 'text', text: emptyText }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'verified fallback' }] }); + ])('empty first Claude response %j falls through to the exact HF fast model', async (emptyText) => { + process.env.AI_FAST_CHAIN = 'claude,hf'; + claude = makeProvider({ behaviors: [{ kind: 'text', text: emptyText }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'verified fallback' }] }); zai = unusedProvider(); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 128, fast: true }); expect(result.text).toBe('verified fallback'); - expect(hf.requestedModels).toEqual(['hf-fast']); expect(claude.requestedModels).toEqual(['claude-fast']); + expect(hf.requestedModels).toEqual(['hf-fast']); expect(result.metrics?.attemptCount).toBe(2); expect(zai.requestedModels).toEqual([]); }); @@ -792,12 +794,12 @@ describe('provider order', () => { expect(claude.requestedModels).toEqual([]); }); - // Claude follows z.ai in both default orders and each chain uses its own + // Claude leads both default orders and each chain uses its own // Claude model (Sonnet answers users, Haiku validates). test.each([ { fast: false, chain: 'smart', model: 'claude-main' }, { fast: true, chain: 'fast', model: 'claude-fast' }, - ])('default $chain chain reaches Claude $model directly after z.ai fails', async ({ fast, chain, model }) => { + ])('default $chain chain reaches Claude $model before z.ai is needed', async ({ fast, chain, model }) => { delete process.env.AI_SMART_CHAIN; delete process.env.AI_FAST_CHAIN; zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'synthetic outage') }] }); @@ -807,8 +809,8 @@ describe('provider order', () => { expect(result.text).toBe('answer from Claude'); expect(result.metrics?.chain).toBe(chain); - expect(result.metrics?.attemptCount).toBe(2); - expect(zai.requestedModels).toHaveLength(1); + expect(result.metrics?.attemptCount).toBe(1); + expect(zai.requestedModels).toHaveLength(0); expect(claude.requestedModels).toEqual([model]); }); @@ -817,8 +819,8 @@ describe('provider order', () => { test('Claude requests omit the sampling temperature that other providers still get', async () => { process.env.AI_SMART_CHAIN = 'zai,claude'; Object.assign(process.env, { CLAUDE_API_TOKEN: 'claude-key', CLAUDE_MODEL: 'claude-main' }); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'synthetic outage') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from Claude' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'synthetic outage') }] }); + zai = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from Claude' }] }); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, temperature: 0 }); @@ -840,8 +842,8 @@ describe('provider order', () => { process.env.AI_SMART_CHAIN = 'zai,claude'; process.env.AI_FAST_CHAIN = 'zai,claude'; Object.assign(process.env, { CLAUDE_API_TOKEN: 'claude-key', CLAUDE_MODEL: model, CLAUDE_FAST_MODEL: model }); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'synthetic outage') }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'summary' }] }); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: apiError(503, 'synthetic outage') }] }); + zai = makeProvider({ behaviors: [{ kind: 'text', text: 'summary' }] }); await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens, fast }); @@ -938,35 +940,35 @@ describe('provider order', () => { describe('aiStreamRound — auto-detecting a live model', () => { beforeEach(() => { resetModelRegistry(); - zai = makeProvider({ behaviors: [{ kind: 'throw', error: ZAI_QUOTA }] }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from Claude' }] }); - hf = unusedProvider(); + claude = makeProvider({ behaviors: [{ kind: 'throw', error: ZAI_QUOTA }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'answer from HF' }] }); + zai = unusedProvider(); }); test('deleted model → /v1/models probed → request retried on the live model and succeeds', async () => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [ { kind: 'throw', error: MODEL_GONE }, - { kind: 'text', text: 'answer from HF' }, + { kind: 'text', text: 'answer from z.ai' }, ], liveModels: ['whisper-large-v3', 'replacement-model'], }); const result = await ask(); - expect(result.text).toBe('answer from HF'); - expect(result.providerUsed).toBe('HF (replacement-model)'); - expect(hf.requestedModels).toEqual(['hf-main', 'replacement-model']); - expect(claude.requestedModels).toEqual([]); - expect(result.metrics?.attemptCount).toBe(3); // z.ai quota + dead HF model + replacement HF request + expect(result.text).toBe('answer from z.ai'); + expect(result.providerUsed).toBe('z.ai (replacement-model)'); + expect(zai.requestedModels).toEqual(['glm-5.1', 'replacement-model']); + expect(hf.requestedModels).toEqual([]); + expect(result.metrics?.attemptCount).toBe(3); // Claude quota + dead z.ai model + replacement z.ai request expect(result.metrics?.fallbackCount).toBe(1); }); test('the resolved model is cached: the next request skips the dead model and the second probe', async () => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [ { kind: 'throw', error: MODEL_GONE }, - { kind: 'text', text: 'answer from HF' }, + { kind: 'text', text: 'answer from z.ai' }, ], liveModels: ['replacement-model'], }); @@ -974,61 +976,61 @@ describe('aiStreamRound — auto-detecting a live model', () => { await ask(); const second = await ask(); - expect(second.text).toBe('answer from HF'); - expect(hf.requestedModels).toEqual([ - 'hf-main', + expect(second.text).toBe('answer from z.ai'); + expect(zai.requestedModels).toEqual([ + 'glm-5.1', 'replacement-model', 'replacement-model', // second request goes straight to the live model ]); - expect(hf.modelsListCalls).toBe(1); + expect(zai.modelsListCalls).toBe(1); }); test('discovery failure degrades gracefully to the next provider', async () => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }], modelsListError: new Error('models endpoint unreachable'), }); const result = await ask(); - expect(result.text).toBe('answer from Claude'); - expect(hf.modelsListCalls).toBe(1); - expect(hf.requestedModels).toEqual(['hf-main']); + expect(result.text).toBe('answer from HF'); + expect(zai.modelsListCalls).toBe(1); + expect(zai.requestedModels).toEqual(['glm-5.1']); }); test('discovery finding nothing usable falls through to the next provider', async () => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }], liveModels: ['whisper-large-v3', 'playai-tts'], }); const result = await ask(); - expect(result.text).toBe('answer from Claude'); - expect(hf.requestedModels).toEqual(['hf-main']); + expect(result.text).toBe('answer from HF'); + expect(zai.requestedModels).toEqual(['glm-5.1']); }); test('a replacement that also dies triggers one re-probe, then falls through', async () => { - hf = makeProvider({ + zai = makeProvider({ behaviors: [{ kind: 'throw', error: MODEL_GONE }], liveModels: ['replacement-model'], }); - await ask(); // caches replacement-model, which also 404s → falls through to Claude + await ask(); // caches replacement-model, which also 404s → falls through to HF const result = await ask(); - expect(result.text).toBe('answer from Claude'); + expect(result.text).toBe('answer from HF'); // First request: dead configured model + retry on the replacement. // Second request: straight to the cached replacement, then one forced re-probe. - expect(hf.modelsListCalls).toBe(2); + expect(zai.modelsListCalls).toBe(2); - // The re-probe found only the model that just died, so HF is recorded as + // The re-probe found only the model that just died, so z.ai is recorded as // having nothing usable. Further requests must NOT probe /v1/models again — // otherwise a provider whose listing lies costs an extra HTTP round trip on // every single user message. await ask(); await ask(); - expect(hf.modelsListCalls).toBe(2); + expect(zai.modelsListCalls).toBe(2); }); }); @@ -1152,17 +1154,17 @@ describe('the chain a round runs on reaches the alert layer', () => { test('request-specific failure does not trigger an account incident from echoed text', async () => { const messages: string[] = []; initProviderAlerts({ botToken: 'synthetic', adminId: 1, send: (text) => messages.push(text), schedule: () => {} }); - process.env.AI_SMART_CHAIN = 'zai,claude'; - zai = makeProvider({ + process.env.AI_SMART_CHAIN = 'claude,hf'; + claude = makeProvider({ behaviors: [ { kind: 'throw', error: apiError(400, 'Bad input text contains: payment required') }, { kind: 'text', text: 'healthy next request' }, ], }); - claude = makeProvider({ behaviors: [{ kind: 'text', text: 'fallback' }] }); + hf = makeProvider({ behaviors: [{ kind: 'text', text: 'fallback' }] }); const options = { messages: [{ role: 'user' as const, content: 'synthetic' }], maxTokens: 80 }; expect((await aiStreamRound(options)).text).toBe('fallback'); expect((await aiStreamRound(options)).text).toBe('healthy next request'); - expect(zai.requestedModels).toHaveLength(2); + expect(claude.requestedModels).toHaveLength(2); expect(messages).toHaveLength(0); }); diff --git a/test/services/ai/streaming-fallback.test.ts b/test/services/ai/streaming-fallback.test.ts index 0d13cbd5..0cc91da6 100644 --- a/test/services/ai/streaming-fallback.test.ts +++ b/test/services/ai/streaming-fallback.test.ts @@ -140,8 +140,8 @@ const { AllProvidersFailedError, aiStreamRound, _resetStreamingUsageCompatibilit describe('aiStreamRound — provider chain fallback', () => { beforeEach(() => { _resetStreamingUsageCompatibilityForTest(); - fakeZai = undefined; fakeClaude = undefined; + fakeZai = undefined; fakeHf = undefined; }); @@ -184,16 +184,16 @@ describe('aiStreamRound — provider chain fallback', () => { '[]', 'null', ])('invalid or non-object tool arguments cannot become executable output: %s', async (args) => { - process.env.AI_SMART_CHAIN = 'claude'; - fakeClaude = buildFakeClient([{ kind: 'tool', id: 'bad-args', name: 'calculate', args }]); + process.env.AI_SMART_CHAIN = 'zai'; + fakeZai = buildFakeClient([{ kind: 'tool', id: 'bad-args', name: 'calculate', args }]); await expect(aiStreamRound({ messages: [], maxTokens: 128 })).rejects.toMatchObject({ failures: [{ emptyResponse: { classification: 'malformed_tools' } }], }); }); test('orphan argument fragments are diagnosed as malformed tool output', async () => { - process.env.AI_SMART_CHAIN = 'claude'; - fakeClaude = { + process.env.AI_SMART_CHAIN = 'zai'; + fakeZai = { chat: { completions: { create: mock(async () => @@ -210,8 +210,8 @@ describe('aiStreamRound — provider chain fallback', () => { }); test('an incomplete tool declaration is not accepted as usable output', async () => { - process.env.AI_SMART_CHAIN = 'claude'; - fakeClaude = buildFakeClient([{ kind: 'tool', id: 'incomplete', name: '', args: '{}' }]); + process.env.AI_SMART_CHAIN = 'zai'; + fakeZai = buildFakeClient([{ kind: 'tool', id: 'incomplete', name: '', args: '{}' }]); await expect(aiStreamRound({ messages: [], maxTokens: 128 })).rejects.toMatchObject({ failures: [{ emptyResponse: { classification: 'malformed_tools', toolFragmentCount: 1, choiceCount: 1 } }], }); @@ -231,7 +231,7 @@ describe('aiStreamRound — provider chain fallback', () => { test('aborting a stalled iterator requests closure without awaiting a stuck return', async () => { const close = mock(() => new Promise>(() => {})); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => ({ @@ -242,7 +242,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([{ kind: 'text', text: 'fallback after cleanup' }]); + fakeZai = buildFakeClient([{ kind: 'text', text: 'fallback after cleanup' }]); const started = performance.now(); const result = await aiStreamRound({ messages: [], maxTokens: 128, providerTimeoutMs: 20 }); expect(result.text).toBe('fallback after cleanup'); @@ -251,7 +251,7 @@ describe('aiStreamRound — provider chain fallback', () => { }); test('retains a terminal finish reason without a delta', async () => { - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => @@ -307,7 +307,7 @@ describe('aiStreamRound — provider chain fallback', () => { }); test('a stalled response body times out and a healthy fallback finishes the same round', async () => { let firstSignal: AbortSignal | undefined; - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async (_params: unknown, opts: { signal: AbortSignal }) => { @@ -320,7 +320,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([{ kind: 'text', text: 'fallback ready' }]); + fakeZai = buildFakeClient([{ kind: 'text', text: 'fallback ready' }]); const visible: string[] = []; const result = await aiStreamRound( { messages: [{ role: 'user', content: 'synthetic' }], maxTokens: 128, providerTimeoutMs: 20 }, @@ -334,16 +334,16 @@ describe('aiStreamRound — provider chain fallback', () => { test('a completed shared deadline starts no provider request', async () => { const signal = AbortSignal.timeout(1); await Bun.sleep(5); - fakeZai = buildFakeClient([{ kind: 'text', text: 'should not run' }]); + fakeClaude = buildFakeClient([{ kind: 'text', text: 'should not run' }]); await expect(aiStreamRound({ messages: [], maxTokens: 128, signal })).rejects.toThrow(); - expect(fakeZai.chat.completions.create).not.toHaveBeenCalled(); + expect(fakeClaude.chat.completions.create).not.toHaveBeenCalled(); }); - test('returns z.ai result on first success', async () => { - fakeZai = buildFakeClient([ - { kind: 'text', text: 'hello from z.ai' }, + test('returns Claude result on first success', async () => { + fakeClaude = buildFakeClient([ + { kind: 'text', text: 'hello from Claude' }, { kind: 'finish', reason: 'stop' }, ]); - fakeClaude = buildFakeClient(() => { + fakeZai = buildFakeClient(() => { throw new Error('claude should not be called'); }); fakeHf = buildFakeClient(() => { @@ -355,14 +355,14 @@ describe('aiStreamRound — provider chain fallback', () => { maxTokens: 100, }); - expect(result.text).toBe('hello from z.ai'); - expect(result.providerUsed).toContain('z.ai'); - expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); - expect(fakeClaude.chat.completions.create).not.toHaveBeenCalled(); + expect(result.text).toBe('hello from Claude'); + expect(result.providerUsed).toContain('Claude'); + expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeZai.chat.completions.create).not.toHaveBeenCalled(); }); test('captures terminal usage-only chunk and requests streaming usage', async () => { - fakeZai = buildFakeClient([ + fakeClaude = buildFakeClient([ { kind: 'text', text: 'measured' }, { kind: 'finish', reason: 'stop' }, { kind: 'usage', prompt: 120, completion: 30, cached: 40, reasoning: 7 }, @@ -376,12 +376,12 @@ describe('aiStreamRound — provider chain fallback', () => { reasoningTokens: 7, }); expect(result.metrics?.firstUsableSinceAttemptMs).toBeNumber(); - expect(fakeZai.chat.completions.create.mock.calls[0]?.[0]?.stream_options).toEqual({ include_usage: true }); + expect(fakeClaude.chat.completions.create.mock.calls[0]?.[0]?.stream_options).toEqual({ include_usage: true }); }); test('retries same provider without usage telemetry when compatibility rejects stream_options', async () => { let calls = 0; - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async (params: { stream_options?: unknown }) => { @@ -412,12 +412,12 @@ describe('aiStreamRound — provider chain fallback', () => { expect(second.text).toBe('compat'); expect(calls).toBe(3); expect(second.metrics?.attemptCount).toBe(1); - expect(fakeZai.chat.completions.create.mock.calls[2]?.[0]?.stream_options).toBeUndefined(); + expect(fakeClaude.chat.completions.create.mock.calls[2]?.[0]?.stream_options).toBeUndefined(); }); - test('falls through to Claude on z.ai 500', async () => { + test('falls through to z.ai on Claude 500', async () => { const apiError = new OpenAI.APIError(500, { error: { message: 'overloaded' } }, 'server error', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -426,8 +426,8 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ - { kind: 'text', text: 'hello from Claude' }, + fakeZai = buildFakeClient([ + { kind: 'text', text: 'hello from z.ai' }, { kind: 'finish', reason: 'stop' }, ]); fakeHf = buildFakeClient(() => { @@ -439,17 +439,17 @@ describe('aiStreamRound — provider chain fallback', () => { maxTokens: 100, }); - expect(result.text).toBe('hello from Claude'); - expect(result.providerUsed).toContain('Claude'); - expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); + expect(result.text).toBe('hello from z.ai'); + expect(result.providerUsed).toContain('z.ai'); expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); }); - test('falls through on EmptyProviderResponseError (z.ai coding endpoint quirk)', async () => { - // z.ai returns 200 OK but no text and no tool calls — streaming adapter - // should throw EmptyProviderResponseError and chain should try Claude. - fakeZai = buildFakeClient([{ kind: 'finish', reason: 'stop' }]); - fakeClaude = buildFakeClient([ + test('falls through on EmptyProviderResponseError (empty primary response)', async () => { + // Claude returns 200 OK but no text and no tool calls — streaming adapter + // should throw EmptyProviderResponseError and chain should try z.ai. + fakeClaude = buildFakeClient([{ kind: 'finish', reason: 'stop' }]); + fakeZai = buildFakeClient([ { kind: 'text', text: 'claude to the rescue' }, { kind: 'finish', reason: 'stop' }, ]); @@ -463,15 +463,15 @@ describe('aiStreamRound — provider chain fallback', () => { }); expect(result.text).toBe('claude to the rescue'); - expect(result.metrics).toMatchObject({ failedProviders: [{ provider: 'zai', model: 'zai-main' }] }); - expect(result.providerUsed).toContain('Claude'); + expect(result.metrics).toMatchObject({ failedProviders: [{ provider: 'claude', model: 'claude-main' }] }); + expect(result.providerUsed).toContain('z.ai'); }); test('all three providers fail → throws an aggregate naming every provider', async () => { const err500 = new OpenAI.APIError(500, { error: { message: 'boom' } }, 'boom', new Headers()); const err503 = new OpenAI.APIError(503, { error: { message: 'down' } }, 'down', new Headers()); const err504 = new OpenAI.APIError(504, { error: { message: 'gateway' } }, 'gateway', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -480,7 +480,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = { + fakeZai = { chat: { completions: { create: mock(async () => { @@ -519,7 +519,7 @@ describe('aiStreamRound — provider chain fallback', () => { // (bad tool template, unsupported parameter). Another provider still can, so // the chain must continue — treating it as fatal took the bot down on 2026-09-01. const badRequest = new OpenAI.APIError(400, { error: { message: 'bad input' } }, 'bad input', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -528,7 +528,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ + fakeZai = buildFakeClient([ { kind: 'text', text: 'claude answered anyway' }, { kind: 'finish', reason: 'stop' }, ]); @@ -542,13 +542,13 @@ describe('aiStreamRound — provider chain fallback', () => { }); expect(result.text).toBe('claude answered anyway'); - expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); }); test('400 with no body falls through to next provider', async () => { // Providers sometimes return 400 with no body for transient issues — treat as retryable. const noBody = new OpenAI.APIError(400, undefined, '400 status code (no body)', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -557,7 +557,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ + fakeZai = buildFakeClient([ { kind: 'text', text: 'claude picked up' }, { kind: 'finish', reason: 'stop' }, ]); @@ -571,7 +571,7 @@ describe('aiStreamRound — provider chain fallback', () => { }); expect(result.text).toBe('claude picked up'); - expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); }); test('400 with no body is retried once on the same provider before falling through', async () => { @@ -582,7 +582,7 @@ describe('aiStreamRound — provider chain fallback', () => { { kind: 'finish', reason: 'stop' }, ]); let calls = 0; - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async (...args: unknown[]) => { @@ -593,7 +593,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient(() => { + fakeZai = buildFakeClient(() => { throw new Error('claude should not be called'); }); fakeHf = buildFakeClient(() => { @@ -607,12 +607,12 @@ describe('aiStreamRound — provider chain fallback', () => { expect(result.text).toBe('zai recovered'); expect(calls).toBe(2); - expect(fakeClaude.chat.completions.create).not.toHaveBeenCalled(); + expect(fakeZai.chat.completions.create).not.toHaveBeenCalled(); }); test('retries a second empty 400 only once, then falls through', async () => { const noBody = new OpenAI.APIError(400, undefined, '400 status code (no body)', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -621,7 +621,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ + fakeZai = buildFakeClient([ { kind: 'text', text: 'claude fallback' }, { kind: 'finish', reason: 'stop' }, ]); @@ -630,17 +630,17 @@ describe('aiStreamRound — provider chain fallback', () => { }); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }); expect(result.text).toBe('claude fallback'); - expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(2); - expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(2); + expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); expect(fakeHf.chat.completions.create).not.toHaveBeenCalled(); - expect(result.metrics?.attemptCount).toBe(3); // z.ai twice + Claude once + expect(result.metrics?.attemptCount).toBe(3); // Claude twice + z.ai once }); test('caller abort never triggers the empty-400 retry', async () => { const controller = new AbortController(); controller.abort(); const noBody = new OpenAI.APIError(400, undefined, '400 status code (no body)', new Headers()); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -649,7 +649,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient(() => { + fakeZai = buildFakeClient(() => { throw new Error('claude should not be called'); }); fakeHf = buildFakeClient(() => { @@ -658,8 +658,8 @@ describe('aiStreamRound — provider chain fallback', () => { await expect( aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100, signal: controller.signal }), ).rejects.toThrow(); - expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(0); - expect(fakeClaude.chat.completions.create).not.toHaveBeenCalled(); + expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(0); + expect(fakeZai.chat.completions.create).not.toHaveBeenCalled(); expect(fakeHf.chat.completions.create).not.toHaveBeenCalled(); }); @@ -670,7 +670,7 @@ describe('aiStreamRound — provider chain fallback', () => { '400 no body mentioned', new Headers(), ); - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -679,7 +679,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ + fakeZai = buildFakeClient([ { kind: 'text', text: 'normal fallback' }, { kind: 'finish', reason: 'stop' }, ]); @@ -688,15 +688,15 @@ describe('aiStreamRound — provider chain fallback', () => { }); const result = await aiStreamRound({ messages: [{ role: 'user', content: 'hi' }], maxTokens: 100 }); expect(result.text).toBe('normal fallback'); - expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); expect(fakeClaude.chat.completions.create).toHaveBeenCalledTimes(1); + expect(fakeZai.chat.completions.create).toHaveBeenCalledTimes(1); }); test('mid-stream failure after text emitted discards the partial text and falls through', async () => { - // z.ai starts streaming, then the iterator throws. The partial text already + // Claude starts streaming, then the iterator throws. The partial text already // shown to the user is discarded via onProviderSwitch and the next provider // answers from scratch — a half-sent sentence is not a reason to fail the turn. - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -709,8 +709,8 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient([ - { kind: 'text', text: 'full answer from Claude' }, + fakeZai = buildFakeClient([ + { kind: 'text', text: 'full answer from z.ai' }, { kind: 'finish', reason: 'stop' }, ]); fakeHf = buildFakeClient(() => { @@ -732,15 +732,15 @@ describe('aiStreamRound — provider chain fallback', () => { }, ); - expect(result.text).toBe('full answer from Claude'); - expect(deltas).toEqual(['partial ', 'full answer from Claude']); + expect(result.text).toBe('full answer from z.ai'); + expect(deltas).toEqual(['partial ', 'full answer from z.ai']); expect(switched).toBe(1); }); test('aggregates tool_calls across chunks even when provider omits index field', async () => { // HF Router historical bug: tool_call deltas arrive without `index`. // Without a fallback the Map collapses both chunks into the same slot. - fakeZai = { + fakeClaude = { chat: { completions: { create: mock(async () => { @@ -773,7 +773,7 @@ describe('aiStreamRound — provider chain fallback', () => { }, }, }; - fakeClaude = buildFakeClient(() => { + fakeZai = buildFakeClient(() => { throw new Error('unreachable'); }); fakeHf = buildFakeClient(() => { diff --git a/test/services/ai/streaming-usage.test.ts b/test/services/ai/streaming-usage.test.ts index 025b7eba..a3994a4d 100644 --- a/test/services/ai/streaming-usage.test.ts +++ b/test/services/ai/streaming-usage.test.ts @@ -168,7 +168,7 @@ function ask(overrides: Partial = {}) { describe('aiStreamRound records one usage row per provider attempt', () => { test('an answered attempt records provider, model, chain, purpose, duration and every reported token count', async () => { - zai = makeProvider({ kind: 'text', text: 'hello', usage: USAGE }); + claude = makeProvider({ kind: 'text', text: 'hello', usage: USAGE }); const before = Date.now(); await ask({ purpose: 'agent', requestId: 'req-42' }); @@ -176,8 +176,8 @@ describe('aiStreamRound records one usage row per provider attempt', () => { const [row, ...rest] = rows(); expect(rest).toEqual([]); expect(row).toMatchObject({ - provider: 'zai', - model: 'glm-5.1', + provider: 'claude', + model: 'claude-sonnet-5-5', chain: 'smart', purpose: 'agent', outcome: 'ok', @@ -196,15 +196,17 @@ describe('aiStreamRound records one usage row per provider attempt', () => { }); test('the fast chain is recorded as fast and an unlabeled caller as unlabeled', async () => { - zai = makeProvider({ kind: 'text', text: 'hello', usage: USAGE }); + claude = makeProvider({ kind: 'text', text: 'hello', usage: USAGE }); await ask({ fast: true }); - expect(rows().map((row) => [row.chain, row.model, row.purpose])).toEqual([['fast', 'glm-5.1-air', 'unlabeled']]); + expect(rows().map((row) => [row.chain, row.model, row.purpose])).toEqual([ + ['fast', 'claude-haiku-5-5', 'unlabeled'], + ]); }); test('a provider that reports no usage leaves its token columns NULL, never 0', async () => { - zai = makeProvider({ kind: 'text', text: 'hello' }); + claude = makeProvider({ kind: 'text', text: 'hello' }); await ask(); @@ -219,7 +221,7 @@ describe('aiStreamRound records one usage row per provider attempt', () => { }); test('a usage object that omits some details keeps those fields NULL and the reported ones as given', async () => { - zai = makeProvider({ + claude = makeProvider({ kind: 'text', text: 'hello', usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }, @@ -236,50 +238,50 @@ describe('aiStreamRound records one usage row per provider attempt', () => { }); test('a fallback leaves a failed row for every provider that failed and an ok row for the one that answered', async () => { - zai = makeProvider({ kind: 'throw', error: apiError(402, 'Insufficient balance') }); - hf = makeProvider({ kind: 'throw', error: apiError(401, 'Invalid API key') }); - claude = makeProvider({ kind: 'text', text: 'answer', usage: USAGE }); + claude = makeProvider({ kind: 'throw', error: apiError(402, 'Insufficient balance') }); + zai = makeProvider({ kind: 'throw', error: apiError(401, 'Invalid API key') }); + hf = makeProvider({ kind: 'text', text: 'answer', usage: USAGE }); const result = await ask({ purpose: 'agent' }); expect(result.text).toBe('answer'); expect(rows().map((row) => [row.provider, row.outcome, row.errorClass, row.httpStatus, row.promptTokens])).toEqual([ - ['zai', 'error', 'quota', 402, null], - ['hf', 'error', 'auth', 401, null], - ['claude', 'ok', null, null, 1200], + ['claude', 'error', 'quota', 402, null], + ['zai', 'error', 'auth', 401, null], + ['hf', 'ok', null, null, 1200], ]); }); test('server errors, rate limits and rejected requests are classified from the status', async () => { - zai = makeProvider({ kind: 'throw', error: apiError(503, 'overloaded') }); - hf = makeProvider({ kind: 'throw', error: apiError(429, 'Rate limit reached for model') }); - claude = makeProvider({ kind: 'throw', error: apiError(400, 'Invalid JSON payload') }); + claude = makeProvider({ kind: 'throw', error: apiError(503, 'overloaded') }); + zai = makeProvider({ kind: 'throw', error: apiError(429, 'Rate limit reached for model') }); + hf = makeProvider({ kind: 'throw', error: apiError(400, 'Invalid JSON payload') }); await expect(ask()).rejects.toThrow('All 3 AI providers failed'); expect(rows().map((row) => [row.provider, row.errorClass, row.httpStatus])).toEqual([ - ['zai', 'server', 503], - ['hf', 'rate_limit', 429], - ['claude', 'bad_request', 400], + ['claude', 'server', 503], + ['zai', 'rate_limit', 429], + ['hf', 'bad_request', 400], ]); }); test('a provider that never answers is a timeout row whose duration is the time spent waiting', async () => { - zai = makeProvider({ kind: 'hang' }); - hf = makeProvider({ kind: 'text', text: 'second' }); + claude = makeProvider({ kind: 'hang' }); + zai = makeProvider({ kind: 'text', text: 'second' }); const result = await ask({ providerTimeoutMs: 40 }); expect(result.text).toBe('second'); const [timedOut, answered] = rows(); - expect(timedOut).toMatchObject({ provider: 'zai', outcome: 'error', errorClass: 'timeout', httpStatus: null }); + expect(timedOut).toMatchObject({ provider: 'claude', outcome: 'error', errorClass: 'timeout', httpStatus: null }); expect(timedOut?.durationMs).toBeGreaterThanOrEqual(30); - expect(answered).toMatchObject({ provider: 'hf', outcome: 'ok' }); + expect(answered).toMatchObject({ provider: 'zai', outcome: 'ok' }); }); test('a caller abort is recorded as aborted and the chain stops there', async () => { const controller = new AbortController(); - zai = makeProvider({ + claude = makeProvider({ kind: 'throw-after', error: new OpenAI.APIUserAbortError(), before: () => controller.abort(), @@ -287,18 +289,18 @@ describe('aiStreamRound records one usage row per provider attempt', () => { await expect(ask({ signal: controller.signal })).rejects.toThrow(); - expect(rows().map((row) => [row.provider, row.outcome, row.errorClass])).toEqual([['zai', 'error', 'aborted']]); + expect(rows().map((row) => [row.provider, row.outcome, row.errorClass])).toEqual([['claude', 'error', 'aborted']]); }); test('a 200 OK with no output is an empty row that still keeps the tokens the provider billed', async () => { - zai = makeProvider({ kind: 'empty', usage: USAGE }); - hf = makeProvider({ kind: 'text', text: 'second' }); + claude = makeProvider({ kind: 'empty', usage: USAGE }); + zai = makeProvider({ kind: 'text', text: 'second' }); await ask(); const [empty, answered] = rows(); expect(empty).toMatchObject({ - provider: 'zai', + provider: 'claude', outcome: 'error', errorClass: 'empty', httpStatus: null, @@ -306,12 +308,12 @@ describe('aiStreamRound records one usage row per provider attempt', () => { completionTokens: 80, reasoningTokens: 30, }); - expect(answered).toMatchObject({ provider: 'hf', outcome: 'ok' }); + expect(answered).toMatchObject({ provider: 'zai', outcome: 'ok' }); }); test('an empty answer without any usage object leaves the tokens NULL', async () => { - zai = makeProvider({ kind: 'empty' }); - hf = makeProvider({ kind: 'text', text: 'second' }); + claude = makeProvider({ kind: 'empty' }); + zai = makeProvider({ kind: 'text', text: 'second' }); await ask(); @@ -319,57 +321,57 @@ describe('aiStreamRound records one usage row per provider attempt', () => { }); test('an open circuit is recorded as a skip with the status that opened it, and sends nothing', async () => { - zai = makeProvider({ kind: 'throw', error: apiError(402, 'Insufficient balance') }); - hf = makeProvider({ kind: 'text', text: 'first' }, { kind: 'text', text: 'second' }); + claude = makeProvider({ kind: 'throw', error: apiError(402, 'Insufficient balance') }); + zai = makeProvider({ kind: 'text', text: 'first' }, { kind: 'text', text: 'second' }); await ask(); await ask(); - expect(zai.requests).toHaveLength(1); + expect(claude.requests).toHaveLength(1); expect(rows().map((row) => [row.provider, row.outcome, row.errorClass, row.skipReason, row.httpStatus])).toEqual([ - ['zai', 'error', 'quota', null, 402], - ['hf', 'ok', null, null, null], - ['zai', 'skipped', null, 'circuit_open', 402], - ['hf', 'ok', null, null, null], + ['claude', 'error', 'quota', null, 402], + ['zai', 'ok', null, null, null], + ['claude', 'skipped', null, 'circuit_open', 402], + ['zai', 'ok', null, null, null], ]); }); test('a provider benched after a request that was too large with the tool catalog is a skip on the next round', async () => { const tools = [{ type: 'function' as const, function: { name: 'list_events', parameters: {} } }]; - zai = makeProvider({ kind: 'throw', error: apiError(413, 'Request too large for model') }); - hf = makeProvider({ kind: 'text', text: 'first' }, { kind: 'text', text: 'second' }); + claude = makeProvider({ kind: 'throw', error: apiError(413, 'Request too large for model') }); + zai = makeProvider({ kind: 'text', text: 'first' }, { kind: 'text', text: 'second' }); await ask({ tools }); await ask({ tools }); - expect(zai.requests).toHaveLength(1); + expect(claude.requests).toHaveLength(1); expect(rows().map((row) => [row.provider, row.outcome, row.errorClass, row.skipReason])).toEqual([ - ['zai', 'error', 'rate_limit', null], - ['hf', 'ok', null, null], - ['zai', 'skipped', null, 'eligibility_bench'], - ['hf', 'ok', null, null], + ['claude', 'error', 'rate_limit', null], + ['zai', 'ok', null, null], + ['claude', 'skipped', null, 'eligibility_bench'], + ['zai', 'ok', null, null], ]); }); test('providers left unattempted when the round deadline passes are recorded as skipped', async () => { let clock = 0; performance.now = () => clock; - zai = makeProvider({ + claude = makeProvider({ kind: 'throw-after', error: apiError(503, 'slow'), before: () => { clock = 29_800; }, }); + zai = makeProvider({ kind: 'throw', error: new Error('must not be called') }); hf = makeProvider({ kind: 'throw', error: new Error('must not be called') }); - claude = makeProvider({ kind: 'throw', error: new Error('must not be called') }); await expect(ask()).rejects.toThrow(); expect(rows().map((row) => [row.provider, row.outcome, row.skipReason])).toEqual([ - ['zai', 'error', null], + ['claude', 'error', null], + ['zai', 'skipped', 'round_deadline'], ['hf', 'skipped', 'round_deadline'], - ['claude', 'skipped', 'round_deadline'], ]); }); @@ -379,8 +381,8 @@ describe('aiStreamRound records one usage row per provider attempt', () => { clock += 12.7; return clock; }; - zai = makeProvider({ kind: 'throw', error: apiError(503, 'down') }); - hf = makeProvider({ kind: 'text', text: 'answer' }); + claude = makeProvider({ kind: 'throw', error: apiError(503, 'down') }); + zai = makeProvider({ kind: 'text', text: 'answer' }); await ask(); @@ -403,7 +405,7 @@ describe('aiStreamRound records one usage row per provider attempt', () => { }); test('stores no text: the row holds only the documented columns', async () => { - zai = makeProvider({ kind: 'text', text: 'SECRET-ANSWER', usage: USAGE }); + claude = makeProvider({ kind: 'text', text: 'SECRET-ANSWER', usage: USAGE }); await ask({ messages: [{ role: 'user', content: 'SECRET-PROMPT' }], userId: 777_000_111 }); @@ -419,7 +421,7 @@ describe('recording never disturbs the answer', () => { throw new Error('database is locked'); }, }); - zai = makeProvider({ kind: 'text', text: 'the answer', usage: USAGE }); + claude = makeProvider({ kind: 'text', text: 'the answer', usage: USAGE }); const result = await ask(); flushAiUsageEvents(); @@ -437,7 +439,7 @@ describe('recording never disturbs the answer', () => { written += events.length; }, }); - zai = makeProvider({ kind: 'text', text: 'the answer' }); + claude = makeProvider({ kind: 'text', text: 'the answer' }); await ask(); @@ -448,7 +450,7 @@ describe('recording never disturbs the answer', () => { test('without a configured sink nothing is recorded and nothing breaks', async () => { configureAiUsageRecording(null); - zai = makeProvider({ kind: 'text', text: 'the answer' }); + claude = makeProvider({ kind: 'text', text: 'the answer' }); expect((await ask()).text).toBe('the answer'); expect([...repo.iterateSince(0)]).toEqual([]); diff --git a/test/services/ai/usage-recorder.test.ts b/test/services/ai/usage-recorder.test.ts index 0d0c7bce..5a7d0f23 100644 --- a/test/services/ai/usage-recorder.test.ts +++ b/test/services/ai/usage-recorder.test.ts @@ -244,7 +244,7 @@ describe('attempt rows carry role, preset and cost', () => { preset: 'deepseek41-flash', } as const; const tokens = { promptTokens: 1000, completionTokens: 500, reasoningTokens: null, cachedTokens: null }; - const OPENROUTER_DEEPSEEK = 'deepseek/deepseek-v4.1-flash'; + const OPENROUTER_DEEPSEEK = 'deepseek/deepseek-v4.1-flash-1017'; test('role and preset come from the slot, and a legacy slot has neither', () => { const row = answeredAttempt(scope, 'openrouter', OPENROUTER_DEEPSEEK, 5, 100, tokens); diff --git a/test/services/ai/usage-report-text.test.ts b/test/services/ai/usage-report-text.test.ts index 85813243..95ad3fa5 100644 --- a/test/services/ai/usage-report-text.test.ts +++ b/test/services/ai/usage-report-text.test.ts @@ -45,7 +45,7 @@ const rows = [ completionTokens: null, }), // A provider retired from the code whose rows remain in the ledger. - event({ provider: 'gemini', model: 'gemini-3-flash', promptTokens: null, completionTokens: null }), + event({ provider: 'retired', model: 'historical-model', promptTokens: null, completionTokens: null }), ]; describe('formatUsageReport', () => { @@ -79,11 +79,11 @@ describe('formatUsageReport', () => { const text = formatUsageReport(summarizeAiUsage(rows, WINDOW), 7, t('en').aiStats); expect(text).toContain('tokens: in 2400 · out 160'); - // gemini reported nothing: dashes and a count of answers without usage, not 0. - const gemini = text.slice(text.indexOf('gemini:gemini-3-flash')); - expect(gemini).toContain('tokens: in — · out —'); - expect(gemini).toContain('answers without usage: 1'); - expect(gemini).not.toMatch(/in 0 /); + // retired reported nothing: dashes and a count of answers without usage, not 0. + const retired = text.slice(text.indexOf('retired:historical-model')); + expect(retired).toContain('tokens: in — · out —'); + expect(retired).toContain('answers without usage: 1'); + expect(retired).not.toMatch(/in 0 /); }); test('lists the provider verdicts before the models', () => { @@ -91,7 +91,7 @@ describe('formatUsageReport', () => { expect(text.indexOf('Providers')).toBeLessThan(text.indexOf('Models')); expect(text).toContain('zai: degraded'); - expect(text).toContain('gemini: healthy'); + expect(text).toContain('retired: healthy'); }); test('says skipped providers were never contacted and what skipped them', () => { diff --git a/test/services/ai/usage-report.test.ts b/test/services/ai/usage-report.test.ts index 13832bc0..e64acae4 100644 --- a/test/services/ai/usage-report.test.ts +++ b/test/services/ai/usage-report.test.ts @@ -236,10 +236,10 @@ describe('summarizeAiUsage — one verdict per provider', () => { expect(summary.providers.map((p) => p.provider)).toEqual(['zai', 'claude', 'hf', 'openrouter', 'retired']); }); - test('a historical gemini ledger row still appears after the configured providers and does not throw', () => { - const summary = summarizeAiUsage([event({ provider: 'gemini', model: 'gemini-3-flash' }), event()], WINDOW); - expect(summary.providers.map((p) => p.provider)).toEqual(['zai', 'claude', 'hf', 'openrouter', 'gemini']); - expect(summary.providers.find((p) => p.provider === 'gemini')).toMatchObject({ verdict: 'healthy', ok: 1 }); + test('a historical retired ledger row still appears after the configured providers and does not throw', () => { + const summary = summarizeAiUsage([event({ provider: 'retired', model: 'historical-model' }), event()], WINDOW); + expect(summary.providers.map((p) => p.provider)).toEqual(['zai', 'claude', 'hf', 'openrouter', 'retired']); + expect(summary.providers.find((p) => p.provider === 'retired')).toMatchObject({ verdict: 'healthy', ok: 1 }); }); test('a historical groq ledger row still appears after the configured providers and does not throw', () => { @@ -249,13 +249,13 @@ describe('summarizeAiUsage — one verdict per provider', () => { expect(summary.models.map((m) => `${m.provider}:${m.model}`)).toContain('groq:openai/gpt-oss-120b'); }); - test('a historical gemini local_budget skip row is counted, not rejected', () => { + test('a historical retired local_budget skip row is counted, not rejected', () => { const summary = summarizeAiUsage( - [event({ provider: 'gemini', model: 'gemini-3-flash', outcome: 'skipped', skipReason: 'local_budget' })], + [event({ provider: 'retired', model: 'historical-model', outcome: 'skipped', skipReason: 'local_budget' })], WINDOW, ); - const gemini = summary.providers.find((p) => p.provider === 'gemini'); - expect(gemini).toMatchObject({ verdict: 'skipped_only', skipped: 1 }); + const retired = summary.providers.find((p) => p.provider === 'retired'); + expect(retired).toMatchObject({ verdict: 'skipped_only', skipped: 1 }); expect(summary.models[0]?.skippedByReason).toEqual({ local_budget: 1 }); }); diff --git a/uv.lock b/uv.lock index 755e9fc9..d09bd476 100644 --- a/uv.lock +++ b/uv.lock @@ -335,7 +335,7 @@ wheels = [ [[package]] name = "hypercalendarbot" -version = "0.3.7" +version = "0.3.8" source = { virtual = "." } dependencies = [ { name = "numpy" },