diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60507182d5..a8f0778069 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,50 @@ jobs: - name: Unit tests run: just test-unit + model-capabilities: + name: Model Capabilities (regen + corpus + schema + buzz-agent tests) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + save-if: ${{ github.event_name != 'pull_request' }} + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + package-manager-cache: false + + - name: Regenerate artifacts + run: node scripts/generate-model-capabilities.mjs + + - name: Diff check — fail if generated files are stale + run: | + if ! git diff --exit-code \ + crates/buzz-agent/src/generated_model_capabilities.rs \ + desktop/src/features/agents/ui/modelCapabilities.ts; then + echo "" + echo "ERROR: Generated model-capability files are stale." + echo "Run: node scripts/generate-model-capabilities.mjs" + echo "Then commit the regenerated files." + exit 1 + fi + echo "✓ All generated files are up to date." + + - name: Run corpus (TS interpreter via --experimental-strip-types) + run: node --experimental-strip-types scripts/run-corpus.mjs + + - name: Validate manifest (schema-negative tests) + run: node --test scripts/test-manifest-validator.mjs + + - name: Run buzz-agent unit tests (normative corpus + generated interpreter) + run: cargo test -p buzz-agent --lib + desktop-core: name: Desktop Core runs-on: ubuntu-latest diff --git a/.github/workflows/model-capability-regen-diff.yml b/.github/workflows/model-capability-regen-diff.yml deleted file mode 100644 index 7d8d706c91..0000000000 --- a/.github/workflows/model-capability-regen-diff.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Model Capability Regenerate-Then-Diff - -on: - pull_request: - paths: - - 'scripts/model-capabilities.json' - - 'scripts/generate-model-capabilities.mjs' - - 'crates/buzz-agent/src/generated_model_capabilities.rs' - - 'desktop/src/features/agents/ui/modelCapabilities.ts' - - 'scripts/generated-model-capabilities-coverage.json' - - '.github/workflows/model-capability-regen-diff.yml' - # Differential harness and fixtures — any change to old/new side or inputs re-runs. - - 'scripts/run-differential.mjs' - - 'scripts/normative-corpus.json' - - 'scripts/catalog-sample-fixture.json' - - 'desktop/src/features/agents/ui/effortTable.fixture.json' - - 'desktop/src/features/agents/ui/buzzAgentConfig.ts' - - 'crates/buzz-agent/src/config.rs' - - 'crates/buzz-agent/src/llm.rs' - push: - branches: [main, release, 'duncan/databricks-model-label-registry'] - -jobs: - regen-diff: - name: Regenerate and diff model capability artifacts - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 - with: - node-version: '22' - package-manager-cache: false - - - name: Regenerate artifacts - run: node scripts/generate-model-capabilities.mjs - - - name: Diff check — fail if generated files are stale - run: | - if ! git diff --exit-code \ - crates/buzz-agent/src/generated_model_capabilities.rs \ - desktop/src/features/agents/ui/modelCapabilities.ts \ - scripts/generated-model-capabilities-coverage.json; then - echo "" - echo "ERROR: Generated model-capability files are stale." - echo "Run: node scripts/generate-model-capabilities.mjs" - echo "Then commit the regenerated files." - exit 1 - fi - echo "✓ All generated files are up to date." - - - name: Run corpus (TS interpreter via --experimental-strip-types) - run: node --experimental-strip-types scripts/run-corpus.mjs - - - name: Validate manifest (schema-negative tests) - run: node --test scripts/test-manifest-validator.mjs - - - name: Run differential harness (old vs new, all input sets) - run: node --experimental-strip-types scripts/run-differential.mjs - - rust-unit-tests: - name: buzz-agent unit tests (normative corpus + behavioral differential) - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Run buzz-agent unit tests - run: cargo test -p buzz-agent --lib diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index f4a033cbc9..0089cc9f72 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -9,7 +9,7 @@ pub const PROTOCOL_VERSION: u32 = 2; /// config is sent in the request body. /// /// Provider support (doc-verified, July 2025): -/// - **Anthropic adaptive**: `low|medium|high|xhigh|max` (model-dependent; see `anthropic_thinking_config`). +/// - **Anthropic adaptive**: `low|medium|high|xhigh|max` (model-dependent; see `anthropic_thinking_config_generated`). /// `none`/`minimal` are not Anthropic values — rejected at startup. /// - **Anthropic manual budget** (claude-3*, opus-4-5): `low|medium|high`; `xhigh`/`max` clamp to high budget. /// - **OpenAI Responses / Chat Completions**: effort support is model-dependent and normalized at @@ -29,7 +29,7 @@ pub enum ThinkingEffort { impl ThinkingEffort { /// Map level to an Anthropic `budget_tokens` value for legacy Claude 3.x / Opus 4.5 models. /// `XHigh` and `Max` clamp to the high budget value; the answer-room reserve of 1024 tokens - /// is applied separately in `anthropic_thinking_config`. + /// is applied separately in `anthropic_thinking_config_generated`. pub fn anthropic_budget_tokens(self) -> u32 { match self { ThinkingEffort::Low => 1_024, @@ -69,388 +69,6 @@ impl ThinkingEffort { } } } - -/// Strip any endpoint-naming prefix from a model name so the family classifiers -/// (`is_manual_budget_model`, `is_adaptive_thinking_model`, etc.) can match on the canonical -/// `claude-*` form regardless of how the model is stored in the Databricks catalog. -/// -/// Rather than maintaining an allowlist of known prefixes, this function finds the first -/// occurrence of a known model-family token (`claude-`, `gpt-`) and drops everything before -/// it. This handles any endpoint naming convention without needing to enumerate prefixes. -/// -/// Examples: -/// - `databricks-claude-fable-5` → `claude-fable-5` -/// - `goose-claude-fable-5` → `claude-fable-5` -/// - `team-x-claude-opus-4-7` → `claude-opus-4-7` -/// - `goose-gpt-5.5` → `gpt-5.5` -/// - `llama-3` → `llama-3` (no family token, returned unchanged) -/// -/// If no family token is present the name is returned unchanged. -pub(crate) fn strip_catalog_prefix(model: &str) -> &str { - const FAMILY_TOKENS: &[&str] = &["claude-", "gpt-"]; - let lower = model.to_ascii_lowercase(); - let first_idx = FAMILY_TOKENS.iter().filter_map(|tok| lower.find(tok)).min(); - match first_idx { - Some(idx) => &model[idx..], - None => model, - } -} - -/// Build the Anthropic thinking/effort request fields for the given model and effort level. -/// -/// API shape selection (per Anthropic extended-thinking support table, -/// https://platform.claude.com/docs/en/build-with-claude/extended-thinking, July 2025): -/// -/// **Adaptive families** — `thinking: {type:"adaptive"}` + `output_config: {effort}`. -/// These models use adaptive thinking; `thinking:{type:"adaptive"}` is required to enable -/// thinking — without it requests run without thinking even when `output_config.effort` is set. -/// Doc-verified (extended-thinking table): Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5.x, Sonnet 4.6. -/// Matched by explicit version strings (no wildcard over version numbers). -/// -/// **Manual-budget families** — `thinking: {type:"enabled", budget_tokens}`. -/// `budget_tokens` is clamped to `min(level_budget, max_output_tokens - 1024)` to preserve -/// at least 1024 answer tokens. If the result is < 1024 (i.e., `max_output_tokens <= 2047`), -/// thinking is omitted entirely with a `warn!`. -/// Doc-verified: claude-3* (legacy), claude-opus-4-5 (effort page: "uses manual thinking"). -/// -/// **Everything else** — omit both fields. This includes unknown/future `claude-*` names -/// not yet in the support table. Safer to omit than to guess an unverified shape. -/// -/// The Databricks `databricks-` and other endpoint-naming prefixes are stripped before -/// matching so that `databricks-claude-opus-4-7`, `goose-claude-fable-5`, and -/// `team-x-claude-opus-4-7` all route to the correct bucket. See `strip_catalog_prefix`. -/// -/// Returns `(thinking_field, output_config_field)` where each is `None` if not applicable. -pub fn anthropic_thinking_config( - effective_model: &str, - effort: ThinkingEffort, - max_output_tokens: u32, -) -> (Option, Option) { - use serde_json::json; - // Normalise the model name for matching: strip any endpoint-naming prefix - // (e.g. "databricks-claude-opus-4-7" → "claude-opus-4-7", - // "goose-claude-fable-5" → "claude-fable-5", - // "team-x-claude-opus-4-7" → "claude-opus-4-7"). - let model = strip_catalog_prefix(effective_model); - - if is_manual_budget_model(model) { - // Manual-budget shape: budget_tokens must be strictly < max_tokens AND must leave - // at least MIN_ANSWER_TOKENS (1024) for the visible answer. The Anthropic API - // requires budget_tokens < max_tokens AND budget_tokens >= 1024. - // - // Clamp: budget = min(level_budget, max_output_tokens - MIN_ANSWER_TOKENS). - // If result < MIN_ANSWER_TOKENS, thinking would starve the answer — omit thinking - // entirely and warn instead of emitting an invalid or answer-starving budget. - const MIN_ANSWER_TOKENS: u32 = 1024; - let level_budget = effort.anthropic_budget_tokens(); - let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS); - let budget = level_budget.min(headroom); - if budget < MIN_ANSWER_TOKENS { - tracing::warn!( - max_output_tokens, - level_budget, - headroom, - "BUZZ_AGENT_THINKING_EFFORT: max_output_tokens too small to fit thinking budget + answer headroom; omitting thinking fields" - ); - return (None, None); - } - ( - Some(json!({ "type": "enabled", "budget_tokens": budget })), - None, - ) - } else if is_adaptive_thinking_model(model) { - // Adaptive families: thinking must be explicitly enabled via type:"adaptive". - // output_config.effort controls the depth. Both fields are required together. - // Apply per-model effort clamping: if the requested level exceeds the model's - // doc-verified maximum, clamp down to the highest supported level with a warning. - let clamped = clamp_adaptive_effort(model, effort); - ( - Some(json!({ "type": "adaptive" })), - Some(json!({ "effort": clamped.anthropic_effort_str() })), - ) - } else { - // Unrecognised or unverified model name — omit both fields rather than guess. - // This includes unknown future claude-* names not yet in the support table. - (None, None) - } -} - -/// Returns true for adaptive Anthropic models that support the `xhigh` effort level. -/// -/// Used by both `clamp_adaptive_effort` (request-time) and `anthropic_efforts_for_model` -/// (UI capability table) to keep xhigh-support classification in a single place. -/// -/// `model` must already have catalog prefixes stripped (via `strip_catalog_prefix`). -fn anthropic_model_supports_xhigh(model: &str) -> bool { - model.starts_with("claude-opus-4-7") - || model.starts_with("claude-opus-4-8") - || model.starts_with("claude-opus-5") - || model.starts_with("claude-sonnet-5") - || model.starts_with("claude-fable-5") - || model.starts_with("claude-mythos-5") -} - -/// Clamp the requested effort level to the highest doc-verified level for the given adaptive model. -/// -/// Doc-verified availability (Anthropic effort page, July 2025): -/// - `max`: Opus 4.8, 4.7, 4.6; Sonnet 5.x, 4.6; Fable 5; Mythos 5; Mythos Preview -/// - `xhigh`: Opus 4.8, 4.7; Sonnet 5.x; Fable 5; Mythos 5 -/// (NOT Opus 4.6, Sonnet 4.6, or Mythos Preview) -/// - `low|medium|high`: all adaptive families -/// -/// If the requested level is not available for the model, clamps down to the highest -/// supported level below the requested one, and logs a warning. This is dynamic (not -/// startup-time) because `session/set_model` can change the model after startup. -/// -/// `model` must already have catalog prefixes stripped (via `strip_catalog_prefix`). -pub fn clamp_adaptive_effort(model: &str, effort: ThinkingEffort) -> ThinkingEffort { - // Models that support all levels including xhigh (and max). - let supports_xhigh = anthropic_model_supports_xhigh(model); - - let clamped = if supports_xhigh { - effort // all levels pass through - } else if effort == ThinkingEffort::XHigh { - // xhigh not available for this model; clamp to high (the highest supported below xhigh). - ThinkingEffort::High - } else { - effort // low/medium/high/max all pass through for the other adaptive families - }; - - if clamped != effort { - tracing::warn!( - model, - requested = effort.openai_effort_str(), - clamped = clamped.openai_effort_str(), - "BUZZ_AGENT_THINKING_EFFORT is not available for this model; clamping to highest supported level" - ); - } - clamped -} - -/// Returns true if `lower_model` contains `token` as a bounded family segment — i.e., the -/// token is immediately followed by end-of-string or a `-` separator (not a digit or letter). -/// -/// This prevents: -/// - `gpt-5.1` from matching `gpt-5.10` (digit follows the `1`) -/// - `gpt-5-1` from matching `gpt-5-1106` (digit follows the `1`) -/// - `gpt-5-4` from matching `gpt-5-4o` (letter follows the `4`) -/// -/// Gateway prefixes (`databricks-`) and date/build suffixes (`-2025-04-01`) are allowed -/// because they start with `-` which is the only permitted boundary character. -fn gpt5_token_matches(lower_model: &str, token: &str) -> bool { - let mut start = 0; - while let Some(pos) = lower_model[start..].find(token) { - let abs = start + pos; - let after = abs + token.len(); - // The character immediately after the token must be end-of-string or '-'. - // Any alphanumeric character (digit OR letter) means this is a longer token, not - // the family we're looking for. - let safe_suffix = lower_model[after..].chars().next().is_none_or(|c| c == '-'); - if safe_suffix { - return true; - } - start = abs + 1; - } - false -} - -/// Like `gpt5_token_matches` but additionally rejects short version-like numeric suffixes — -/// used for the base `gpt-5` / `gpt5` token to avoid false-matching unrecognized versions. -/// -/// After a `-` separator: -/// - `-…` e.g. `-pro` → **accepted** (capability suffix, no digits) -/// - `digit_run == 1-3` AND the char right after the digits is a **letter** e.g. `-4o` → -/// **accepted** (real variant shape: digit + letter) -/// - `digit_run == 1-3` AND the char after the digits is end-of-string, `-`, `.`, or other -/// separator e.g. `-10`, `-10-preview` → **rejected** (version-like suffix) -/// - `digit_run >= 4` regardless of what follows e.g. `-1106`, `-1106-preview`, `-0514` → -/// **accepted** (date/build segment) -fn gpt5_base_matches(lower_model: &str, token: &str) -> bool { - let mut start = 0; - while let Some(pos) = lower_model[start..].find(token) { - let abs = start + pos; - let after = abs + token.len(); - let rest = &lower_model[after..]; - let safe_suffix = if rest.is_empty() { - // End of string — clean boundary. - true - } else if let Some(tail) = rest.strip_prefix('-') { - // Count leading digits in the suffix component. - let digit_run: usize = tail.chars().take_while(|c| c.is_ascii_digit()).count(); - if digit_run == 0 { - // No leading digit (e.g. '-pro'): capability suffix → accepted. - true - } else if digit_run >= 4 { - // 4+ digit run (e.g. '-1106', '-1106-preview', '-0514'): date/build → accepted. - true - } else { - // 1-3 digit run: accepted only if the char right after the digits is a letter - // (real variant shape like '-4o'). Separator/EOS after short digits is - // version-like (e.g. '-10', '-10-preview') → rejected. - tail[digit_run..] - .chars() - .next() - .is_some_and(|c| c.is_ascii_alphabetic()) - } - } else { - // Dot, letter, or other non-hyphen character directly after token → not base. - false - }; - if safe_suffix { - return true; - } - start = abs + 1; - } - false -} - -/// Returns the set of `reasoning.effort` values supported by a given OpenAI model family. -/// -/// Doc-verified availability (OpenAI model pages, July 2025): -/// -/// | Model | Supported effort values | -/// |-------------|-------------------------------------------| -/// | gpt-5-pro | `high` only | -/// | gpt-5.6 | `none, low, medium, high, xhigh, max` | -/// | gpt-5.5 | `none, low, medium, high, xhigh` | -/// | gpt-5.4 | `none, low, medium, high, xhigh` | -/// | gpt-5.1 | `none, low, medium, high` | -/// | gpt-5 (base)| `minimal, low, medium, high` | -/// | unknown | not doc-verified — `max` clamps to `xhigh` | -/// -/// Note the `none` vs `minimal` split: `gpt-5` (base) supports `minimal` but not `none`; -/// `gpt-5.1`/`gpt-5.4`/`gpt-5.5`/`gpt-5.6` support `none` but not `minimal`. These are matched via -/// nearest-supported fallback in `normalize_effort_for_openai_route`. -/// -/// Match order: `-pro` variant checked before versioned strings to prevent `gpt-5-pro` from -/// falling into the `gpt-5` base bucket (substring "gpt-5" is shared). -/// -/// `model` is a raw model name (may include Databricks gateway prefixes or date suffixes). -/// Unknown models return `None` — callers pass through values except `max`, which clamps to -/// `xhigh` until support is confirmed. -/// Versioned tokens use `gpt5_token_matches` (end-of-string or `-` boundary, blocking digit -/// and letter continuations). The base token uses `gpt5_base_matches`, which additionally -/// rejects short `-<1-3 digit>` suffixes that look like two-digit version numbers. -fn openai_efforts_for_model(model: &str) -> Option<&'static [ThinkingEffort]> { - // Effort ordered from lowest to highest for each family. - const GPT5_PRO: &[ThinkingEffort] = &[ThinkingEffort::High]; - const GPT5_6: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - const GPT5_5_AND_5_4: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ]; - const GPT5_1: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ]; - const GPT5_BASE: &[ThinkingEffort] = &[ - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ]; - - let lower = model.to_ascii_lowercase(); - // Check gpt-5-pro before gpt-5.5 / gpt-5.4 etc. to avoid the `-pro` name - // matching the base "gpt-5" prefix first. - if gpt5_token_matches(&lower, "gpt-5-pro") || gpt5_token_matches(&lower, "gpt5-pro") { - Some(GPT5_PRO) - } else if gpt5_token_matches(&lower, "gpt-5.6") - || gpt5_token_matches(&lower, "gpt5.6") - || gpt5_token_matches(&lower, "gpt-5-6") - || gpt5_token_matches(&lower, "gpt5-6") - { - Some(GPT5_6) - } else if gpt5_token_matches(&lower, "gpt-5.5") - || gpt5_token_matches(&lower, "gpt5.5") - || gpt5_token_matches(&lower, "gpt-5-5") - || gpt5_token_matches(&lower, "gpt5-5") - || gpt5_token_matches(&lower, "gpt-5.4") - || gpt5_token_matches(&lower, "gpt5.4") - || gpt5_token_matches(&lower, "gpt-5-4") - || gpt5_token_matches(&lower, "gpt5-4") - { - // gpt-5.5 and gpt-5.4 share the same effort availability table. - Some(GPT5_5_AND_5_4) - } else if gpt5_token_matches(&lower, "gpt-5.1") - || gpt5_token_matches(&lower, "gpt5.1") - || gpt5_token_matches(&lower, "gpt-5-1") - || gpt5_token_matches(&lower, "gpt5-1") - { - Some(GPT5_1) - } else if gpt5_base_matches(&lower, "gpt-5") || gpt5_base_matches(&lower, "gpt5") { - // Base gpt-5 (no version suffix matching any of the above). - Some(GPT5_BASE) - } else { - // Unknown model — not doc-verified; server validates. - None - } -} - -/// Returns the effort capability set for a given Anthropic model. -/// -/// This is the single production source of truth for Anthropic family routing. -/// Both `anthropic_thinking_config` (request-time) and the effort-table UI -/// (`valid_effort_values_for_provider_model`, via its Anthropic branch) must -/// derive their behaviour from this helper so the two stay in sync. -/// -/// Returns `(valid_values, default)` where: -/// - `valid_values` is the static slice of `ThinkingEffort` values accepted -/// by this model family's effort dropdown. -/// - `default` is `None` for manual-budget models (no semantic default — -/// user must choose) or `Some(High)` for adaptive families. -/// -/// `model` must already have catalog prefixes stripped (via `strip_catalog_prefix`). -pub fn anthropic_efforts_for_model( - model: &str, -) -> (&'static [ThinkingEffort], Option) { - const MANUAL: &[ThinkingEffort] = &[ - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ]; - const ADAPTIVE_XHIGH: &[ThinkingEffort] = &[ - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - const ADAPTIVE_NO_XHIGH: &[ThinkingEffort] = &[ - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::Max, - ]; - - if is_manual_budget_model(model) { - return (MANUAL, None); - } - if is_adaptive_thinking_model(model) { - // Reuse `anthropic_model_supports_xhigh` (the single source of truth - // shared with `clamp_adaptive_effort`) — no side-effects, no duplication. - if anthropic_model_supports_xhigh(model) { - return (ADAPTIVE_XHIGH, Some(ThinkingEffort::High)); - } else { - return (ADAPTIVE_NO_XHIGH, Some(ThinkingEffort::High)); - } - } - // Unknown Anthropic model — assume full adaptive (xhigh-capable) as a safe default. - (ADAPTIVE_XHIGH, Some(ThinkingEffort::High)) -} - /// Resolve the nearest supported effort level for a given OpenAI model. /// /// When the requested effort is not in the model's supported set, falls back to the @@ -520,37 +138,6 @@ fn resolve_openai_effort( resolved } -/// Normalize the effort value for an OpenAI-shaped request body (Chat Completions or Responses). -/// -/// Per-model effort availability is applied for doc-verified OpenAI model families. A requested -/// level not in the model's supported set is substituted with the nearest supported level (see -/// `resolve_openai_effort` for preference order). For unknown/unverified models, `max` is clamped -/// to `xhigh` because its support cannot be confirmed; all other values pass through unchanged. -/// -/// Applies to pure-OpenAI request paths AND DBv2 OpenAI-shaped routes. -/// -/// Doc-verified model table (July 2025): -/// - `gpt-5-pro`: `high` only -/// - `gpt-5.6`: `none, low, medium, high, xhigh, max` -/// - `gpt-5.5`, `gpt-5.4`: `none, low, medium, high, xhigh` -/// - `gpt-5.1`: `none, low, medium, high` -/// - `gpt-5` (base): `minimal, low, medium, high` -/// - unknown: `max` clamps to `xhigh`; other values pass through -pub fn normalize_effort_for_openai_route(effort: ThinkingEffort, model: &str) -> ThinkingEffort { - match openai_efforts_for_model(model) { - Some(supported) => resolve_openai_effort(model, effort, supported), - None if effort == ThinkingEffort::Max => { - tracing::warn!( - requested = "max", - resolved = "xhigh", - "BUZZ_AGENT_THINKING_EFFORT=max not confirmed for unknown OpenAI model; clamping to xhigh" - ); - ThinkingEffort::XHigh - } - None => effort, - } -} - /// Normalize the effort value for an Anthropic-shaped request body (Messages API). /// /// Anthropic-shaped bodies (`anthropic_body`) do not have a `none` or `minimal` concept — @@ -577,12 +164,10 @@ pub fn normalize_effort_for_anthropic_route(effort: ThinkingEffort) -> Option Option { - // Adaptive shape: clamp effort downward to the highest supported level. - // Uses the generated supported_efforts (the manifest-owned authority) rather - // than the legacy clamp_adaptive_effort hand table. + // Adaptive shape: clamp effort downward to the highest supported level + // using the generated supported_efforts (the manifest-owned authority). let clamped = cap .supported_efforts .iter() @@ -725,90 +307,6 @@ pub fn anthropic_thinking_config_generated( } } } - -/// Old DatabricksV2-scoped Anthropic thinking config — kept as a differential shim. -/// -/// Production code uses `anthropic_thinking_config_generated` instead. -/// This hard-codes `"databricks_v2"` and uses the legacy `clamp_adaptive_effort` hand table. -#[cfg(test)] -pub(crate) fn _old_anthropic_thinking_config_for_databricks_v2( - raw_model: &str, - effort: ThinkingEffort, - max_output_tokens: u32, -) -> (Option, Option) { - use crate::generated_model_capabilities::{resolve_model_capabilities, ThinkingMode}; - use serde_json::json; - - match resolve_model_capabilities("databricks_v2", raw_model).thinking_mode { - ThinkingMode::ManualBudget => { - const MIN_ANSWER_TOKENS: u32 = 1024; - let level_budget = effort.anthropic_budget_tokens(); - let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS); - let budget = level_budget.min(headroom); - if budget < MIN_ANSWER_TOKENS { - return (None, None); - } - ( - Some(json!({ "type": "enabled", "budget_tokens": budget })), - None, - ) - } - ThinkingMode::Adaptive => { - let model = strip_catalog_prefix(raw_model); - let clamped = clamp_adaptive_effort(model, effort); - ( - Some(json!({ "type": "adaptive" })), - Some(json!({ "effort": clamped.anthropic_effort_str() })), - ) - } - ThinkingMode::OmitFields | ThinkingMode::None | ThinkingMode::NotApplicable => (None, None), - } -} - -/// Returns true for Claude model families that use manual thinking budgets (doc-verified, July 2025). -/// -/// Source: https://platform.claude.com/docs/en/build-with-claude/extended-thinking (support table) -/// - claude-3*: legacy manual budget (all Claude 3.x variants). -/// - claude-opus-4-5: effort page states "uses manual thinking, where effort works alongside -/// the thinking token budget" — manual bucket, not adaptive. -/// -/// `model` must already have catalog prefixes stripped (via `strip_catalog_prefix`). -fn is_manual_budget_model(model: &str) -> bool { - model.starts_with("claude-3") || model == "claude-opus-4-5" -} - -/// Returns true for Claude model families that use adaptive thinking (doc-verified, July 2025). -/// -/// Sources: https://platform.claude.com/docs/en/build-with-claude/extended-thinking (support table) -/// https://platform.claude.com/docs/en/build-with-claude/effort (effort page) -/// -/// Adaptive thinking models (always-on or default-on): -/// Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5.x, Sonnet 4.6, -/// Fable 5 (always-on), Mythos 5 (always-on), Mythos Preview (default-on). -/// -/// Note: Opus 4.5 is NOT in this bucket — it uses manual budget (see `is_manual_budget_model`). -/// No prefix wildcards over version numbers; each entry is doc-verified explicitly. -/// -/// `model` must already have catalog prefixes stripped (via `strip_catalog_prefix`). -fn is_adaptive_thinking_model(model: &str) -> bool { - // Exact version strings for Opus 4.x adaptive models (4.6, 4.7, 4.8). - // Opus 4.5 is excluded — manual budget only. - model.starts_with("claude-opus-4-6") - || model.starts_with("claude-opus-4-7") - || model.starts_with("claude-opus-4-8") - || model.starts_with("claude-opus-5") - // Sonnet 5.x (any patch/date suffix after "claude-sonnet-5"). - || model.starts_with("claude-sonnet-5") - // Sonnet 4.6 exactly (not Sonnet 4.5 or earlier — not in the adaptive table). - || model.starts_with("claude-sonnet-4-6") - // Fable 5 and Mythos 5 (always-on adaptive thinking, July 2025). - || model.starts_with("claude-fable-5") - || model.starts_with("claude-mythos-5") - // Mythos Preview (default-on adaptive thinking, July 2025). - // Note: xhigh is NOT available on Mythos Preview — clamp_adaptive_effort handles this. - || model.starts_with("claude-mythos-preview") -} - /// Parse `BUZZ_AGENT_THINKING_EFFORT`. Pure (env-free) for testability. pub fn parse_thinking_effort(raw: Option<&str>) -> Result, String> { match raw.map(|s| s.trim().to_ascii_lowercase()).as_deref() { @@ -1168,7 +666,7 @@ impl Config { // // OpenAI, Databricks, and DatabricksV2 defer effort validation to request-time routing: // availability is model-dependent, and `session/set_model` can change the effective model - // after startup. `normalize_effort_for_openai_route` / `normalize_effort_for_anthropic_route` + // after startup. `normalize_effort_for_provider` / `normalize_effort_for_anthropic_route` // apply route-aware normalization in `llm.rs` when building each request. if let Some(effort) = self.thinking_effort { let is_pure_anthropic = matches!(self.provider, Provider::Anthropic); @@ -1340,32 +838,6 @@ fn parse_hook_servers(raw: Option<&str>) -> HookServers { HookServers::Only(names) } -// --------------------------------------------------------------------------- -// Test-only re-exports: let llm.rs tests call private classifiers without -// duplicating them. These wrappers are cfg(test)-only and intentionally thin. -// --------------------------------------------------------------------------- - -#[cfg(test)] -pub(crate) fn is_manual_budget_model_for_test(model: &str) -> bool { - is_manual_budget_model(model) -} - -#[cfg(test)] -pub(crate) fn is_adaptive_thinking_model_for_test(model: &str) -> bool { - is_adaptive_thinking_model(model) -} - -/// Mirror of the `tests::valid_effort_values_for_provider_model` helper in config's -/// own test module, promoted to a module-level cfg(test) function so llm.rs tests -/// can call it without re-implementing the logic. -#[cfg(test)] -pub(crate) fn valid_effort_values_for_provider_model_for_test( - provider: &str, - model: &str, -) -> (Vec<&'static str>, Option<&'static str>) { - tests::valid_effort_values_for_provider_model(provider, model) -} - #[cfg(test)] mod tests { use super::*; @@ -1656,647 +1128,166 @@ mod tests { assert!(ThinkingEffort::High < ThinkingEffort::XHigh); assert!(ThinkingEffort::XHigh < ThinkingEffort::Max); } - - // ---- anthropic_thinking_config helper — per-family tests ---- + // ---- normalize_effort_for_databricks_v2 regression tests (F1 corrections) ---- + // These pin the exact behavior Paul's pre-review probes checked. The key invariant: + // normalize_effort_for_databricks_v2 must resolve against the generated supported_efforts + // (which carries exact-record F1 corrections), NOT the old hand table. #[test] - fn anthropic_thinking_config_claude3_emits_budget_tokens() { - // Claude 3.x → `thinking.budget_tokens`; clamped to min(level_budget, max_output - 1024). - // max_output_tokens = 4096: headroom = 4096 - 1024 = 3072; High budget (32768) → 3072. - let (thinking, output_config) = - anthropic_thinking_config("claude-3-7-sonnet-20250219", ThinkingEffort::High, 4096); - let t = thinking.expect("thinking field must be present for claude-3"); - assert_eq!(t["type"], "enabled"); - assert_eq!(t["budget_tokens"], 3072); // capped: min(32768, 4096-1024) - assert!( - output_config.is_none(), - "output_config must be absent for claude-3" + fn normalize_effort_for_databricks_v2_gpt_5_5_xhigh_clamps_to_high() { + // F1 correction: databricks-gpt-5-5 generated supported_efforts = [low, medium, high]. + // XHigh is outside the supported set → nearest supported is High. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::XHigh, "databricks-gpt-5-5"), + ThinkingEffort::High, + "databricks-gpt-5-5 XHigh must clamp to High (F1 correction: supported=[low,medium,high])" ); } #[test] - fn anthropic_thinking_config_claude3_omits_thinking_when_max_output_too_small() { - // max_output_tokens = 2047: headroom = 2047 - 1024 = 1023 < 1024 → omit thinking. - let (thinking, output_config) = - anthropic_thinking_config("claude-3-7-sonnet-20250219", ThinkingEffort::High, 2047); - assert!( - thinking.is_none(), - "thinking must be omitted when max_output_tokens - 1024 < 1024 (budget would starve answer)" + fn normalize_effort_for_databricks_v2_gpt_5_5_none_clamps_to_low() { + // F1 correction: databricks-gpt-5-5 supported_efforts = [low, medium, high]. + // None is outside the set → nearest supported is Low. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::None, "databricks-gpt-5-5"), + ThinkingEffort::Low, + "databricks-gpt-5-5 None must clamp to Low (F1 correction: supported=[low,medium,high])" ); - assert!(output_config.is_none()); - } - - #[test] - fn anthropic_thinking_config_claude3_emits_thinking_at_boundary_2048() { - // max_output_tokens = 2048: headroom = 2048 - 1024 = 1024 ≥ 1024 → emit budget = 1024. - let (thinking, _) = - anthropic_thinking_config("claude-3-7-sonnet-20250219", ThinkingEffort::High, 2048); - let t = thinking.expect("thinking must be present when max_output_tokens = 2048"); - assert_eq!(t["budget_tokens"], 1024); // min(32768, 2048-1024) = 1024 - } - - #[test] - fn anthropic_thinking_config_claude3_budget_uncapped_when_fits() { - // High budget fits comfortably under a large max_output_tokens. - let (thinking, _) = - anthropic_thinking_config("claude-3-7-sonnet-20250219", ThinkingEffort::High, 65_536); - let t = thinking.unwrap(); - assert_eq!(t["budget_tokens"], 32_768); - } - - #[test] - fn anthropic_thinking_config_opus_4_8_emits_adaptive_and_effort() { - // Opus 4.8 — adaptive family. Requires thinking:{type:"adaptive"} to enable thinking. - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-8", ThinkingEffort::High, 32_768); - let t = thinking.expect("thinking must be present for claude-opus-4-8"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-opus-4-8"); - assert_eq!(oc["effort"], "high"); - } - - #[test] - fn anthropic_thinking_config_opus_4_7_emits_adaptive_and_effort() { - // Opus 4.7 — adaptive family. - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-7", ThinkingEffort::Medium, 32_768); - let t = thinking.expect("thinking must be present for claude-opus-4-7"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-opus-4-7"); - assert_eq!(oc["effort"], "medium"); } #[test] - fn anthropic_thinking_config_sonnet_5_emits_adaptive_and_effort() { - // Sonnet 5 — adaptive family. - let (thinking, output_config) = - anthropic_thinking_config("claude-sonnet-5-20250901", ThinkingEffort::Low, 32_768); - let t = thinking.expect("thinking must be present for claude-sonnet-5"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-sonnet-5"); - assert_eq!(oc["effort"], "low"); + fn normalize_effort_for_databricks_v2_gpt_5_5_in_range_passes_through() { + // Values within the corrected set must pass through unchanged. + for effort in [ + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ] { + assert_eq!( + normalize_effort_for_databricks_v2(effort, "databricks-gpt-5-5"), + effort, + "databricks-gpt-5-5 {effort:?} is in supported set, must pass through" + ); + } } #[test] - fn anthropic_thinking_config_sonnet_4_6_emits_adaptive_and_effort() { - // Sonnet 4.6 — adaptive family. Docs explicitly list "Combine effort with adaptive thinking." - let (thinking, output_config) = - anthropic_thinking_config("claude-sonnet-4-6", ThinkingEffort::High, 32_768); - let t = thinking.expect("thinking must be present for claude-sonnet-4-6"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-sonnet-4-6"); - assert_eq!(oc["effort"], "high"); + fn normalize_effort_for_databricks_v2_gpt_5_6_sol_max_passes_through() { + // databricks-gpt-5-6-sol F1 adoption: [low, medium, high, max] — max is supported. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::Max, "databricks-gpt-5-6-sol"), + ThinkingEffort::Max, + "databricks-gpt-5-6-sol Max must pass through (F1: supported includes max)" + ); } #[test] - fn anthropic_thinking_config_opus_4_5_emits_manual_budget() { - // Opus 4.5 — manual budget (NOT adaptive; effort page: "uses manual thinking"). - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-5", ThinkingEffort::High, 65_536); - let t = thinking.expect("thinking must be present for claude-opus-4-5"); - assert_eq!(t["type"], "enabled"); - assert_eq!(t["budget_tokens"], 32_768); // High budget fits under 65536 - assert!( - output_config.is_none(), - "output_config must be absent for claude-opus-4-5 (manual budget)" + fn resolve_provider_openrouter_with_key() { + assert_eq!( + resolve_provider(Some("openrouter"), None, None, Some("sk-or-123")).unwrap(), + Provider::OpenRouter ); } #[test] - fn anthropic_thinking_config_opus_4_5_budget_capped() { - // Opus 4.5 manual budget is clamped to min(level_budget, max_output_tokens - 1024). - // max_output_tokens = 4096: headroom = 4096 - 1024 = 3072; High budget (32768) → 3072. - let (thinking, _) = - anthropic_thinking_config("claude-opus-4-5", ThinkingEffort::High, 4096); - let t = thinking.unwrap(); - assert_eq!(t["budget_tokens"], 3072); // min(32768, 4096-1024) + fn resolve_provider_openrouter_missing_key() { + let err = resolve_provider(Some("openrouter"), None, None, None).unwrap_err(); + assert!(err.contains("OPENROUTER_API_KEY")); } - #[test] - fn anthropic_thinking_config_opus_4_5_omits_thinking_when_max_output_1025() { - // max_output_tokens = 1025: headroom = 1025 - 1024 = 1 < 1024 → omit thinking. - let (thinking, _) = - anthropic_thinking_config("claude-opus-4-5", ThinkingEffort::High, 1025); - assert!( - thinking.is_none(), - "thinking must be omitted when max_output_tokens - 1024 < 1024" - ); + // ---- Config::validate() — Anthropic effort gate ---- + // Pinned behavioral coverage for the pure-Anthropic none|minimal rejection + // at config.rs:672-681. This gate is reached from Config::from_env() at :558. + // A regression (e.g. predicate accidentally inverted) makes these tests fail + // while the cargo-test suite remains otherwise green. + + /// Minimal Config that passes all validate() invariants (Anthropic provider). + /// Tests set only the fields under scrutiny before calling validate(). + fn cfg_anthropic_for_validate() -> Config { + Config { + provider: Provider::Anthropic, + system_prompt: String::new(), + api_key: "sk-ant-key".into(), + model: "claude-opus-4-8".into(), + base_url: "https://api.anthropic.com".into(), + anthropic_api_version: "2023-06-01".into(), + openai_api: OpenAiApi::Auto, + prefer_mesh_for_auto: false, + max_rounds: 10, + max_output_tokens: 32_768, + llm_timeout: Duration::from_secs(10), + tool_timeout: Duration::from_secs(10), + mcp_init_timeout: Duration::from_secs(10), + mcp_max_restart_attempts: 1, + mcp_restart_base_ms: 100, + mcp_restart_max_ms: 1_000, + max_sessions: 1, + max_line_bytes: 4 * 1024 * 1024, + max_history_bytes: 16 * 1024 * 1024, + max_tool_result_text_bytes: 50 * 1024, + max_context_tokens: 200_000, + max_handoffs: 1, + max_parallel_tools: 1, + hook_timeout: Duration::from_secs(1), + stop_max_rejections: 0, + require_reply: false, + hook_servers: HookServers::None, + hints_enabled: false, + thinking_effort: None, + prompt_caching: false, + } } #[test] - fn anthropic_thinking_config_manual_budget_low_emits_1024_when_fits() { - // Low budget (1024 tokens) exactly fits when max_output_tokens = 2048. - // headroom = 2048 - 1024 = 1024; min(1024, 1024) = 1024 ≥ 1024 → emit. - let (thinking, _) = - anthropic_thinking_config("claude-3-7-sonnet-20250219", ThinkingEffort::Low, 2048); - let t = thinking.expect("Low budget (1024) must be emitted when max_output_tokens = 2048"); - assert_eq!(t["budget_tokens"], 1024); + fn validate_rejects_none_and_minimal_for_pure_anthropic() { + for effort in [ThinkingEffort::None, ThinkingEffort::Minimal] { + let mut cfg = cfg_anthropic_for_validate(); + cfg.thinking_effort = Some(effort); + let err = cfg.validate().unwrap_err(); + assert!( + err.contains("not valid for Anthropic providers"), + "effort={effort:?}: expected rejection, got {err:?}" + ); + } } #[test] - fn anthropic_thinking_config_unknown_claude_omits_both_fields() { - // An unknown/future "claude-*" name that is not in the allowlist → omit both fields. - // This prevents sending an unverified shape to an unrecognized model. - // Includes Opus 4.9 (future version), which is NOT in the doc-verified adaptive list. - for model in &[ - "claude-haiku-4-5", - "claude-sonnet-4-5", - "claude-unknown-9-1", - "claude-future-model", - "claude-opus-4-9", + fn validate_accepts_valid_efforts_for_pure_anthropic() { + for effort in [ + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ThinkingEffort::XHigh, + ThinkingEffort::Max, ] { - let (thinking, output_config) = - anthropic_thinking_config(model, ThinkingEffort::High, 32_768); - assert!( - thinking.is_none(), - "thinking must be absent for unverified claude model: {model}" - ); + let mut cfg = cfg_anthropic_for_validate(); + cfg.thinking_effort = Some(effort); assert!( - output_config.is_none(), - "output_config must be absent for unverified claude model: {model}" + cfg.validate().is_ok(), + "effort={effort:?}: expected Ok, got error" ); } } + // ---- normalize_effort_for_anthropic_route() matrix ---- + // Pinned coverage for the none|minimal → None mapping at config.rs:151-164. + // Used by the DBv2 Anthropic arm (llm.rs:216). A regression that lets + // none/minimal reach the wire as an Anthropic effort level fails these tests. + #[test] - fn anthropic_thinking_config_non_claude_omits_both_fields() { - // Non-Anthropic model names (gpt-5, llama, etc.) → omit both fields. - let (thinking, output_config) = - anthropic_thinking_config("gpt-4o-mini", ThinkingEffort::High, 32_768); - assert!( - thinking.is_none(), - "thinking must be absent for non-claude model" + fn normalize_effort_for_anthropic_route_omits_none_and_minimal() { + assert_eq!( + normalize_effort_for_anthropic_route(ThinkingEffort::None), + None ); - assert!( - output_config.is_none(), - "output_config must be absent for non-claude model" + assert_eq!( + normalize_effort_for_anthropic_route(ThinkingEffort::Minimal), + None ); } #[test] - fn anthropic_thinking_config_databricks_prefix_stripped_for_claude3() { - // Databricks gateway prefixes like "databricks-claude-3-..." must be stripped. - let (thinking, output_config) = - anthropic_thinking_config("databricks-claude-3-5-sonnet", ThinkingEffort::Low, 8_192); - let t = thinking.expect("thinking must be present after stripping databricks- prefix"); - assert_eq!(t["type"], "enabled"); - assert!(output_config.is_none()); - } - - #[test] - fn anthropic_thinking_config_databricks_prefix_stripped_for_opus_4_7() { - // Databricks gateway prefix stripping applies to adaptive Claude families too. - let (thinking, output_config) = - anthropic_thinking_config("databricks-claude-opus-4-7", ThinkingEffort::High, 32_768); - let t = thinking - .expect("thinking:{type:adaptive} must be present for databricks-claude-opus-4-7"); - assert_eq!(t["type"], "adaptive"); - let oc = - output_config.expect("output_config must be present for databricks-claude-opus-4-7"); - assert_eq!(oc["effort"], "high"); - } - - #[test] - fn anthropic_thinking_config_databricks_prefix_stripped_for_opus_4_8() { - // Databricks gateway prefix stripping applies to Opus 4.8 too. - let (thinking, output_config) = - anthropic_thinking_config("databricks-claude-opus-4-8", ThinkingEffort::Medium, 32_768); - let t = thinking - .expect("thinking:{type:adaptive} must be present for databricks-claude-opus-4-8"); - assert_eq!(t["type"], "adaptive"); - let oc = - output_config.expect("output_config must be present for databricks-claude-opus-4-8"); - assert_eq!(oc["effort"], "medium"); - } - - #[test] - fn anthropic_thinking_config_goose_prefix_stripped_for_fable_5() { - // "goose-" catalog prefix must be stripped so goose-claude-fable-5 routes to - // the adaptive + xhigh/max bucket, not the "unknown model → (None, None)" path. - let (thinking, output_config) = - anthropic_thinking_config("goose-claude-fable-5", ThinkingEffort::Max, 32_768); - let t = - thinking.expect("thinking:{type:adaptive} must be present for goose-claude-fable-5"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for goose-claude-fable-5"); - assert_eq!(oc["effort"], "max"); - } - - #[test] - fn anthropic_thinking_config_goose_prefix_stripped_for_sonnet_5() { - // Adaptive xhigh model via goose- prefix. - let (thinking, output_config) = - anthropic_thinking_config("goose-claude-sonnet-5", ThinkingEffort::XHigh, 32_768); - let t = - thinking.expect("thinking:{type:adaptive} must be present for goose-claude-sonnet-5"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for goose-claude-sonnet-5"); - assert_eq!(oc["effort"], "xhigh"); - } - - #[test] - fn anthropic_thinking_config_arbitrary_prefix_stripped_for_opus_4_7() { - // team-x-claude-opus-4-7: first claude- token at index 7 → strips "team-x-" - // Verifies the arbitrary-prefix normalization reaches anthropic_thinking_config - // end-to-end: UI exposes max as valid, and runtime must honor it. - let (thinking, output_config) = - anthropic_thinking_config("team-x-claude-opus-4-7", ThinkingEffort::Max, 32_768); - let t = - thinking.expect("thinking:{type:adaptive} must be present for team-x-claude-opus-4-7"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for team-x-claude-opus-4-7"); - assert_eq!(oc["effort"], "max"); - } - - // ---- clamp_adaptive_effort — per-model clamping tests ---- - - #[test] - fn clamp_adaptive_effort_xhigh_passes_through_for_opus_4_7() { - // Opus 4.7 supports xhigh — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-7", ThinkingEffort::XHigh), - ThinkingEffort::XHigh - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_passes_through_for_opus_4_8() { - // Opus 4.8 supports xhigh — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-8", ThinkingEffort::XHigh), - ThinkingEffort::XHigh - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_passes_through_for_sonnet_5() { - // Sonnet 5 supports xhigh — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-sonnet-5-20250901", ThinkingEffort::XHigh), - ThinkingEffort::XHigh - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_clamped_to_high_for_opus_4_6() { - // Opus 4.6 does NOT support xhigh (only low/medium/high/max) — clamp to high. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-6", ThinkingEffort::XHigh), - ThinkingEffort::High - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_clamped_to_high_for_sonnet_4_6() { - // Sonnet 4.6 does NOT support xhigh — clamp to high. - assert_eq!( - clamp_adaptive_effort("claude-sonnet-4-6", ThinkingEffort::XHigh), - ThinkingEffort::High - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_opus_4_6() { - // Opus 4.6 supports max — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-6", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_opus_4_7() { - // Opus 4.7 supports max — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-7", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_opus_4_8() { - // Opus 4.8 supports max — no clamping. - assert_eq!( - clamp_adaptive_effort("claude-opus-4-8", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn clamp_adaptive_effort_low_medium_high_never_clamped() { - // low/medium/high pass through for all adaptive models. - for model in &[ - "claude-opus-4-6", - "claude-opus-4-7", - "claude-opus-4-8", - "claude-sonnet-5-20250901", - "claude-sonnet-4-6", - ] { - for effort in [ - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ] { - assert_eq!( - clamp_adaptive_effort(model, effort), - effort, - "model={model} effort={effort:?}" - ); - } - } - } - - // ---- anthropic_thinking_config — xhigh/max body-shape assertions ---- - - #[test] - fn anthropic_thinking_config_opus_4_8_xhigh_emits_xhigh_effort() { - // Opus 4.8 supports xhigh; output_config.effort must be "xhigh". - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-8", ThinkingEffort::XHigh, 32_768); - let t = thinking.expect("thinking must be present for claude-opus-4-8"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-opus-4-8"); - assert_eq!(oc["effort"], "xhigh"); - } - - #[test] - fn anthropic_thinking_config_opus_4_8_max_emits_max_effort() { - // Opus 4.8 supports max; output_config.effort must be "max". - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-8", ThinkingEffort::Max, 32_768); - let t = thinking.expect("thinking must be present for claude-opus-4-8"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-opus-4-8"); - assert_eq!(oc["effort"], "max"); - } - - #[test] - fn anthropic_thinking_config_opus_4_7_xhigh_emits_xhigh_effort() { - // Opus 4.7 supports xhigh. - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-7", ThinkingEffort::XHigh, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.unwrap(); - assert_eq!(oc["effort"], "xhigh"); - } - - #[test] - fn anthropic_thinking_config_opus_4_6_xhigh_clamps_to_high() { - // Opus 4.6 does NOT support xhigh → clamp to high. - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-6", ThinkingEffort::XHigh, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.unwrap(); - assert_eq!( - oc["effort"], "high", - "xhigh must clamp to high for claude-opus-4-6" - ); - } - - #[test] - fn anthropic_thinking_config_opus_4_6_max_passes_through() { - // Opus 4.6 supports max — passes through without clamping. - let (thinking, output_config) = - anthropic_thinking_config("claude-opus-4-6", ThinkingEffort::Max, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.unwrap(); - assert_eq!(oc["effort"], "max"); - } - - #[test] - fn anthropic_thinking_config_manual_bucket_xhigh_clamps_to_high_budget() { - // Manual-budget models (claude-3*, opus-4-5): xhigh clamps to high budget (32_768). - for model in &["claude-3-7-sonnet-20250219", "claude-opus-4-5"] { - let (thinking, output_config) = - anthropic_thinking_config(model, ThinkingEffort::XHigh, 65_536); - let t = thinking.expect("thinking must be present"); - assert_eq!(t["type"], "enabled"); - assert_eq!( - t["budget_tokens"], 32_768, - "xhigh must clamp to high budget for manual model {model}" - ); - assert!(output_config.is_none()); - } - } - - #[test] - fn anthropic_thinking_config_manual_bucket_max_clamps_to_high_budget() { - // Manual-budget models: max also clamps to high budget (32_768). - let (thinking, _) = - anthropic_thinking_config("claude-opus-4-5", ThinkingEffort::Max, 65_536); - let t = thinking.unwrap(); - assert_eq!(t["type"], "enabled"); - assert_eq!(t["budget_tokens"], 32_768); - } - - // ---- provider-level validation tests ---- - - /// Build a minimal Config with the given provider and thinking_effort, bypassing from_env(). - /// Uses `Config::for_discovery` as a base and patches the fields we care about. - fn make_config_for_validation( - provider: Provider, - thinking_effort: Option, - ) -> Config { - let mut cfg = Config::for_discovery(provider, "key".into(), "https://example.com".into()); - cfg.model = "some-model".into(); - cfg.thinking_effort = thinking_effort; - // for_discovery sets max_output_tokens=1 and max_context_tokens=200_001 which satisfies - // the context > output constraint. Adjust to something valid for further checks. - cfg.max_output_tokens = 1024; - cfg.max_context_tokens = 200_000 + 1024; - // Restore mandatory positive values that for_discovery zeroes out. - cfg.mcp_max_restart_attempts = 1; - cfg.mcp_restart_base_ms = 1; - cfg.mcp_restart_max_ms = 1; - cfg.max_parallel_tools = 1; - cfg.llm_timeout = Duration::from_secs(1); - cfg.tool_timeout = Duration::from_secs(1); - cfg.mcp_init_timeout = Duration::from_secs(1); - cfg - } - - #[test] - fn validate_rejects_none_effort_for_anthropic() { - let cfg = make_config_for_validation(Provider::Anthropic, Some(ThinkingEffort::None)); - let err = cfg.validate().unwrap_err(); - assert!( - err.contains("BUZZ_AGENT_THINKING_EFFORT=none"), - "error must name the value: {err}" - ); - assert!( - err.contains("not valid for Anthropic"), - "error must name the provider: {err}" - ); - assert!( - err.contains("low|medium|high|xhigh|max"), - "error must name allowed values: {err}" - ); - } - - #[test] - fn validate_rejects_minimal_effort_for_anthropic() { - let cfg = make_config_for_validation(Provider::Anthropic, Some(ThinkingEffort::Minimal)); - let err = cfg.validate().unwrap_err(); - assert!(err.contains("BUZZ_AGENT_THINKING_EFFORT=minimal"), "{err}"); - assert!(err.contains("not valid for Anthropic"), "{err}"); - } - - #[test] - fn validate_accepts_all_efforts_for_databricks_v2() { - // DatabricksV2 dispatches across Anthropic/OpenAI/MLflow routes at request build time. - // No effort value is invalid for all three routes — startup rejects none. - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ] { - let cfg = make_config_for_validation(Provider::DatabricksV2, Some(effort)); - assert!( - cfg.validate().is_ok(), - "DatabricksV2 must accept {effort:?} at startup (route-aware normalization at request build)" - ); - } - } - - #[test] - fn validate_accepts_all_efforts_for_openai() { - // OpenAI effort support is model-dependent and normalized at request build time. - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ] { - let cfg = make_config_for_validation(Provider::OpenAi, Some(effort)); - assert!( - cfg.validate().is_ok(), - "OpenAI must accept {effort:?} at startup (route-aware normalization at request build)" - ); - } - } - - #[test] - fn validate_accepts_all_efforts_for_databricks() { - // Legacy Databricks effort support is model-dependent and normalized at request build time. - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ] { - let cfg = make_config_for_validation(Provider::Databricks, Some(effort)); - assert!( - cfg.validate().is_ok(), - "Databricks must accept {effort:?} at startup (route-aware normalization at request build)" - ); - } - } - - #[test] - fn validate_accepts_xhigh_for_anthropic() { - // xhigh is valid for Anthropic providers — model-level clamping is dynamic. - let cfg = make_config_for_validation(Provider::Anthropic, Some(ThinkingEffort::XHigh)); - assert!( - cfg.validate().is_ok(), - "xhigh must be accepted at startup for Anthropic" - ); - } - - #[test] - fn validate_accepts_max_for_anthropic() { - // max is valid for Anthropic providers. - let cfg = make_config_for_validation(Provider::Anthropic, Some(ThinkingEffort::Max)); - assert!(cfg.validate().is_ok(), "max must be accepted for Anthropic"); - } - - #[test] - fn validate_accepts_xhigh_for_openai() { - // xhigh is valid for OpenAI providers (server-validated per-model). - let cfg = make_config_for_validation(Provider::OpenAi, Some(ThinkingEffort::XHigh)); - assert!(cfg.validate().is_ok(), "xhigh must be accepted for OpenAI"); - } - - #[test] - fn validate_accepts_none_and_minimal_for_openai() { - // none/minimal are valid OpenAI effort values. - let cfg_none = make_config_for_validation(Provider::OpenAi, Some(ThinkingEffort::None)); - assert!( - cfg_none.validate().is_ok(), - "none must be accepted for OpenAI" - ); - let cfg_minimal = - make_config_for_validation(Provider::OpenAi, Some(ThinkingEffort::Minimal)); - assert!( - cfg_minimal.validate().is_ok(), - "minimal must be accepted for OpenAI" - ); - } - - // ---- normalize_effort_for_openai_route ---- - - #[test] - fn normalize_openai_route_clamps_max_to_xhigh() { - // Use an unknown model so only the max→xhigh clamp fires, not per-model logic. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::Max, "llama-4"), - ThinkingEffort::XHigh - ); - } - - #[test] - fn normalize_openai_route_passes_through_all_other_values_for_unknown_model() { - // Unknown/unverified models pass through unchanged (server-validated). - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ] { - assert_eq!( - normalize_effort_for_openai_route(effort, "unknown-future-model"), - effort, - "normalize_effort_for_openai_route must pass through {effort:?} for unknown model" - ); - } - } - - // ---- normalize_effort_for_anthropic_route ---- - - #[test] - fn normalize_anthropic_route_none_yields_none() { - assert_eq!( - normalize_effort_for_anthropic_route(ThinkingEffort::None), - None, - "none must yield None (omit thinking fields)" - ); - } - - #[test] - fn normalize_anthropic_route_minimal_yields_none() { - assert_eq!( - normalize_effort_for_anthropic_route(ThinkingEffort::Minimal), - None, - "minimal must yield None (omit thinking fields)" - ); - } - - #[test] - fn normalize_anthropic_route_passes_through_valid_values() { + fn normalize_effort_for_anthropic_route_passes_through_valid_levels() { for effort in [ ThinkingEffort::Low, ThinkingEffort::Medium, @@ -2307,735 +1298,8 @@ mod tests { assert_eq!( normalize_effort_for_anthropic_route(effort), Some(effort), - "normalize_effort_for_anthropic_route must pass through {effort:?}" + "effort={effort:?}: expected Some({effort:?}), got None" ); } } - - // ---- F2: Fable 5 / Mythos 5 / Mythos Preview adaptive thinking ---- - - #[test] - fn anthropic_thinking_config_fable_5_emits_adaptive_and_effort() { - // Fable 5 — always-on adaptive thinking. - let (thinking, output_config) = - anthropic_thinking_config("claude-fable-5", ThinkingEffort::High, 32_768); - let t = thinking.expect("thinking must be present for claude-fable-5"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-fable-5"); - assert_eq!(oc["effort"], "high"); - } - - #[test] - fn anthropic_thinking_config_mythos_5_emits_adaptive_and_effort() { - // Mythos 5 — always-on adaptive thinking. - let (thinking, output_config) = - anthropic_thinking_config("claude-mythos-5", ThinkingEffort::Medium, 32_768); - let t = thinking.expect("thinking must be present for claude-mythos-5"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-mythos-5"); - assert_eq!(oc["effort"], "medium"); - } - - #[test] - fn anthropic_thinking_config_mythos_preview_emits_adaptive_and_effort() { - // Mythos Preview — default-on adaptive thinking. - let (thinking, output_config) = - anthropic_thinking_config("claude-mythos-preview", ThinkingEffort::Low, 32_768); - let t = thinking.expect("thinking must be present for claude-mythos-preview"); - assert_eq!(t["type"], "adaptive"); - let oc = output_config.expect("output_config must be present for claude-mythos-preview"); - assert_eq!(oc["effort"], "low"); - } - - #[test] - fn clamp_adaptive_effort_xhigh_passes_through_for_fable_5() { - // Fable 5 supports xhigh. - assert_eq!( - clamp_adaptive_effort("claude-fable-5", ThinkingEffort::XHigh), - ThinkingEffort::XHigh - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_passes_through_for_mythos_5() { - // Mythos 5 supports xhigh. - assert_eq!( - clamp_adaptive_effort("claude-mythos-5", ThinkingEffort::XHigh), - ThinkingEffort::XHigh - ); - } - - #[test] - fn clamp_adaptive_effort_xhigh_clamped_to_high_for_mythos_preview() { - // Mythos Preview does NOT support xhigh — clamp to high. - assert_eq!( - clamp_adaptive_effort("claude-mythos-preview", ThinkingEffort::XHigh), - ThinkingEffort::High - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_fable_5() { - // Fable 5 supports max. - assert_eq!( - clamp_adaptive_effort("claude-fable-5", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_mythos_5() { - // Mythos 5 supports max. - assert_eq!( - clamp_adaptive_effort("claude-mythos-5", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn clamp_adaptive_effort_max_passes_through_for_mythos_preview() { - // Mythos Preview supports max. - assert_eq!( - clamp_adaptive_effort("claude-mythos-preview", ThinkingEffort::Max), - ThinkingEffort::Max - ); - } - - #[test] - fn anthropic_thinking_config_fable_5_xhigh_emits_xhigh() { - let (thinking, output_config) = - anthropic_thinking_config("claude-fable-5", ThinkingEffort::XHigh, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - assert_eq!(output_config.unwrap()["effort"], "xhigh"); - } - - #[test] - fn anthropic_thinking_config_mythos_5_xhigh_emits_xhigh() { - let (thinking, output_config) = - anthropic_thinking_config("claude-mythos-5", ThinkingEffort::XHigh, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - assert_eq!(output_config.unwrap()["effort"], "xhigh"); - } - - #[test] - fn anthropic_thinking_config_mythos_preview_xhigh_clamps_to_high() { - // Mythos Preview does NOT support xhigh → clamp to high. - let (thinking, output_config) = - anthropic_thinking_config("claude-mythos-preview", ThinkingEffort::XHigh, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - assert_eq!( - output_config.unwrap()["effort"], - "high", - "xhigh must clamp to high for claude-mythos-preview" - ); - } - - #[test] - fn anthropic_thinking_config_fable_5_max_passes_through() { - let (thinking, output_config) = - anthropic_thinking_config("claude-fable-5", ThinkingEffort::Max, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - assert_eq!(output_config.unwrap()["effort"], "max"); - } - - #[test] - fn anthropic_thinking_config_mythos_preview_max_passes_through() { - let (thinking, output_config) = - anthropic_thinking_config("claude-mythos-preview", ThinkingEffort::Max, 32_768); - let t = thinking.unwrap(); - assert_eq!(t["type"], "adaptive"); - assert_eq!(output_config.unwrap()["effort"], "max"); - } - - // ---- openai_efforts_for_model / normalize_effort_for_openai_route per-model table ---- - - #[test] - fn openai_efforts_for_model_gpt5_pro_high_only() { - // gpt-5-pro: high only — any other value must be substituted. - let supported = openai_efforts_for_model("gpt-5-pro").expect("gpt-5-pro must be in table"); - assert_eq!( - supported, - &[ThinkingEffort::High], - "gpt-5-pro supports only high" - ); - } - - #[test] - fn openai_efforts_for_model_gpt5_6_includes_max() { - let expected: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - - for model in ["gpt-5.6", "gpt-5.6-sol", "gpt-5-6-sol", "goose-gpt-5-6-sol"] { - assert_eq!( - openai_efforts_for_model(model), - Some(expected), - "{model} must match the gpt-5.6 effort table" - ); - } - } - - #[test] - fn openai_efforts_for_model_gpt5_5_includes_xhigh() { - let supported = openai_efforts_for_model("gpt-5.5").expect("gpt-5.5 must be in table"); - assert!( - supported.contains(&ThinkingEffort::XHigh), - "gpt-5.5 must support xhigh" - ); - assert!( - supported.contains(&ThinkingEffort::None), - "gpt-5.5 must support none" - ); - } - - #[test] - fn openai_efforts_for_model_gpt5_1_excludes_xhigh_and_minimal() { - let supported = openai_efforts_for_model("gpt-5.1").expect("gpt-5.1 must be in table"); - assert!( - !supported.contains(&ThinkingEffort::XHigh), - "gpt-5.1 must NOT support xhigh" - ); - assert!( - !supported.contains(&ThinkingEffort::Minimal), - "gpt-5.1 must NOT support minimal" - ); - assert!( - supported.contains(&ThinkingEffort::None), - "gpt-5.1 must support none" - ); - } - - #[test] - fn openai_efforts_for_model_gpt5_base_excludes_none_includes_minimal() { - let supported = openai_efforts_for_model("gpt-5").expect("gpt-5 base must be in table"); - assert!( - !supported.contains(&ThinkingEffort::None), - "gpt-5 base must NOT support none" - ); - assert!( - supported.contains(&ThinkingEffort::Minimal), - "gpt-5 base must support minimal" - ); - } - - #[test] - fn openai_efforts_for_model_unknown_returns_none() { - // Unknown models are not doc-verified — caller treats as server-validated pass-through. - assert!(openai_efforts_for_model("llama-4").is_none()); - assert!(openai_efforts_for_model("claude-opus-4-8").is_none()); - assert!(openai_efforts_for_model("gpt-4o").is_none()); - } - - // ---- Boundary-safe matching: version digits must not false-match longer versions ---- - - #[test] - fn openai_efforts_for_model_boundary_dated_base_ids_are_not_versioned() { - // gpt-5-1106: the "-1" is not version 5.1 — it's a date segment on the base model. - // Must fall through to base table, not gpt-5.1. - let result = openai_efforts_for_model("gpt-5-1106"); - let base = openai_efforts_for_model("gpt-5").unwrap(); - assert_eq!( - result, - Some(base), - "gpt-5-1106 must match base table (not gpt-5.1): got {result:?}" - ); - // Crucially, must NOT support None (that's a gpt-5.1 property, not base). - assert!( - !result.unwrap().contains(&ThinkingEffort::None), - "gpt-5-1106 must NOT support none — base table only has minimal" - ); - } - - #[test] - fn openai_efforts_for_model_boundary_gpt5_4o_is_base_not_5_4() { - // gpt-5-4o: the "-4" could false-match the gpt-5.4 family, but "4o" is a - // capability suffix on the base gpt-5 model, not version 5.4. - // Must fall through to base table. - let result = openai_efforts_for_model("gpt-5-4o"); - let base = openai_efforts_for_model("gpt-5").unwrap(); - assert_eq!( - result, - Some(base), - "gpt-5-4o must match base table (not gpt-5.4): got {result:?}" - ); - // Crucially, must NOT support XHigh (that's a gpt-5.4 property, not base). - assert!( - !result.unwrap().contains(&ThinkingEffort::XHigh), - "gpt-5-4o must NOT support xhigh — that's a gpt-5.4 property and would 400" - ); - } - - #[test] - fn openai_efforts_for_model_boundary_multi_digit_versions_pass_through() { - // Dotted two-digit versions (gpt-5.10, gpt5.10, gpt-5.50) must not match any known - // single-digit family — the digit boundary check on dotted tokens blocks them. - // These return None (server-validated pass-through). - assert!( - openai_efforts_for_model("gpt-5.10").is_none(), - "gpt-5.10 must pass through (unknown future model)" - ); - assert!( - openai_efforts_for_model("gpt5.10").is_none(), - "gpt5.10 must pass through (unknown future model)" - ); - assert!( - openai_efforts_for_model("gpt-5.50").is_none(), - "gpt-5.50 must pass through (not gpt-5.5)" - ); - // Dash two-digit versions (gpt-5-10, databricks-gpt-5-10) look like short numeric - // version segments and must also pass through as unknown — not bucketed as base. - assert!( - openai_efforts_for_model("gpt-5-10").is_none(), - "gpt-5-10 must pass through (short numeric suffix = potential unrecognized version)" - ); - assert!( - openai_efforts_for_model("databricks-gpt-5-10").is_none(), - "databricks-gpt-5-10 must pass through (short numeric suffix)" - ); - // Short numeric suffix + textual continuation (e.g. a hypothetical 'gpt-5.10-preview') - // must also pass through — the digit count (1-3) determines version-like, regardless of - // what follows. - assert!( - openai_efforts_for_model("gpt-5-10-preview").is_none(), - "gpt-5-10-preview must pass through (short numeric version suffix with text tail)" - ); - assert!( - openai_efforts_for_model("databricks-gpt-5-10-preview").is_none(), - "databricks-gpt-5-10-preview must pass through (short numeric version suffix with text tail)" - ); - } - - #[test] - fn openai_efforts_for_model_boundary_date_segment_with_suffix_is_base() { - // 4+ digit date segment followed by a textual suffix must still resolve to the base - // table — the date length (>=4) determines it's a build/date, not a version number. - let result = openai_efforts_for_model("gpt-5-1106-preview"); - assert!( - result.is_some(), - "gpt-5-1106-preview must match base table (4-digit date segment)" - ); - let supported = result.unwrap(); - assert!( - supported.contains(&ThinkingEffort::Minimal), - "gpt-5-1106-preview (base) must support minimal" - ); - assert!( - !supported.contains(&ThinkingEffort::None), - "gpt-5-1106-preview (base) must NOT support none" - ); - assert!( - !supported.contains(&ThinkingEffort::XHigh), - "gpt-5-1106-preview (base) must NOT support xhigh" - ); - } - - #[test] - fn openai_efforts_for_model_boundary_databricks_prefixed_still_matches() { - // Databricks-prefixed names (gateway forwarding) must still resolve to the right table. - let result = openai_efforts_for_model("databricks-gpt-5-5"); - assert_eq!( - result, - openai_efforts_for_model("gpt-5.5"), - "databricks-gpt-5-5 must match gpt-5.5 family table" - ); - } - - #[test] - fn openai_efforts_for_model_boundary_date_suffixed_still_matches() { - // Date-suffixed names (e.g. gpt-5.1-2025-04-01) must still resolve to the right family. - let result = openai_efforts_for_model("gpt-5.1-2025-04-01"); - assert_eq!( - result, - openai_efforts_for_model("gpt-5.1"), - "gpt-5.1-2025-04-01 must match gpt-5.1 family table" - ); - } - - #[test] - fn openai_efforts_for_model_pro_before_base_gpt5() { - // gpt-5-pro must match the -pro table, not the base gpt-5 table. - let pro = openai_efforts_for_model("gpt-5-pro").unwrap(); - let base = openai_efforts_for_model("gpt-5").unwrap(); - assert_ne!( - pro, base, - "gpt-5-pro and gpt-5 base must hit different table entries" - ); - assert_eq!(pro, &[ThinkingEffort::High]); - } - - #[test] - fn normalize_openai_route_gpt5_pro_high_passes_through() { - // gpt-5-pro: high is the only supported value → high passes through unchanged. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::High, "gpt-5-pro"), - ThinkingEffort::High - ); - } - - #[test] - fn normalize_openai_route_gpt5_pro_anything_but_high_becomes_high() { - // gpt-5-pro: any effort other than high must resolve to high. - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::XHigh, - ] { - assert_eq!( - normalize_effort_for_openai_route(effort, "gpt-5-pro"), - ThinkingEffort::High, - "gpt-5-pro: {effort:?} must resolve to high" - ); - } - } - - #[test] - fn normalize_openai_route_gpt5_base_none_becomes_minimal() { - // gpt-5 base supports minimal but not none. none → minimal (peer fallback). - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::None, "gpt-5"), - ThinkingEffort::Minimal, - "gpt-5 base: none must fall back to minimal (peer)" - ); - } - - #[test] - fn normalize_openai_route_passes_max_through_for_gpt5_6() { - for model in ["gpt-5.6", "gpt-5-6-sol"] { - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::Max, model), - ThinkingEffort::Max, - "{model} must preserve max" - ); - } - } - - #[test] - fn normalize_openai_route_gpt5_5_max_becomes_xhigh() { - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.5"), - ThinkingEffort::XHigh, - "gpt-5.5 must clamp max to xhigh" - ); - } - - #[test] - fn normalize_openai_route_gpt5_5_minimal_becomes_none() { - // gpt-5.5 supports none but not minimal. minimal → none (peer fallback). - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::Minimal, "gpt-5.5"), - ThinkingEffort::None, - "gpt-5.5: minimal must fall back to none (peer)" - ); - } - - #[test] - fn normalize_openai_route_gpt5_1_xhigh_becomes_high() { - // gpt-5.1 does not support xhigh → nearest supported below xhigh is high. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::XHigh, "gpt-5.1"), - ThinkingEffort::High, - "gpt-5.1: xhigh must resolve to high" - ); - } - - #[test] - fn normalize_openai_route_gpt5_4_xhigh_passes_through() { - // gpt-5.4 supports xhigh → pass through unchanged. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::XHigh, "gpt-5.4"), - ThinkingEffort::XHigh - ); - } - - #[test] - fn normalize_openai_route_gpt5_5_xhigh_passes_through() { - // gpt-5.5 supports xhigh → pass through unchanged. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::XHigh, "gpt-5.5"), - ThinkingEffort::XHigh - ); - } - - #[test] - fn normalize_openai_route_gpt5_dash_suffix_variants_match_correctly() { - // Databricks-prefixed or date-suffixed names must still hit the right family. - // "gpt-5.5" and "gpt-5-5" are treated identically; ditto for other families. - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::XHigh, "gpt-5-5"), - ThinkingEffort::XHigh, - "gpt-5-5 (dash) must match gpt-5.5 table" - ); - assert_eq!( - normalize_effort_for_openai_route(ThinkingEffort::None, "gpt-5-1"), - ThinkingEffort::None, - "gpt-5-1 (dash) must match gpt-5.1 table" - ); - } - - #[test] - fn normalize_openai_route_unknown_model_passthrough() { - // Unknown models: all values pass through without substitution (server-validated). - for effort in [ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ] { - assert_eq!( - normalize_effort_for_openai_route(effort, "llama-4"), - effort, - "unknown model: {effort:?} must pass through unchanged" - ); - } - } - - // ---- effort-table fixture sync guard ---------------------------------------- - // - // Loads `effortTable.fixture.json` (the single source of truth shared with - // the TS test in `buzzAgentConfig.test.mjs`) and verifies that this Rust - // implementation produces the same valid-effort-value sets and default values - // as the TS `getProviderEffortConfig` function. - // - // Drift (a new model family added to one side but not the other) fails CI here - // before it can silently diverge in production. - // ───────────────────────────────────────────────────────────────────────────── - - /// Compute the valid effort values for a provider/model pair, mirroring - /// `getProviderEffortConfig` in `buzzAgentConfig.ts`. - /// - /// Returns `(valid_values, default_value)` where `default_value` is `None` - /// for Anthropic manual-budget models (TS `defaultValue: null`), otherwise - /// `Some("medium")` or `Some("high")`. - pub(super) fn valid_effort_values_for_provider_model( - provider: &str, - model: &str, - ) -> (Vec<&'static str>, Option<&'static str>) { - const ALL_7: &[&str] = &["none", "minimal", "low", "medium", "high", "xhigh", "max"]; - const ALL_EXCEPT_MAX: &[&str] = &["none", "minimal", "low", "medium", "high", "xhigh"]; - const GPT5_PRO: &[&str] = &["high"]; - const GPT5_1: &[&str] = &["none", "low", "medium", "high"]; - - let p = provider.to_ascii_lowercase(); - // Canonicalize provider aliases — mirrors the production path and TS - // PROVIDER_ALIASES so this shim stays in sync with the fixture. - let p = match p.as_str() { - "openai-compat" => "openai".to_owned(), - "databricks-v2" => "databricks_v2".to_owned(), - _ => p, - }; - // Strip arbitrary endpoint-naming prefix before model matching, mirroring TS and - // strip_catalog_prefix: find the first known family token (claude-, gpt-) and - // drop everything before it. Handles any catalog naming convention. - let raw_model = model.trim(); - let lower_raw = raw_model.to_ascii_lowercase(); - const FAMILY_TOKENS: &[&str] = &["claude-", "gpt-"]; - let first_idx = FAMILY_TOKENS - .iter() - .filter_map(|tok| lower_raw.find(tok)) - .min(); - let stripped = match first_idx { - Some(idx) => &raw_model[idx..], - None => raw_model, - }; - let m = stripped.to_ascii_lowercase(); - - // Thin adapter: converts production helper output to the string-based - // return type used by this function. - fn anthropic_result(m: &str) -> (Vec<&'static str>, Option<&'static str>) { - let (values, default) = anthropic_efforts_for_model(m); - let strs: Vec<&'static str> = values.iter().map(|e| e.openai_effort_str()).collect(); - (strs, default.map(|e| e.openai_effort_str())) - } - - fn openai_result(m: &str) -> (Vec<&'static str>, Option<&'static str>) { - if let Some(values) = openai_efforts_for_model(m) { - let strs: Vec<&'static str> = - values.iter().map(|e| e.openai_effort_str()).collect(); - // Determine default from the family. - let default_val = if strs == GPT5_PRO { - Some("high") - } else if strs == GPT5_1 { - Some("none") - } else { - Some("medium") - }; - (strs, default_val) - } else { - // Unknown model → all-except-max, default medium. - (ALL_EXCEPT_MAX.to_vec(), Some("medium")) - } - } - - if p == "anthropic" { - return anthropic_result(&m); - } - if p == "openai" { - return openai_result(&m); - } - if p == "databricks_v2" { - if m.starts_with("claude-") { - return anthropic_result(&m); - } - // gpt-5 family check mirrors gpt5FamilyModel in TS. - let is_gpt5 = gpt5_token_matches(&m, "gpt-5-pro") - || gpt5_token_matches(&m, "gpt5-pro") - || gpt5_token_matches(&m, "gpt-5.6") - || gpt5_token_matches(&m, "gpt5.6") - || gpt5_token_matches(&m, "gpt-5-6") - || gpt5_token_matches(&m, "gpt5-6") - || gpt5_token_matches(&m, "gpt-5.5") - || gpt5_token_matches(&m, "gpt5.5") - || gpt5_token_matches(&m, "gpt-5.4") - || gpt5_token_matches(&m, "gpt5.4") - || gpt5_token_matches(&m, "gpt-5.1") - || gpt5_token_matches(&m, "gpt5.1") - || gpt5_base_matches(&m, "gpt-5") - || gpt5_base_matches(&m, "gpt5"); - if is_gpt5 { - return openai_result(&m); - } - if !m.is_empty() { - // Concrete non-claude, non-gpt5: MLflow path → all-except-max. - return openai_result(&m); - } - // Blank model: route unknown, all-7. - return (ALL_7.to_vec(), Some("medium")); - } - if p == "databricks" { - return openai_result(&m); - } - if p == "openrouter" { - return (ALL_7.to_vec(), Some("medium")); - } - // Unknown/empty provider → all-7, default medium. - (ALL_7.to_vec(), Some("medium")) - } - - #[derive(serde::Deserialize)] - struct FixtureEntry { - note: Option, - provider: String, - model: String, - #[serde(rename = "validValues")] - valid_values: Vec, - #[serde(rename = "defaultValue")] - default_value: Option, - } - - #[test] - fn effort_table_fixture_matches_rust_implementation() { - let fixture_json = - include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); - let entries: Vec = - serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); - - assert!( - !entries.is_empty(), - "fixture must contain at least one entry" - ); - - for entry in &entries { - let label = entry.note.as_deref().unwrap_or(entry.model.as_str()); - let (valid_values, default_value) = - valid_effort_values_for_provider_model(&entry.provider, &entry.model); - - let expected: Vec<&str> = entry.valid_values.iter().map(String::as_str).collect(); - assert_eq!( - valid_values, expected, - "validValues mismatch for fixture entry \"{label}\" \ - (provider={}, model={}): Rust side has {valid_values:?}, \ - fixture expects {expected:?}", - entry.provider, entry.model, - ); - - let expected_default: Option<&str> = entry.default_value.as_deref(); - assert_eq!( - default_value, expected_default, - "defaultValue mismatch for fixture entry \"{label}\" \ - (provider={}, model={}): Rust side has {default_value:?}, \ - fixture expects {expected_default:?}", - entry.provider, entry.model, - ); - } - } - - // ---- normalize_effort_for_databricks_v2 regression tests (F1 corrections) ---- - // These pin the exact behavior Paul's pre-review probes checked. The key invariant: - // normalize_effort_for_databricks_v2 must resolve against the generated supported_efforts - // (which carries exact-record F1 corrections), NOT the old hand table. - - #[test] - fn normalize_effort_for_databricks_v2_gpt_5_5_xhigh_clamps_to_high() { - // F1 correction: databricks-gpt-5-5 generated supported_efforts = [low, medium, high]. - // XHigh is outside the supported set → nearest supported is High. - assert_eq!( - normalize_effort_for_databricks_v2(ThinkingEffort::XHigh, "databricks-gpt-5-5"), - ThinkingEffort::High, - "databricks-gpt-5-5 XHigh must clamp to High (F1 correction: supported=[low,medium,high])" - ); - } - - #[test] - fn normalize_effort_for_databricks_v2_gpt_5_5_none_clamps_to_low() { - // F1 correction: databricks-gpt-5-5 supported_efforts = [low, medium, high]. - // None is outside the set → nearest supported is Low. - assert_eq!( - normalize_effort_for_databricks_v2(ThinkingEffort::None, "databricks-gpt-5-5"), - ThinkingEffort::Low, - "databricks-gpt-5-5 None must clamp to Low (F1 correction: supported=[low,medium,high])" - ); - } - - #[test] - fn normalize_effort_for_databricks_v2_gpt_5_5_in_range_passes_through() { - // Values within the corrected set must pass through unchanged. - for effort in [ - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ] { - assert_eq!( - normalize_effort_for_databricks_v2(effort, "databricks-gpt-5-5"), - effort, - "databricks-gpt-5-5 {effort:?} is in supported set, must pass through" - ); - } - } - - #[test] - fn normalize_effort_for_databricks_v2_gpt_5_6_sol_max_passes_through() { - // databricks-gpt-5-6-sol F1 adoption: [low, medium, high, max] — max is supported. - assert_eq!( - normalize_effort_for_databricks_v2(ThinkingEffort::Max, "databricks-gpt-5-6-sol"), - ThinkingEffort::Max, - "databricks-gpt-5-6-sol Max must pass through (F1: supported includes max)" - ); - } - - #[test] - fn resolve_provider_openrouter_with_key() { - assert_eq!( - resolve_provider(Some("openrouter"), None, None, Some("sk-or-123")).unwrap(), - Provider::OpenRouter - ); - } - - #[test] - fn resolve_provider_openrouter_missing_key() { - let err = resolve_provider(Some("openrouter"), None, None, None).unwrap_err(); - assert!(err.contains("OPENROUTER_API_KEY")); - } } diff --git a/crates/buzz-agent/src/generated_model_capabilities.rs b/crates/buzz-agent/src/generated_model_capabilities.rs index 9f238e79ea..219c22e351 100644 --- a/crates/buzz-agent/src/generated_model_capabilities.rs +++ b/crates/buzz-agent/src/generated_model_capabilities.rs @@ -1336,7 +1336,6 @@ pub const DATABRICKS_MODEL_NAMES: &[(&str, &str)] = &[ /// Returns true if `model` contains `token` at a word boundary (end-of-string or "-"). /// Does not match if followed immediately by a digit or letter. -/// Mirrors gpt5_token_matches in config.rs. fn gpt5_token_matches_rs(model: &str, token: &str) -> bool { let lower = model; let tok_lower = token; diff --git a/crates/buzz-agent/src/generated_model_capabilities_tests.rs b/crates/buzz-agent/src/generated_model_capabilities_tests.rs index 670280ba70..ce8e8e8c54 100644 --- a/crates/buzz-agent/src/generated_model_capabilities_tests.rs +++ b/crates/buzz-agent/src/generated_model_capabilities_tests.rs @@ -8,7 +8,9 @@ //! cross-interpreter conformance gate; the JS runner executes the same file. //! 2. Handwritten supplement tests — adversarial cases and completeness checks that //! benefit from Rust-specific assertion ergonomics. -//! 3. Per-interpreter mutation evidence — see scripts/MUTATION_EVIDENCE.md. +//! 3. Per-interpreter mutation evidence — all 7 mutations were killed by both interpreters +//! (2026-07-31, Phase 2). The mutation runner was deleted in Phase 3; see +//! `scripts/MODEL_CAPABILITIES.md` for the historical record. #[cfg(test)] mod shared_corpus_tests { diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index b8b438d84b..9ebf539314 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1116,50 +1116,6 @@ fn is_responses_required_error(body: &str) -> bool { || b.contains("use the responses api") } -/// OpenAI-family code names used by the OLD segment-based route classifier. -/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. -/// Production routing now delegates to `resolve_model_capabilities` (see -/// `databricks_v2_route_for_model` below). -#[cfg(test)] -const _OLD_DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; - -/// Anthropic (Claude) family and release code names used by the OLD classifier. -/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. -#[cfg(test)] -const _OLD_DATABRICKS_V2_CLAUDE_NAMES: &[&str] = - &["claude", "opus", "sonnet", "haiku", "mythos", "fable"]; - -/// Split a Databricks v2 endpoint name into its lowercase alphanumeric segments, -/// breaking on any non-alphanumeric delimiter (`-`, `_`, `.`, `/`, …). E.g. -/// `Databricks-Claude-Opus-5` -> `["databricks", "claude", "opus", "5"]`. -/// Used by the old classifier (differential harness). Phase 3 removes this. -#[cfg(test)] -fn model_name_segments(model: &str) -> Vec { - model - .split(|c: char| !c.is_ascii_alphanumeric()) - .filter(|s| !s.is_empty()) - .map(str::to_ascii_lowercase) - .collect() -} - -/// OLD segment-based route classifier — preserved for the Phase-2 differential -/// harness. Production routing now delegates to `databricks_v2_route_for_model`. -/// Phase 3 removes this function. -#[cfg(test)] -fn _old_databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { - let segments = model_name_segments(model); - let has_named_segment = - |names: &[&str]| segments.iter().any(|seg| names.contains(&seg.as_str())); - let is_gpt_family = segments.iter().any(|seg| seg.starts_with("gpt")); - if is_gpt_family || has_named_segment(_OLD_DATABRICKS_V2_OPENAI_CODE_NAMES) { - DatabricksV2Route::OpenAiResponses - } else if has_named_segment(_OLD_DATABRICKS_V2_CLAUDE_NAMES) { - DatabricksV2Route::AnthropicMessages - } else { - DatabricksV2Route::MlflowChatCompletions - } -} - /// Returns the Databricks v2 wire route for a model name. /// /// Phase 2 cutover: delegates to `resolve_model_capabilities` from the generated @@ -3373,871 +3329,6 @@ mod tests { } } - /// Phase-2 comprehensive differential: old hand-coded logic vs generated capability module, - /// covering all three normative input sets (effortTable.fixture.json, normative-corpus.json, - /// catalog-sample-fixture.json) and all axes the old Rust code owned: - /// - supported_efforts / default_effort - /// - databricks_v2_wire_route (databricks_v2 entries only) - /// - thinking_mode (Anthropic and Anthropic-routed DatabricksV2 entries) - /// - /// Allowlist is axis-scoped: each entry covers (provider, raw_model_id, axis). - /// Any declared allowlist entry that never suppresses a divergence is a stale entry - /// and causes the test to FAIL (mirrors JS harness semantics). - #[test] - fn comprehensive_differential_old_vs_new_all_inputs() { - use crate::config::{ - is_adaptive_thinking_model_for_test, is_manual_budget_model_for_test, - strip_catalog_prefix as config_strip_catalog_prefix, - valid_effort_values_for_provider_model_for_test, - }; - use crate::generated_model_capabilities::{ - resolve_model_capabilities, DatabricksV2Route as GenRoute, - ThinkingMode as GenThinkingMode, - }; - use std::collections::HashSet; - - // ----------------------------------------------------------------------- - // Axis-scoped allowlist: (provider, raw_model_id, axis) - // Each entry documents an intentional divergence from the old hand tables. - // ----------------------------------------------------------------------- - #[derive(Debug)] - struct AllowlistEntry { - provider: &'static str, - raw_model_id: &'static str, - axis: &'static str, - reason: &'static str, - } - let allowlist: &[AllowlistEntry] = &[ - // Phase 1 ADOPT: models.dev payload d5a4974c advertises [low,medium,high]; - // old code returns [none,low,medium,high,xhigh]. - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-5", - axis: "supported_efforts", - reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", - }, - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-4-mini", - axis: "supported_efforts", - reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", - }, - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-4-nano", - axis: "supported_efforts", - reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", - }, - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-6-sol", - axis: "supported_efforts", - reason: "Phase 1 ADOPT: models.dev [low,medium,high,max]; old [none,low,medium,high,xhigh,max]", - }, - // Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route. - // Old config.rs effort table (pre-segment logic) classified goose-opus-5 as MLflow; - // old llm.rs segment classifier already routed it to AnthropicMessages. The - // manifest adopts the llm.rs (correct) view. Effort axis diverges because the old - // config.rs table assumed MLflow (openai-shaped), not Anthropic adaptive. - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "goose-opus-5", - axis: "supported_efforts", - reason: "Phase 1 F1: old config.rs rated it MLflow; manifest adopts anthropic adaptive", - }, - AllowlistEntry { - provider: "databricks_v2", - raw_model_id: "goose-opus-5", - axis: "default_effort", - reason: "Phase 1 F1: old config.rs had no default for this model; manifest adopts anthropic adaptive High", - }, - // Blank Anthropic model: manifest assumes adaptive (forward-compatible default); - // old is_adaptive_thinking_model("") and is_manual_budget_model("") both return false - // → OmitFields. The manifest's stance (adaptive fallback for blank provider) is - // intentional and matches the corpus expectation. - AllowlistEntry { - provider: "anthropic", - raw_model_id: "", - axis: "thinking_mode", - reason: "Manifest adopts adaptive fallback for blank Anthropic model; old code returns OmitFields", - }, - ]; - - // Track which allowlist entries are actually exercised. - let mut allowlist_hits: HashSet<(&str, &str, &str)> = HashSet::new(); - let mut divergences: Vec = Vec::new(); - - let is_allowlisted = |provider: &str, - model: &str, - axis: &str, - hits: &mut HashSet<(&str, &str, &str)>| { - for entry in allowlist { - if entry.provider == provider && entry.raw_model_id == model && entry.axis == axis { - hits.insert((entry.provider, entry.raw_model_id, entry.axis)); - return true; - } - } - false - }; - - // ----------------------------------------------------------------------- - // Derive "old" thinking_mode from hand-coded classifiers - // ----------------------------------------------------------------------- - let old_thinking_mode = |provider: &str, raw_model: &str, old_route: DatabricksV2Route| { - let is_anthropic_route = provider == "anthropic" - || (provider == "databricks_v2" - && old_route == DatabricksV2Route::AnthropicMessages); - if !is_anthropic_route { - return GenThinkingMode::None; - } - let model = config_strip_catalog_prefix(raw_model); - if is_manual_budget_model_for_test(model) { - GenThinkingMode::ManualBudget - } else if is_adaptive_thinking_model_for_test(model) { - GenThinkingMode::Adaptive - } else { - GenThinkingMode::OmitFields - } - }; - - // ----------------------------------------------------------------------- - // Per-entry check function - // ----------------------------------------------------------------------- - let mut check = |label: &str, - provider: &str, - raw_model: &str, - hits: &mut HashSet<(&str, &str, &str)>| { - // Canonicalize provider aliases so both sides of the differential - // operate on the same provider string (mirrors production and TS). - let provider = match provider { - "openai-compat" => "openai", - "databricks-v2" => "databricks_v2", - other => other, - }; - let new_cap = resolve_model_capabilities(provider, raw_model); - let (old_efforts, old_default) = - valid_effort_values_for_provider_model_for_test(provider, raw_model); - - // --- supported_efforts --- - let new_efforts: Vec<&'static str> = new_cap - .supported_efforts - .iter() - .map(|e| e.openai_effort_str()) - .collect(); - if new_efforts != old_efforts - && !is_allowlisted(provider, raw_model, "supported_efforts", hits) - { - divergences.push(format!( - "DIVERGE supported_efforts [{label}] provider={provider} model={raw_model:?}: old={old_efforts:?} new={new_efforts:?}" - )); - } - - // --- default_effort --- - let new_default: Option<&'static str> = - new_cap.default_effort.map(|e| e.openai_effort_str()); - if new_default != old_default - && !is_allowlisted(provider, raw_model, "default_effort", hits) - { - divergences.push(format!( - "DIVERGE default_effort [{label}] provider={provider} model={raw_model:?}: old={old_default:?} new={new_default:?}" - )); - } - - // --- databricks_v2_wire_route (databricks_v2 only) --- - if provider == "databricks_v2" { - let old_route = _old_databricks_v2_route_for_model(raw_model); - let new_route_gen = &new_cap.databricks_v2_wire_route; - let new_route = match new_route_gen { - GenRoute::OpenAiResponses => DatabricksV2Route::OpenAiResponses, - GenRoute::AnthropicMessages => DatabricksV2Route::AnthropicMessages, - GenRoute::MlflowChatCompletions - | GenRoute::RouteUnknown - | GenRoute::NotApplicable => DatabricksV2Route::MlflowChatCompletions, - }; - if new_route != old_route - && !is_allowlisted(provider, raw_model, "databricks_v2_wire_route", hits) - { - divergences.push(format!( - "DIVERGE databricks_v2_wire_route [{label}] model={raw_model:?}: old={old_route:?} new={new_route:?}" - )); - } - - // --- thinking_mode (databricks_v2 Anthropic-routed models) --- - let old_tm = old_thinking_mode(provider, raw_model, old_route); - if new_cap.thinking_mode != old_tm - && !is_allowlisted(provider, raw_model, "thinking_mode", hits) - { - divergences.push(format!( - "DIVERGE thinking_mode [{label}] provider={provider} model={raw_model:?}: old={old_tm:?} new={:?}", - new_cap.thinking_mode - )); - } - } else if provider == "anthropic" { - // thinking_mode for pure Anthropic - let old_tm = - old_thinking_mode(provider, raw_model, DatabricksV2Route::AnthropicMessages); - if new_cap.thinking_mode != old_tm - && !is_allowlisted(provider, raw_model, "thinking_mode", hits) - { - divergences.push(format!( - "DIVERGE thinking_mode [{label}] provider={provider} model={raw_model:?}: old={old_tm:?} new={:?}", - new_cap.thinking_mode - )); - } - } - }; - - // ----------------------------------------------------------------------- - // Input set 1: effortTable.fixture.json (36 entries) - // ----------------------------------------------------------------------- - #[derive(serde::Deserialize)] - struct FixtureEntry { - note: Option, - provider: String, - model: String, - } - let fixture_json = - include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); - let fixture: Vec = - serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); - for entry in &fixture { - let label = format!("fixture:{}", entry.note.as_deref().unwrap_or(&entry.model)); - check(&label, &entry.provider, &entry.model, &mut allowlist_hits); - } - - // ----------------------------------------------------------------------- - // Input set 2: normative-corpus.json (45 entries) - // ----------------------------------------------------------------------- - #[derive(serde::Deserialize)] - struct CorpusEntry { - // Group-header entries carry a `_group` string field; test-vector - // entries do not. We skip group headers (provider/raw_model_id absent). - #[serde(rename = "_group")] - group: Option, - id: Option, - provider: Option, - raw_model_id: Option, - } - let corpus_json = include_str!("../../../scripts/normative-corpus.json"); - let corpus: Vec = - serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); - for entry in &corpus { - if entry.group.is_some() { - // Group-header row — skip. - continue; - } - let (Some(provider), Some(model)) = (&entry.provider, &entry.raw_model_id) else { - continue; - }; - let label = format!("corpus:{}", entry.id.as_deref().unwrap_or(model.as_str())); - check(&label, provider, model, &mut allowlist_hits); - } - - // ----------------------------------------------------------------------- - // Input set 3: catalog-sample-fixture.json (databricks_v2 only) - // ----------------------------------------------------------------------- - #[derive(serde::Deserialize)] - struct CatalogEntry { - name: String, - } - #[derive(serde::Deserialize)] - struct CatalogFixture { - endpoints: Vec, - } - let catalog_json = include_str!("../../../scripts/catalog-sample-fixture.json"); - let catalog: CatalogFixture = - serde_json::from_str(catalog_json).expect("catalog fixture must be valid JSON"); - for entry in &catalog.endpoints { - let label = format!("catalog:{}", entry.name); - check(&label, "databricks_v2", &entry.name, &mut allowlist_hits); - } - - // ----------------------------------------------------------------------- - // Stale allowlist entries — any declared entry that never fired is a bug - // ----------------------------------------------------------------------- - let mut stale: Vec = Vec::new(); - for entry in allowlist { - if !allowlist_hits.contains(&(entry.provider, entry.raw_model_id, entry.axis)) { - stale.push(format!( - "STALE_ALLOWLIST provider={} model={} axis={} reason={}", - entry.provider, entry.raw_model_id, entry.axis, entry.reason - )); - } - } - - let mut failures = divergences.clone(); - failures.extend(stale); - - assert!( - failures.is_empty(), - "Comprehensive differential found {} failure(s):\n{}", - failures.len(), - failures.join("\n") - ); - - // Report summary (visible with --nocapture). - let total_entries = fixture.len() - + corpus - .iter() - .filter(|e| e.group.is_none() && e.provider.is_some()) - .count() - + catalog.endpoints.len(); - println!( - "Comprehensive differential: {} input entries, {} allowlist slots exercised/{}, 0 unexpected divergences", - total_entries, - allowlist_hits.len(), - allowlist.len(), - ); - } - - /// Phase-2 behavioral differential: drives the actual production normalization - /// functions against the old shims over all committed inputs. - /// - /// This test catches the class of defect found at `305627e32`: a record-level - /// differential passes (the generated record is correct) while the production - /// function diverges (it delegates to the old hand table instead of the record). - /// - /// For every input that hits a provider with an OpenAI-shaped normalization policy - /// (databricks_v2 with OpenAiStandard / OpenAiClampMaxToXHigh), this test drives - /// `normalize_effort_for_databricks_v2(effort, raw_model)` across all 7 requested - /// effort levels and compares against `normalize_effort_for_openai_route(effort, stripped)`. - /// - /// For Anthropic-routed inputs (databricks_v2 with NormalizationPolicy::None), this - /// test compares `anthropic_thinking_config_generated("databricks_v2", ...)` against - /// `_old_anthropic_thinking_config_for_databricks_v2(...)` for each non-None effort. - /// - /// Allowlist entries cover intentional behavioral divergences (F1 corrections); - /// stale entries fail the test. - #[test] - fn behavioral_differential_production_functions_match_old_shims() { - use crate::config::{ - _old_anthropic_thinking_config_for_databricks_v2, anthropic_thinking_config_generated, - normalize_effort_for_databricks_v2, normalize_effort_for_openai_route, - strip_catalog_prefix as config_strip_catalog_prefix, - }; - use crate::generated_model_capabilities::{ - resolve_model_capabilities, NormalizationPolicy, - }; - use std::collections::HashSet; - - const MAX_OUTPUT_TOKENS: u32 = 32_768; - - // All 7 effort levels in ordinal order. - const ALL_EFFORTS: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - - // Axis-scoped allowlist mirroring the record differential. - // "normalization_result" = effort normalization output diverges. - // "thinking_shape" = thinking request JSON shape diverges. - #[derive(Debug)] - struct BehavAllowlistEntry { - raw_model_id: &'static str, - axis: &'static str, - reason: &'static str, - } - // Only databricks_v2 entries are probed here; provider is implicitly databricks_v2. - let allowlist: &[BehavAllowlistEntry] = &[ - // F1 corrections: generated supported_efforts differs from old hand table. - // normalize_effort_for_databricks_v2 now resolves against generated supported_efforts - // → old shim's clamping of none→none, xhigh→xhigh is replaced by none→low, xhigh→high. - BehavAllowlistEntry { - raw_model_id: "databricks-gpt-5-5", - axis: "normalization_result", - reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", - }, - BehavAllowlistEntry { - raw_model_id: "databricks-gpt-5-4-mini", - axis: "normalization_result", - reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", - }, - BehavAllowlistEntry { - raw_model_id: "databricks-gpt-5-4-nano", - axis: "normalization_result", - reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", - }, - BehavAllowlistEntry { - raw_model_id: "databricks-gpt-5-6-sol", - axis: "normalization_result", - reason: "F1 ADOPT: generated [low,medium,high,max]; old table admits none+xhigh", - }, - ]; - - let mut allowlist_hits: HashSet<(&str, &str)> = HashSet::new(); - let mut divergences: Vec = Vec::new(); - - let is_allowlisted = |model: &str, axis: &str, hits: &mut HashSet<(&str, &str)>| { - for entry in allowlist { - if entry.raw_model_id == model && entry.axis == axis { - hits.insert((entry.raw_model_id, entry.axis)); - return true; - } - } - false - }; - - // --- Collect all databricks_v2 inputs from the three committed sets --- - #[derive(serde::Deserialize)] - struct FixtureEntry { - note: Option, - provider: String, - model: String, - } - #[derive(serde::Deserialize)] - struct CorpusEntry { - #[serde(rename = "_group")] - group: Option, - id: Option, - provider: Option, - raw_model_id: Option, - } - #[derive(serde::Deserialize)] - struct CatalogEntry { - name: String, - } - #[derive(serde::Deserialize)] - struct CatalogFixture { - endpoints: Vec, - } - - let mut inputs: Vec<(String, String)> = Vec::new(); // (label, raw_model_id) for databricks_v2 - - let fixture_json = - include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); - let fixture: Vec = - serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); - for e in &fixture { - if e.provider == "databricks_v2" { - let label = format!("fixture:{}", e.note.as_deref().unwrap_or(&e.model)); - inputs.push((label, e.model.clone())); - } - } - - let corpus_json = include_str!("../../../scripts/normative-corpus.json"); - let corpus: Vec = - serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); - for e in &corpus { - if e.group.is_some() { - continue; - } - if let (Some(prov), Some(model)) = (&e.provider, &e.raw_model_id) { - if prov == "databricks_v2" { - let label = format!("corpus:{}", e.id.as_deref().unwrap_or(model.as_str())); - inputs.push((label, model.clone())); - } - } - } - - let catalog_json = include_str!("../../../scripts/catalog-sample-fixture.json"); - let catalog: CatalogFixture = - serde_json::from_str(catalog_json).expect("catalog fixture must be valid JSON"); - for e in &catalog.endpoints { - let label = format!("catalog:{}", e.name); - inputs.push((label, e.name.clone())); - } - - // --- Behavioral probe for each input --- - for (label, raw_model) in &inputs { - let cap = resolve_model_capabilities("databricks_v2", raw_model); - - match cap.normalization_policy { - NormalizationPolicy::OpenAiStandard - | NormalizationPolicy::OpenAiClampMaxToXHigh => { - // Probe all 7 effort levels through the production normalization function - // vs the old shim. - let stripped = config_strip_catalog_prefix(raw_model); - let mut any_divergence = false; - for &effort in ALL_EFFORTS { - let new_result = normalize_effort_for_databricks_v2(effort, raw_model); - let old_result = normalize_effort_for_openai_route(effort, stripped); - if new_result != old_result { - any_divergence = true; - } - } - if any_divergence - && !is_allowlisted(raw_model, "normalization_result", &mut allowlist_hits) - { - // Collect per-effort details for the error message. - let details: Vec = ALL_EFFORTS - .iter() - .filter_map(|&effort| { - let new_result = - normalize_effort_for_databricks_v2(effort, raw_model); - let old_result = - normalize_effort_for_openai_route(effort, stripped); - if new_result != old_result { - Some(format!( - " {} → old={} new={}", - effort.openai_effort_str(), - old_result.openai_effort_str(), - new_result.openai_effort_str() - )) - } else { - None - } - }) - .collect(); - divergences.push(format!( - "BEHAVIORAL_DIVERGE normalization_result [{label}] model={raw_model:?}:\n{}", - details.join("\n") - )); - } - } - NormalizationPolicy::None => { - // Anthropic-routed: compare thinking config shape for each non-None effort. - let mut any_divergence = false; - for &effort in ALL_EFFORTS { - if effort == ThinkingEffort::None || effort == ThinkingEffort::Minimal { - continue; // omit-thinking cases: both produce (None, None), no shape to compare - } - let new_shape = anthropic_thinking_config_generated( - "databricks_v2", - raw_model, - effort, - MAX_OUTPUT_TOKENS, - ); - let old_shape = _old_anthropic_thinking_config_for_databricks_v2( - raw_model, - effort, - MAX_OUTPUT_TOKENS, - ); - if new_shape != old_shape { - any_divergence = true; - } - } - if any_divergence - && !is_allowlisted(raw_model, "thinking_shape", &mut allowlist_hits) - { - let details: Vec = ALL_EFFORTS - .iter() - .filter_map(|&effort| { - if effort == ThinkingEffort::None - || effort == ThinkingEffort::Minimal - { - return None; - } - let new_shape = anthropic_thinking_config_generated( - "databricks_v2", - raw_model, - effort, - MAX_OUTPUT_TOKENS, - ); - let old_shape = _old_anthropic_thinking_config_for_databricks_v2( - raw_model, - effort, - MAX_OUTPUT_TOKENS, - ); - if new_shape != old_shape { - Some(format!( - " effort={}: old={:?} new={:?}", - effort.openai_effort_str(), - old_shape, - new_shape - )) - } else { - None - } - }) - .collect(); - divergences.push(format!( - "BEHAVIORAL_DIVERGE thinking_shape [{label}] model={raw_model:?}:\n{}", - details.join("\n") - )); - } - } - } - } - - // Stale allowlist: any declared entry that never fired is a bug. - let mut stale: Vec = Vec::new(); - for entry in allowlist { - if !allowlist_hits.contains(&(entry.raw_model_id, entry.axis)) { - stale.push(format!( - "STALE_ALLOWLIST model={} axis={} reason={}", - entry.raw_model_id, entry.axis, entry.reason - )); - } - } - - let mut failures = divergences.clone(); - failures.extend(stale); - - assert!( - failures.is_empty(), - "Behavioral differential found {} failure(s):\n{}", - failures.len(), - failures.join("\n") - ); - - println!( - "Behavioral differential: {} databricks_v2 inputs probed, {} behavioral allowlist slots exercised/{}, 0 unexpected divergences", - inputs.len(), - allowlist_hits.len(), - allowlist.len(), - ); - } - - /// Behavioral differential for `normalize_effort_for_provider` — the production - /// authority for pure OpenAI and legacy Databricks effort normalization. - /// - /// This test catches a provider-generic repeat of the `305627e32` defect class: - /// a record-level differential passes (the generated record is correct) while the - /// production function diverges (delegates to the old hand table instead of the - /// record). The existing behavioral differential above covers `databricks_v2`; this - /// test covers `openai` and `databricks` routes, including the `openai-compat` - /// alias that the TS canonicalizer resolves to `openai` (Thufir P3 action 1). - /// - /// For every corpus entry with provider in {openai, databricks, openai-compat}, - /// this drives `normalize_effort_for_provider(canonical_provider, model, effort)` - /// and `normalize_effort_for_openai_route(effort, model)` across all 7 effort - /// levels and asserts they agree. No allowlist is expected — these functions are - /// definitionally aligned and any divergence is a bug. - #[test] - fn behavioral_differential_normalize_effort_for_provider() { - use crate::config::{normalize_effort_for_openai_route, normalize_effort_for_provider}; - - const ALL_EFFORTS: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - - #[derive(serde::Deserialize)] - struct CorpusEntry { - #[serde(rename = "_group")] - group: Option, - id: Option, - provider: Option, - raw_model_id: Option, - } - - let corpus_json = include_str!("../../../scripts/normative-corpus.json"); - let corpus: Vec = - serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); - - // Collect (label, canonical_provider, raw_model_id) for openai/databricks/openai-compat. - let mut inputs: Vec<(String, &'static str, String)> = Vec::new(); - for e in &corpus { - if e.group.is_some() { - continue; - } - let (prov, model) = match (&e.provider, &e.raw_model_id) { - (Some(p), Some(m)) => (p.as_str(), m.as_str()), - _ => continue, - }; - let canonical: &'static str = match prov { - "openai" | "openai-compat" => "openai", - "databricks" => "databricks", - _ => continue, // databricks_v2 and others are covered by the other differential - }; - let label = format!( - "corpus:{} (raw_provider={})", - e.id.as_deref().unwrap_or(model), - prov - ); - inputs.push((label, canonical, model.to_owned())); - } - - assert!( - !inputs.is_empty(), - "No openai/databricks/openai-compat inputs found in normative corpus" - ); - - let mut divergences: Vec = Vec::new(); - - for (label, canonical_provider, raw_model) in &inputs { - let mut per_effort: Vec = Vec::new(); - for &effort in ALL_EFFORTS { - let new_result = - normalize_effort_for_provider(canonical_provider, raw_model, effort); - let old_result = normalize_effort_for_openai_route(effort, raw_model); - if new_result != old_result { - per_effort.push(format!( - " {} → old={} new={}", - effort.openai_effort_str(), - old_result.openai_effort_str(), - new_result.openai_effort_str() - )); - } - } - if !per_effort.is_empty() { - divergences.push(format!( - "BEHAVIORAL_DIVERGE normalize_effort_for_provider [{label}] model={raw_model:?} provider={canonical_provider:?}:\n{}", - per_effort.join("\n") - )); - } - } - - assert!( - divergences.is_empty(), - "behavioral_differential_normalize_effort_for_provider found {} failure(s):\n{}", - divergences.len(), - divergences.join("\n") - ); - - println!( - "behavioral_differential_normalize_effort_for_provider: {} openai/databricks corpus inputs probed, 0 divergences", - inputs.len() - ); - } - - /// Behavioral shape differential for the pure Anthropic route. - /// - /// Drives `anthropic_thinking_config_generated("anthropic", raw_model, effort, …)` - /// against the prior pure-Anthropic authority `anthropic_thinking_config(raw_model, …)` - /// for every corpus entry with `provider == "anthropic"` across all seven effort levels. - /// - /// This completes the provider-general scope of the authorized corrective pass: the - /// existing differential covers databricks_v2; the normalize-effort differential covers - /// openai/databricks/openai-compat; this test covers the Anthropic thinking-config path. - /// - /// An F1 allowlist entry covers the blank-model corpus entries: the old hand-table returns - /// `(None, None)` for an unrecognized (empty) model, while the generated manifest explicitly - /// classifies blank Anthropic models as adaptive (the intentional Phase-2 behavior). - #[test] - fn behavioral_differential_anthropic_route() { - use crate::config::{anthropic_thinking_config, anthropic_thinking_config_generated}; - use std::collections::HashSet; - - const MAX_OUTPUT_TOKENS: u32 = 32_768; - - const ALL_EFFORTS: &[ThinkingEffort] = &[ - ThinkingEffort::None, - ThinkingEffort::Minimal, - ThinkingEffort::Low, - ThinkingEffort::Medium, - ThinkingEffort::High, - ThinkingEffort::XHigh, - ThinkingEffort::Max, - ]; - - // F1 allowlist: intentional divergences between the generated manifest and the old - // hand-table. Stale entries (that never fire) fail the test. - struct AllowlistEntry { - raw_model_id: &'static str, - reason: &'static str, - } - let allowlist: &[AllowlistEntry] = &[ - // F1 ADOPT: generated manifest classifies blank Anthropic model as adaptive - // (corpus entries anthropic-unknown-blank and anthropic-blank-adaptive-full); - // old anthropic_thinking_config returns (None, None) for unrecognized models. - AllowlistEntry { - raw_model_id: "", - reason: "F1 ADOPT: generated assumes adaptive for blank Anthropic model; old hand-table returned (None, None)", - }, - ]; - let mut allowlist_hits: HashSet<&str> = HashSet::new(); - - #[derive(serde::Deserialize)] - struct CorpusEntry { - #[serde(rename = "_group")] - group: Option, - id: Option, - provider: Option, - raw_model_id: Option, - } - - let corpus_json = include_str!("../../../scripts/normative-corpus.json"); - let corpus: Vec = - serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); - - // Collect (label, raw_model_id) for provider == "anthropic". - let mut inputs: Vec<(String, String)> = Vec::new(); - for e in &corpus { - if e.group.is_some() { - continue; - } - let (prov, model) = match (&e.provider, &e.raw_model_id) { - (Some(p), Some(m)) => (p.as_str(), m.as_str()), - _ => continue, - }; - if prov == "anthropic" { - let label = format!("corpus:{}", e.id.as_deref().unwrap_or(model)); - inputs.push((label, model.to_owned())); - } - } - - assert!( - !inputs.is_empty(), - "No anthropic inputs found in normative corpus" - ); - - let mut divergences: Vec = Vec::new(); - - for (label, raw_model) in &inputs { - let mut per_effort: Vec = Vec::new(); - for &effort in ALL_EFFORTS { - let new_shape = anthropic_thinking_config_generated( - "anthropic", - raw_model, - effort, - MAX_OUTPUT_TOKENS, - ); - let old_shape = anthropic_thinking_config(raw_model, effort, MAX_OUTPUT_TOKENS); - if new_shape != old_shape { - per_effort.push(format!( - " effort={}: old={:?} new={:?}", - effort.openai_effort_str(), - old_shape, - new_shape - )); - } - } - if !per_effort.is_empty() { - // Check allowlist before treating as a divergence. - let is_allowlisted = allowlist - .iter() - .any(|e| e.raw_model_id == raw_model.as_str()); - if is_allowlisted { - allowlist_hits.insert(raw_model.as_str()); - } else { - divergences.push(format!( - "BEHAVIORAL_DIVERGE anthropic_route [{label}] model={raw_model:?}:\n{}", - per_effort.join("\n") - )); - } - } - } - - // Stale allowlist: any declared entry that never fired is a bug. - let mut stale: Vec = Vec::new(); - for entry in allowlist { - if !allowlist_hits.contains(entry.raw_model_id) { - stale.push(format!( - "STALE_ALLOWLIST model={} reason={}", - entry.raw_model_id, entry.reason - )); - } - } - - let mut failures = divergences.clone(); - failures.extend(stale); - - assert!( - failures.is_empty(), - "behavioral_differential_anthropic_route found {} failure(s):\n{}", - failures.len(), - failures.join("\n") - ); - - println!( - "behavioral_differential_anthropic_route: {} anthropic corpus inputs probed, {} F1 allowlist slots exercised/{}, 0 unexpected divergences", - inputs.len(), - allowlist_hits.len(), - allowlist.len(), - ); - } - #[test] fn parse_responses_rejects_malformed_function_arguments() { let v = serde_json::json!({ @@ -4760,6 +3851,30 @@ mod tests { assert_eq!(body["output_config"]["effort"], "max"); } + #[test] + fn anthropic_body_opus_4_6_xhigh_clamps_to_high() { + // Production-seam clamp: Opus 4.6 supported_efforts=[low,medium,high,max] — xhigh is + // absent. The adaptive clamp (config.rs:284-296) must find the highest supported ≤ xhigh, + // which is high (max > xhigh in the ThinkingEffort ordering). A regression that removes + // the clamp or breaks the ordering would emit "xhigh" or omit the field entirely. + let mut c = cfg(Provider::Anthropic); + c.max_output_tokens = 32_768; + let body = anthropic_body( + &c, + "system", + &[HistoryItem::User("hi".into())], + &[], + "claude-opus-4-6", + Some(ThinkingEffort::XHigh), + "anthropic", + ); + assert_eq!(body["thinking"]["type"], "adaptive"); + assert_eq!( + body["output_config"]["effort"], "high", + "XHigh on Opus 4.6 must clamp to high (xhigh not in supported_efforts)" + ); + } + #[test] fn openai_body_emits_xhigh_effort() { // xhigh is a valid OpenAI effort value — must pass through. @@ -4816,173 +3931,6 @@ mod tests { assert_eq!(body["reasoning"]["effort"], "minimal"); } - // ---- DatabricksV2 route-aware effort normalization (body-level assertions) ---- - // - // These tests verify the body shape produced by the body builders when passed - // a pre-normalized effort value. The effort is pre-normalized here via the old - // helper (normalize_effort_for_openai_route) to produce the expected clamped value, - // mirroring what normalize_effort_for_databricks_v2 would return for these models - // (OpenAiStandard policy → delegates to normalize_effort_for_openai_route). - - #[test] - fn dbv2_openai_route_max_effort_clamped_to_xhigh_in_responses_body() { - // DBv2 GPT-5.5 route: max → clamped to xhigh (OpenAiStandard policy). - // Pre-normalize via normalize_effort_for_openai_route (same as what - // normalize_effort_for_databricks_v2 delegates to for OpenAiStandard). - let clamped = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.5"); - let body = responses_body( - &cfg_responses(), - "system", - &[HistoryItem::User("hi".into())], - &[], - "gpt-5.5", - Some(clamped), - ); - assert_eq!( - body["reasoning"]["effort"], "xhigh", - "DBv2 GPT-5.5 route: max must be clamped to xhigh before responses_body" - ); - } - - #[test] - fn dbv2_openai_route_max_effort_passes_through_for_gpt5_6() { - let normalized = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.6-sol"); - let body = responses_body( - &cfg_responses(), - "system", - &[HistoryItem::User("hi".into())], - &[], - "gpt-5.6-sol", - Some(normalized), - ); - assert_eq!( - body["reasoning"]["effort"], "max", - "DBv2 GPT-5.6 route must serialize max to the Responses API" - ); - } - - #[test] - fn dbv2_mlflow_route_max_effort_clamped_to_xhigh_in_openai_body() { - // DBv2 MLflow route (unknown model): max → clamped to xhigh by normalize_effort_for_openai_route. - // Unknown models pass through after the max→xhigh clamp. - let clamped = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "llama-4"); - let body = openai_body( - &cfg(Provider::OpenAi), - "system", - &[HistoryItem::User("hi".into())], - &[], - "llama-4", - Some(clamped), - ); - assert_eq!( - body["reasoning_effort"], "xhigh", - "DBv2 MLflow route: max must be clamped to xhigh before openai_body" - ); - } - - #[test] - fn dbv2_openai_route_none_minimal_pass_through_in_responses_body() { - // Verify that supported values pass through for the respective model families. - // gpt-5.5 supports none (but not minimal); gpt-5 base supports minimal (but not none). - let none_normalized = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::None, "gpt-5.5"); - assert_eq!( - none_normalized, - ThinkingEffort::None, - "OpenAI normalizer must not touch none for gpt-5.5" - ); - let minimal_normalized = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::Minimal, "gpt-5"); - assert_eq!( - minimal_normalized, - ThinkingEffort::Minimal, - "OpenAI normalizer must not touch minimal for gpt-5 base" - ); - let body = responses_body( - &cfg_responses(), - "system", - &[HistoryItem::User("hi".into())], - &[], - "gpt-5.5", - Some(none_normalized), - ); - assert_eq!( - body["reasoning"]["effort"], "none", - "DBv2 GPT-5.5 route: none must be emitted as-is" - ); - } - - #[test] - fn dbv2_claude_route_none_effort_omits_thinking_fields() { - // DBv2 Claude route: none → normalize_effort_for_anthropic_route returns None → omit. - let normalized = crate::config::normalize_effort_for_anthropic_route(ThinkingEffort::None); - assert_eq!( - normalized, None, - "Anthropic normalizer must return None for ThinkingEffort::None" - ); - let mut c = cfg(Provider::Anthropic); - c.max_output_tokens = 32_768; - let body = anthropic_body( - &c, - "system", - &[HistoryItem::User("hi".into())], - &[], - "claude-opus-4-8", - normalized, // None → omit thinking fields - "anthropic", - ); - assert!( - body.get("thinking").is_none(), - "DBv2 Claude route: none effort must omit thinking fields" - ); - assert!( - body.get("output_config").is_none(), - "DBv2 Claude route: none effort must omit output_config" - ); - } - - #[test] - fn dbv2_route_switch_max_body_level_simulation() { - // Body-level simulation of a session/set_model switch from a Claude model to a GPT-5 - // model when thinking_effort=max. Calls body builders and normalizers directly (not - // through the ACP session/set_model path or DatabricksV2 dispatch) to verify the - // correct output shape for each side of the route switch. - // Before the switch: Claude route → max passes through as Anthropic "max". - // After the switch: GPT-5 route → max clamped to xhigh. - let mut c = cfg(Provider::Anthropic); - c.max_output_tokens = 32_768; - - // Before switch: claude-opus-4-8 with effort=max → adaptive shape, effort="max" - let (thinking_before, oc_before) = crate::config::anthropic_thinking_config( - "claude-opus-4-8", - ThinkingEffort::Max, - 32_768, - ); - assert_eq!(thinking_before.unwrap()["type"], "adaptive"); - assert_eq!(oc_before.unwrap()["effort"], "max"); - - // After switch to GPT-5.5 route: normalize max → xhigh for responses_body - // (gpt-5.5 supports xhigh, so the clamp result is xhigh, not further reduced) - let clamped = - crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.5"); - assert_eq!(clamped, ThinkingEffort::XHigh); - let body_after = responses_body( - &cfg_responses(), - "system", - &[HistoryItem::User("hi".into())], - &[], - "gpt-5.5", - Some(clamped), - ); - assert_eq!( - body_after["reasoning"]["effort"], "xhigh", - "After set_model to GPT-5.5: max must be clamped to xhigh" - ); - } - /// Regression: a connection that is accepted and then dropped before any /// HTTP response bytes are written surfaces as a reqwest request-class /// error (not `is_connect()`, not `is_timeout()`). The retry predicate diff --git a/desktop/src/features/agents/ui/buzzAgentConfig.ts b/desktop/src/features/agents/ui/buzzAgentConfig.ts index d4a982b315..8638af47c0 100644 --- a/desktop/src/features/agents/ui/buzzAgentConfig.ts +++ b/desktop/src/features/agents/ui/buzzAgentConfig.ts @@ -1,10 +1,8 @@ /** * Source-of-truth constants for buzz-agent model-tuning configuration knobs. * - * Phase 2b pass 2: getProviderEffortConfig() is now generated-backed (thin - * wrapper over resolveModelCapabilities()). The legacy hand-table implementation - * is preserved as getProviderEffortConfig_oldHandTable() for the differential - * harness only — nothing user-facing imports the _old shim. Phase 3 retires it. + * `getProviderEffortConfig()` is generated-backed (thin wrapper over + * `resolveModelCapabilities()`). */ import { canonicalizeProvider } from "../lib/formatAgentModelLabel.ts"; import { resolveModelCapabilities } from "./modelCapabilities.ts"; @@ -50,10 +48,6 @@ export type ThinkingEffortValue = * thinking configuration entirely (i.e. "Inherit" is the natural default). * This applies to Anthropic manual-budget models where the effort level maps * to a budget_tokens count — there is no "default effort level" in the API. - * - * Mirrors the model-family tables in `crates/buzz-agent/src/config.rs` - * (`openai_efforts_for_model`, `is_manual_budget_model`, - * `is_adaptive_thinking_model`, `clamp_adaptive_effort`). Keep in sync. */ export type ProviderEffortConfig = { validValues: ReadonlyArray; @@ -61,8 +55,6 @@ export type ProviderEffortConfig = { defaultValue: ThinkingEffortValue | null; }; -const ALL_VALUES = BUZZ_AGENT_THINKING_EFFORT_VALUES; - /** * Returns the valid thinking-effort values and semantic default for the * given provider and optional model string, resolved from the generated @@ -91,237 +83,6 @@ export function getProviderEffortConfig( }; } -/** - * Legacy hand-table implementation — differential harness shim only. - * - * Preserved for the run-differential.mjs old-vs-new comparison until Phase 3 - * retires it. Nothing user-facing should import this name. - * - * @deprecated Use getProviderEffortConfig() (generated-backed) instead. - */ -export function getProviderEffortConfig_oldHandTable( - providerId: string, - model?: string, -): ProviderEffortConfig { - const provider = providerId.toLowerCase(); - // Strip arbitrary endpoint-naming prefix before model-family matching. - // Find the first occurrence of a known family token and drop everything before it. - // e.g. "goose-claude-fable-5" → "claude-fable-5" - // "team-x-gpt-5.5" → "gpt-5.5" - // "databricks-claude-3" → "claude-3" - // "claude-opus-4-7" → "claude-opus-4-7" (no prefix to strip) - const rawModel = (model ?? "").trim().toLowerCase(); - const FAMILY_TOKENS = ["claude-", "gpt-"] as const; - const firstFamilyIdx = Math.min( - ...FAMILY_TOKENS.map((tok) => { - const idx = rawModel.indexOf(tok); - return idx === -1 ? Infinity : idx; - }), - ); - const m = - firstFamilyIdx === Infinity ? rawModel : rawModel.slice(firstFamilyIdx); - - if (provider === "anthropic") { - return anthropicConfig(m); - } - if (provider === "openai") { - return openaiConfig(m); - } - if (provider === "databricks_v2") { - // Route by model family: claude* → Anthropic tables, gpt-5* → OpenAI tables. - // Non-Claude concrete models (e.g. llama-3) go through MlflowChatCompletions, - // which applies normalize_effort_for_openai_route → clamps max to xhigh. - // Route them through openaiConfig to exclude max. Only blank/unknown model - // uses the all-7 fallback (can't know the route without a concrete model). - if (m.startsWith("claude-")) { - return anthropicConfig(m); - } - if (gpt5FamilyModel(m)) { - return openaiConfig(m); - } - if (m.length > 0) { - // Concrete non-Claude, non-GPT model → MLflow path clamps max → xhigh. - return openaiConfig(m); - } - // Blank model — route unknown, show all 7. - return { validValues: ALL_VALUES, defaultValue: "medium" }; - } - if (provider === "databricks") { - // databricks v1 uses OpenAI Chat Completions wire format. - return openaiConfig(m); - } - if (provider === "openrouter") { - return { validValues: ALL_VALUES, defaultValue: "medium" }; - } - // openai-compat, unknown, empty — all values, default medium. - return { validValues: ALL_VALUES, defaultValue: "medium" }; -} - -// --------------------------------------------------------------------------- -// Anthropic family tables -// --------------------------------------------------------------------------- - -function anthropicConfig(m: string): ProviderEffortConfig { - // Manual-budget models: claude-3* and claude-opus-4-5. - // These use budget_tokens — there is no "default effort level" in the API. - if (m.startsWith("claude-3") || m === "claude-opus-4-5") { - return { - validValues: ["low", "medium", "high"], - defaultValue: null, - }; - } - // Adaptive models that support xhigh: opus-4-7+, sonnet-5.x, fable-5, mythos-5. - // mirrors clamp_adaptive_effort supports_xhigh check. - if ( - m.startsWith("claude-opus-4-7") || - m.startsWith("claude-opus-4-8") || - m.startsWith("claude-sonnet-5") || - m.startsWith("claude-fable-5") || - m.startsWith("claude-mythos-5") - ) { - return { - validValues: ["low", "medium", "high", "xhigh", "max"], - defaultValue: "high", - }; - } - // Adaptive models that do NOT support xhigh: opus-4-6, sonnet-4-6, mythos-preview. - if ( - m.startsWith("claude-opus-4-6") || - m.startsWith("claude-sonnet-4-6") || - m.startsWith("claude-mythos-preview") - ) { - return { - validValues: ["low", "medium", "high", "max"], - defaultValue: "high", - }; - } - // Unknown Anthropic model — assume adaptive with full support. - return { - validValues: ["low", "medium", "high", "xhigh", "max"], - defaultValue: "high", - }; -} - -// --------------------------------------------------------------------------- -// OpenAI family tables — mirrors openai_efforts_for_model in config.rs -// --------------------------------------------------------------------------- - -/** - * Returns true if `m` contains a GPT-5 family token at a word boundary - * (not immediately followed by a digit or letter). Mirrors - * `gpt5_token_matches` / `gpt5_base_matches` in config.rs. - */ -function gpt5TokenMatches(m: string, token: string): boolean { - let start = 0; - while (true) { - const idx = m.indexOf(token, start); - if (idx === -1) return false; - const afterIdx = idx + token.length; - const afterChar = afterIdx < m.length ? m[afterIdx] : ""; - // Boundary: end-of-string or a `-` separator (not a digit or letter). - if (afterChar === "" || afterChar === "-") return true; - start = afterIdx; - } -} - -/** Like gpt5TokenMatches but also rejects short -<1-3 digit> suffixes (e.g. -5, -10). */ -function gpt5BaseMatches(m: string, token: string): boolean { - let start = 0; - while (true) { - const idx = m.indexOf(token, start); - if (idx === -1) return false; - const afterIdx = idx + token.length; - const suffix = m.slice(afterIdx); - if (suffix === "") return true; - if (!suffix.startsWith("-")) { - start = afterIdx; - continue; - } - // Has a `-` suffix — check if it looks like a 1-3 digit version number. - const dashRest = suffix.slice(1); - if (/^\d{1,3}(?:[^a-z\d]|$)/i.test(dashRest)) { - start = afterIdx; - continue; - } - return true; - } -} - -/** Returns true if the model string belongs to any GPT-5 family. */ -function gpt5FamilyModel(m: string): boolean { - return ( - gpt5TokenMatches(m, "gpt-5-pro") || - gpt5TokenMatches(m, "gpt5-pro") || - gpt5TokenMatches(m, "gpt-5.6") || - gpt5TokenMatches(m, "gpt5.6") || - gpt5TokenMatches(m, "gpt-5-6") || - gpt5TokenMatches(m, "gpt5-6") || - gpt5TokenMatches(m, "gpt-5.5") || - gpt5TokenMatches(m, "gpt5.5") || - gpt5TokenMatches(m, "gpt-5.4") || - gpt5TokenMatches(m, "gpt5.4") || - gpt5TokenMatches(m, "gpt-5.1") || - gpt5TokenMatches(m, "gpt5.1") || - gpt5BaseMatches(m, "gpt-5") || - gpt5BaseMatches(m, "gpt5") - ); -} - -function openaiConfig(m: string): ProviderEffortConfig { - // Check -pro before versioned suffixes (gpt-5-pro contains "gpt-5"). - if (gpt5TokenMatches(m, "gpt-5-pro") || gpt5TokenMatches(m, "gpt5-pro")) { - return { validValues: ["high"], defaultValue: "high" }; - } - if ( - gpt5TokenMatches(m, "gpt-5.6") || - gpt5TokenMatches(m, "gpt5.6") || - gpt5TokenMatches(m, "gpt-5-6") || - gpt5TokenMatches(m, "gpt5-6") - ) { - return { - validValues: ["none", "low", "medium", "high", "xhigh", "max"], - defaultValue: "medium", - }; - } - if ( - gpt5TokenMatches(m, "gpt-5.5") || - gpt5TokenMatches(m, "gpt5.5") || - gpt5TokenMatches(m, "gpt-5-5") || - gpt5TokenMatches(m, "gpt5-5") || - gpt5TokenMatches(m, "gpt-5.4") || - gpt5TokenMatches(m, "gpt5.4") || - gpt5TokenMatches(m, "gpt-5-4") || - gpt5TokenMatches(m, "gpt5-4") - ) { - return { - validValues: ["none", "low", "medium", "high", "xhigh"], - defaultValue: "medium", - }; - } - if ( - gpt5TokenMatches(m, "gpt-5.1") || - gpt5TokenMatches(m, "gpt5.1") || - gpt5TokenMatches(m, "gpt-5-1") || - gpt5TokenMatches(m, "gpt5-1") - ) { - return { - validValues: ["none", "low", "medium", "high"], - defaultValue: "none", - }; - } - if (gpt5BaseMatches(m, "gpt-5") || gpt5BaseMatches(m, "gpt5")) { - return { - validValues: ["minimal", "low", "medium", "high"], - defaultValue: "medium", - }; - } - // Unknown OpenAI model — conservative fallback; max is enabled only for families whose table includes it. - return { - validValues: ["none", "minimal", "low", "medium", "high", "xhigh"], - defaultValue: "medium", - }; -} - /** * Returns true when the given runtime id is buzz-agent, which is the only * runtime that supports the tier-1 model-tuning knobs above. @@ -329,7 +90,3 @@ function openaiConfig(m: string): ProviderEffortConfig { export function isBuzzAgentRuntime(runtimeId: string): boolean { return runtimeId === "buzz-agent"; } - -// --------------------------------------------------------------------------- -// Differential harness support -// --------------------------------------------------------------------------- diff --git a/desktop/src/features/agents/ui/effortTable.fixture.json b/desktop/src/features/agents/ui/effortTable.fixture.json deleted file mode 100644 index 3225d6038a..0000000000 --- a/desktop/src/features/agents/ui/effortTable.fixture.json +++ /dev/null @@ -1,261 +0,0 @@ -[ - { - "note": "Anthropic manual-budget: claude-3 family", - "provider": "anthropic", - "model": "claude-3-7-sonnet-20250219", - "validValues": ["low", "medium", "high"], - "defaultValue": null - }, - { - "note": "Anthropic manual-budget: claude-opus-4-5", - "provider": "anthropic", - "model": "claude-opus-4-5", - "validValues": ["low", "medium", "high"], - "defaultValue": null - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-opus-4-7", - "provider": "anthropic", - "model": "claude-opus-4-7", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-opus-4-8", - "provider": "anthropic", - "model": "claude-opus-4-8", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-sonnet-5", - "provider": "anthropic", - "model": "claude-sonnet-5-20260101", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-fable-5", - "provider": "anthropic", - "model": "claude-fable-5", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-opus-5", - "provider": "anthropic", - "model": "claude-opus-5", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive xhigh-capable: claude-mythos-5", - "provider": "anthropic", - "model": "claude-mythos-5", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive no-xhigh: claude-opus-4-6", - "provider": "anthropic", - "model": "claude-opus-4-6", - "validValues": ["low", "medium", "high", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive no-xhigh: claude-sonnet-4-6", - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "validValues": ["low", "medium", "high", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic adaptive no-xhigh: claude-mythos-preview", - "provider": "anthropic", - "model": "claude-mythos-preview", - "validValues": ["low", "medium", "high", "max"], - "defaultValue": "high" - }, - { - "note": "Anthropic unknown model: blank \u2014 assume full adaptive", - "provider": "anthropic", - "model": "", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "OpenAI gpt-5-pro: high only", - "provider": "openai", - "model": "gpt-5-pro", - "validValues": ["high"], - "defaultValue": "high" - }, - { - "note": "OpenAI gpt-5.6: none/low/medium/high/xhigh/max", - "provider": "openai", - "model": "gpt-5.6", - "validValues": ["none", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "OpenAI gpt-5.5: none/low/medium/high/xhigh", - "provider": "openai", - "model": "gpt-5.5", - "validValues": ["none", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "OpenAI gpt-5.4: same table as gpt-5.5", - "provider": "openai", - "model": "gpt-5.4", - "validValues": ["none", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "OpenAI gpt-5.1: none/low/medium/high", - "provider": "openai", - "model": "gpt-5.1", - "validValues": ["none", "low", "medium", "high"], - "defaultValue": "none" - }, - { - "note": "OpenAI gpt-5 base: minimal/low/medium/high", - "provider": "openai", - "model": "gpt-5", - "validValues": ["minimal", "low", "medium", "high"], - "defaultValue": "medium" - }, - { - "note": "OpenAI unknown model (gpt-4o): all-except-max", - "provider": "openai", - "model": "gpt-4o", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "OpenAI empty model: all-except-max", - "provider": "openai", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "DatabricksV2 claude route (claude-opus-4-7): xhigh-capable anthropic table", - "provider": "databricks_v2", - "model": "claude-opus-4-7", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "DatabricksV2 claude route with databricks- prefix stripped", - "provider": "databricks_v2", - "model": "databricks-claude-opus-4-7", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "DatabricksV2 gpt-5.6-sol route: OpenAI max-capable table", - "provider": "databricks_v2", - "model": "gpt-5.6-sol", - "validValues": ["none", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "DatabricksV2 gpt-5-6-sol route: dashed OpenAI max-capable table", - "provider": "databricks_v2", - "model": "gpt-5-6-sol", - "validValues": ["none", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "DatabricksV2 gpt-5.4 route: OpenAI gpt-5.5/5.4 table", - "provider": "databricks_v2", - "model": "gpt-5.4", - "validValues": ["none", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "DatabricksV2 gpt-5.1 with databricks- prefix: OpenAI gpt-5.1 table", - "provider": "databricks_v2", - "model": "databricks-gpt-5.1", - "validValues": ["none", "low", "medium", "high"], - "defaultValue": "none" - }, - { - "note": "DatabricksV2 concrete non-claude non-gpt5 (llama-3): MLflow path, all-except-max", - "provider": "databricks_v2", - "model": "llama-3", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "DatabricksV2 blank model: route unknown, all-7", - "provider": "databricks_v2", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "databricks v1: routes like openai unknown, all-except-max", - "provider": "databricks", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "openai-compat: canonicalizes to openai, empty model → all-except-max with medium default", - "provider": "openai-compat", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "openai-compat/gpt-5-pro: canonicalizes to openai, gpt-5-pro → [high] only", - "provider": "openai-compat", - "model": "gpt-5-pro", - "validValues": ["high"], - "defaultValue": "high" - }, - { - "note": "openrouter: all-7 with medium default", - "provider": "openrouter", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "empty provider: all-7 with medium default", - "provider": "", - "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], - "defaultValue": "medium" - }, - { - "note": "databricks_v2 goose-claude-fable-5: strips goose- prefix, routes anthropic adaptive+xhigh, max valid", - "provider": "databricks_v2", - "model": "goose-claude-fable-5", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "databricks_v2 goose-gpt-5.5: strips goose- prefix, routes openai gpt-5.5 table (none+low-xhigh, no minimal)", - "provider": "databricks_v2", - "model": "goose-gpt-5.5", - "validValues": ["none", "low", "medium", "high", "xhigh"], - "defaultValue": "medium" - }, - { - "note": "databricks_v2 goose-claude-sonnet-5: strips goose- prefix, routes anthropic adaptive+xhigh", - "provider": "databricks_v2", - "model": "goose-claude-sonnet-5", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - }, - { - "note": "databricks_v2 arbitrary prefix team-x-claude-opus-4-7: strips to claude-opus-4-7, routes anthropic adaptive+xhigh, max valid", - "provider": "databricks_v2", - "model": "team-x-claude-opus-4-7", - "validValues": ["low", "medium", "high", "xhigh", "max"], - "defaultValue": "high" - } -] diff --git a/desktop/src/features/agents/ui/effortTable.fixture.test.mjs b/desktop/src/features/agents/ui/effortTable.fixture.test.mjs deleted file mode 100644 index c63b94915e..0000000000 --- a/desktop/src/features/agents/ui/effortTable.fixture.test.mjs +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Effort-table sync guard: TS side. - * - * Loads the checked-in fixture and asserts that `getProviderEffortConfig` - * matches every entry. Drift between `buzzAgentConfig.ts` and the fixture - * (e.g. a new model family added to one side but not the other) fails CI. - * The companion Rust test in `crates/buzz-agent/src/config.rs` mirrors - * this check so both sides of the mirror must stay in sync. - */ - -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import test from "node:test"; -import { fileURLToPath } from "node:url"; -import path from "node:path"; - -import { getProviderEffortConfig } from "./buzzAgentConfig.ts"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const fixture = JSON.parse( - readFileSync(path.join(__dirname, "effortTable.fixture.json"), "utf8"), -); - -for (const entry of fixture) { - const { - note, - provider, - model, - validValues: expectedValidValues, - defaultValue: expectedDefault, - } = entry; - const label = note ?? `${provider}/${model}`; - - test(`effort fixture: ${label}`, () => { - const { validValues, defaultValue } = getProviderEffortConfig( - provider, - model, - ); - - assert.deepEqual( - [...validValues], - expectedValidValues, - `validValues mismatch for "${label}"`, - ); - - assert.equal( - defaultValue, - expectedDefault, - `defaultValue mismatch for "${label}"`, - ); - }); -} diff --git a/scripts/MODELS_DEV_RECONCILIATION.md b/scripts/MODELS_DEV_RECONCILIATION.md deleted file mode 100644 index 59722e97e5..0000000000 --- a/scripts/MODELS_DEV_RECONCILIATION.md +++ /dev/null @@ -1,115 +0,0 @@ -# models.dev Reasoning Options Reconciliation Table - -**Source queried**: https://models.dev/api.json (2026-07-31)
-**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0`
-**Policy (plan v4 §Behavior policy)**: models.dev `reasoning_options` become exact overrides. -Each divergence from the current family rule result is reconciled here: either (a) adopted as an -intentional correction or (b) rejected with a curation note. - -**Verbatim source snapshot**: `scripts/catalog-sample-fixture.json` — verbatim `id`, `name`, and -nested `reasoning_options` objects captured from the live API without transformation. -Re-verify hash: `curl -s https://models.dev/api.json | sha256sum` - -## Divergences - -### `databricks-gpt-5-4-mini` - -| | Current family rule (gpt5-4) | models.dev | Disposition | -|---|---|---|---| -| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | - -**Rationale**: The Databricks AI Gateway v2 endpoint for `databricks-gpt-5-4-mini` explicitly -advertises only `[low, medium, high]` in its `reasoning_options`. The family rule's `none` and -`xhigh` are derived from the upstream OpenAI GPT-5.4 spec, which this Databricks endpoint does -not expose. Provider-advertised wins per plan F1 policy. - -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
-**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"`
-**Test vector**: `resolver-exact-raw-id-hit` in `scripts/normative-corpus.json` - ---- - -### `databricks-gpt-5-4-nano` - -| | Current family rule (gpt5-4) | models.dev | Disposition | -|---|---|---|---| -| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | - -**Rationale**: Same as `databricks-gpt-5-4-mini`. The nano variant exposes the same restricted -effort set. Provider-advertised wins. - -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
-**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-nano"` - ---- - -### `databricks-gpt-5-6-sol` - -| | Current family rule (gpt5-6) | models.dev | Disposition | -|---|---|---|---| -| `supported_efforts` | `[none, low, medium, high, xhigh, max]` | `[low, medium, high, max]` | **ADOPT** | - -**Rationale**: The Databricks AI Gateway v2 endpoint for `databricks-gpt-5-6-sol` advertises only -`[low, medium, high, max]` in its `reasoning_options`. The family rule's `none` and `xhigh` are -derived from the upstream OpenAI GPT-5.6 spec, which this Databricks endpoint does not expose. -Provider-advertised wins per plan F1 policy. - -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]`
-**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-6-sol"` - ---- - -### `databricks-gpt-5-5` - -| | Current family rule (gpt5-5) | models.dev | Disposition | -|---|---|---|---| -| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | - -**Rationale**: The Databricks AI Gateway v2 endpoint for `databricks-gpt-5-5` advertises only -`[low, medium, high]` in its `reasoning_options`. The family rule's `none` and `xhigh` are -derived from the upstream OpenAI GPT-5.5 spec, which this Databricks endpoint does not expose. -Provider-advertised wins per plan F1 policy. - -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
-**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-5"` - ---- - -### `databricks-claude-opus-4-7` - -| | Current family rule (anthropic-adaptive-xhigh-opus-4-7) | models.dev | Disposition | -|---|---|---|---| -| `reasoning_options` type | effort-based | `budget_tokens` | **NO EFFORT DIVERGENCE** | - -**Rationale**: models.dev advertises `reasoning_options=[{"type":"budget_tokens","min":1024}]` — -a different capability axis (extended thinking token budget), not an effort-level selector. -There is no effort divergence to reconcile. The effort capabilities for this model come from the -`anthropic-adaptive-xhigh-opus-4-7` family rule (Anthropic extended-thinking support table). - -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]`
-**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-claude-opus-4-7"` - ---- - -## Non-divergences (confirmed consistent) - -The following models were checked against models.dev or provider docs and found consistent with -the manifest family rules. No exact records needed. - -| Model family | Source | Checked against | Status | -|---|---|---|---| -| `claude-opus-4-7` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-opus-4-8` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-sonnet-5.*` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-fable-5` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-mythos-5` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-opus-4-6` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-sonnet-4-6` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-mythos-preview` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `claude-3*` | [https://platform.claude.com/docs/en/build-with-claude/extended-thinking](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) | Anthropic extended-thinking support table (July 2025) | ✓ Consistent | -| `gpt-5-pro` | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | -| `gpt-5.6` | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | -| `gpt-5.5` | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | -| `gpt-5.4` | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | -| `gpt-5.1` | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | -| `gpt-5` (base) | [https://platform.openai.com/docs/guides/reasoning](https://platform.openai.com/docs/guides/reasoning) | OpenAI reasoning guide (July 2025) | ✓ Consistent | diff --git a/scripts/MODEL_CAPABILITIES_SCHEMA.md b/scripts/MODEL_CAPABILITIES.md similarity index 52% rename from scripts/MODEL_CAPABILITIES_SCHEMA.md rename to scripts/MODEL_CAPABILITIES.md index 98f12a6813..944637cd69 100644 --- a/scripts/MODEL_CAPABILITIES_SCHEMA.md +++ b/scripts/MODEL_CAPABILITIES.md @@ -1,11 +1,19 @@ -# Model Capabilities Manifest — Schema Reference +# Model Capabilities Manifest -**Source of truth**: `scripts/model-capabilities.json` -**Generator**: `scripts/generate-model-capabilities.mjs` +**Source of truth**: `scripts/model-capabilities.json` +**Generator**: `scripts/generate-model-capabilities.mjs` **Emitted artifacts**: - `crates/buzz-agent/src/generated_model_capabilities.rs` - `desktop/src/features/agents/ui/modelCapabilities.ts` -- `scripts/generated-model-capabilities-coverage.json` (test fixture) + +## How to regenerate + +```sh +node scripts/generate-model-capabilities.mjs +``` + +CI regenerates and diffs on every PR that touches the manifest, generator, or generated +files. Any stale generated file fails the `model-capabilities` job in `ci.yml`. ## Resolver contract (plan v4) @@ -38,10 +46,10 @@ from multiple tiers. | Value | Meaning | |-------|---------| -| `manual-budget` | `thinking:{type:"enabled", budget_tokens}` — claude-3*, claude-opus-4-5 | -| `adaptive` | `thinking:{type:"adaptive"}` + `output_config:{effort}` — opus-4-6+, sonnet-4-6+, etc. | -| `omit-fields` | Unknown Anthropic model — omit thinking fields rather than guess request shape | -| `none` | Non-Anthropic-routed model — thinking fields not applicable | +| `manual-budget` | `thinking:{type:"enabled", budget_tokens}` -- claude-3*, claude-opus-4-5 | +| `adaptive` | `thinking:{type:"adaptive"}` + `output_config:{effort}` -- opus-4-6+, sonnet-4-6+, etc. | +| `omit-fields` | Unknown Anthropic model -- omit thinking fields rather than guess request shape | +| `none` | Non-Anthropic-routed model -- thinking fields not applicable | | `not-applicable` | Provider does not use Anthropic thinking API | ### `databricks_v2_wire_route` values @@ -55,7 +63,7 @@ Transport for pure OpenAI, legacy Databricks, and OpenRouter is selected by `Ope | `openai-responses` | `/ai-gateway/openai/v1/responses` | | `anthropic-messages` | `/ai-gateway/anthropic/v1/messages` | | `mlflow-chat` | `/ai-gateway/mlflow/v1/chat/completions` | -| `route-unknown` | DBv2 blank model — route not yet determinable | +| `route-unknown` | DBv2 blank model -- route not yet determinable | | `not-applicable` | Not a DBv2 provider | ## Family rule match kinds @@ -84,20 +92,87 @@ divergence from family rule results is reconciled against provider docs and eith - (a) **adopted** as an intentional correction with its own test + exact record, or - (b) **rejected** with a curation note in the exact record. -See reconciliation table: `scripts/MODELS_DEV_RECONCILIATION.md`. +### models.dev reconciliation table + +**Source queried**: https://models.dev/api.json (2026-07-31) +**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0` +**Verbatim source snapshot SHA-256** (catalog-sample-fixture.json, deleted in Phase 3): +`dc4092a04392f258bea65de2cef53cb1902dce1779dc2b1b2e21fb56774f2d78` + +#### `databricks-gpt-5-4-mini` + +| | Family rule (gpt5-4) | models.dev | Disposition | +|---|---|---|---| +| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | + +Provider-advertised wins per plan F1 policy. Source: providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options +(retrieved 2026-07-31). + +#### `databricks-gpt-5-4-nano` + +| | Family rule (gpt5-4) | models.dev | Disposition | +|---|---|---|---| +| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | + +Same as `databricks-gpt-5-4-mini`. + +#### `databricks-gpt-5-6-sol` + +| | Family rule (gpt5-6) | models.dev | Disposition | +|---|---|---|---| +| `supported_efforts` | `[none, low, medium, high, xhigh, max]` | `[low, medium, high, max]` | **ADOPT** | + +Provider-advertised wins. Source: providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options +(retrieved 2026-07-31). + +#### `databricks-gpt-5-5` + +| | Family rule (gpt5-5) | models.dev | Disposition | +|---|---|---|---| +| `supported_efforts` | `[none, low, medium, high, xhigh]` | `[low, medium, high]` | **ADOPT** | + +Provider-advertised wins. Source: providers.databricks.models["databricks-gpt-5-5"].reasoning_options +(retrieved 2026-07-31). + +#### `databricks-claude-opus-4-7` + +| | Family rule | models.dev | Disposition | +|---|---|---|---| +| `reasoning_options` type | effort-based | `budget_tokens` | **NO EFFORT DIVERGENCE** | + +models.dev advertises a different capability axis (extended thinking token budget), not an +effort-level selector. No effort divergence to reconcile. Source: providers.databricks.models +["databricks-claude-opus-4-7"].reasoning_options (retrieved 2026-07-31). + +### Non-divergences (confirmed consistent) + +| Model family | Source | Status | +|---|---|---| +| `claude-opus-4-7`, `claude-opus-4-8` | Anthropic extended-thinking docs (July 2025) | ok | +| `claude-sonnet-5.*`, `claude-fable-5`, `claude-mythos-5` | Anthropic extended-thinking docs (July 2025) | ok | +| `claude-opus-4-6`, `claude-sonnet-4-6`, `claude-mythos-preview` | Anthropic extended-thinking docs (July 2025) | ok | +| `claude-3*` | Anthropic extended-thinking docs (July 2025) | ok | +| `gpt-5-pro`, `gpt-5.6`, `gpt-5.5`, `gpt-5.4`, `gpt-5.1`, `gpt-5` | OpenAI reasoning guide (July 2025) | ok | + +## Mutation evidence (historical record) + +Mutation testing was run at Phase 2 completion (2026-07-31). 7 generator mutations were applied +in isolation against both TS and Rust interpreters. All 7 were killed by both interpreters (7/7). +The mutation runner (`scripts/run-mutation-evidence.mjs`) was deleted in Phase 3; the normative +corpus (`scripts/normative-corpus.json`) that kills these mutations continues to run in CI. ## Adding a new model family 1. Add a `family_rules` entry with a new unique `id`, appropriate `match_kind`, `providers`, `match_priority`, and all capability axes. 2. Run `node scripts/generate-model-capabilities.mjs` to regenerate artifacts. -3. CI `model-capability-regen-diff` job verifies byte-clean regeneration. +3. CI verifies byte-clean regeneration. 4. The normative corpus (`scripts/normative-corpus.json`) may need new vectors. ## Adding an exact model override 1. Add an `exact_records` entry with `provider` + `raw_model_id` (the full raw ID, no prefix stripping). Include a `_reconciliation` note and doc citation. -2. Run `node scripts/generate-model-capabilities.mjs` — completeness validator will fail if any +2. Run `node scripts/generate-model-capabilities.mjs` -- completeness validator will fail if any axis cannot be resolved. 3. Regenerate and commit. diff --git a/scripts/MUTATION_EVIDENCE.md b/scripts/MUTATION_EVIDENCE.md deleted file mode 100644 index 555a1878e5..0000000000 --- a/scripts/MUTATION_EVIDENCE.md +++ /dev/null @@ -1,45 +0,0 @@ -# Model-Capability Manifest — Mutation Evidence - -**Interpreter coverage**: both generated interpreters are exercised per mutation fault. -- **TypeScript**: `scripts/run-corpus.mjs` imports `resolveModelCapabilities()` from - `desktop/src/features/agents/ui/modelCapabilities.ts` via `--experimental-strip-types`. -- **Rust**: `cargo test -p buzz-agent -- generated_model_capabilities::tests::shared_corpus_tests` - deserializes and executes every vector in `scripts/normative-corpus.json` against - `resolve_model_capabilities()`. - -## How to reproduce - -```sh -# Runs generator mutations; exercises both TS and Rust interpreters per fault -node --experimental-strip-types scripts/run-mutation-evidence.mjs - -# Run interpreters independently: -node --experimental-strip-types scripts/run-corpus.mjs -cargo test -p buzz-agent -- generated_model_capabilities::tests::shared_corpus_tests -``` - -## Mutation run results (both interpreters) - -All 7 mutations applied in isolation; manifest restored after each run. -Each mutation must be detected (killed) by **both** interpreters for it to count as covered. - -| ID | Mutation | Expected killer(s) | TS | Rust | -|----|----------|--------------------|----|------| -| M1 | Reduce `claude-opus-4-7` `supported_efforts` to `[low,medium,high]` (drops xhigh+max) | `anthropic-claude-opus-4-7`, `dbv2-claude-prefix-stripped`, `dbv2-claude-route-anthropic-messages` | **killed ✓** | **killed ✓** | -| M2 | Add `xhigh` to `gpt5-base` `supported_efforts` | `openai-gpt5-base`, `openai-gpt5-1106-should-not-match-base`, `openai-gpt5-4o-matches-base`, `openai-gpt5-date-suffix` | **killed ✓** | **killed ✓** | -| M3 | Change `gpt5-1` `default_effort` to `"high"` instead of `"none"` | `openai-gpt5.1` | **killed ✓** | **killed ✓** | -| M4 | Swap `dbv2-claude-code-names-segment` route from `anthropic-messages` to `openai-responses` | `dbv2-goose-opus-5-is-anthropic` | **killed ✓** | **killed ✓** | -| M5 | Remove all three DBv2 segment rules | `dbv2-goose-opus-5-is-anthropic`, `dbv2-consolidated-llama-not-sol`, `dbv2-terraform-coder-not-terra` | **killed ✓** | **killed ✓** | -| M6 | Change `databricks_v2` concrete-unknown fallback route from `mlflow-chat` to `openai-responses` | `dbv2-concrete-unknown-mlflow-no-max` | **killed ✓** | **killed ✓** | -| M7 | Remove `xhigh` from `gpt5-4` `supported_efforts` | `resolver-prefixed-alias-misses-exact` | **killed ✓** | **killed ✓** | - -**Summary: 7/7 mutations killed in both TS and Rust interpreters.** - -## Coverage gaps - -- Provider fallback mutations for `anthropic`, `openai`, `databricks`, `openrouter`, and - `_default` are not individually mutated. These are covered by explicit fallback vectors - in the corpus for `anthropic`, `openai`, and `databricks_v2`. -- Rust mutations are run by recompiling the mutated generated file per fault (via `cargo - test` after `node generate-model-capabilities.mjs`). Compile time is acceptable for - offline mutation runs; CI only runs the already-compiled shared corpus harness. diff --git a/scripts/catalog-sample-fixture.json b/scripts/catalog-sample-fixture.json deleted file mode 100644 index 06b0f89390..0000000000 --- a/scripts/catalog-sample-fixture.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "_comment": "Verbatim models.dev snapshot for differential harness (plan v4 §Oracle). Contains exact records captured from the live API for exact-override entries. Verbatim: name and reasoning_options are reproduced without transformation.", - "_source_url": "https://models.dev/api.json", - "_retrieval_date": "2026-07-31", - "_payload_sha256": "d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0", - "_retrieval_note": "Full payload SHA-256 computed over the raw response body of GET https://models.dev/api.json (no transforms). Re-verify: curl -s https://models.dev/api.json | sha256sum", - "_models_dev_records": { - "databricks-gpt-5-5": { - "id": "databricks-gpt-5-5", - "name": "GPT-5.5", - "reasoning_options": [ - { - "type": "effort", - "values": [ - "low", - "medium", - "high" - ] - } - ] - }, - "databricks-gpt-5-4-mini": { - "id": "databricks-gpt-5-4-mini", - "name": "GPT-5.4 mini", - "reasoning_options": [ - { - "type": "effort", - "values": [ - "low", - "medium", - "high" - ] - } - ] - }, - "databricks-gpt-5-4-nano": { - "id": "databricks-gpt-5-4-nano", - "name": "GPT-5.4 nano", - "reasoning_options": [ - { - "type": "effort", - "values": [ - "low", - "medium", - "high" - ] - } - ] - }, - "databricks-gpt-5-6-sol": { - "id": "databricks-gpt-5-6-sol", - "name": "GPT-5.6 Sol", - "reasoning_options": [ - { - "type": "effort", - "values": [ - "low", - "medium", - "high", - "max" - ] - } - ] - }, - "databricks-claude-opus-4-7": { - "id": "databricks-claude-opus-4-7", - "name": "Claude Opus 4.7", - "reasoning_options": [ - { - "type": "budget_tokens", - "min": 1024 - } - ] - } - }, - "endpoints": [ - { - "name": "databricks-gpt-5-5", - "note": "DATABRICKS_V2_KNOWN_MODELS entry; gpt5-5 family; openai-responses route" - }, - { - "name": "databricks-gpt-5-4-mini", - "note": "exact record; models.dev override: low|medium|high (not family rule none+xhigh)" - }, - { - "name": "databricks-gpt-5-4-nano", - "note": "exact record; models.dev override: low|medium|high" - }, - { - "name": "databricks-gpt-5-6-sol", - "note": "exact record; models.dev source: low|medium|high|max (adopted as-is)" - }, - { - "name": "databricks-claude-opus-4-7", - "note": "DATABRICKS_V2_KNOWN_MODELS entry; anthropic adaptive xhigh-capable; anthropic-messages route" - }, - { - "name": "goose-claude-fable-5", - "note": "goose- prefix stripped; claude-fable-5 → anthropic adaptive xhigh-capable; anthropic-messages" - }, - { - "name": "goose-claude-sonnet-5-20260101", - "note": "goose- prefix stripped; claude-sonnet-5 family; anthropic adaptive xhigh-capable" - }, - { - "name": "goose-opus-5", - "note": "'opus' segment → anthropic-messages route; effort: fallback (prefix-stripped alias 'opus-5' not recognized Claude family)" - }, - { - "name": "consolidated-llama", - "note": "segment test: 'sol' is substring of 'consolidated', NOT a segment → mlflow-chat" - }, - { - "name": "terraform-coder", - "note": "segment test: 'terra' is prefix of 'terraform', NOT a segment → mlflow-chat" - }, - { - "name": "corpus-reranker", - "note": "segment test: 'opus' is NOT a segment of 'corpus-reranker' → mlflow-chat" - }, - { - "name": "octopus-model", - "note": "segment test: 'opus' is NOT a segment of 'octopus-model' → mlflow-chat" - }, - { - "name": "llama-3-70b", - "note": "concrete non-Claude non-GPT → mlflow-chat; effort: all-except-max" - }, - { - "name": "", - "note": "blank model → route-unknown; all 7 efforts; default medium" - } - ] -} diff --git a/scripts/generate-model-capabilities.mjs b/scripts/generate-model-capabilities.mjs index 88ce365a0a..1ac36cba0d 100644 --- a/scripts/generate-model-capabilities.mjs +++ b/scripts/generate-model-capabilities.mjs @@ -5,7 +5,6 @@ * Reads `scripts/model-capabilities.json` and emits: * - `crates/buzz-agent/src/generated_model_capabilities.rs` * - `desktop/src/features/agents/ui/modelCapabilities.ts` - * - `scripts/generated-model-capabilities-coverage.json` (snapshot/drift fixture — full-table resolver output, diff-checked by CI) * * The generator performs three ordered resolution steps (resolver contract, plan v4): * 1. Provider-qualified raw exact lookup — key is (provider, raw_model_id), matched @@ -570,68 +569,6 @@ function getProviderFallback(provider, isBlank) { }; } -// --------------------------------------------------------------------------- -// Build full-table snapshot/drift fixture -// Every (provider, model) pair that can be reached by any manifest rule is resolved -// and written here. CI diffs this against the committed copy — any resolver output -// change for any input shows up as a diff, catching silent behavior shifts. -// --------------------------------------------------------------------------- - -const allEntries = []; - -// All family rule canonical model IDs -for (const rule of manifest.family_rules) { - for (const provider of rule.providers) { - const result = resolve(provider, rule.match_value); - allEntries.push({ - note: `family rule ${rule.id} / provider ${provider}`, - provider, - model: rule.match_value, - resolved: result, - }); - // Also test aliases - for (const alias of rule.match_aliases ?? []) { - const r2 = resolve(provider, alias); - allEntries.push({ - note: `family rule ${rule.id} alias ${alias} / provider ${provider}`, - provider, - model: alias, - resolved: r2, - }); - } - } -} - -// All exact_records -for (const rec of manifest.exact_records ?? []) { - const result = resolve(rec.provider, rec.raw_model_id); - allEntries.push({ - note: `exact record ${rec.provider}::${rec.raw_model_id}`, - provider: rec.provider, - model: rec.raw_model_id, - resolved: result, - }); -} - -// All provider fallbacks (blank + concrete unknown examples) -for (const [provider] of Object.entries(manifest.provider_fallbacks)) { - if (provider === "_default") continue; - const blankResult = resolve(provider, ""); - allEntries.push({ - note: `fallback ${provider} blank`, - provider, - model: "", - resolved: blankResult, - }); - const unknownResult = resolve(provider, "some-unknown-model-xyz"); - allEntries.push({ - note: `fallback ${provider} concrete_unknown`, - provider, - model: "some-unknown-model-xyz", - resolved: unknownResult, - }); -} - // --------------------------------------------------------------------------- // Rust code generation // --------------------------------------------------------------------------- @@ -1079,7 +1016,6 @@ const rustGpt5Helpers = ` /// Returns true if \`model\` contains \`token\` at a word boundary (end-of-string or "-"). /// Does not match if followed immediately by a digit or letter. -/// Mirrors gpt5_token_matches in config.rs. fn gpt5_token_matches_rs(model: &str, token: &str) -> bool { let lower = model; let tok_lower = token; @@ -1484,13 +1420,6 @@ const outputs = [ content: tsContent, label: "TypeScript", }, - { - path: outputDirOverride - ? join(outputDirOverride, "generated-model-capabilities-coverage.json") - : join(repoRoot, "scripts", "generated-model-capabilities-coverage.json"), - content: JSON.stringify(allEntries, null, 2) + "\n", - label: "Coverage snapshot/drift fixture", - }, ]; let checkFailed = false; diff --git a/scripts/generated-model-capabilities-coverage.json b/scripts/generated-model-capabilities-coverage.json deleted file mode 100644 index 22cf77db20..0000000000 --- a/scripts/generated-model-capabilities-coverage.json +++ /dev/null @@ -1,2743 +0,0 @@ -[ - { - "note": "family rule anthropic-manual-budget-claude3 / provider anthropic", - "provider": "anthropic", - "model": "claude-3", - "resolved": { - "registry_label": null, - "thinking_mode": "manual-budget", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": null, - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-manual-budget-claude3", - "rule_priority": 10, - "normalized_alias": "claude-3", - "raw_model_id": "claude-3" - } - } - }, - { - "note": "family rule anthropic-manual-budget-claude3 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-3", - "resolved": { - "registry_label": null, - "thinking_mode": "manual-budget", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": null, - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-manual-budget-claude3", - "rule_priority": 10, - "normalized_alias": "claude-3", - "raw_model_id": "claude-3" - } - } - }, - { - "note": "family rule anthropic-manual-budget-opus-4-5 / provider anthropic", - "provider": "anthropic", - "model": "claude-opus-4-5", - "resolved": { - "registry_label": "Claude Opus 4.5", - "thinking_mode": "manual-budget", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": null, - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-manual-budget-opus-4-5", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-5", - "raw_model_id": "claude-opus-4-5" - } - } - }, - { - "note": "family rule anthropic-manual-budget-opus-4-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-opus-4-5", - "resolved": { - "registry_label": "Claude Opus 4.5", - "thinking_mode": "manual-budget", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": null, - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-manual-budget-opus-4-5", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-5", - "raw_model_id": "claude-opus-4-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-4-7 / provider anthropic", - "provider": "anthropic", - "model": "claude-opus-4-7", - "resolved": { - "registry_label": "Claude Opus 4.7", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-4-7", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-7", - "raw_model_id": "claude-opus-4-7" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-4-7 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-opus-4-7", - "resolved": { - "registry_label": "Claude Opus 4.7", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-4-7", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-7", - "raw_model_id": "claude-opus-4-7" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-4-8 / provider anthropic", - "provider": "anthropic", - "model": "claude-opus-4-8", - "resolved": { - "registry_label": "Claude Opus 4.8", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-4-8", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-8", - "raw_model_id": "claude-opus-4-8" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-4-8 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-opus-4-8", - "resolved": { - "registry_label": "Claude Opus 4.8", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-4-8", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-8", - "raw_model_id": "claude-opus-4-8" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-5 / provider anthropic", - "provider": "anthropic", - "model": "claude-opus-5", - "resolved": { - "registry_label": "Claude Opus 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-5", - "rule_priority": 10, - "normalized_alias": "claude-opus-5", - "raw_model_id": "claude-opus-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-opus-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-opus-5", - "resolved": { - "registry_label": "Claude Opus 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-opus-5", - "rule_priority": 10, - "normalized_alias": "claude-opus-5", - "raw_model_id": "claude-opus-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-sonnet-5 / provider anthropic", - "provider": "anthropic", - "model": "claude-sonnet-5", - "resolved": { - "registry_label": "Claude Sonnet 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-sonnet-5", - "rule_priority": 10, - "normalized_alias": "claude-sonnet-5", - "raw_model_id": "claude-sonnet-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-sonnet-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-sonnet-5", - "resolved": { - "registry_label": "Claude Sonnet 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-sonnet-5", - "rule_priority": 10, - "normalized_alias": "claude-sonnet-5", - "raw_model_id": "claude-sonnet-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-fable-5 / provider anthropic", - "provider": "anthropic", - "model": "claude-fable-5", - "resolved": { - "registry_label": "Claude Fable 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-fable-5", - "rule_priority": 10, - "normalized_alias": "claude-fable-5", - "raw_model_id": "claude-fable-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-fable-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-fable-5", - "resolved": { - "registry_label": "Claude Fable 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-fable-5", - "rule_priority": 10, - "normalized_alias": "claude-fable-5", - "raw_model_id": "claude-fable-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-mythos-5 / provider anthropic", - "provider": "anthropic", - "model": "claude-mythos-5", - "resolved": { - "registry_label": "Claude Mythos 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-mythos-5", - "rule_priority": 10, - "normalized_alias": "claude-mythos-5", - "raw_model_id": "claude-mythos-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-xhigh-mythos-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-mythos-5", - "resolved": { - "registry_label": "Claude Mythos 5", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-xhigh-mythos-5", - "rule_priority": 10, - "normalized_alias": "claude-mythos-5", - "raw_model_id": "claude-mythos-5" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-opus-4-6 / provider anthropic", - "provider": "anthropic", - "model": "claude-opus-4-6", - "resolved": { - "registry_label": "Claude Opus 4.6", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-opus-4-6", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-6", - "raw_model_id": "claude-opus-4-6" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-opus-4-6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-opus-4-6", - "resolved": { - "registry_label": "Claude Opus 4.6", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-opus-4-6", - "rule_priority": 10, - "normalized_alias": "claude-opus-4-6", - "raw_model_id": "claude-opus-4-6" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-sonnet-4-6 / provider anthropic", - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "resolved": { - "registry_label": "Claude Sonnet 4.6", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-sonnet-4-6", - "rule_priority": 10, - "normalized_alias": "claude-sonnet-4-6", - "raw_model_id": "claude-sonnet-4-6" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-sonnet-4-6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-sonnet-4-6", - "resolved": { - "registry_label": "Claude Sonnet 4.6", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-sonnet-4-6", - "rule_priority": 10, - "normalized_alias": "claude-sonnet-4-6", - "raw_model_id": "claude-sonnet-4-6" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-mythos-preview / provider anthropic", - "provider": "anthropic", - "model": "claude-mythos-preview", - "resolved": { - "registry_label": "Claude Mythos Preview", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-mythos-preview", - "rule_priority": 10, - "normalized_alias": "claude-mythos-preview", - "raw_model_id": "claude-mythos-preview" - } - } - }, - { - "note": "family rule anthropic-adaptive-no-xhigh-mythos-preview / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude-mythos-preview", - "resolved": { - "registry_label": "Claude Mythos Preview", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "anthropic-adaptive-no-xhigh-mythos-preview", - "rule_priority": 10, - "normalized_alias": "claude-mythos-preview", - "raw_model_id": "claude-mythos-preview" - } - } - }, - { - "note": "family rule openai-gpt5-pro / provider openai", - "provider": "openai", - "model": "gpt-5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt-5-pro", - "raw_model_id": "gpt-5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-pro alias gpt5-pro / provider openai", - "provider": "openai", - "model": "gpt5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt5-pro", - "raw_model_id": "gpt5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-pro / provider databricks", - "provider": "databricks", - "model": "gpt-5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt-5-pro", - "raw_model_id": "gpt-5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-pro alias gpt5-pro / provider databricks", - "provider": "databricks", - "model": "gpt5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt5-pro", - "raw_model_id": "gpt5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-pro / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt-5-pro", - "raw_model_id": "gpt-5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-pro alias gpt5-pro / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5-pro", - "resolved": { - "registry_label": "GPT-5 Pro", - "thinking_mode": "none", - "supported_efforts": [ - "high" - ], - "default_effort": "high", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-pro", - "rule_priority": 20, - "normalized_alias": "gpt5-pro", - "raw_model_id": "gpt5-pro" - } - } - }, - { - "note": "family rule openai-gpt5-6 / provider openai", - "provider": "openai", - "model": "gpt-5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5.6", - "raw_model_id": "gpt-5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5.6 / provider openai", - "provider": "openai", - "model": "gpt5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5.6", - "raw_model_id": "gpt5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt-5-6 / provider openai", - "provider": "openai", - "model": "gpt-5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5-6", - "raw_model_id": "gpt-5-6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5-6 / provider openai", - "provider": "openai", - "model": "gpt5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5-6", - "raw_model_id": "gpt5-6" - } - } - }, - { - "note": "family rule openai-gpt5-6 / provider databricks", - "provider": "databricks", - "model": "gpt-5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5.6", - "raw_model_id": "gpt-5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5.6 / provider databricks", - "provider": "databricks", - "model": "gpt5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5.6", - "raw_model_id": "gpt5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt-5-6 / provider databricks", - "provider": "databricks", - "model": "gpt-5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5-6", - "raw_model_id": "gpt-5-6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5-6 / provider databricks", - "provider": "databricks", - "model": "gpt5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5-6", - "raw_model_id": "gpt5-6" - } - } - }, - { - "note": "family rule openai-gpt5-6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5.6", - "raw_model_id": "gpt-5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5.6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5.6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5.6", - "raw_model_id": "gpt5.6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt-5-6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt-5-6", - "raw_model_id": "gpt-5-6" - } - } - }, - { - "note": "family rule openai-gpt5-6 alias gpt5-6 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5-6", - "resolved": { - "registry_label": "GPT-5.6", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-6", - "rule_priority": 15, - "normalized_alias": "gpt5-6", - "raw_model_id": "gpt5-6" - } - } - }, - { - "note": "family rule openai-gpt5-5 / provider openai", - "provider": "openai", - "model": "gpt-5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5.5", - "raw_model_id": "gpt-5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5.5 / provider openai", - "provider": "openai", - "model": "gpt5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5.5", - "raw_model_id": "gpt5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt-5-5 / provider openai", - "provider": "openai", - "model": "gpt-5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5-5", - "raw_model_id": "gpt-5-5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5-5 / provider openai", - "provider": "openai", - "model": "gpt5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5-5", - "raw_model_id": "gpt5-5" - } - } - }, - { - "note": "family rule openai-gpt5-5 / provider databricks", - "provider": "databricks", - "model": "gpt-5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5.5", - "raw_model_id": "gpt-5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5.5 / provider databricks", - "provider": "databricks", - "model": "gpt5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5.5", - "raw_model_id": "gpt5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt-5-5 / provider databricks", - "provider": "databricks", - "model": "gpt-5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5-5", - "raw_model_id": "gpt-5-5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5-5 / provider databricks", - "provider": "databricks", - "model": "gpt5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5-5", - "raw_model_id": "gpt5-5" - } - } - }, - { - "note": "family rule openai-gpt5-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5.5", - "raw_model_id": "gpt-5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5.5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5.5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5.5", - "raw_model_id": "gpt5.5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt-5-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt-5-5", - "raw_model_id": "gpt-5-5" - } - } - }, - { - "note": "family rule openai-gpt5-5 alias gpt5-5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-5", - "rule_priority": 15, - "normalized_alias": "gpt5-5", - "raw_model_id": "gpt5-5" - } - } - }, - { - "note": "family rule openai-gpt5-4 / provider openai", - "provider": "openai", - "model": "gpt-5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5.4", - "raw_model_id": "gpt-5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5.4 / provider openai", - "provider": "openai", - "model": "gpt5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5.4", - "raw_model_id": "gpt5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt-5-4 / provider openai", - "provider": "openai", - "model": "gpt-5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5-4", - "raw_model_id": "gpt-5-4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5-4 / provider openai", - "provider": "openai", - "model": "gpt5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5-4", - "raw_model_id": "gpt5-4" - } - } - }, - { - "note": "family rule openai-gpt5-4 / provider databricks", - "provider": "databricks", - "model": "gpt-5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5.4", - "raw_model_id": "gpt-5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5.4 / provider databricks", - "provider": "databricks", - "model": "gpt5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5.4", - "raw_model_id": "gpt5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt-5-4 / provider databricks", - "provider": "databricks", - "model": "gpt-5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5-4", - "raw_model_id": "gpt-5-4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5-4 / provider databricks", - "provider": "databricks", - "model": "gpt5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5-4", - "raw_model_id": "gpt5-4" - } - } - }, - { - "note": "family rule openai-gpt5-4 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5.4", - "raw_model_id": "gpt-5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5.4 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5.4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5.4", - "raw_model_id": "gpt5.4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt-5-4 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt-5-4", - "raw_model_id": "gpt-5-4" - } - } - }, - { - "note": "family rule openai-gpt5-4 alias gpt5-4 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5-4", - "resolved": { - "registry_label": "GPT-5.4", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-4", - "rule_priority": 15, - "normalized_alias": "gpt5-4", - "raw_model_id": "gpt5-4" - } - } - }, - { - "note": "family rule openai-gpt5-1 / provider openai", - "provider": "openai", - "model": "gpt-5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5.1", - "raw_model_id": "gpt-5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5.1 / provider openai", - "provider": "openai", - "model": "gpt5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5.1", - "raw_model_id": "gpt5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt-5-1 / provider openai", - "provider": "openai", - "model": "gpt-5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5-1", - "raw_model_id": "gpt-5-1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5-1 / provider openai", - "provider": "openai", - "model": "gpt5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5-1", - "raw_model_id": "gpt5-1" - } - } - }, - { - "note": "family rule openai-gpt5-1 / provider databricks", - "provider": "databricks", - "model": "gpt-5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5.1", - "raw_model_id": "gpt-5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5.1 / provider databricks", - "provider": "databricks", - "model": "gpt5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5.1", - "raw_model_id": "gpt5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt-5-1 / provider databricks", - "provider": "databricks", - "model": "gpt-5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5-1", - "raw_model_id": "gpt-5-1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5-1 / provider databricks", - "provider": "databricks", - "model": "gpt5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5-1", - "raw_model_id": "gpt5-1" - } - } - }, - { - "note": "family rule openai-gpt5-1 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5.1", - "raw_model_id": "gpt-5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5.1 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5.1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5.1", - "raw_model_id": "gpt5.1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt-5-1 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt-5-1", - "raw_model_id": "gpt-5-1" - } - } - }, - { - "note": "family rule openai-gpt5-1 alias gpt5-1 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5-1", - "resolved": { - "registry_label": "GPT-5.1", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "low", - "medium", - "high" - ], - "default_effort": "none", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-1", - "rule_priority": 15, - "normalized_alias": "gpt5-1", - "raw_model_id": "gpt5-1" - } - } - }, - { - "note": "family rule openai-gpt5-base / provider openai", - "provider": "openai", - "model": "gpt-5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt-5", - "raw_model_id": "gpt-5" - } - } - }, - { - "note": "family rule openai-gpt5-base alias gpt5 / provider openai", - "provider": "openai", - "model": "gpt5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt5", - "raw_model_id": "gpt5" - } - } - }, - { - "note": "family rule openai-gpt5-base / provider databricks", - "provider": "databricks", - "model": "gpt-5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt-5", - "raw_model_id": "gpt-5" - } - } - }, - { - "note": "family rule openai-gpt5-base alias gpt5 / provider databricks", - "provider": "databricks", - "model": "gpt5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "not-applicable", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt5", - "raw_model_id": "gpt5" - } - } - }, - { - "note": "family rule openai-gpt5-base / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt-5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt-5", - "raw_model_id": "gpt-5" - } - } - }, - { - "note": "family rule openai-gpt5-base alias gpt5 / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt5", - "resolved": { - "registry_label": "GPT-5", - "thinking_mode": "none", - "supported_efforts": [ - "minimal", - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "family", - "rule_id": "openai-gpt5-base", - "rule_priority": 10, - "normalized_alias": "gpt5", - "raw_model_id": "gpt5" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment / provider databricks_v2", - "provider": "databricks_v2", - "model": "claude", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "claude", - "raw_model_id": "claude" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment alias opus / provider databricks_v2", - "provider": "databricks_v2", - "model": "opus", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "opus", - "raw_model_id": "opus" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment alias sonnet / provider databricks_v2", - "provider": "databricks_v2", - "model": "sonnet", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "sonnet", - "raw_model_id": "sonnet" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment alias haiku / provider databricks_v2", - "provider": "databricks_v2", - "model": "haiku", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "haiku", - "raw_model_id": "haiku" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment alias mythos / provider databricks_v2", - "provider": "databricks_v2", - "model": "mythos", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "mythos", - "raw_model_id": "mythos" - } - } - }, - { - "note": "family rule dbv2-claude-code-names-segment alias fable / provider databricks_v2", - "provider": "databricks_v2", - "model": "fable", - "resolved": { - "registry_label": null, - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "family", - "rule_id": "dbv2-claude-code-names-segment", - "rule_priority": 5, - "normalized_alias": "fable", - "raw_model_id": "fable" - } - } - }, - { - "note": "family rule dbv2-gpt-code-names-segment / provider databricks_v2", - "provider": "databricks_v2", - "model": "gpt", - "resolved": { - "registry_label": null, - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "family", - "rule_id": "dbv2-gpt-code-names-segment", - "rule_priority": 5, - "normalized_alias": "gpt", - "raw_model_id": "gpt" - } - } - }, - { - "note": "family rule dbv2-sol-luna-terra-segment / provider databricks_v2", - "provider": "databricks_v2", - "model": "sol", - "resolved": { - "registry_label": null, - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "family", - "rule_id": "dbv2-sol-luna-terra-segment", - "rule_priority": 5, - "normalized_alias": "sol", - "raw_model_id": "sol" - } - } - }, - { - "note": "family rule dbv2-sol-luna-terra-segment alias luna / provider databricks_v2", - "provider": "databricks_v2", - "model": "luna", - "resolved": { - "registry_label": null, - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "family", - "rule_id": "dbv2-sol-luna-terra-segment", - "rule_priority": 5, - "normalized_alias": "luna", - "raw_model_id": "luna" - } - } - }, - { - "note": "family rule dbv2-sol-luna-terra-segment alias terra / provider databricks_v2", - "provider": "databricks_v2", - "model": "terra", - "resolved": { - "registry_label": null, - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "family", - "rule_id": "dbv2-sol-luna-terra-segment", - "rule_priority": 5, - "normalized_alias": "terra", - "raw_model_id": "terra" - } - } - }, - { - "note": "exact record databricks_v2::databricks-gpt-5-4-mini", - "provider": "databricks_v2", - "model": "databricks-gpt-5-4-mini", - "resolved": { - "registry_label": "GPT-5.4 Mini", - "thinking_mode": "none", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "exact", - "exact_key": "databricks_v2::databricks-gpt-5-4-mini", - "registry_label": "exact_record", - "supported_efforts": "exact_record", - "databricks_v2_wire_route": "family:openai-gpt5-4@15", - "thinking_mode": "family:openai-gpt5-4@15", - "normalization_policy": "family:openai-gpt5-4@15", - "default_effort": "family:openai-gpt5-4@15" - } - } - }, - { - "note": "exact record databricks_v2::databricks-gpt-5-4-nano", - "provider": "databricks_v2", - "model": "databricks-gpt-5-4-nano", - "resolved": { - "registry_label": "GPT-5.4 Nano", - "thinking_mode": "none", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "exact", - "exact_key": "databricks_v2::databricks-gpt-5-4-nano", - "registry_label": "exact_record", - "supported_efforts": "exact_record", - "databricks_v2_wire_route": "family:openai-gpt5-4@15", - "thinking_mode": "family:openai-gpt5-4@15", - "normalization_policy": "family:openai-gpt5-4@15", - "default_effort": "family:openai-gpt5-4@15" - } - } - }, - { - "note": "exact record databricks_v2::databricks-gpt-5-6-sol", - "provider": "databricks_v2", - "model": "databricks-gpt-5-6-sol", - "resolved": { - "registry_label": "GPT-5.6 Sol", - "thinking_mode": "none", - "supported_efforts": [ - "low", - "medium", - "high", - "max" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "exact", - "exact_key": "databricks_v2::databricks-gpt-5-6-sol", - "registry_label": "exact_record", - "supported_efforts": "exact_record", - "databricks_v2_wire_route": "family:openai-gpt5-6@15", - "thinking_mode": "family:openai-gpt5-6@15", - "normalization_policy": "family:openai-gpt5-6@15", - "default_effort": "family:openai-gpt5-6@15" - } - } - }, - { - "note": "exact record databricks_v2::databricks-gpt-5-5", - "provider": "databricks_v2", - "model": "databricks-gpt-5-5", - "resolved": { - "registry_label": "GPT-5.5", - "thinking_mode": "none", - "supported_efforts": [ - "low", - "medium", - "high" - ], - "default_effort": "medium", - "databricks_v2_wire_route": "openai-responses", - "normalization_policy": "openai-standard", - "_provenance": { - "source": "exact", - "exact_key": "databricks_v2::databricks-gpt-5-5", - "registry_label": "exact_record", - "supported_efforts": "exact_record", - "databricks_v2_wire_route": "family:openai-gpt5-5@15", - "thinking_mode": "family:openai-gpt5-5@15", - "normalization_policy": "family:openai-gpt5-5@15", - "default_effort": "family:openai-gpt5-5@15" - } - } - }, - { - "note": "exact record databricks_v2::databricks-claude-opus-4-7", - "provider": "databricks_v2", - "model": "databricks-claude-opus-4-7", - "resolved": { - "registry_label": "Claude Opus 4.7", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "databricks_v2_wire_route": "anthropic-messages", - "normalization_policy": "none", - "_provenance": { - "source": "exact", - "exact_key": "databricks_v2::databricks-claude-opus-4-7", - "registry_label": "exact_record", - "supported_efforts": "family:anthropic-adaptive-xhigh-opus-4-7@10", - "databricks_v2_wire_route": "family:anthropic-adaptive-xhigh-opus-4-7@10", - "thinking_mode": "family:anthropic-adaptive-xhigh-opus-4-7@10", - "normalization_policy": "family:anthropic-adaptive-xhigh-opus-4-7@10", - "default_effort": "family:anthropic-adaptive-xhigh-opus-4-7@10" - } - } - }, - { - "note": "fallback anthropic blank", - "provider": "anthropic", - "model": "", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "adaptive", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "normalization_policy": "none", - "_provenance": { - "source": "fallback", - "provider": "anthropic", - "state": "blank" - }, - "registry_label": null - } - }, - { - "note": "fallback anthropic concrete_unknown", - "provider": "anthropic", - "model": "some-unknown-model-xyz", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "omit-fields", - "supported_efforts": [ - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "high", - "normalization_policy": "none", - "_provenance": { - "source": "fallback", - "provider": "anthropic", - "state": "concrete_unknown" - }, - "registry_label": null - } - }, - { - "note": "fallback openai blank", - "provider": "openai", - "model": "", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "openai", - "state": "blank" - }, - "registry_label": null - } - }, - { - "note": "fallback openai concrete_unknown", - "provider": "openai", - "model": "some-unknown-model-xyz", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "openai", - "state": "concrete_unknown" - }, - "registry_label": null - } - }, - { - "note": "fallback databricks_v2 blank", - "provider": "databricks_v2", - "model": "", - "resolved": { - "databricks_v2_wire_route": "route-unknown", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "databricks_v2", - "state": "blank" - }, - "registry_label": null - } - }, - { - "note": "fallback databricks_v2 concrete_unknown", - "provider": "databricks_v2", - "model": "some-unknown-model-xyz", - "resolved": { - "databricks_v2_wire_route": "mlflow-chat", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "databricks_v2", - "state": "concrete_unknown" - }, - "registry_label": null - } - }, - { - "note": "fallback databricks blank", - "provider": "databricks", - "model": "", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "databricks", - "state": "blank" - }, - "registry_label": null - } - }, - { - "note": "fallback databricks concrete_unknown", - "provider": "databricks", - "model": "some-unknown-model-xyz", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "default_effort": "medium", - "normalization_policy": "openai-clamp-max-to-xhigh", - "_provenance": { - "source": "fallback", - "provider": "databricks", - "state": "concrete_unknown" - }, - "registry_label": null - } - }, - { - "note": "fallback openrouter blank", - "provider": "openrouter", - "model": "", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "normalization_policy": "none", - "_provenance": { - "source": "fallback", - "provider": "openrouter", - "state": "blank" - }, - "registry_label": null - } - }, - { - "note": "fallback openrouter concrete_unknown", - "provider": "openrouter", - "model": "some-unknown-model-xyz", - "resolved": { - "databricks_v2_wire_route": "not-applicable", - "thinking_mode": "none", - "supported_efforts": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh", - "max" - ], - "default_effort": "medium", - "normalization_policy": "none", - "_provenance": { - "source": "fallback", - "provider": "openrouter", - "state": "concrete_unknown" - }, - "registry_label": null - } - } -] diff --git a/scripts/run-differential.mjs b/scripts/run-differential.mjs deleted file mode 100755 index a55d2aa035..0000000000 --- a/scripts/run-differential.mjs +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env node -/** - * Phase-2 differential harness — compare old buzzAgentConfig.ts effort logic with - * the new generated modelCapabilities.ts interpreter over: - * 1. The 36-entry effortTable.fixture.json (cross-boundary Rust/TS fixture) - * 2. The 45-vector normative corpus (scripts/normative-corpus.json) - * 3. The catalog-sample fixture (scripts/catalog-sample-fixture.json) - * - * Equality is required except for entries in the committed allowlist of intentional - * F1 corrections (models.dev provider-capability reconciliations). - * - * Usage: node --experimental-strip-types scripts/run-differential.mjs [--verbose] - * Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence or unexercised allowlist entry. - */ - -import { readFileSync } from "node:fs"; -import { join, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = join(__dirname, ".."); -const VERBOSE = process.argv.includes("--verbose"); - -// --------------------------------------------------------------------------- -// Import both interpreters -// --------------------------------------------------------------------------- - -// NEW: generated capability module -const { resolveModelCapabilities: resolveNew } = await import( - join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelCapabilities.ts") -); - -// OLD: buzzAgentConfig.ts effort config -const { getProviderEffortConfig_oldHandTable: getOldEffortConfig } = await import( - join(repoRoot, "desktop", "src", "features", "agents", "ui", "buzzAgentConfig.ts") -); - -// --------------------------------------------------------------------------- -// Intentional corrections allowlist (Phase 1 F1 reconciliations) -// Each entry: { provider, raw_model_id, reason } -// --------------------------------------------------------------------------- -const ALLOWLIST = [ - { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-5", - axes: ["supported_efforts"], - reason: "Phase 1 ADOPT: models.dev d5a4974c advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", - }, - { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-4-mini", - axes: ["supported_efforts"], - reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", - }, - { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-4-nano", - axes: ["supported_efforts"], - reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", - }, - { - provider: "databricks_v2", - raw_model_id: "databricks-gpt-5-6-sol", - axes: ["supported_efforts"], - reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high,max]; old returns [none,low,medium,high,xhigh,max]", - }, - { - provider: "databricks_v2", - raw_model_id: "goose-opus-5", - axes: ["supported_efforts", "default_effort"], - reason: "Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route; old config.rs disagreed with llm.rs (corpus note dbv2-goose-opus-5-is-anthropic). Generated adopts anthropic adaptive-xhigh capabilities consistent with the wire route.", - }, -]; - -// Track which allowlist entries are actually exercised (suppressed a divergence). -// Keyed as "provider:raw_model_id:axis". -const allowlistHits = new Set(); - -function isAllowlisted(provider, rawModelId, axis) { - const entry = ALLOWLIST.find( - (e) => - e.provider === provider && - e.raw_model_id === rawModelId && - e.axes.includes(axis), - ); - if (entry) { - allowlistHits.add(`${provider}:${rawModelId}:${axis}`); - return true; - } - return false; -} - -// --------------------------------------------------------------------------- -// Comparison helpers -// --------------------------------------------------------------------------- - -/** - * Compare effort axes from both interpreters for one (provider, model) pair. - * Returns array of divergence objects. - */ -function compareEffortAxes(provider, model) { - const newResult = resolveNew(provider, model); - const oldResult = getOldEffortConfig(provider, model); - - const divergences = []; - - // supported_efforts - const newEfforts = newResult.supportedEfforts ?? []; - const oldEfforts = oldResult?.validValues ?? []; - if (JSON.stringify(newEfforts) !== JSON.stringify(oldEfforts)) { - if (!isAllowlisted(provider, model, "supported_efforts")) { - divergences.push({ - axis: "supported_efforts", - old: oldEfforts, - new: newEfforts, - }); - } - } - - // default_effort - const newDefault = newResult.defaultEffort ?? null; - const oldDefault = oldResult?.defaultValue ?? null; - if (newDefault !== oldDefault) { - if (!isAllowlisted(provider, model, "default_effort")) { - divergences.push({ - axis: "default_effort", - old: oldDefault, - new: newDefault, - }); - } - } - - return divergences; -} - -// --------------------------------------------------------------------------- -// Test suites -// --------------------------------------------------------------------------- - -let totalChecks = 0; -let totalDivergences = 0; - -function runCheck(label, provider, model) { - totalChecks++; - const divs = compareEffortAxes(provider, model); - if (divs.length > 0) { - totalDivergences += divs.length; - for (const d of divs) { - console.error( - `DIVERGE [${label}] provider=${provider} model=${model} axis=${d.axis}\n` + - ` old: ${JSON.stringify(d.old)}\n` + - ` new: ${JSON.stringify(d.new)}`, - ); - } - } else if (VERBOSE) { - console.log(`OK [${label}] provider=${provider} model=${model}`); - } -} - -// 1. effortTable.fixture.json -console.log("--- effortTable.fixture.json ---"); -const fixture = JSON.parse( - readFileSync( - join(repoRoot, "desktop", "src", "features", "agents", "ui", "effortTable.fixture.json"), - "utf8", - ), -); -for (const entry of fixture) { - if (!entry.provider) continue; - runCheck("fixture", entry.provider, entry.model ?? ""); -} - -// 2. normative-corpus.json (effort axes only) -console.log("--- normative-corpus.json ---"); -const corpus = JSON.parse( - readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"), -); -for (const entry of corpus) { - if (entry._group) continue; - if (!entry.provider || !entry.expect) continue; - if (!entry.expect.supported_efforts && !entry.expect.default_effort) continue; - runCheck("corpus", entry.provider, entry.raw_model_id ?? ""); -} - -// 3. catalog-sample-fixture.json (exact records from pinned models.dev payload) -console.log("--- catalog-sample-fixture.json ---"); -const catalogFixture = JSON.parse( - readFileSync(join(repoRoot, "scripts", "catalog-sample-fixture.json"), "utf8"), -); -for (const ep of catalogFixture.endpoints ?? []) { - if (!ep.name) continue; - // All catalog endpoints are databricks_v2 provider - runCheck("catalog-sample", "databricks_v2", ep.name); -} - -// --------------------------------------------------------------------------- -// Summary -// --------------------------------------------------------------------------- - -// Count total allowlist axis slots expected to be hit -const totalAllowlistSlots = ALLOWLIST.reduce((n, e) => n + e.axes.length, 0); -const allowlistHitCount = allowlistHits.size; - -// Detect stale allowlist entries (declared but never actually suppressed a divergence) -const staleEntries = []; -for (const entry of ALLOWLIST) { - for (const axis of entry.axes) { - const key = `${entry.provider}:${entry.raw_model_id}:${axis}`; - if (!allowlistHits.has(key)) { - staleEntries.push({ ...entry, axis }); - } - } -} - -console.log( - `\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${allowlistHitCount}/${totalAllowlistSlots} allowlist slots exercised`, -); - -if (staleEntries.length > 0) { - for (const e of staleEntries) { - console.error( - `STALE_ALLOWLIST provider=${e.provider} model=${e.raw_model_id} axis=${e.axis} — entry never fired; remove or update it`, - ); - } -} - -if (totalDivergences > 0) { - console.error( - `FAIL: ${totalDivergences} unexpected divergence(s) — see output above`, - ); - process.exit(1); -} else if (staleEntries.length > 0) { - console.error( - `FAIL: ${staleEntries.length} stale allowlist entry(ies) — entries that never suppress a divergence mask future regressions`, - ); - process.exit(1); -} else { - console.log("PASS: old and new effort logic agree on all non-allowlisted entries"); -} diff --git a/scripts/run-mutation-evidence.mjs b/scripts/run-mutation-evidence.mjs deleted file mode 100755 index 6083f56d66..0000000000 --- a/scripts/run-mutation-evidence.mjs +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env node -/** - * Per-interpreter mutation evidence runner. - * - * Introduces deliberate resolver faults into the manifest, regenerates artifacts, - * and verifies the shared normative corpus detects every fault in BOTH the generated - * TypeScript interpreter (via --experimental-strip-types import) and the Rust - * interpreter (via cargo test shared-corpus harness). Exits 0 if all mutations are - * killed in both interpreters; exits 1 if any survive. - * - * Usage: - * node --experimental-strip-types scripts/run-mutation-evidence.mjs [--verbose] - * - * This script is non-CI (run manually to generate MUTATION_EVIDENCE.md). It writes - * its findings to stdout in a format suitable for copy-paste into the evidence doc. - * - * Mutations applied (each in isolation, manifest restored after each run): - * M1: Swap anthropic-adaptive-xhigh-opus-4-7 efforts from [low,medium,high,xhigh,max] - * to [low,medium,high] — kills corpus vectors that check xhigh/max. - * M2: Change gpt5-base supported_efforts to include "xhigh" — kills vectors that - * check gpt5-base resolves minimal-only, not xhigh. - * M3: Change openai-gpt5-1 default_effort to "high" instead of "none" — kills - * the gpt5.1 corpus vector that checks default_effort=none. - * M4: Swap databricks_v2_wire_route in dbv2-claude-code-names-segment from - * "anthropic-messages" to "openai-responses" — kills segment-route corpus vectors. - * M5: Remove all three DBv2 segment rules — kills goose-opus-5 and terraform/consolidated - * segment collision vectors. - * M6: Change databricks_v2 concrete_unknown fallback route from "mlflow-chat" to - * "openai-responses" — kills dbv2-concrete-unknown-mlflow-no-max vector. - * M7: Change gpt5-4 supported_efforts to remove "xhigh" — kills - * resolver-prefixed-alias-misses-exact vector. - */ - -import { readFileSync, writeFileSync } from "node:fs"; -import { join, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { execSync, spawnSync } from "node:child_process"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = join(__dirname, ".."); -const VERBOSE = process.argv.includes("--verbose"); - -const manifestPath = join(repoRoot, "scripts", "model-capabilities.json"); -const generatorPath = join(repoRoot, "scripts", "generate-model-capabilities.mjs"); -const jsRunnerPath = join(repoRoot, "scripts", "run-corpus.mjs"); - -const originalManifest = readFileSync(manifestPath, "utf8"); - -/** - * Run the TS corpus and return: - * { kind: "passed" } — all vectors pass (mutation survived) - * { kind: "killed", output } — nonzero exit AND at least one expectedKiller ID - * appears in stdout/stderr ("FAIL " line) - * { kind: "error", output } — nonzero exit but NO expected corpus output - * (import error, missing file, syntax error, etc.) - */ -function runTsCorpus(expectedKillers) { - let result; - try { - result = spawnSync( - process.execPath, - ["--experimental-strip-types", jsRunnerPath], - { cwd: repoRoot, encoding: "utf8" }, - ); - } catch (e) { - return { kind: "error", output: `spawn error: ${e.message}` }; - } - if (result.status === 0) return { kind: "passed" }; - const output = (result.stdout ?? "") + (result.stderr ?? ""); - // A genuine corpus kill produces "FAIL " lines. - // An infrastructure failure (import error, syntax error) produces no such lines. - const hasCorpusFailure = expectedKillers.some((id) => output.includes(`FAIL ${id}`)); - if (hasCorpusFailure) return { kind: "killed", output }; - return { kind: "error", output }; -} - -/** - * Run the Rust corpus and return: - * { kind: "passed" } — all vectors pass - * { kind: "killed", output } — nonzero exit AND at least one expectedKiller ID - * appears in the panic output ("[]" format) - * { kind: "error", output } — nonzero exit but NO expected corpus output - * (compile error, missing cargo, linker error, etc.) - */ -function runRustCorpus(expectedKillers) { - let result; - try { - result = spawnSync( - "cargo", - [ - "test", - "-p", "buzz-agent", - "--", - "generated_model_capabilities::tests::shared_corpus_tests", - "--nocapture", - ], - { cwd: repoRoot, encoding: "utf8", env: { ...process.env, RUST_BACKTRACE: "0" } }, - ); - } catch (e) { - return { kind: "error", output: `spawn error: ${e.message}` }; - } - if (result.status === 0) return { kind: "passed" }; - const output = (result.stdout ?? "") + (result.stderr ?? ""); - // The Rust corpus runner panics with "[] : got..." messages. - const hasCorpusFailure = expectedKillers.some((id) => output.includes(`[${id}]`)); - if (hasCorpusFailure) return { kind: "killed", output }; - return { kind: "error", output }; -} - -function regen() { - execSync(`"${process.execPath}" "${generatorPath}"`, { - cwd: repoRoot, - stdio: VERBOSE ? "inherit" : "pipe", - }); -} - -function restore() { - writeFileSync(manifestPath, originalManifest, "utf8"); -} - -function applyMutation(mutFn) { - const manifest = JSON.parse(originalManifest); - mutFn(manifest); - writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8"); -} - -const mutations = [ - { - id: "M1", - description: "Reduce claude-opus-4-7 supported_efforts to [low,medium,high] (drops xhigh+max)", - expectedKillers: ["anthropic-claude-opus-4-7", "dbv2-claude-prefix-stripped", "dbv2-claude-route-anthropic-messages"], - mutate(manifest) { - const rule = manifest.family_rules.find(r => r.id === "anthropic-adaptive-xhigh-opus-4-7"); - rule.supported_efforts = ["low", "medium", "high"]; - }, - }, - { - id: "M2", - description: "Add xhigh to gpt5-base supported_efforts [minimal,low,medium,high,xhigh]", - expectedKillers: ["openai-gpt5-base", "openai-gpt5-1106-should-not-match-base", "openai-gpt5-4o-matches-base", "openai-gpt5-date-suffix"], - mutate(manifest) { - const rule = manifest.family_rules.find(r => r.id === "openai-gpt5-base"); - rule.supported_efforts = ["minimal", "low", "medium", "high", "xhigh"]; - }, - }, - { - id: "M3", - description: "Change gpt5-1 default_effort to 'high' instead of 'none'", - expectedKillers: ["openai-gpt5.1"], - mutate(manifest) { - const rule = manifest.family_rules.find(r => r.id === "openai-gpt5-1"); - rule.default_effort = "high"; - }, - }, - { - id: "M4", - description: "Swap dbv2-claude-code-names-segment route from anthropic-messages to openai-responses", - expectedKillers: ["dbv2-goose-opus-5-is-anthropic"], - mutate(manifest) { - const rule = manifest.family_rules.find(r => r.id === "dbv2-claude-code-names-segment"); - rule.databricks_v2_wire_route = "openai-responses"; - }, - }, - { - id: "M5", - description: "Remove all three DBv2 segment rules (dbv2-claude-code-names-segment, dbv2-gpt-code-names-segment, dbv2-sol-luna-terra-segment)", - expectedKillers: ["dbv2-goose-opus-5-is-anthropic", "dbv2-consolidated-llama-not-sol", "dbv2-terraform-coder-not-terra"], - mutate(manifest) { - manifest.family_rules = manifest.family_rules.filter( - r => !["dbv2-claude-code-names-segment", "dbv2-gpt-code-names-segment", "dbv2-sol-luna-terra-segment"].includes(r.id) - ); - }, - }, - { - id: "M6", - description: "Change databricks_v2 concrete_unknown fallback route from mlflow-chat to openai-responses", - expectedKillers: ["dbv2-concrete-unknown-mlflow-no-max"], - mutate(manifest) { - manifest.provider_fallbacks.databricks_v2.concrete_unknown.databricks_v2_wire_route = "openai-responses"; - }, - }, - { - id: "M7", - description: "Remove xhigh from gpt5-4 supported_efforts [none,low,medium,high]", - expectedKillers: ["resolver-prefixed-alias-misses-exact"], - mutate(manifest) { - const rule = manifest.family_rules.find(r => r.id === "openai-gpt5-4"); - rule.supported_efforts = ["none", "low", "medium", "high"]; - }, - }, -]; - -let allKilled = true; -const results = []; - -for (const mut of mutations) { - process.stdout.write(` ${mut.id}: ${mut.description}\n`); - try { - applyMutation(mut.mutate); - regen(); - - // TS interpreter - process.stdout.write(` TS ... `); - const tsResult = runTsCorpus(mut.expectedKillers); - const tsKilled = tsResult.kind === "killed"; - const tsError = tsResult.kind === "error"; - if (tsKilled) { - process.stdout.write("killed ✓\n"); - } else if (tsError) { - process.stdout.write(`ERROR (infrastructure failure — not a corpus kill)\n`); - if (VERBOSE) process.stdout.write(` ${tsResult.output}\n`); - } else { - process.stdout.write("SURVIVED ✗\n"); - if (VERBOSE) process.stdout.write(` ${tsResult.output ?? ""}\n`); - } - - // Rust interpreter - process.stdout.write(` Rust... `); - const rustResult = runRustCorpus(mut.expectedKillers); - const rustKilled = rustResult.kind === "killed"; - const rustError = rustResult.kind === "error"; - if (rustKilled) { - process.stdout.write("killed ✓\n"); - } else if (rustError) { - process.stdout.write(`ERROR (infrastructure failure — not a corpus kill)\n`); - if (VERBOSE) process.stdout.write(` ${rustResult.output}\n`); - } else { - process.stdout.write("SURVIVED ✗\n"); - if (VERBOSE) process.stdout.write(` ${rustResult.output ?? ""}\n`); - } - - const killed = tsKilled && rustKilled; - if (!killed) allKilled = false; - results.push({ ...mut, killed, tsKilled, rustKilled, tsError, rustError }); - } catch (e) { - process.stdout.write(` ERROR: ${e.message}\n`); - allKilled = false; - results.push({ ...mut, killed: false, tsKilled: false, rustKilled: false, tsError: true, rustError: true, output: e.message }); - } finally { - restore(); - regen(); // restore generated files - } -} - -console.log(""); -const killed = results.filter(r => r.killed).length; -const errored = results.filter(r => r.tsError || r.rustError).length; -console.log(`Mutation results: ${killed}/${results.length} killed (both interpreters)` + - (errored > 0 ? `, ${errored} ERROR (infrastructure failure — see output above)` : "")); - -if (!allKilled) { - const hasErrors = results.some(r => r.tsError || r.rustError); - if (hasErrors) { - console.error("ERROR: Infrastructure failures prevented some mutations from being verified as killed."); - console.error(" Run with --verbose to see the full output for ERROR entries."); - } - console.error("ERROR: Some mutations survived — corpus does not kill all resolver faults."); - process.exit(1); -} - -console.log("All mutations killed in both TS and Rust interpreters.");