From ab3aeae2aedbcb9de33d2daf4d08c691b426208f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 16 Nov 2026 14:30:45 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=B1=20Weekly=20static=20model=20list?= =?UTF-8?q?=20freshness=20check=20and=20list-shape=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .github/workflows/static-model-freshness.yml + script: compare static fallbacks and OpenRouter suggestions with OpenRouter /models and npm release metadata; keep one tracking issue; no secrets - TestStaticModelListsShape: non-empty, no duplicates, Go/JS in sync - docs: gateway-backed agents inherit the gateway's live model list Signed-off-by: Andy Anderson --- .../scripts/check-static-model-freshness.py | 196 ++++++++++++++++++ .github/workflows/static-model-freshness.yml | 94 +++++++++ .../added-static-model-freshness-check.md | 1 + src/docs/agent-configuration.md | 7 + .../static_model_lists_shape_test.go | 91 ++++++++ 5 files changed, 389 insertions(+) create mode 100755 .github/scripts/check-static-model-freshness.py create mode 100644 .github/workflows/static-model-freshness.yml create mode 100644 changelog.d/added-static-model-freshness-check.md create mode 100644 src/pkg/dashboard/static_model_lists_shape_test.go diff --git a/.github/scripts/check-static-model-freshness.py b/.github/scripts/check-static-model-freshness.py new file mode 100755 index 0000000000..a1e1553481 --- /dev/null +++ b/.github/scripts/check-static-model-freshness.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Compare hive's static CLI model lists against public, keyless sources. + +Run weekly by .github/workflows/static-model-freshness.yml. Prints a Markdown +drift report to stdout (empty body sections are omitted) and exits 0; the +workflow opens or updates ONE tracking issue with it. + +Sources (all public, no auth): + * OpenRouter GET https://openrouter.ai/api/v1/models. Used for: + - the curated OpenRouter suggestions (crux/openrouter SuggestedModels and + DefaultModel), which ARE OpenRouter ids: a missing id is drift; + - the Claude, Gemini and OpenAI/Codex/Copilot static fallbacks, mapped + onto their OpenRouter equivalents (anthropic/, google/, openai/). These + are NOT the CLIs' own catalogs, so a miss is a hint to look, and a + newer model in the same family that the static list lacks is reported + as "newer upstream". + * npm registry (registry.npmjs.org//latest) for the CLIs that ship as + npm packages: the latest published version, as context for whether a + CLI release may have changed its model list since the snapshot. + +Not covered (need auth or have no machine-readable list): the Claude CLI's +own /v1/models (OAuth), the Copilot SDK catalog (Copilot login), the Codex +app-server catalog (ChatGPT login), Gemini CLI's internal list, Kiro (pi), +agy, omp, goose providers and bob. Those are probed live at runtime by the +dashboard; their static lists are fallbacks and are checked only for shape +(non-empty, no duplicates, Go/JS in sync) by the Go unit tests. +""" +import json +import re +import sys +import urllib.request + +OPENROUTER_MODELS = "https://openrouter.ai/api/v1/models" +NPM_LATEST = "https://registry.npmjs.org/{pkg}/latest" +TIMEOUT_S = 30 +USER_AGENT = "hive-static-model-freshness" + +# Package names of CLIs whose model lists hive snapshots. +NPM_PACKAGES = { + "claude": "@anthropic-ai/claude-code", + "copilot": "@github/copilot", + "codex": "@openai/codex", + "gemini": "@google/gemini-cli", +} + +CLI_MODELS_GO = "src/pkg/dashboard/cli_models.go" +OPENROUTER_GO = "src/crux/openrouter/openrouter.go" + +# Static Go lists checked against OpenRouter, with the OpenRouter provider +# prefix their ids map onto. +GO_LISTS = { + "claudeStaticModels": "anthropic", + "copilotStaticModels": None, # mixed vendors; prefix inferred per id + "codexStaticModels": "openai", + "geminiStaticModels": "google", +} + +ALIASES = {"opus", "sonnet", "haiku", "auto", "default"} + + +def fetch_json(url): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=TIMEOUT_S) as resp: + return json.load(resp) + + +def go_string_list(src, name): + m = re.search(r"var " + re.escape(name) + r" = \[\]string\{(.*?)\n\}", src, re.S) + if not m: + return [] + body = re.sub(r"//[^\n]*", "", m.group(1)) + return re.findall(r'"([^"]+)"', body) + + +def openrouter_suggestions(src): + ids = re.findall(r'\{ID: "([^"]+)", Label:', src) + m = re.search(r'const DefaultModel = "([^"]+)"', src) + return ids, (m.group(1) if m else "") + + +def vendor_for(model_id): + if model_id.startswith("claude-"): + return "anthropic" + if model_id.startswith("gemini-"): + return "google" + if model_id.startswith(("gpt-", "o3", "o4", "codex-")): + return "openai" + return None + + +def norm(model_id): + """Canonical comparison key: no date suffix, dots and dashes unified.""" + s = model_id.lower().split("/")[-1].split(":")[0] + s = re.sub(r"-\d{8}$", "", s) + return s.replace(".", "-") + + +def family(key): + """(family, version) where family is the name around the version. + + claude-sonnet-5-5 -> (('claude-sonnet', ''), (5, 5)); + gemini-2-5-flash -> (('gemini', 'flash'), (2, 5)). Variants with a + different tail (flash-lite, flash-image) are different families, so only + a genuinely newer version of the same line is reported. + """ + m = re.match(r"^([a-z]+(?:-[a-z]+)*?)-(\d+(?:-\d+)*)(?:-(.*))?$", key) + if not m: + return None, () + return (m.group(1), m.group(3) or ""), tuple(int(x) for x in m.group(2).split("-")) + + +def main(): + with open(CLI_MODELS_GO, encoding="utf-8") as f: + cli_src = f.read() + with open(OPENROUTER_GO, encoding="utf-8") as f: + or_src = f.read() + + sections = [] + drift = False + try: + catalog = fetch_json(OPENROUTER_MODELS).get("data", []) + except Exception as exc: # noqa: BLE001 — report, never fail the cron + catalog = [] + sections.append(f"### OpenRouter catalog unavailable\n\n`{OPENROUTER_MODELS}` failed: `{exc}`. Nothing was compared against it this week.") + live_ids = {m.get("id", "") for m in catalog if m.get("id")} + by_vendor = {} + for mid in live_ids: + if "/" not in mid or mid.startswith("~"): + continue + vendor, rest = mid.split("/", 1) + by_vendor.setdefault(vendor, set()).add(norm(rest)) + + if live_ids: + ids, default = openrouter_suggestions(or_src) + missing = [i for i in ids + ([default] if default else []) if i not in live_ids] + if missing: + drift = True + sections.append("### OpenRouter suggestions no longer in the catalog\n\n" + "`crux/openrouter` SuggestedModels / DefaultModel (the dashboard already hides these at runtime; refresh the list and `testdata/models.json`):\n\n" + + "\n".join(f"- `{i}`" for i in sorted(set(missing)))) + + for name, prefix in GO_LISTS.items(): + ids = [i for i in go_string_list(cli_src, name) if i not in ALIASES] + if not ids: + sections.append(f"### `{name}` could not be parsed\n\nThe checker found no ids; update the parser or the list.") + continue + not_found, newer = [], set() + have = {norm(i) for i in ids} + fams = {} + for i in ids: + vendor = prefix or vendor_for(i) + if not vendor: + continue + key = norm(i) + if key not in by_vendor.get(vendor, set()): + not_found.append(i) + fam, ver = family(key) + if fam: + fams.setdefault((vendor, fam), max(fams.get((vendor, fam), ()), ver)) + for (vendor, fam), top in fams.items(): + for key in by_vendor.get(vendor, set()): + f2, v2 = family(key) + if f2 == fam and v2 > top and key not in have: + newer.add(f"{vendor}/{key}") + lines = [] + if newer: + lines.append("Newer models of the same family on OpenRouter that the list lacks:\n\n" + "\n".join(f"- `{n}`" for n in sorted(newer))) + if not_found: + lines.append("Ids with no OpenRouter counterpart (may be CLI-only aliases; check the CLI's own catalog):\n\n" + "\n".join(f"- `{n}`" for n in sorted(not_found))) + if lines: + drift = True + sections.append(f"### `{name}` ({CLI_MODELS_GO})\n\n" + "\n\n".join(lines)) + + versions = [] + for cli, pkg in NPM_PACKAGES.items(): + try: + versions.append(f"- {cli}: `{pkg}` latest `{fetch_json(NPM_LATEST.format(pkg=pkg)).get('version', '?')}`") + except Exception as exc: # noqa: BLE001 + versions.append(f"- {cli}: `{pkg}` lookup failed (`{exc}`)") + + if not drift: + print("NO_DRIFT") + print("## Static model list freshness\n") + print("Weekly comparison of hive's static model lists against public sources. " + "Static lists are fallbacks: Claude, Copilot, Codex, Gemini, goose, agy and omp are probed live at runtime, " + "and agents on a gateway inherit the gateway's live `/v1/models` list.\n") + print("\n\n".join(sections) if sections else "No drift found against OpenRouter this week.") + print("\n### CLI releases (npm, for context)\n\n" + "\n".join(versions)) + print("\n### Not checked\n\nNeeds auth or has no machine-readable list: the Claude CLI's `/v1/models` (OAuth), the Copilot SDK catalog, " + "the Codex app-server catalog, Kiro (pi), agy, omp, goose providers, bob. The Go unit test `TestStaticModelListsShape` still checks every list is " + "non-empty, duplicate-free and matches `static/index.html`.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/static-model-freshness.yml b/.github/workflows/static-model-freshness.yml new file mode 100644 index 0000000000..ab86f4eeec --- /dev/null +++ b/.github/workflows/static-model-freshness.yml @@ -0,0 +1,94 @@ +name: Static model list freshness + +# Weekly drift check for hive's STATIC model lists: the fallbacks every probed +# CLI serves when live discovery fails (claude, copilot, codex, gemini) and the +# curated OpenRouter suggestions/default in crux/openrouter. +# +# .github/scripts/check-static-model-freshness.py compares them against +# public, keyless sources only: +# - OpenRouter GET https://openrouter.ai/api/v1/models: the curated +# OpenRouter ids directly, and the Claude/Gemini/OpenAI fallbacks mapped +# onto anthropic/, google/ and openai/ ("newer model of the same line +# upstream", "no upstream counterpart"); +# - the npm registry's latest version of each CLI package, as context. +# Sources that need auth (Claude OAuth /v1/models, the Copilot SDK catalog, +# Codex app-server, Kiro, agy, omp, goose, bob) are NOT checked here; the +# report says so. Their lists are shape-checked by the Go unit test +# TestStaticModelListsShape (non-empty, no duplicates, Go/JS in sync). +# +# The report goes into ONE tracking issue (label static-model-freshness), +# created on first drift and edited in place afterwards; when nothing drifts +# an open issue is updated to say so, and no issue is opened. No repository +# secrets are used: only the workflow's own GITHUB_TOKEN, for the issue. + +on: + schedule: + - cron: '17 7 * * 1' + workflow_dispatch: + inputs: + dry_run: + description: 'Print the report only; do not open or edit the tracking issue' + type: boolean + default: false + +permissions: + contents: read + issues: write + +concurrency: + group: static-model-freshness + cancel-in-progress: false + +env: + TRACKING_LABEL: static-model-freshness + TRACKING_TITLE: 'Static model lists: weekly freshness report' + +jobs: + check: + name: compare static model lists with public sources + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: v5 + persist-credentials: false + + - name: Build the drift report + id: report + run: | + set -euo pipefail + python3 -I .github/scripts/check-static-model-freshness.py > "$RUNNER_TEMP/raw.md" + if head -n 1 "$RUNNER_TEMP/raw.md" | grep -qx 'NO_DRIFT'; then + echo "drift=false" >> "$GITHUB_OUTPUT" + tail -n +2 "$RUNNER_TEMP/raw.md" > "$RUNNER_TEMP/report.md" + else + echo "drift=true" >> "$GITHUB_OUTPUT" + cp "$RUNNER_TEMP/raw.md" "$RUNNER_TEMP/report.md" + fi + { + echo + echo "_Updated by [${GITHUB_WORKFLOW}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) on $(date -u +%Y-%m-%d)._" + } >> "$RUNNER_TEMP/report.md" + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Open or update the tracking issue + if: ${{ !inputs.dry_run }} + env: + GH_TOKEN: ${{ github.token }} + DRIFT: ${{ steps.report.outputs.drift }} + run: | + set -euo pipefail + gh label create "$TRACKING_LABEL" --repo "$GITHUB_REPOSITORY" --color c5def5 \ + --description "Weekly static model list drift report" 2>/dev/null || true + number="$(gh issue list --repo "$GITHUB_REPOSITORY" --label "$TRACKING_LABEL" --state open \ + --json number --jq '.[0].number // empty')" + if [ -n "$number" ]; then + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/report.md" + echo "Updated #$number (drift=$DRIFT)" + elif [ "$DRIFT" = "true" ]; then + gh issue create --repo "$GITHUB_REPOSITORY" --title "$TRACKING_TITLE" \ + --label "$TRACKING_LABEL" --body-file "$RUNNER_TEMP/report.md" + else + echo "No drift and no open tracking issue; nothing to do." + fi diff --git a/changelog.d/added-static-model-freshness-check.md b/changelog.d/added-static-model-freshness-check.md new file mode 100644 index 0000000000..4c206b1610 --- /dev/null +++ b/changelog.d/added-static-model-freshness-check.md @@ -0,0 +1 @@ +- A weekly workflow compares hive's static model fallbacks and curated OpenRouter suggestions against OpenRouter's public catalog, plus npm release metadata, and keeps one tracking issue listing any drift. A unit test now fails on empty or duplicate static lists and on Go/JS copies that disagree. diff --git a/src/docs/agent-configuration.md b/src/docs/agent-configuration.md index 4c8f7f7854..e730be44ed 100644 --- a/src/docs/agent-configuration.md +++ b/src/docs/agent-configuration.md @@ -748,6 +748,13 @@ Agents can inspect peer panes with `hive-panes [lines]` when the deployment incl Every discovery probe is best-effort: a failed or absent probe falls back to a current static list, so a model dropdown is never empty. Fallback entries are marked "unverified" in the UI. +An agent whose backend is a model gateway (OpenRouter, LiteLLM, vLLM, llm-d, watsonx or a custom OpenAI-compatible endpoint) does not use a CLI's static list at all. It inherits the gateway's live `/v1/models` list: the model dropdown, the Default Model picker and auto-heal all read what the gateway currently serves. The same holds for any CLI you point at a gateway, because its model is then whatever id the gateway accepts. Only the CLI-native backends (claude, copilot, codex, gemini, goose, agy, omp, bob, pi) ever fall back to a static list. + +Static lists drift as vendors ship models, so two checks keep them honest: + +- `TestStaticModelListsShape` (in `src/pkg/dashboard`) fails a PR if any static list is empty or has duplicates, or if a first-paint copy in `static/index.html` disagrees with its Go list. +- The weekly [Static model list freshness](../../.github/workflows/static-model-freshness.yml) workflow compares the lists with public, keyless sources. The OpenRouter catalog is used directly for the curated OpenRouter suggestions, and for Claude, Gemini and OpenAI ids mapped onto `anthropic/`, `google/` and `openai/`. npm release versions of the CLIs are included as context. The workflow keeps one tracking issue (label `static-model-freshness`) up to date. Catalogs that need auth (Claude OAuth `/v1/models`, the Copilot SDK, the Codex app server, Kiro, agy, omp, goose, bob) are not compared; the CLIs are probed live at runtime instead. + ### Model pinning and auto-switching Each agent shows a 📌 pin on its CLI and its model in the dashboard. The semantics are deliberately narrow — a model changes out from under you in exactly two cases: diff --git a/src/pkg/dashboard/static_model_lists_shape_test.go b/src/pkg/dashboard/static_model_lists_shape_test.go new file mode 100644 index 0000000000..14bfaa62d1 --- /dev/null +++ b/src/pkg/dashboard/static_model_lists_shape_test.go @@ -0,0 +1,91 @@ +package dashboard + +import ( + "sort" + "testing" +) + +// Every static model list hive ships is a fallback a dropdown can land on, so +// each must be non-empty and duplicate-free, and the copies baked into +// static/index.html must match the Go lists the server serves. The weekly +// static-model-freshness workflow compares the CONTENTS against public +// sources; this test pins the SHAPE. +func TestStaticModelListsShape(t *testing.T) { + lists := map[string][]string{ + "claudeStaticModels": claudeStaticModels, + "claudePinnedCLIModels": claudePinnedCLIModels, + "claudeAlwaysIncludeModels": claudeAlwaysIncludeModels, + "copilotStaticModels": copilotStaticModels, + "copilotPinnedCLIModels": copilotPinnedCLIModels, + "copilotAlwaysIncludeModels": copilotAlwaysIncludeModels, + "codexStaticModels": codexStaticModels, + "geminiStaticModels": geminiStaticModels, + "geminiPinnedCLIModels": geminiPinnedCLIModels, + "bobStaticModels": bobStaticModels, + "piKiroStaticModels": piKiroStaticModels, + "agyStaticModels": agyStaticModels, + "ompStaticModels": ompStaticModels, + "inferenceStaticModelAliases": inferenceStaticModelAliases, + } + for provider, models := range gooseProviderStaticModels { + lists["gooseProviderStaticModels["+provider+"]"] = models + } + names := make([]string, 0, len(lists)) + for name := range lists { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + assertModelListShape(t, name, lists[name]) + } + + html := indexHTML(t) + // Lists the browser shows before /api/config/backends answers. These are + // exact copies of the Go fallbacks, order included. + for goName, jsConst := range map[string]string{ + "claudeStaticModels": "CLAUDE_CLI_MODELS", + "geminiStaticModels": "GEMINI_CLI_MODELS", + "codexStaticModels": "CODEX_CLI_MODELS", + "agyStaticModels": "AGY_CLI_MODELS", + "ompStaticModels": "OMP_CLI_MODELS", + "bobStaticModels": "BOB_CLI_MODELS", + } { + got := htmlModelValues(t, html, jsConst) + assertModelListShape(t, jsConst, got) + if !equalStrings(got, lists[goName]) { + t.Errorf("static/index.html %s = %v, want %s = %v (keep cli_models*.go and backends-tokens.js in sync)", jsConst, got, goName, lists[goName]) + } + } + // Copilot's first-paint list is a curated subset (it leads with auto and + // the picker model); every id it offers must be one the CLI accepts. + copilotJS := htmlModelValues(t, html, "COPILOT_CLI_MODELS") + assertModelListShape(t, "COPILOT_CLI_MODELS", copilotJS) + pinned := map[string]bool{} + for _, m := range copilotPinnedCLIModels { + pinned[m] = true + } + for _, m := range copilotJS { + if !pinned[m] { + t.Errorf("COPILOT_CLI_MODELS offers %q, which is not in copilotPinnedCLIModels", m) + } + } + assertModelListShape(t, "GOOSE_CLI_MODELS", htmlModelValues(t, html, "GOOSE_CLI_MODELS")) +} + +func assertModelListShape(t *testing.T, name string, models []string) { + t.Helper() + if len(models) == 0 { + t.Errorf("%s is empty: a dropdown falling back to it would offer nothing", name) + return + } + seen := map[string]bool{} + for _, m := range models { + if m == "" { + t.Errorf("%s contains an empty id", name) + } + if seen[m] { + t.Errorf("%s lists %q twice", name, m) + } + seen[m] = true + } +}