Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
196 changes: 196 additions & 0 deletions .github/scripts/check-static-model-freshness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Compare hive's static CLI model lists against public, keyless sources.

Run weekly by .github/workflows/static-model-freshness.yml. Prints a Markdown
drift report to stdout (empty body sections are omitted) and exits 0; the
workflow opens or updates ONE tracking issue with it.

Sources (all public, no auth):
* OpenRouter GET https://openrouter.ai/api/v1/models. Used for:
- the curated OpenRouter suggestions (crux/openrouter SuggestedModels and
DefaultModel), which ARE OpenRouter ids: a missing id is drift;
- the Claude, Gemini and OpenAI/Codex/Copilot static fallbacks, mapped
onto their OpenRouter equivalents (anthropic/, google/, openai/). These
are NOT the CLIs' own catalogs, so a miss is a hint to look, and a
newer model in the same family that the static list lacks is reported
as "newer upstream".
* npm registry (registry.npmjs.org/<pkg>/latest) for the CLIs that ship as
npm packages: the latest published version, as context for whether a
CLI release may have changed its model list since the snapshot.

Not covered (need auth or have no machine-readable list): the Claude CLI's
own /v1/models (OAuth), the Copilot SDK catalog (Copilot login), the Codex
app-server catalog (ChatGPT login), Gemini CLI's internal list, Kiro (pi),
agy, omp, goose providers and bob. Those are probed live at runtime by the
dashboard; their static lists are fallbacks and are checked only for shape
(non-empty, no duplicates, Go/JS in sync) by the Go unit tests.
"""
import json
import re
import sys
import urllib.request

OPENROUTER_MODELS = "https://openrouter.ai/api/v1/models"
NPM_LATEST = "https://registry.npmjs.org/{pkg}/latest"
TIMEOUT_S = 30
USER_AGENT = "hive-static-model-freshness"

# Package names of CLIs whose model lists hive snapshots.
NPM_PACKAGES = {
"claude": "@anthropic-ai/claude-code",
"copilot": "@github/copilot",
"codex": "@openai/codex",
"gemini": "@google/gemini-cli",
}

CLI_MODELS_GO = "src/pkg/dashboard/cli_models.go"
OPENROUTER_GO = "src/crux/openrouter/openrouter.go"

# Static Go lists checked against OpenRouter, with the OpenRouter provider
# prefix their ids map onto.
GO_LISTS = {
"claudeStaticModels": "anthropic",
"copilotStaticModels": None, # mixed vendors; prefix inferred per id
"codexStaticModels": "openai",
"geminiStaticModels": "google",
}

ALIASES = {"opus", "sonnet", "haiku", "auto", "default"}


def fetch_json(url):
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "application/json"})
with urllib.request.urlopen(req, timeout=TIMEOUT_S) as resp:
return json.load(resp)


def go_string_list(src, name):
m = re.search(r"var " + re.escape(name) + r" = \[\]string\{(.*?)\n\}", src, re.S)
if not m:
return []
body = re.sub(r"//[^\n]*", "", m.group(1))
return re.findall(r'"([^"]+)"', body)


def openrouter_suggestions(src):
ids = re.findall(r'\{ID: "([^"]+)", Label:', src)
m = re.search(r'const DefaultModel = "([^"]+)"', src)
return ids, (m.group(1) if m else "")


def vendor_for(model_id):
if model_id.startswith("claude-"):
return "anthropic"
if model_id.startswith("gemini-"):
return "google"
if model_id.startswith(("gpt-", "o3", "o4", "codex-")):
return "openai"
return None


def norm(model_id):
"""Canonical comparison key: no date suffix, dots and dashes unified."""
s = model_id.lower().split("/")[-1].split(":")[0]
s = re.sub(r"-\d{8}$", "", s)
return s.replace(".", "-")


def family(key):
"""(family, version) where family is the name around the version.

claude-sonnet-5-5 -> (('claude-sonnet', ''), (5, 5));
gemini-2-5-flash -> (('gemini', 'flash'), (2, 5)). Variants with a
different tail (flash-lite, flash-image) are different families, so only
a genuinely newer version of the same line is reported.
"""
m = re.match(r"^([a-z]+(?:-[a-z]+)*?)-(\d+(?:-\d+)*)(?:-(.*))?$", key)
if not m:
return None, ()
return (m.group(1), m.group(3) or ""), tuple(int(x) for x in m.group(2).split("-"))


def main():
with open(CLI_MODELS_GO, encoding="utf-8") as f:
cli_src = f.read()
with open(OPENROUTER_GO, encoding="utf-8") as f:
or_src = f.read()

sections = []
drift = False
try:
catalog = fetch_json(OPENROUTER_MODELS).get("data", [])
except Exception as exc: # noqa: BLE001 — report, never fail the cron
catalog = []
sections.append(f"### OpenRouter catalog unavailable\n\n`{OPENROUTER_MODELS}` failed: `{exc}`. Nothing was compared against it this week.")
live_ids = {m.get("id", "") for m in catalog if m.get("id")}
by_vendor = {}
for mid in live_ids:
if "/" not in mid or mid.startswith("~"):
continue
vendor, rest = mid.split("/", 1)
by_vendor.setdefault(vendor, set()).add(norm(rest))

if live_ids:
ids, default = openrouter_suggestions(or_src)
missing = [i for i in ids + ([default] if default else []) if i not in live_ids]
if missing:
drift = True
sections.append("### OpenRouter suggestions no longer in the catalog\n\n"
"`crux/openrouter` SuggestedModels / DefaultModel (the dashboard already hides these at runtime; refresh the list and `testdata/models.json`):\n\n"
+ "\n".join(f"- `{i}`" for i in sorted(set(missing))))

for name, prefix in GO_LISTS.items():
ids = [i for i in go_string_list(cli_src, name) if i not in ALIASES]
if not ids:
sections.append(f"### `{name}` could not be parsed\n\nThe checker found no ids; update the parser or the list.")
continue
not_found, newer = [], set()
have = {norm(i) for i in ids}
fams = {}
for i in ids:
vendor = prefix or vendor_for(i)
if not vendor:
continue
key = norm(i)
if key not in by_vendor.get(vendor, set()):
not_found.append(i)
fam, ver = family(key)
if fam:
fams.setdefault((vendor, fam), max(fams.get((vendor, fam), ()), ver))
for (vendor, fam), top in fams.items():
for key in by_vendor.get(vendor, set()):
f2, v2 = family(key)
if f2 == fam and v2 > top and key not in have:
newer.add(f"{vendor}/{key}")
lines = []
if newer:
lines.append("Newer models of the same family on OpenRouter that the list lacks:\n\n" + "\n".join(f"- `{n}`" for n in sorted(newer)))
if not_found:
lines.append("Ids with no OpenRouter counterpart (may be CLI-only aliases; check the CLI's own catalog):\n\n" + "\n".join(f"- `{n}`" for n in sorted(not_found)))
if lines:
drift = True
sections.append(f"### `{name}` ({CLI_MODELS_GO})\n\n" + "\n\n".join(lines))

versions = []
for cli, pkg in NPM_PACKAGES.items():
try:
versions.append(f"- {cli}: `{pkg}` latest `{fetch_json(NPM_LATEST.format(pkg=pkg)).get('version', '?')}`")
except Exception as exc: # noqa: BLE001
versions.append(f"- {cli}: `{pkg}` lookup failed (`{exc}`)")

if not drift:
print("NO_DRIFT")
print("## Static model list freshness\n")
print("Weekly comparison of hive's static model lists against public sources. "
"Static lists are fallbacks: Claude, Copilot, Codex, Gemini, goose, agy and omp are probed live at runtime, "
"and agents on a gateway inherit the gateway's live `/v1/models` list.\n")
print("\n\n".join(sections) if sections else "No drift found against OpenRouter this week.")
print("\n### CLI releases (npm, for context)\n\n" + "\n".join(versions))
print("\n### Not checked\n\nNeeds auth or has no machine-readable list: the Claude CLI's `/v1/models` (OAuth), the Copilot SDK catalog, "
"the Codex app-server catalog, Kiro (pi), agy, omp, goose providers, bob. The Go unit test `TestStaticModelListsShape` still checks every list is "
"non-empty, duplicate-free and matches `static/index.html`.")
return 0


if __name__ == "__main__":
sys.exit(main())
94 changes: 94 additions & 0 deletions .github/workflows/static-model-freshness.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
name: Static model list freshness

# Weekly drift check for hive's STATIC model lists: the fallbacks every probed
# CLI serves when live discovery fails (claude, copilot, codex, gemini) and the
# curated OpenRouter suggestions/default in crux/openrouter.
#
# .github/scripts/check-static-model-freshness.py compares them against
# public, keyless sources only:
# - OpenRouter GET https://openrouter.ai/api/v1/models: the curated
# OpenRouter ids directly, and the Claude/Gemini/OpenAI fallbacks mapped
# onto anthropic/, google/ and openai/ ("newer model of the same line
# upstream", "no upstream counterpart");
# - the npm registry's latest version of each CLI package, as context.
# Sources that need auth (Claude OAuth /v1/models, the Copilot SDK catalog,
# Codex app-server, Kiro, agy, omp, goose, bob) are NOT checked here; the
# report says so. Their lists are shape-checked by the Go unit test
# TestStaticModelListsShape (non-empty, no duplicates, Go/JS in sync).
#
# The report goes into ONE tracking issue (label static-model-freshness),
# created on first drift and edited in place afterwards; when nothing drifts
# an open issue is updated to say so, and no issue is opened. No repository
# secrets are used: only the workflow's own GITHUB_TOKEN, for the issue.

on:
schedule:
- cron: '17 7 * * 1'
workflow_dispatch:
inputs:
dry_run:
description: 'Print the report only; do not open or edit the tracking issue'
type: boolean
default: false

permissions:
contents: read
issues: write

concurrency:
group: static-model-freshness
cancel-in-progress: false

env:
TRACKING_LABEL: static-model-freshness
TRACKING_TITLE: 'Static model lists: weekly freshness report'

jobs:
check:
name: compare static model lists with public sources
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: v5
persist-credentials: false

- name: Build the drift report
id: report
run: |
set -euo pipefail
python3 -I .github/scripts/check-static-model-freshness.py > "$RUNNER_TEMP/raw.md"
if head -n 1 "$RUNNER_TEMP/raw.md" | grep -qx 'NO_DRIFT'; then
echo "drift=false" >> "$GITHUB_OUTPUT"
tail -n +2 "$RUNNER_TEMP/raw.md" > "$RUNNER_TEMP/report.md"
else
echo "drift=true" >> "$GITHUB_OUTPUT"
cp "$RUNNER_TEMP/raw.md" "$RUNNER_TEMP/report.md"
fi
{
echo
echo "_Updated by [${GITHUB_WORKFLOW}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) on $(date -u +%Y-%m-%d)._"
} >> "$RUNNER_TEMP/report.md"
cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"

- name: Open or update the tracking issue
if: ${{ !inputs.dry_run }}
env:
GH_TOKEN: ${{ github.token }}
DRIFT: ${{ steps.report.outputs.drift }}
run: |
set -euo pipefail
gh label create "$TRACKING_LABEL" --repo "$GITHUB_REPOSITORY" --color c5def5 \
--description "Weekly static model list drift report" 2>/dev/null || true
number="$(gh issue list --repo "$GITHUB_REPOSITORY" --label "$TRACKING_LABEL" --state open \
--json number --jq '.[0].number // empty')"
if [ -n "$number" ]; then
gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/report.md"
echo "Updated #$number (drift=$DRIFT)"
elif [ "$DRIFT" = "true" ]; then
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TRACKING_TITLE" \
--label "$TRACKING_LABEL" --body-file "$RUNNER_TEMP/report.md"
else
echo "No drift and no open tracking issue; nothing to do."
fi
1 change: 1 addition & 0 deletions changelog.d/added-static-model-freshness-check.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- A weekly workflow compares hive's static model fallbacks and curated OpenRouter suggestions against OpenRouter's public catalog, plus npm release metadata, and keeps one tracking issue listing any drift. A unit test now fails on empty or duplicate static lists and on Go/JS copies that disagree.
7 changes: 7 additions & 0 deletions src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,13 @@ Agents can inspect peer panes with `hive-panes [lines]` when the deployment incl

Every discovery probe is best-effort: a failed or absent probe falls back to a current static list, so a model dropdown is never empty. Fallback entries are marked "unverified" in the UI.

An agent whose backend is a model gateway (OpenRouter, LiteLLM, vLLM, llm-d, watsonx or a custom OpenAI-compatible endpoint) does not use a CLI's static list at all. It inherits the gateway's live `/v1/models` list: the model dropdown, the Default Model picker and auto-heal all read what the gateway currently serves. The same holds for any CLI you point at a gateway, because its model is then whatever id the gateway accepts. Only the CLI-native backends (claude, copilot, codex, gemini, goose, agy, omp, bob, pi) ever fall back to a static list.

Static lists drift as vendors ship models, so two checks keep them honest:

- `TestStaticModelListsShape` (in `src/pkg/dashboard`) fails a PR if any static list is empty or has duplicates, or if a first-paint copy in `static/index.html` disagrees with its Go list.
- The weekly [Static model list freshness](../../.github/workflows/static-model-freshness.yml) workflow compares the lists with public, keyless sources. The OpenRouter catalog is used directly for the curated OpenRouter suggestions, and for Claude, Gemini and OpenAI ids mapped onto `anthropic/`, `google/` and `openai/`. npm release versions of the CLIs are included as context. The workflow keeps one tracking issue (label `static-model-freshness`) up to date. Catalogs that need auth (Claude OAuth `/v1/models`, the Copilot SDK, the Codex app server, Kiro, agy, omp, goose, bob) are not compared; the CLIs are probed live at runtime instead.

### Model pinning and auto-switching

Each agent shows a 📌 pin on its CLI and its model in the dashboard. The semantics are deliberately narrow — a model changes out from under you in exactly two cases:
Expand Down
91 changes: 91 additions & 0 deletions src/pkg/dashboard/static_model_lists_shape_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package dashboard

import (
"sort"
"testing"
)

// Every static model list hive ships is a fallback a dropdown can land on, so
// each must be non-empty and duplicate-free, and the copies baked into
// static/index.html must match the Go lists the server serves. The weekly
// static-model-freshness workflow compares the CONTENTS against public
// sources; this test pins the SHAPE.
func TestStaticModelListsShape(t *testing.T) {
lists := map[string][]string{
"claudeStaticModels": claudeStaticModels,
"claudePinnedCLIModels": claudePinnedCLIModels,
"claudeAlwaysIncludeModels": claudeAlwaysIncludeModels,
"copilotStaticModels": copilotStaticModels,
"copilotPinnedCLIModels": copilotPinnedCLIModels,
"copilotAlwaysIncludeModels": copilotAlwaysIncludeModels,
"codexStaticModels": codexStaticModels,
"geminiStaticModels": geminiStaticModels,
"geminiPinnedCLIModels": geminiPinnedCLIModels,
"bobStaticModels": bobStaticModels,
"piKiroStaticModels": piKiroStaticModels,
"agyStaticModels": agyStaticModels,
"ompStaticModels": ompStaticModels,
"inferenceStaticModelAliases": inferenceStaticModelAliases,
}
for provider, models := range gooseProviderStaticModels {
lists["gooseProviderStaticModels["+provider+"]"] = models
}
names := make([]string, 0, len(lists))
for name := range lists {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
assertModelListShape(t, name, lists[name])
}

html := indexHTML(t)
// Lists the browser shows before /api/config/backends answers. These are
// exact copies of the Go fallbacks, order included.
for goName, jsConst := range map[string]string{
"claudeStaticModels": "CLAUDE_CLI_MODELS",
"geminiStaticModels": "GEMINI_CLI_MODELS",
"codexStaticModels": "CODEX_CLI_MODELS",
"agyStaticModels": "AGY_CLI_MODELS",
"ompStaticModels": "OMP_CLI_MODELS",
"bobStaticModels": "BOB_CLI_MODELS",
} {
got := htmlModelValues(t, html, jsConst)
assertModelListShape(t, jsConst, got)
if !equalStrings(got, lists[goName]) {
t.Errorf("static/index.html %s = %v, want %s = %v (keep cli_models*.go and backends-tokens.js in sync)", jsConst, got, goName, lists[goName])
}
}
// Copilot's first-paint list is a curated subset (it leads with auto and
// the picker model); every id it offers must be one the CLI accepts.
copilotJS := htmlModelValues(t, html, "COPILOT_CLI_MODELS")
assertModelListShape(t, "COPILOT_CLI_MODELS", copilotJS)
pinned := map[string]bool{}
for _, m := range copilotPinnedCLIModels {
pinned[m] = true
}
for _, m := range copilotJS {
if !pinned[m] {
t.Errorf("COPILOT_CLI_MODELS offers %q, which is not in copilotPinnedCLIModels", m)
}
}
assertModelListShape(t, "GOOSE_CLI_MODELS", htmlModelValues(t, html, "GOOSE_CLI_MODELS"))
}

func assertModelListShape(t *testing.T, name string, models []string) {
t.Helper()
if len(models) == 0 {
t.Errorf("%s is empty: a dropdown falling back to it would offer nothing", name)
return
}
seen := map[string]bool{}
for _, m := range models {
if m == "" {
t.Errorf("%s contains an empty id", name)
}
if seen[m] {
t.Errorf("%s lists %q twice", name, m)
}
seen[m] = true
}
}
Loading