Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
339 changes: 314 additions & 25 deletions .github/scripts/check-static-model-freshness.py

Large diffs are not rendered by default.

83 changes: 83 additions & 0 deletions .github/scripts/claude-model-freshness.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/usr/bin/env bash
# Builds the weekly Claude model freshness change in the working tree (#18754).
# Run by .github/workflows/claude-model-freshness.yml; see its header.
#
# 1. fetches OpenRouter's public model catalogue once (both steps below read
# the same snapshot);
# 2. installs the Claude Code version pinned in src/Dockerfile exactly the
# way the image does (npm --ignore-scripts, then the vendor's install.cjs)
# so the candidates can be probed against it, unauthenticated;
# 3. check-static-model-freshness.py --apply claude: candidates, the probe
# gate, and the edits to claudePinnedCLIModels, claudeStaticModels,
# CLAUDE_CLI_MODELS and cli-model-requirements.txt;
# 4. for the accepted ids: `pricetable --add` (new modelPrices rows, only
# through its plausibility rules) and `make dashboard-gen`.
#
# Usage: claude-model-freshness.sh <out dir>
# Env: DRY_RUN=1 prints every edit and writes nothing.
# SOURCE_URL overrides the catalogue URL.
# Out: <out>/report.md (the PR body), <out>/accepted.txt (one id per line;
# empty when nothing is added), <out>/pin.txt.
set -euo pipefail

out="${1:?usage: claude-model-freshness.sh <out dir>}"
mkdir -p "$out"
out="$(cd "$out" && pwd)"
cd "$(git rev-parse --show-toplevel)"

source_url="${SOURCE_URL:-https://openrouter.ai/api/v1/models}"
catalog="$out/openrouter.json"
curl -fsSL --retry 3 --max-time 60 -H 'Accept: application/json' -A hive-claude-model-freshness \
"$source_url" -o "$catalog"

pin="$(sed -n 's/^ARG CLAUDE_CODE_VERSION=\([^ ]*\).*/\1/p' src/Dockerfile | head -1)"
[ -n "$pin" ] || { echo "no ARG CLAUDE_CODE_VERSION in src/Dockerfile" >&2; exit 1; }
printf '%s\n' "$pin" > "$out/pin.txt"

# Without the binary nothing is added: every candidate is reported under
# "needs a check" instead (check-static-model-freshness.py gate_candidates).
claude_bin=""
prefix="$out/claude-code"
if npm install --prefix "$prefix" --ignore-scripts --no-fund --no-audit \
"@anthropic-ai/claude-code@${pin}" >&2 \
&& node "$prefix/node_modules/@anthropic-ai/claude-code/install.cjs" >&2 \
&& "$prefix/node_modules/.bin/claude" --version >&2; then
claude_bin="$prefix/node_modules/.bin/claude"
else
echo "::warning::Could not install Claude Code ${pin} to probe; no model will be added this run."
fi

dry=()
[ "${DRY_RUN:-}" = "1" ] && dry=(--dry-run)
probe=()
[ -n "$claude_bin" ] && probe=(--claude-bin "$claude_bin")

python3 -I .github/scripts/check-static-model-freshness.py --apply claude \
--catalog "$catalog" --accepted-out "$out/accepted.txt" "${probe[@]}" "${dry[@]}" > "$out/lists.md"
cp "$out/lists.md" "$out/report.md"

if [ ! -s "$out/accepted.txt" ]; then
echo "no Claude model to add"
exit 0
fi

ids="$(paste -sd, "$out/accepted.txt")"
(cd src && go run ./internal/tools/pricetable --source "$catalog" --add "$ids" \
--report "$out/prices.md" "${dry[@]}")
{ echo; cat "$out/prices.md"; } >> "$out/report.md"

if [ "${DRY_RUN:-}" != "1" ]; then
make dashboard-gen
make dashboard-gen-check
test -z "$(gofmt -l src/pkg/tokens/pricing.go src/pkg/dashboard/cli_models.go)" \
|| { echo "::error::generated Go files are not gofmt-clean" >&2; exit 1; }
{
echo
echo "### Model lists against the CLI pins"
echo
echo '```'
HIVE_MODEL_CHECK_STRICT=0 bash src/scripts/check-cli-model-lists.sh 2>&1 || true
echo '```'
} >> "$out/report.md"
fi
echo "Claude models to add: $ids"
2 changes: 1 addition & 1 deletion .github/scripts/integration-governance.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000;
const MAX_AUTHOR_BATCH = 50;
// This literal is inside the enrolled .github closure. A caller-supplied
// preparedBy.sourceSHA must never choose a new executable verifier tree.
const TRUSTED_SCRIPTS_TREE = 'c269cb813e67f341946dedd7ba9ab967a0530831';
const TRUSTED_SCRIPTS_TREE = '22b8f6c54d4b00a9d1193a5e2416591691177970';
const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4';
const sha = value => crypto.createHash('sha256').update(value).digest('hex');
const git = (args, cwd = process.cwd()) =>
Expand Down
217 changes: 217 additions & 0 deletions .github/scripts/test-claude-model-freshness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Hermetic tests for check-static-model-freshness.py --apply claude (#18754).

No network and no real Claude Code: the catalog is a fixture and the pinned
CLI is a fake script that answers --version and the unauthenticated probe.

python3 .github/scripts/test-claude-model-freshness.py
"""

import importlib.util
import os
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import unittest

HERE = os.path.dirname(os.path.abspath(__file__))
SCRIPT = os.path.join(HERE, "check-static-model-freshness.py")
FIXTURES = os.path.join(HERE, "testdata", "claude-model-freshness")

spec = importlib.util.spec_from_file_location("freshness", SCRIPT)
freshness = importlib.util.module_from_spec(spec)
spec.loader.exec_module(freshness)

LAYOUT = {
"cli_models.go.txt": freshness.CLI_MODELS_GO,
"backends-tokens.js.txt": freshness.BACKENDS_JS,
"cli-model-requirements.txt": freshness.REQUIREMENTS,
"Dockerfile.txt": freshness.DOCKERFILE,
}

FAKE_CLAUDE = """#!/usr/bin/env python3
import sys
args = sys.argv[1:]
if args == ["--version"]:
print({version!r} + " (Claude Code)")
sys.exit(0)
model = args[args.index("--model") + 1]
if {mark!r} and (model in {unknown!r} or model.startswith("claude-hive-probe")):
print('[claude-code:unrecognized_model] {{"model":"' + model + '","query_source":"sdk"}}')
print("Not logged in \\u00b7 Please run /login")
sys.exit(1)
"""


def read(path):
with open(path, encoding="utf-8") as fh:
return fh.read()


def load_catalog(name):
import json
with open(os.path.join(FIXTURES, name), encoding="utf-8") as fh:
return json.load(fh)["data"]


def go_list(src, name):
return freshness.go_string_list(src, name)


def js_values(src):
m = re.search(r"const CLAUDE_CLI_MODELS = \[\n(.*?)\n\s*\];", src, re.S)
return re.findall(r"value: '([^']+)'", m.group(1))


class Base(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="claude-freshness-test-")
self.root = os.path.join(self.tmp, "repo")
for fixture, rel in LAYOUT.items():
dst = os.path.join(self.root, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(os.path.join(FIXTURES, fixture), dst)

def tearDown(self):
shutil.rmtree(self.tmp, ignore_errors=True)

def fake_claude(self, unknown=(), version="2.1.326", mark=True):
path = os.path.join(self.tmp, "claude")
with open(path, "w", encoding="utf-8") as fh:
fh.write(FAKE_CLAUDE.format(version=version, unknown=list(unknown), mark=mark))
os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR)
return path

def snapshot(self):
return {rel: read(os.path.join(self.root, rel)) for rel in LAYOUT.values()}

def run_cli(self, *extra, catalog="openrouter.json"):
accepted = os.path.join(self.tmp, "accepted.txt")
proc = subprocess.run([sys.executable, "-I", SCRIPT, "--apply", "claude", "--root", self.root,
"--catalog", os.path.join(FIXTURES, catalog), "--accepted-out", accepted, *extra],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120)
self.assertEqual(proc.returncode, 0, proc.stdout)
return proc.stdout, read(accepted).split()


class Candidates(Base):
def test_same_line_newer_only_dashed_and_labelled(self):
src = read(os.path.join(self.root, freshness.CLI_MODELS_GO))
got = freshness.claude_candidates(src, load_catalog("openrouter.json"))
self.assertEqual([(c["cli_id"], c["source_id"], c["label"]) for c in got], [
("claude-fable-5-1", "anthropic/claude-fable-5.1", "Fable 5.1"),
("claude-fable-5-2", "anthropic/claude-fable-5.2", "Fable 5.2"),
("claude-haiku-5-5", "anthropic/claude-haiku-5.5", "Haiku 5.5"),
])

def test_excluded(self):
src = read(os.path.join(self.root, freshness.CLI_MODELS_GO))
ids = {c["cli_id"] for c in freshness.claude_candidates(src, load_catalog("openrouter.json"))}
for never in ("claude-mythos-5", "claude-mythos-5-5-preview", # limited access
"claude-sonnet-6", # expiration_date set
"claude-opus-4-1", # older than the list
"claude-opus-5-5", # already listed
"claude-opus-6-fast", "claude-3-7-sonnet"): # not a plain line id
self.assertNotIn(never, ids)

def test_dotted_to_dashed(self):
self.assertEqual(freshness.norm("anthropic/claude-fable-5.2"), "claude-fable-5-2")
self.assertEqual(freshness.claude_line("claude-fable-5-2"), ("fable", (5, 2)))
self.assertIsNone(freshness.claude_line("claude-mythos-5"))
self.assertEqual(freshness.claude_label("haiku", (5, 5)), "Haiku 5.5")


class Apply(Base):
def test_dry_run_prints_every_edit_and_writes_nothing(self):
before = self.snapshot()
out, accepted = self.run_cli("--dry-run", "--claude-bin", self.fake_claude(unknown=["claude-fable-5-2"]))
self.assertEqual(accepted, ["claude-fable-5-1", "claude-haiku-5-5"])
self.assertEqual(self.snapshot(), before, "a dry run must not write")
self.assertIn("**Dry run:** nothing was written.", out)
for cid, label in (("claude-fable-5-1", "Fable 5.1"), ("claude-haiku-5-5", "Haiku 5.5")):
section = out.split(f"#### {label} (`{cid}`")[1].split("####")[0].split("###")[0]
for want in (f"`claudePinnedCLIModels` gains `\"{cid}\"`",
f"`claudeStaticModels` gains `\"{cid}\"`",
f"`CLAUDE_CLI_MODELS` gains `{{ value: '{cid}', label: '{label}' }},`",
"`src/pkg/dashboard/static/index.html`: rebuilt with `make dashboard-gen`",
f"`claude {cid} 2.1.326`",
"`modelPrices` row from the OpenRouter quote, through `pricetable --add`"):
self.assertIn(want, section, f"{cid}: missing edit {want!r}\n{out}")
needs = out.split("### Needs a check against the Claude Code changelog (1)")[1]
self.assertIn("Fable 5.2 (`claude-fable-5-2`, from `anthropic/claude-fable-5.2`): the pinned Claude Code does not recognise this id", needs)
self.assertNotIn("#### Fable 5.2", out, "an id the pinned CLI does not recognise is never added")

def test_apply_edits_in_sync_and_idempotent(self):
claude = self.fake_claude(unknown=["claude-fable-5-2"])
_, accepted = self.run_cli("--claude-bin", claude)
self.assertEqual(accepted, ["claude-fable-5-1", "claude-haiku-5-5"])
go = read(os.path.join(self.root, freshness.CLI_MODELS_GO))
js = read(os.path.join(self.root, freshness.BACKENDS_JS))
reqs = read(os.path.join(self.root, freshness.REQUIREMENTS))
want = ["claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-1", "claude-fable-5",
"claude-haiku-5-5", "claude-haiku-4-5-20251001", "opus", "sonnet", "haiku"]
self.assertEqual(go_list(go, "claudeStaticModels"), want)
self.assertEqual(go_list(go, "claudePinnedCLIModels"), want)
self.assertEqual(js_values(js), want, "JS list must stay an exact copy of claudeStaticModels")
self.assertEqual(go_list(go, "geminiStaticModels"), ["gemini-2.5-pro"], "other lists untouched")
self.assertIn(" { value: 'claude-fable-5-1', label: 'Fable 5.1' },\n", js)
self.assertIn('\t"claude-haiku-5-5",\n', go)
self.assertTrue(reqs.endswith("claude claude-fable-5-1 2.1.326\nclaude claude-haiku-5-5 2.1.326\n"), reqs)

after = self.snapshot()
out, again = self.run_cli("--claude-bin", claude)
self.assertEqual(again, [])
self.assertEqual(self.snapshot(), after, "a second run with the same candidates changes nothing")
self.assertIn("### Added (0)", out)

def test_insert_before_aliases_when_line_is_new_to_the_list(self):
lines = ['\t"claude-opus-5-5",', "\t// aliases", '\t"opus",']
got = freshness.insert_listed(lines, "claude-haiku-5-5", lambda ind: f'{ind}"claude-haiku-5-5",', freshness.GO_ID)
self.assertEqual(got, ['\t"claude-opus-5-5",', '\t"claude-haiku-5-5",', "\t// aliases", '\t"opus",'])
self.assertIsNone(freshness.insert_listed(got, "claude-haiku-5-5", lambda ind: "", freshness.GO_ID))

def test_unreliable_probe_adds_nothing(self):
before = self.snapshot()
out, accepted = self.run_cli("--claude-bin", self.fake_claude(mark=False))
self.assertEqual(accepted, [])
self.assertEqual(self.snapshot(), before)
self.assertIn("probe controls failed", out)
self.assertIn("### Needs a check against the Claude Code changelog (3)", out)

def test_wrong_version_adds_nothing(self):
before = self.snapshot()
out, accepted = self.run_cli("--claude-bin", self.fake_claude(version="2.1.311"))
self.assertEqual(accepted, [])
self.assertEqual(self.snapshot(), before)
self.assertIn("not the pinned `2.1.326`", out)

def test_no_binary_adds_nothing(self):
before = self.snapshot()
out, accepted = self.run_cli()
self.assertEqual(accepted, [])
self.assertEqual(self.snapshot(), before)
self.assertIn("no pinned Claude Code binary", out)

def test_no_candidates_no_edits(self):
before = self.snapshot()
out, accepted = self.run_cli("--claude-bin", self.fake_claude(), catalog="openrouter-current.json")
self.assertEqual(accepted, [])
self.assertEqual(self.snapshot(), before)
self.assertIn("Gate: no candidates.", out)
self.assertNotIn("Needs a check", out)


class Report(unittest.TestCase):
def test_compare_list_shared_with_report(self):
by_vendor = freshness.index_catalog(load_catalog("openrouter.json"))
not_found, newer = freshness.compare_list(["claude-fable-5", "claude-opus-9"], "anthropic", by_vendor)
self.assertEqual(not_found, ["claude-opus-9"])
self.assertIn(("anthropic", "claude-fable-5-2"), newer)
self.assertIn(("anthropic", "claude-fable-5-1"), newer)


if __name__ == "__main__":
unittest.main(verbosity=2)
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM scratch
ARG CODEX_VERSION=0.169.0
ARG CLAUDE_CODE_VERSION=2.1.326
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Miniature of backends-tokens.js for test-claude-model-freshness.py.
const CLAUDE_CLI_MODELS = [
{ value: 'claude-opus-5-5', label: 'Opus 5.5' },
{ value: 'claude-sonnet-5-5', label: 'Sonnet 5.5' },
{ value: 'claude-fable-5', label: 'Fable 5' },
{ value: 'claude-haiku-4-5-20251001', label: 'Haiku 4.5' },
{ value: 'opus', label: 'Opus (alias)' },
{ value: 'sonnet', label: 'Sonnet (alias)' },
{ value: 'haiku', label: 'Haiku (alias)' },
];
const GEMINI_CLI_MODELS = [
{ value: 'gemini-2.5-pro', label: 'Gemini 2.5 Pro' },
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# <cli> <model id> <first CLI version that accepts it>
claude claude-sonnet-5-5 2.1.284
31 changes: 31 additions & 0 deletions .github/scripts/testdata/claude-model-freshness/cli_models.go.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package dashboard

// Miniature of src/pkg/dashboard/cli_models.go for
// test-claude-model-freshness.py: the two Claude lists --apply claude edits.

var claudeStaticModels = []string{
"claude-opus-5-5",
"claude-sonnet-5-5",
"claude-fable-5",
"claude-haiku-4-5-20251001",
// Bare aliases the Claude CLI also accepts (see claudeAlwaysIncludeModels).
"opus",
"sonnet",
"haiku",
}

var geminiStaticModels = []string{
"gemini-2.5-pro",
}

// claudePinnedCLIModels is the closed model set accepted by the Claude Code
// version pinned in src/Dockerfile.
var claudePinnedCLIModels = []string{
"claude-opus-5-5",
"claude-sonnet-5-5",
"claude-fable-5",
"claude-haiku-4-5-20251001",
"opus",
"sonnet",
"haiku",
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"data": [
{"id": "anthropic/claude-opus-5.5"},
{"id": "anthropic/claude-fable-5"},
{"id": "anthropic/claude-haiku-4.5"},
{"id": "anthropic/claude-mythos-6"}
]
}
19 changes: 19 additions & 0 deletions .github/scripts/testdata/claude-model-freshness/openrouter.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"data": [
{"id": "anthropic/claude-opus-5.5"},
{"id": "anthropic/claude-opus-4.1"},
{"id": "anthropic/claude-fable-5"},
{"id": "anthropic/claude-fable-5.1"},
{"id": "anthropic/claude-fable-5.2"},
{"id": "anthropic/claude-fable-5.2:batch"},
{"id": "anthropic/claude-haiku-5.5"},
{"id": "anthropic/claude-mythos-5"},
{"id": "anthropic/claude-mythos-5.5-preview"},
{"id": "anthropic/claude-sonnet-6", "expiration_date": "2026-12-01"},
{"id": "anthropic/claude-opus-6-fast"},
{"id": "anthropic/claude-3.7-sonnet"},
{"id": "~anthropic/claude-opus-latest"},
{"id": "google/gemini-9-pro"},
{"id": null}
]
}
Loading
Loading