From 888179a749ff5657b6fbb84562999dab3165e862 Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:06:48 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Weekly=20PR=20adding=20new=20Cl?= =?UTF-8?q?aude=20models=20the=20pinned=20Claude=20Code=20accepts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds .github/workflows/claude-model-freshness.yml: every Monday it finds Claude ids on OpenRouter newer than the dashboard's Claude lists (same comparison as the static-model-freshness report), probes each with the Claude Code version pinned in src/Dockerfile (unauthenticated, empty HOME: the CLI prints an unrecognized_model diagnostic for ids it does not know), and for recognised ids edits claudePinnedCLIModels, claudeStaticModels, CLAUDE_CLI_MODELS, static/index.html, cli-model-requirements.txt and modelPrices (via the new `pricetable --add`, which applies the existing plausibility rules). One needs-human PR on bot/claude-model-freshness, updated in place; no accepted model means no branch and no PR. Refs #18754 Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcWm7qFmFJp26r9LPdmi3t --- .../scripts/check-static-model-freshness.py | 339 ++++++++++++++++-- .github/scripts/claude-model-freshness.sh | 83 +++++ .../scripts/test-claude-model-freshness.py | 217 +++++++++++ .../claude-model-freshness/Dockerfile.txt | 3 + .../backends-tokens.js.txt | 13 + .../cli-model-requirements.txt | 2 + .../claude-model-freshness/cli_models.go.txt | 31 ++ .../openrouter-current.json | 8 + .../claude-model-freshness/openrouter.json | 19 + .github/workflows/claude-model-freshness.yml | 255 +++++++++++++ .../added-18754-claude-model-freshness.md | 1 + src/docs/cli-pins.md | 11 + src/internal/tools/pricetable/add.go | 318 ++++++++++++++++ src/internal/tools/pricetable/add_test.go | 190 ++++++++++ src/internal/tools/pricetable/main.go | 13 + src/internal/tools/pricetable/reconcile.go | 16 +- src/internal/tools/pricetable/table.go | 5 +- .../pricetable/testdata/openrouter-add.json | 10 + src/scripts/test-check-cli-model-lists.sh | 11 + 19 files changed, 1518 insertions(+), 27 deletions(-) create mode 100755 .github/scripts/claude-model-freshness.sh create mode 100755 .github/scripts/test-claude-model-freshness.py create mode 100644 .github/scripts/testdata/claude-model-freshness/Dockerfile.txt create mode 100644 .github/scripts/testdata/claude-model-freshness/backends-tokens.js.txt create mode 100644 .github/scripts/testdata/claude-model-freshness/cli-model-requirements.txt create mode 100644 .github/scripts/testdata/claude-model-freshness/cli_models.go.txt create mode 100644 .github/scripts/testdata/claude-model-freshness/openrouter-current.json create mode 100644 .github/scripts/testdata/claude-model-freshness/openrouter.json create mode 100644 .github/workflows/claude-model-freshness.yml create mode 100644 changelog.d/added-18754-claude-model-freshness.md create mode 100644 src/internal/tools/pricetable/add.go create mode 100644 src/internal/tools/pricetable/add_test.go create mode 100644 src/internal/tools/pricetable/testdata/openrouter-add.json diff --git a/.github/scripts/check-static-model-freshness.py b/.github/scripts/check-static-model-freshness.py index a1e1553481..8a2299d56c 100755 --- a/.github/scripts/check-static-model-freshness.py +++ b/.github/scripts/check-static-model-freshness.py @@ -25,9 +25,13 @@ dashboard; their static lists are fallbacks and are checked only for shape (non-empty, no duplicates, Go/JS in sync) by the Go unit tests. """ +import argparse import json +import os import re +import subprocess import sys +import tempfile import urllib.request OPENROUTER_MODELS = "https://openrouter.ai/api/v1/models" @@ -109,6 +113,311 @@ def family(key): return (m.group(1), m.group(3) or ""), tuple(int(x) for x in m.group(2).split("-")) +def index_catalog(catalog): + """{vendor: {normalised key}} for every plain OpenRouter id.""" + by_vendor = {} + for m in catalog: + mid = m.get("id") or "" + if "/" not in mid or mid.startswith("~"): + continue + vendor, rest = mid.split("/", 1) + by_vendor.setdefault(vendor, set()).add(norm(rest)) + return by_vendor + + +def compare_list(ids, prefix, by_vendor): + """The comparison behind the report, for one static list. + + Returns (not_found, newer): the list's ids with no catalog counterpart, + and the (vendor, key) pairs the catalog has in the same family at a + higher version than the list's newest entry of that family. + """ + not_found, newer = [], set() + have = {norm(i) for i in ids} + fams = {} + for i in ids: + vendor = prefix or vendor_for(i) + if not vendor: + continue + key = norm(i) + if key not in by_vendor.get(vendor, set()): + not_found.append(i) + fam, ver = family(key) + if fam: + fams.setdefault((vendor, fam), max(fams.get((vendor, fam), ()), ver)) + for (vendor, fam), top in fams.items(): + for key in by_vendor.get(vendor, set()): + f2, v2 = family(key) + if f2 == fam and v2 > top and key not in have: + newer.add((vendor, key)) + return not_found, newer + + +# ---- --apply claude: add newer Claude models the pinned CLI accepts (#18754) ---- +# +# Run weekly by .github/workflows/claude-model-freshness.yml. The candidates +# are the same comparison the report makes (compare_list over +# claudePinnedCLIModels and claudeStaticModels against OpenRouter's anthropic/ +# ids), narrowed to the four Claude lines. Each candidate is then put to the +# Claude Code version pinned in src/Dockerfile, offline and without +# credentials: an unauthenticated `claude --model -p x` prints an +# `unrecognized_model` diagnostic for an id the CLI's built-in model table +# does not know, and nothing of the kind for one it does, before it stops at +# "Not logged in". The "version X or newer is required" refusal (#8417) comes +# from the API after login, so it cannot be probed without credentials. Two +# controls run first (a listed id must be recognised, a made-up id must not); +# if either fails, the CLI's output can no longer be read and every candidate +# goes under "needs a check" instead of being added. + +CLAUDE_LINES = ("fable", "opus", "sonnet", "haiku") +CLAUDE_KEY = re.compile(r"^claude-(" + "|".join(CLAUDE_LINES) + r")-(\d+(?:-\d+)*)$") +CLAUDE_GO_LISTS = ("claudePinnedCLIModels", "claudeStaticModels") +CLAUDE_JS_CONST = "CLAUDE_CLI_MODELS" +BACKENDS_JS = "src/pkg/dashboard/static-src/js/backends-tokens.js" +REQUIREMENTS = "src/scripts/cli-model-requirements.txt" +DOCKERFILE = "src/Dockerfile" +PROBE_NEGATIVE = "claude-hive-probe-0-0" +PROBE_TIMEOUT_S = 90 +UNRECOGNIZED_MARKERS = ("unrecognized_model", "doesn't recognize the model") +REFUSED = re.compile(r"version \S+ or newer is required", re.I) +GO_ID = re.compile(r'^\s*"([^"]+)",') +JS_ID = re.compile(r"^\s*\{\s*value:\s*'([^']+)'") + + +def claude_line(key): + """('fable', (5, 2)) for claude-fable-5-2; None for anything else.""" + m = CLAUDE_KEY.match(key) + if not m: + return None + return m.group(1), tuple(int(x) for x in m.group(2).split("-")) + + +def claude_label(line, version): + return f"{line.capitalize()} {'.'.join(str(v) for v in version)}" + + +def retired(model): + """OpenRouter sets expiration_date on a model it has scheduled for + withdrawal; one already retired or on its way out is never added.""" + return bool(model.get("expiration_date")) + + +def claude_candidates(cli_src, catalog): + """Claude ids on OpenRouter newer than the lists' newest of the same line. + + Mythos (limited access), retired ids, variants (":thinking") and router + aliases ("~anthropic/...") are never candidates. Each candidate carries + the Claude CLI's dashed spelling: anthropic/claude-fable-5.2 -> + claude-fable-5-2. + """ + listed = [] + for name in CLAUDE_GO_LISTS: + listed += [i for i in go_string_list(cli_src, name) if i not in ALIASES] + usable = [m for m in catalog + if (m.get("id") or "").startswith("anthropic/") + and ":" not in m["id"] + and "mythos" not in m["id"].lower() + and not retired(m)] + source = {norm(m["id"]): m["id"] for m in usable} + _, newer = compare_list(listed, "anthropic", index_catalog(usable)) + out = [] + for _, key in newer: + lv = claude_line(key) + if not lv: + continue + out.append({"source_id": source[key], "cli_id": key, "line": lv[0], + "version": lv[1], "label": claude_label(*lv)}) + return sorted(out, key=lambda c: (CLAUDE_LINES.index(c["line"]), c["version"])) + + +def probe_claude(claude_bin, model): + """Classify the pinned CLI's offline answer for one --model value.""" + with tempfile.TemporaryDirectory(prefix="claude-probe-") as home: + # Only PATH and an empty HOME: no API key, no OAuth token, no config. + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": home, "LANG": "C.UTF-8"} + try: + proc = subprocess.run([claude_bin, "--model", model, "-p", "x"], cwd=home, env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, timeout=PROBE_TIMEOUT_S) + except (OSError, subprocess.TimeoutExpired) as exc: + return "error", str(exc) + out = proc.stdout or "" + if any(marker in out for marker in UNRECOGNIZED_MARKERS): + return "unrecognized", out + if REFUSED.search(out): + return "refused", out + return "recognized", out + + +def claude_version(claude_bin): + """The binary's `--version` output, or the error running it.""" + try: + return subprocess.run([claude_bin, "--version"], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, timeout=PROBE_TIMEOUT_S).stdout.strip() + except (OSError, subprocess.TimeoutExpired) as exc: + return f"error: {exc}" + + +def gate_candidates(candidates, claude_bin, known_id, pin, probe=probe_claude): + """Split candidates into (accepted, needs_check, note).""" + if not candidates: + return [], [], "no candidates" + if not claude_bin: + note = "no pinned Claude Code binary was available to probe" + return [], [(c, note) for c in candidates], note + version = claude_version(claude_bin) + if not re.match(re.escape(pin) + r"\b", version): + note = f"the binary to probe reports `{version[:80]}`, not the pinned `{pin}`" + return [], [(c, note) for c in candidates], note + pos, _ = probe(claude_bin, known_id) + neg, _ = probe(claude_bin, PROBE_NEGATIVE) + if pos != "recognized" or neg != "unrecognized": + note = (f"probe controls failed (listed id `{known_id}` -> {pos}, made-up id `{PROBE_NEGATIVE}` -> {neg}), " + "so the pinned CLI's offline answer cannot be trusted this run") + return [], [(c, note) for c in candidates], note + accepted, needs = [], [] + for c in candidates: + verdict, out = probe(claude_bin, c["cli_id"]) + if verdict == "recognized": + accepted.append(c) + elif verdict == "unrecognized": + needs.append((c, "the pinned Claude Code does not recognise this id")) + elif verdict == "refused": + needs.append((c, "the pinned Claude Code refuses it: " + REFUSED.search(out).group(0))) + else: + needs.append((c, "probe error: " + out.strip()[:200])) + note = (f"each id probed with the pinned CLI, unauthenticated; controls passed (listed `{known_id}` recognised, " + f"made-up `{PROBE_NEGATIVE}` not)") + return accepted, needs, note + + +def insert_listed(lines, cli_id, render, id_re): + """Insert render(indent) into a list body for cli_id; None when present. + + Placement: before the first entry of the same Claude line with a lower + version, else after that line's last entry, else before the bare aliases + (and the comment above them), else at the end. Go and JS lists get the + same placement, so lists that matched before still match. + """ + ids = [(m.group(1) if (m := id_re.match(ln)) else None) for ln in lines] + if cli_id in ids: + return None + line, version = claude_line(cli_id) + same = [] + for i, x in enumerate(ids): + lv = claude_line(norm(x)) if x else None + if lv and lv[0] == line: + same.append((i, lv[1])) + at = next((i for i, v in same if v < version), None) + if at is None and same: + at = same[-1][0] + 1 + if at is None: + at = next((i for i, x in enumerate(ids) if x in ALIASES), len(lines)) + while 0 < at < len(lines) and lines[at - 1].strip().startswith("//"): + at -= 1 + sample = next((lines[i] for i, x in enumerate(ids) if x), "\t") + indent = re.match(r"\s*", sample).group(0) + return lines[:at] + [render(indent)] + lines[at:] + + +def edit_block(src, pattern, cli_id, render, id_re): + """Apply insert_listed to the list body the pattern's group 2 spans.""" + m = re.search(pattern, src, re.S) + if not m: + raise ValueError(f"list not found: {pattern}") + new = insert_listed(m.group(2).split("\n"), cli_id, render, id_re) + if new is None: + return src, False + return src[:m.start(2)] + "\n".join(new) + src[m.end(2):], True + + +def apply_claude(root, catalog, claude_bin, dry_run): + """Gate the candidates and make the list edits. Returns (report, accepted).""" + def path(rel): + return os.path.join(root, rel) + + def read(rel): + with open(path(rel), encoding="utf-8") as f: + return f.read() + + go_src, js_src, reqs = read(CLI_MODELS_GO), read(BACKENDS_JS), read(REQUIREMENTS) + pin_m = re.search(r"^ARG CLAUDE_CODE_VERSION=(\S+)", read(DOCKERFILE), re.M) + if not pin_m: + raise ValueError(f"no ARG CLAUDE_CODE_VERSION in {DOCKERFILE}") + pin = pin_m.group(1) + known = next(i for i in go_string_list(go_src, "claudePinnedCLIModels") if i not in ALIASES) + candidates = claude_candidates(go_src, catalog) + accepted, needs, note = gate_candidates(candidates, claude_bin, known, pin) + + added = [] + for c in accepted: + cid, edits = c["cli_id"], [] + for name in CLAUDE_GO_LISTS: + go_src, changed = edit_block(go_src, r"(var " + re.escape(name) + r" = \[\]string\{\n)(.*?)(\n\})", + cid, lambda ind, cid=cid: f'{ind}"{cid}",', GO_ID) + if changed: + edits.append(f"`{CLI_MODELS_GO}`: `{name}` gains `\"{cid}\"`") + js_line = f"{{ value: '{cid}', label: '{c['label']}' }}," + js_src, changed = edit_block(js_src, r"(const " + CLAUDE_JS_CONST + r" = \[\n)(.*?)(\n\s*\];)", + cid, lambda ind, t=js_line: ind + t, JS_ID) + if changed: + edits.append(f"`{BACKENDS_JS}`: `{CLAUDE_JS_CONST}` gains `{js_line}`") + edits.append("`src/pkg/dashboard/static/index.html`: rebuilt with `make dashboard-gen`") + if not any(ln.split()[:2] == ["claude", cid] for ln in reqs.splitlines()): + req_line = f"claude {cid} {pin}" + reqs = reqs + ("" if reqs.endswith("\n") else "\n") + req_line + "\n" + edits.append(f"`{REQUIREMENTS}`: `{req_line}`") + edits.append("`src/pkg/tokens/pricing.go`: `modelPrices` row from the OpenRouter quote, through " + "`pricetable --add` and its plausibility rules (see the price section)") + added.append((c, edits)) + + if accepted and not dry_run: + for rel, body in ((CLI_MODELS_GO, go_src), (BACKENDS_JS, js_src), (REQUIREMENTS, reqs)): + with open(path(rel), "w", encoding="utf-8") as f: + f.write(body) + + lines = ["## Claude model freshness", "", + f"Pinned Claude Code: `{pin}` (`{DOCKERFILE}`). Gate: {note}.", ""] + if dry_run: + lines += ["**Dry run:** nothing was written.", ""] + lines += [f"### Added ({len(added)})", ""] + if not added: + lines += ["No model added.", ""] + for c, edits in added: + lines += [f"#### {c['label']} (`{c['cli_id']}`, from `{c['source_id']}`)", ""] + lines += [f"- {e}" for e in edits] + [""] + if needs: + lines += [f"### Needs a check against the Claude Code changelog ({len(needs)})", "", + "Newer than the lists on OpenRouter, but not shown to be accepted by the pinned Claude Code, so not added:", ""] + lines += [f"- {c['label']} (`{c['cli_id']}`, from `{c['source_id']}`): {why}" for c, why in needs] + [""] + return "\n".join(lines), [c["cli_id"] for c in accepted] + + +def main_apply(args): + if args.catalog: + with open(args.catalog, encoding="utf-8") as f: + catalog = json.load(f).get("data", []) + else: + catalog = fetch_json(OPENROUTER_MODELS).get("data", []) + report, accepted = apply_claude(args.root, catalog, args.claude_bin, args.dry_run) + print(report) + if args.accepted_out: + with open(args.accepted_out, "w", encoding="utf-8") as f: + f.write("".join(i + "\n" for i in accepted)) + return 0 + + +def parse_args(argv): + ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + ap.add_argument("--apply", choices=["claude"], help="edit the Claude lists for newer models the pinned CLI accepts") + ap.add_argument("--catalog", help="OpenRouter /api/v1/models JSON file instead of fetching it") + ap.add_argument("--claude-bin", help="the pinned Claude Code binary to probe (without it nothing is added)") + ap.add_argument("--root", default=".", help="repository root (default: .)") + ap.add_argument("--dry-run", action="store_true", help="print the edits; write nothing") + ap.add_argument("--accepted-out", help="write the added CLI ids here, one per line") + return ap.parse_args(argv) + + def main(): with open(CLI_MODELS_GO, encoding="utf-8") as f: cli_src = f.read() @@ -123,12 +432,7 @@ def main(): catalog = [] sections.append(f"### OpenRouter catalog unavailable\n\n`{OPENROUTER_MODELS}` failed: `{exc}`. Nothing was compared against it this week.") live_ids = {m.get("id", "") for m in catalog if m.get("id")} - by_vendor = {} - for mid in live_ids: - if "/" not in mid or mid.startswith("~"): - continue - vendor, rest = mid.split("/", 1) - by_vendor.setdefault(vendor, set()).add(norm(rest)) + by_vendor = index_catalog(catalog) if live_ids: ids, default = openrouter_suggestions(or_src) @@ -144,24 +448,8 @@ def main(): if not ids: sections.append(f"### `{name}` could not be parsed\n\nThe checker found no ids; update the parser or the list.") continue - not_found, newer = [], set() - have = {norm(i) for i in ids} - fams = {} - for i in ids: - vendor = prefix or vendor_for(i) - if not vendor: - continue - key = norm(i) - if key not in by_vendor.get(vendor, set()): - not_found.append(i) - fam, ver = family(key) - if fam: - fams.setdefault((vendor, fam), max(fams.get((vendor, fam), ()), ver)) - for (vendor, fam), top in fams.items(): - for key in by_vendor.get(vendor, set()): - f2, v2 = family(key) - if f2 == fam and v2 > top and key not in have: - newer.add(f"{vendor}/{key}") + not_found, newer_keys = compare_list(ids, prefix, by_vendor) + newer = {f"{vendor}/{key}" for vendor, key in newer_keys} lines = [] if newer: lines.append("Newer models of the same family on OpenRouter that the list lacks:\n\n" + "\n".join(f"- `{n}`" for n in sorted(newer))) @@ -193,4 +481,5 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + _args = parse_args(sys.argv[1:]) + sys.exit(main_apply(_args) if _args.apply else main()) diff --git a/.github/scripts/claude-model-freshness.sh b/.github/scripts/claude-model-freshness.sh new file mode 100755 index 0000000000..c6275881f0 --- /dev/null +++ b/.github/scripts/claude-model-freshness.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Builds the weekly Claude model freshness change in the working tree (#18754). +# Run by .github/workflows/claude-model-freshness.yml; see its header. +# +# 1. fetches OpenRouter's public model catalogue once (both steps below read +# the same snapshot); +# 2. installs the Claude Code version pinned in src/Dockerfile exactly the +# way the image does (npm --ignore-scripts, then the vendor's install.cjs) +# so the candidates can be probed against it, unauthenticated; +# 3. check-static-model-freshness.py --apply claude: candidates, the probe +# gate, and the edits to claudePinnedCLIModels, claudeStaticModels, +# CLAUDE_CLI_MODELS and cli-model-requirements.txt; +# 4. for the accepted ids: `pricetable --add` (new modelPrices rows, only +# through its plausibility rules) and `make dashboard-gen`. +# +# Usage: claude-model-freshness.sh +# Env: DRY_RUN=1 prints every edit and writes nothing. +# SOURCE_URL overrides the catalogue URL. +# Out: /report.md (the PR body), /accepted.txt (one id per line; +# empty when nothing is added), /pin.txt. +set -euo pipefail + +out="${1:?usage: claude-model-freshness.sh }" +mkdir -p "$out" +out="$(cd "$out" && pwd)" +cd "$(git rev-parse --show-toplevel)" + +source_url="${SOURCE_URL:-https://openrouter.ai/api/v1/models}" +catalog="$out/openrouter.json" +curl -fsSL --retry 3 --max-time 60 -H 'Accept: application/json' -A hive-claude-model-freshness \ + "$source_url" -o "$catalog" + +pin="$(sed -n 's/^ARG CLAUDE_CODE_VERSION=\([^ ]*\).*/\1/p' src/Dockerfile | head -1)" +[ -n "$pin" ] || { echo "no ARG CLAUDE_CODE_VERSION in src/Dockerfile" >&2; exit 1; } +printf '%s\n' "$pin" > "$out/pin.txt" + +# Without the binary nothing is added: every candidate is reported under +# "needs a check" instead (check-static-model-freshness.py gate_candidates). +claude_bin="" +prefix="$out/claude-code" +if npm install --prefix "$prefix" --ignore-scripts --no-fund --no-audit \ + "@anthropic-ai/claude-code@${pin}" >&2 \ + && node "$prefix/node_modules/@anthropic-ai/claude-code/install.cjs" >&2 \ + && "$prefix/node_modules/.bin/claude" --version >&2; then + claude_bin="$prefix/node_modules/.bin/claude" +else + echo "::warning::Could not install Claude Code ${pin} to probe; no model will be added this run." +fi + +dry=() +[ "${DRY_RUN:-}" = "1" ] && dry=(--dry-run) +probe=() +[ -n "$claude_bin" ] && probe=(--claude-bin "$claude_bin") + +python3 -I .github/scripts/check-static-model-freshness.py --apply claude \ + --catalog "$catalog" --accepted-out "$out/accepted.txt" "${probe[@]}" "${dry[@]}" > "$out/lists.md" +cp "$out/lists.md" "$out/report.md" + +if [ ! -s "$out/accepted.txt" ]; then + echo "no Claude model to add" + exit 0 +fi + +ids="$(paste -sd, "$out/accepted.txt")" +(cd src && go run ./internal/tools/pricetable --source "$catalog" --add "$ids" \ + --report "$out/prices.md" "${dry[@]}") +{ echo; cat "$out/prices.md"; } >> "$out/report.md" + +if [ "${DRY_RUN:-}" != "1" ]; then + make dashboard-gen + make dashboard-gen-check + test -z "$(gofmt -l src/pkg/tokens/pricing.go src/pkg/dashboard/cli_models.go)" \ + || { echo "::error::generated Go files are not gofmt-clean" >&2; exit 1; } + { + echo + echo "### Model lists against the CLI pins" + echo + echo '```' + HIVE_MODEL_CHECK_STRICT=0 bash src/scripts/check-cli-model-lists.sh 2>&1 || true + echo '```' + } >> "$out/report.md" +fi +echo "Claude models to add: $ids" diff --git a/.github/scripts/test-claude-model-freshness.py b/.github/scripts/test-claude-model-freshness.py new file mode 100755 index 0000000000..bed60a6847 --- /dev/null +++ b/.github/scripts/test-claude-model-freshness.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Hermetic tests for check-static-model-freshness.py --apply claude (#18754). + +No network and no real Claude Code: the catalog is a fixture and the pinned +CLI is a fake script that answers --version and the unauthenticated probe. + + python3 .github/scripts/test-claude-model-freshness.py +""" + +import importlib.util +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "check-static-model-freshness.py") +FIXTURES = os.path.join(HERE, "testdata", "claude-model-freshness") + +spec = importlib.util.spec_from_file_location("freshness", SCRIPT) +freshness = importlib.util.module_from_spec(spec) +spec.loader.exec_module(freshness) + +LAYOUT = { + "cli_models.go.txt": freshness.CLI_MODELS_GO, + "backends-tokens.js.txt": freshness.BACKENDS_JS, + "cli-model-requirements.txt": freshness.REQUIREMENTS, + "Dockerfile.txt": freshness.DOCKERFILE, +} + +FAKE_CLAUDE = """#!/usr/bin/env python3 +import sys +args = sys.argv[1:] +if args == ["--version"]: + print({version!r} + " (Claude Code)") + sys.exit(0) +model = args[args.index("--model") + 1] +if {mark!r} and (model in {unknown!r} or model.startswith("claude-hive-probe")): + print('[claude-code:unrecognized_model] {{"model":"' + model + '","query_source":"sdk"}}') +print("Not logged in \\u00b7 Please run /login") +sys.exit(1) +""" + + +def read(path): + with open(path, encoding="utf-8") as fh: + return fh.read() + + +def load_catalog(name): + import json + with open(os.path.join(FIXTURES, name), encoding="utf-8") as fh: + return json.load(fh)["data"] + + +def go_list(src, name): + return freshness.go_string_list(src, name) + + +def js_values(src): + m = re.search(r"const CLAUDE_CLI_MODELS = \[\n(.*?)\n\s*\];", src, re.S) + return re.findall(r"value: '([^']+)'", m.group(1)) + + +class Base(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="claude-freshness-test-") + self.root = os.path.join(self.tmp, "repo") + for fixture, rel in LAYOUT.items(): + dst = os.path.join(self.root, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copyfile(os.path.join(FIXTURES, fixture), dst) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def fake_claude(self, unknown=(), version="2.1.326", mark=True): + path = os.path.join(self.tmp, "claude") + with open(path, "w", encoding="utf-8") as fh: + fh.write(FAKE_CLAUDE.format(version=version, unknown=list(unknown), mark=mark)) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + return path + + def snapshot(self): + return {rel: read(os.path.join(self.root, rel)) for rel in LAYOUT.values()} + + def run_cli(self, *extra, catalog="openrouter.json"): + accepted = os.path.join(self.tmp, "accepted.txt") + proc = subprocess.run([sys.executable, "-I", SCRIPT, "--apply", "claude", "--root", self.root, + "--catalog", os.path.join(FIXTURES, catalog), "--accepted-out", accepted, *extra], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=120) + self.assertEqual(proc.returncode, 0, proc.stdout) + return proc.stdout, read(accepted).split() + + +class Candidates(Base): + def test_same_line_newer_only_dashed_and_labelled(self): + src = read(os.path.join(self.root, freshness.CLI_MODELS_GO)) + got = freshness.claude_candidates(src, load_catalog("openrouter.json")) + self.assertEqual([(c["cli_id"], c["source_id"], c["label"]) for c in got], [ + ("claude-fable-5-1", "anthropic/claude-fable-5.1", "Fable 5.1"), + ("claude-fable-5-2", "anthropic/claude-fable-5.2", "Fable 5.2"), + ("claude-haiku-5-5", "anthropic/claude-haiku-5.5", "Haiku 5.5"), + ]) + + def test_excluded(self): + src = read(os.path.join(self.root, freshness.CLI_MODELS_GO)) + ids = {c["cli_id"] for c in freshness.claude_candidates(src, load_catalog("openrouter.json"))} + for never in ("claude-mythos-5", "claude-mythos-5-5-preview", # limited access + "claude-sonnet-6", # expiration_date set + "claude-opus-4-1", # older than the list + "claude-opus-5-5", # already listed + "claude-opus-6-fast", "claude-3-7-sonnet"): # not a plain line id + self.assertNotIn(never, ids) + + def test_dotted_to_dashed(self): + self.assertEqual(freshness.norm("anthropic/claude-fable-5.2"), "claude-fable-5-2") + self.assertEqual(freshness.claude_line("claude-fable-5-2"), ("fable", (5, 2))) + self.assertIsNone(freshness.claude_line("claude-mythos-5")) + self.assertEqual(freshness.claude_label("haiku", (5, 5)), "Haiku 5.5") + + +class Apply(Base): + def test_dry_run_prints_every_edit_and_writes_nothing(self): + before = self.snapshot() + out, accepted = self.run_cli("--dry-run", "--claude-bin", self.fake_claude(unknown=["claude-fable-5-2"])) + self.assertEqual(accepted, ["claude-fable-5-1", "claude-haiku-5-5"]) + self.assertEqual(self.snapshot(), before, "a dry run must not write") + self.assertIn("**Dry run:** nothing was written.", out) + for cid, label in (("claude-fable-5-1", "Fable 5.1"), ("claude-haiku-5-5", "Haiku 5.5")): + section = out.split(f"#### {label} (`{cid}`")[1].split("####")[0].split("###")[0] + for want in (f"`claudePinnedCLIModels` gains `\"{cid}\"`", + f"`claudeStaticModels` gains `\"{cid}\"`", + f"`CLAUDE_CLI_MODELS` gains `{{ value: '{cid}', label: '{label}' }},`", + "`src/pkg/dashboard/static/index.html`: rebuilt with `make dashboard-gen`", + f"`claude {cid} 2.1.326`", + "`modelPrices` row from the OpenRouter quote, through `pricetable --add`"): + self.assertIn(want, section, f"{cid}: missing edit {want!r}\n{out}") + needs = out.split("### Needs a check against the Claude Code changelog (1)")[1] + self.assertIn("Fable 5.2 (`claude-fable-5-2`, from `anthropic/claude-fable-5.2`): the pinned Claude Code does not recognise this id", needs) + self.assertNotIn("#### Fable 5.2", out, "an id the pinned CLI does not recognise is never added") + + def test_apply_edits_in_sync_and_idempotent(self): + claude = self.fake_claude(unknown=["claude-fable-5-2"]) + _, accepted = self.run_cli("--claude-bin", claude) + self.assertEqual(accepted, ["claude-fable-5-1", "claude-haiku-5-5"]) + go = read(os.path.join(self.root, freshness.CLI_MODELS_GO)) + js = read(os.path.join(self.root, freshness.BACKENDS_JS)) + reqs = read(os.path.join(self.root, freshness.REQUIREMENTS)) + want = ["claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-1", "claude-fable-5", + "claude-haiku-5-5", "claude-haiku-4-5-20251001", "opus", "sonnet", "haiku"] + self.assertEqual(go_list(go, "claudeStaticModels"), want) + self.assertEqual(go_list(go, "claudePinnedCLIModels"), want) + self.assertEqual(js_values(js), want, "JS list must stay an exact copy of claudeStaticModels") + self.assertEqual(go_list(go, "geminiStaticModels"), ["gemini-2.5-pro"], "other lists untouched") + self.assertIn(" { value: 'claude-fable-5-1', label: 'Fable 5.1' },\n", js) + self.assertIn('\t"claude-haiku-5-5",\n', go) + self.assertTrue(reqs.endswith("claude claude-fable-5-1 2.1.326\nclaude claude-haiku-5-5 2.1.326\n"), reqs) + + after = self.snapshot() + out, again = self.run_cli("--claude-bin", claude) + self.assertEqual(again, []) + self.assertEqual(self.snapshot(), after, "a second run with the same candidates changes nothing") + self.assertIn("### Added (0)", out) + + def test_insert_before_aliases_when_line_is_new_to_the_list(self): + lines = ['\t"claude-opus-5-5",', "\t// aliases", '\t"opus",'] + got = freshness.insert_listed(lines, "claude-haiku-5-5", lambda ind: f'{ind}"claude-haiku-5-5",', freshness.GO_ID) + self.assertEqual(got, ['\t"claude-opus-5-5",', '\t"claude-haiku-5-5",', "\t// aliases", '\t"opus",']) + self.assertIsNone(freshness.insert_listed(got, "claude-haiku-5-5", lambda ind: "", freshness.GO_ID)) + + def test_unreliable_probe_adds_nothing(self): + before = self.snapshot() + out, accepted = self.run_cli("--claude-bin", self.fake_claude(mark=False)) + self.assertEqual(accepted, []) + self.assertEqual(self.snapshot(), before) + self.assertIn("probe controls failed", out) + self.assertIn("### Needs a check against the Claude Code changelog (3)", out) + + def test_wrong_version_adds_nothing(self): + before = self.snapshot() + out, accepted = self.run_cli("--claude-bin", self.fake_claude(version="2.1.311")) + self.assertEqual(accepted, []) + self.assertEqual(self.snapshot(), before) + self.assertIn("not the pinned `2.1.326`", out) + + def test_no_binary_adds_nothing(self): + before = self.snapshot() + out, accepted = self.run_cli() + self.assertEqual(accepted, []) + self.assertEqual(self.snapshot(), before) + self.assertIn("no pinned Claude Code binary", out) + + def test_no_candidates_no_edits(self): + before = self.snapshot() + out, accepted = self.run_cli("--claude-bin", self.fake_claude(), catalog="openrouter-current.json") + self.assertEqual(accepted, []) + self.assertEqual(self.snapshot(), before) + self.assertIn("Gate: no candidates.", out) + self.assertNotIn("Needs a check", out) + + +class Report(unittest.TestCase): + def test_compare_list_shared_with_report(self): + by_vendor = freshness.index_catalog(load_catalog("openrouter.json")) + not_found, newer = freshness.compare_list(["claude-fable-5", "claude-opus-9"], "anthropic", by_vendor) + self.assertEqual(not_found, ["claude-opus-9"]) + self.assertIn(("anthropic", "claude-fable-5-2"), newer) + self.assertIn(("anthropic", "claude-fable-5-1"), newer) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/scripts/testdata/claude-model-freshness/Dockerfile.txt b/.github/scripts/testdata/claude-model-freshness/Dockerfile.txt new file mode 100644 index 0000000000..b80276b9c0 --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/Dockerfile.txt @@ -0,0 +1,3 @@ +FROM scratch +ARG CODEX_VERSION=0.169.0 +ARG CLAUDE_CODE_VERSION=2.1.326 diff --git a/.github/scripts/testdata/claude-model-freshness/backends-tokens.js.txt b/.github/scripts/testdata/claude-model-freshness/backends-tokens.js.txt new file mode 100644 index 0000000000..4338a9e297 --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/backends-tokens.js.txt @@ -0,0 +1,13 @@ + // Miniature of backends-tokens.js for test-claude-model-freshness.py. + const CLAUDE_CLI_MODELS = [ + { value: 'claude-opus-5-5', label: 'Opus 5.5' }, + { value: 'claude-sonnet-5-5', label: 'Sonnet 5.5' }, + { value: 'claude-fable-5', label: 'Fable 5' }, + { value: 'claude-haiku-4-5-20251001', label: 'Haiku 4.5' }, + { value: 'opus', label: 'Opus (alias)' }, + { value: 'sonnet', label: 'Sonnet (alias)' }, + { value: 'haiku', label: 'Haiku (alias)' }, + ]; + const GEMINI_CLI_MODELS = [ + { value: 'gemini-2.5-pro', label: 'Gemini 2.5 Pro' }, + ]; diff --git a/.github/scripts/testdata/claude-model-freshness/cli-model-requirements.txt b/.github/scripts/testdata/claude-model-freshness/cli-model-requirements.txt new file mode 100644 index 0000000000..2a2675d972 --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/cli-model-requirements.txt @@ -0,0 +1,2 @@ +# +claude claude-sonnet-5-5 2.1.284 diff --git a/.github/scripts/testdata/claude-model-freshness/cli_models.go.txt b/.github/scripts/testdata/claude-model-freshness/cli_models.go.txt new file mode 100644 index 0000000000..e1a2937d69 --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/cli_models.go.txt @@ -0,0 +1,31 @@ +package dashboard + +// Miniature of src/pkg/dashboard/cli_models.go for +// test-claude-model-freshness.py: the two Claude lists --apply claude edits. + +var claudeStaticModels = []string{ + "claude-opus-5-5", + "claude-sonnet-5-5", + "claude-fable-5", + "claude-haiku-4-5-20251001", + // Bare aliases the Claude CLI also accepts (see claudeAlwaysIncludeModels). + "opus", + "sonnet", + "haiku", +} + +var geminiStaticModels = []string{ + "gemini-2.5-pro", +} + +// claudePinnedCLIModels is the closed model set accepted by the Claude Code +// version pinned in src/Dockerfile. +var claudePinnedCLIModels = []string{ + "claude-opus-5-5", + "claude-sonnet-5-5", + "claude-fable-5", + "claude-haiku-4-5-20251001", + "opus", + "sonnet", + "haiku", +} diff --git a/.github/scripts/testdata/claude-model-freshness/openrouter-current.json b/.github/scripts/testdata/claude-model-freshness/openrouter-current.json new file mode 100644 index 0000000000..c0eb7f9fc6 --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/openrouter-current.json @@ -0,0 +1,8 @@ +{ + "data": [ + {"id": "anthropic/claude-opus-5.5"}, + {"id": "anthropic/claude-fable-5"}, + {"id": "anthropic/claude-haiku-4.5"}, + {"id": "anthropic/claude-mythos-6"} + ] +} diff --git a/.github/scripts/testdata/claude-model-freshness/openrouter.json b/.github/scripts/testdata/claude-model-freshness/openrouter.json new file mode 100644 index 0000000000..811ffa5b0e --- /dev/null +++ b/.github/scripts/testdata/claude-model-freshness/openrouter.json @@ -0,0 +1,19 @@ +{ + "data": [ + {"id": "anthropic/claude-opus-5.5"}, + {"id": "anthropic/claude-opus-4.1"}, + {"id": "anthropic/claude-fable-5"}, + {"id": "anthropic/claude-fable-5.1"}, + {"id": "anthropic/claude-fable-5.2"}, + {"id": "anthropic/claude-fable-5.2:batch"}, + {"id": "anthropic/claude-haiku-5.5"}, + {"id": "anthropic/claude-mythos-5"}, + {"id": "anthropic/claude-mythos-5.5-preview"}, + {"id": "anthropic/claude-sonnet-6", "expiration_date": "2026-12-01"}, + {"id": "anthropic/claude-opus-6-fast"}, + {"id": "anthropic/claude-3.7-sonnet"}, + {"id": "~anthropic/claude-opus-latest"}, + {"id": "google/gemini-9-pro"}, + {"id": null} + ] +} diff --git a/.github/workflows/claude-model-freshness.yml b/.github/workflows/claude-model-freshness.yml new file mode 100644 index 0000000000..8617fe6fbf --- /dev/null +++ b/.github/workflows/claude-model-freshness.yml @@ -0,0 +1,255 @@ +name: Claude model freshness + +# Weekly PR that adds new Claude models to the dashboard's lists and price +# table once the Claude Code version pinned in src/Dockerfile accepts them +# (#18754). Adding a model used to take five hand edits nobody was told to +# make, so the lists fell behind (Fable 5.1/5.2 and Haiku 5.5, #18753). +# +# .github/scripts/claude-model-freshness.sh does the work: +# - candidates: Claude ids on OpenRouter's public catalogue +# (https://openrouter.ai/api/v1/models, no key) newer than the lists' +# newest of the same line (Fable, Opus, Sonnet, Haiku) — the comparison +# static-model-freshness.yml reports, via the same script. Mythos and ids +# OpenRouter has scheduled for withdrawal are never candidates; +# - gate: each candidate is put to the PINNED Claude Code, installed the way +# the image installs it and run unauthenticated (empty HOME, no key). The +# CLI prints an `unrecognized_model` diagnostic for an id its built-in +# model table does not know, before it stops at "Not logged in"; two +# controls (a listed id, a made-up id) must behave as expected or nothing +# is added. The "version X or newer is required" refusal (#8417) comes +# from the API after login, so it cannot be probed without credentials. +# A candidate not shown to be accepted is listed under "needs a check" +# and never added; +# - edits per accepted model: claudePinnedCLIModels (the closed allowlist, +# #8418), claudeStaticModels, CLAUDE_CLI_MODELS with a label, +# static/index.html via `make dashboard-gen`, a cli-model-requirements.txt +# line at the pin (#9805), and a modelPrices row from `pricetable --add`, +# which applies the price refresh's plausibility rules and leaves out (and +# reports) a quote that fails them. +# +# Its own workflow rather than a job in static-model-freshness.yml (which only +# needs issues: write) or price-table-refresh.yml (which reprices existing +# rows on its own branch): the write permissions stay on the job that opens +# this PR, and a model addition and a repricing are reviewed separately. +# +# One PR on bot/claude-model-freshness, force-pushed and edited in place by +# later runs until merged; no candidates accepted means no branch and no PR. +# Never auto-merged: model and price changes reach every hive, so it carries +# needs-human. Credentials as in price-table-refresh.yml: TOPUP_PUSH_TOKEN +# when set (so the PR gets normal pull_request CI runs), else GITHUB_TOKEN. +# +# On a pull_request touching this machinery the selftest job runs the +# hermetic tests and a DRY RUN of the whole pipeline against the live +# catalogue and the real pin; it has read-only permissions and opens nothing. + +on: + schedule: + - cron: '47 7 * * 1' + workflow_dispatch: + inputs: + base: + description: 'Branch whose lists to update and to open the PR against' + type: string + default: v5 + dry_run: + description: 'Print every edit; do not push a branch or open a PR' + type: boolean + default: false + pull_request: + paths: + - '.github/workflows/claude-model-freshness.yml' + - '.github/scripts/claude-model-freshness.sh' + - '.github/scripts/check-static-model-freshness.py' + - '.github/scripts/test-claude-model-freshness.py' + - '.github/scripts/testdata/claude-model-freshness/**' + - 'src/internal/tools/pricetable/**' + +permissions: + contents: read + +concurrency: + group: claude-model-freshness-${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.base || 'v5' }} + cancel-in-progress: false + +env: + BASE_BRANCH: ${{ inputs.base || 'v5' }} + HEAD_BRANCH: bot/claude-model-freshness + +jobs: + selftest: + name: self-test and dry run + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Hermetic tests (no network) + run: | + set -euo pipefail + bash -n .github/scripts/claude-model-freshness.sh + python3 .github/scripts/test-claude-model-freshness.py + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: src/go.mod + cache-dependency-path: | + src/go.sum + src/crux/go.sum + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + + - name: Dry run against the live catalogue and the pinned CLI + env: + DRY_RUN: '1' + run: | + set -euo pipefail + bash .github/scripts/claude-model-freshness.sh "$RUNNER_TEMP/cmf" + cat "$RUNNER_TEMP/cmf/report.md" >> "$GITHUB_STEP_SUMMARY" + cat "$RUNNER_TEMP/cmf/report.md" + test -z "$(git status --porcelain)" || { echo "::error::the dry run changed the working tree"; git status --short; exit 1; } + + refresh: + name: add new Claude models + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.BASE_BRANCH }} + fetch-depth: 0 + token: ${{ secrets.TOPUP_PUSH_TOKEN != '' && secrets.TOPUP_PUSH_TOKEN || github.token }} + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: src/go.mod + cache-dependency-path: | + src/go.sum + src/crux/go.sum + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + + - name: Find, gate and apply new Claude models + id: apply + env: + DRY_RUN: ${{ inputs.dry_run == true && '1' || '' }} + run: | + set -euo pipefail + bash .github/scripts/claude-model-freshness.sh "$RUNNER_TEMP/cmf" + cat "$RUNNER_TEMP/cmf/report.md" >> "$GITHUB_STEP_SUMMARY" + if [ -s "$RUNNER_TEMP/cmf/accepted.txt" ] && [ -n "$(git status --porcelain)" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + echo "pin=$(cat "$RUNNER_TEMP/cmf/pin.txt")" >> "$GITHUB_OUTPUT" + echo "ids=$(paste -sd, "$RUNNER_TEMP/cmf/accepted.txt")" >> "$GITHUB_OUTPUT" + + - name: Nothing to do + if: steps.apply.outputs.changed != 'true' + env: + DRY_RUN: ${{ inputs.dry_run == true && '1' || '' }} + run: | + if [ "$DRY_RUN" = "1" ]; then + echo "Dry run: the edits above were printed; no branch or PR is created." + else + echo "No Claude model to add; no branch, no PR." + fi + cat "$RUNNER_TEMP/cmf/report.md" + + - name: Find the open PR + if: steps.apply.outputs.changed == 'true' + id: existing + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + number="$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" --head "$HEAD_BRANCH" \ + --state open --json number --jq '.[0].number // empty')" + echo "number=${number}" >> "$GITHUB_OUTPUT" + [ -z "$number" ] || echo "will refresh open PR #${number}" + + - name: Commit the additions + if: steps.apply.outputs.changed == 'true' + env: + IDS: ${{ steps.apply.outputs.ids }} + PIN: ${{ steps.apply.outputs.pin }} + PR_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$HEAD_BRANCH" + # One fragment per run; drop any earlier unmerged one on this branch. + while IFS= read -r -d '' fragment; do + if ! git cat-file -e "origin/${BASE_BRANCH}:${fragment}" 2>/dev/null; then + git rm -q -- "$fragment" + fi + done < <(git ls-files -z -- 'changelog.d/added-claude-models-*.md') + date="$(date -u +%Y-%m-%d)" + printf -- '- The Claude model dropdowns and price table now include %s, which the pinned Claude Code %s accepts.\n' \ + "$(printf '%s' "$IDS" | sed 's/,/, /g')" "$PIN" > "changelog.d/added-claude-models-${date}.md" + git add src/pkg/dashboard/cli_models.go src/pkg/dashboard/static-src/js/backends-tokens.js \ + src/pkg/dashboard/static/index.html src/scripts/cli-model-requirements.txt \ + src/pkg/tokens/pricing.go changelog.d + { + echo "✨ dashboard: add Claude models ${IDS}" + echo + echo "Automated by claude-model-freshness.yml: each id is newer than the" + echo "Claude lists on OpenRouter and recognised by the pinned Claude Code" + echo "${PIN}; see the PR body for the gate and the price quotes." + } > "$RUNNER_TEMP/commit-msg.txt" + git commit -s -F "$RUNNER_TEMP/commit-msg.txt" + # Same reasoning as price-table-refresh.yml and #11412 in + # cli-pin-bump.yml: a PAT-opened PR squashes under the PAT owner, + # so add their sign-off too. + if [ -n "$PR_TOKEN" ]; then + owner="$(GH_TOKEN="$PR_TOKEN" gh api user --jq '"\(.login) \(.id)"')" + login="${owner% *}" + id="${owner##* }" + if [ -z "$login" ] || [ -z "$id" ] || [ "$login" = "$id" ]; then + echo "::error::Could not resolve the TOPUP_PUSH_TOKEN owner; a PR it opens would squash to a mismatched sign-off." + exit 1 + fi + git commit --amend --no-edit \ + --trailer "Signed-off-by: ${login} <${id}+${login}@users.noreply.github.com>" + fi + git push --force origin "$HEAD_BRANCH" + + - name: Open or refresh the PR + if: steps.apply.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN != '' && secrets.TOPUP_PUSH_TOKEN || github.token }} + IDS: ${{ steps.apply.outputs.ids }} + EXISTING: ${{ steps.existing.outputs.number }} + run: | + set -euo pipefail + title="✨ dashboard: add Claude models ${IDS}" + { + cat "$RUNNER_TEMP/cmf/report.md" + echo + echo "### Review" + echo + echo "- [ ] Each added id is in the Claude Code changelog for the pinned version or earlier" + echo "- [ ] Each new price row matches Anthropic's price page (OpenRouter mirrors list price but can lag or round)" + echo "- [ ] Anything under *Needs a check* or *Held for review* is followed up by hand" + echo + echo "Generated by [claude-model-freshness](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) with \`.github/scripts/claude-model-freshness.sh\`; the branch is force-pushed by later runs until merged. Not auto-merged: a human reviews model and price changes." + } > "$RUNNER_TEMP/pr-body.md" + if [ -n "$EXISTING" ]; then + gh pr edit "$EXISTING" --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md" --add-label needs-human + echo "refreshed PR #${EXISTING}" + else + gh pr create --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" --head "$HEAD_BRANCH" \ + --title "$title" --body-file "$RUNNER_TEMP/pr-body.md" --label needs-human + fi diff --git a/changelog.d/added-18754-claude-model-freshness.md b/changelog.d/added-18754-claude-model-freshness.md new file mode 100644 index 0000000000..ac526f1ce3 --- /dev/null +++ b/changelog.d/added-18754-claude-model-freshness.md @@ -0,0 +1 @@ +- A weekly workflow (`claude-model-freshness.yml`) now opens one `needs-human` PR adding new Claude models to the dashboard's model lists, `cli-model-requirements.txt` and the price table, once the Claude Code version pinned in the image accepts them; models the pinned CLI does not recognise are listed for a manual check instead of added, and price rows come only through the price refresh's plausibility rules (`pricetable --add`) (#18754). diff --git a/src/docs/cli-pins.md b/src/docs/cli-pins.md index c1aa9c421b..be752f488a 100644 --- a/src/docs/cli-pins.md +++ b/src/docs/cli-pins.md @@ -110,6 +110,17 @@ A pin change can leave Hive's model lists behind. Two layers cover it: `HIVE_MODEL_CHECK_STRICT=1` to make drift fail. - Each bump PR body carries a checklist for what cannot be automated: allowlists/fallbacks, `static/index.html`, and `src/pkg/tokens/pricing.go`. +- For Claude, `.github/workflows/claude-model-freshness.yml` does those edits + weekly (#18754). It takes the Claude ids on OpenRouter's public catalogue + that are newer than the lists' newest of the same line (Fable, Opus, Sonnet, + Haiku; never Mythos, never an id OpenRouter is withdrawing) and probes each + with the pinned Claude Code, unauthenticated: the CLI prints an + `unrecognized_model` diagnostic for an id its model table does not know. + Recognised ids go into `claudePinnedCLIModels`, `claudeStaticModels`, + `CLAUDE_CLI_MODELS`, `static/index.html`, a requirements line at the pin, + and a `modelPrices` row from `pricetable --add` (same plausibility rules as + the price refresh). Everything else is listed under "needs a check" in one + PR on `bot/claude-model-freshness`, which carries `needs-human`. The PR then runs the normal gates. The docker workflow builds both images for linux/amd64 and linux/arm64, where every hash-verified layer runs diff --git a/src/internal/tools/pricetable/add.go b/src/internal/tools/pricetable/add.go new file mode 100644 index 0000000000..6bd38b557b --- /dev/null +++ b/src/internal/tools/pricetable/add.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "fmt" + "go/format" + "math/big" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/pkg/tokens" +) + +// addition is a modelPrices row --add proposes for a model the table lacks. +type addition struct { + Key string + SourceID string + // Sibling is the existing row of the same model line the quote was + // compared against; empty when the table has none. + Sibling string + Fields map[string]*big.Rat +} + +// addResult is the outcome of one --add run. +type addResult struct { + Added []addition + // Present ids already have a row; --add never reprices (the weekly + // reconcile does that). + Present []string + // Missing ids have no usable quote in the source. + Missing []unverified + // Held ids have a quote that tripped a plausibility rule. + Held []suspect +} + +// versionedKey splits a normalised key that ends in a dashed version into the +// model line and the version numbers: claude-fable-5-1 -> (claude-fable, [5 1]). +var versionedKey = regexp.MustCompile(`^(.*?[a-z])-(\d+(?:-\d+)*)$`) + +// planAdditions decides, for each requested id, whether a new row can be +// added from the catalogue. A quote must pass the same plausibility rules a +// price move must pass: the move rules against the newest existing row of the +// same model line (a new model priced more than 5× away from its predecessor +// looks like a catalogue defect), and the ratio rules on its own. +func planAdditions(t *table, c *catalogue, ids []string) *addResult { + r := &addResult{} + seen := map[string]bool{} + for _, id := range ids { + key := tokens.NormalizeModelID(id) + if key == "" || seen[key] { + continue + } + seen[key] = true + if _, ok := t.entry(key); ok { + r.Present = append(r.Present, key) + continue + } + if dup, ok := c.Ambiguous[key]; ok { + r.Missing = append(r.Missing, unverified{key, "several catalogue ids normalise to this key: " + strings.Join(dup, ", ")}) + continue + } + q, ok := c.Quotes[key] + if !ok { + r.Missing = append(r.Missing, unverified{key, "not listed by the source; add the row by hand from the vendor's price page"}) + continue + } + fields, rules := proposeRow(q) + sib, hasSib := siblingEntry(t, key) + p := make(map[string]fieldChange, len(priceFields)) + for _, name := range priceFields { + old := fields[name] + if hasSib { + old = sib.Fields[name].Value + } + p[name] = fieldChange{Old: old, New: fields[name]} + } + if hasSib { + ch := change{Key: key, SourceID: q.SourceID, Fields: map[string]fieldChange{}} + for name, fc := range p { + if fc.Old.Cmp(fc.New) != 0 { + ch.Fields[name] = fc + } + } + rules = append(rules, moveRules(ch)...) + } + rules = append(rules, shapeRules(p)...) + if len(rules) > 0 { + r.Held = append(r.Held, suspect{Key: key, SourceID: q.SourceID, Fields: p, Rules: rules}) + continue + } + a := addition{Key: key, SourceID: q.SourceID, Fields: fields} + if hasSib { + a.Sibling = sib.Key + } + r.Added = append(r.Added, a) + } + sort.Slice(r.Added, func(i, j int) bool { return r.Added[i].Key < r.Added[j].Key }) + sort.Slice(r.Held, func(i, j int) bool { return r.Held[i].Key < r.Held[j].Key }) + sort.Slice(r.Missing, func(i, j int) bool { return r.Missing[i].Key < r.Missing[j].Key }) + sort.Strings(r.Present) + return r +} + +// proposeRow turns a quote into the four table fields. Input and output come +// from the source. Cache read must be published: a new row has no current +// value to keep, so a missing rate is a rule, not a guess. Cache write +// follows the table's documented convention (input rate) unless the source +// publishes a write premium. +func proposeRow(q quote) (map[string]*big.Rat, []string) { + var rules []string + if q.Input.Sign() <= 0 || q.Output.Sign() <= 0 { + rules = append(rules, "input and output must both be published and above zero") + } + read := q.CacheRead + if read == nil { + rules = append(rules, "source publishes no cache-read rate") + read = q.Input + } + write := q.Input + if q.CacheWrite != nil && q.CacheWrite.Cmp(q.Input) >= 0 { + write = q.CacheWrite + } + return map[string]*big.Rat{ + fieldInput: q.Input, + fieldOutput: q.Output, + fieldCacheRead: read, + fieldCacheWrite: write, + }, rules +} + +// siblingEntry is the existing row with the highest version in key's model +// line (claude-fable-5-2 -> the newest claude-fable- row). +func siblingEntry(t *table, key string) (entry, bool) { + m := versionedKey.FindStringSubmatch(key) + if m == nil { + return entry{}, false + } + var best entry + var bestVer []int + for _, e := range t.Entries { + em := versionedKey.FindStringSubmatch(e.Key) + if em == nil || em[1] != m[1] { + continue + } + v := versionInts(em[2]) + if bestVer == nil || versionLess(bestVer, v) { + best, bestVer = e, v + } + } + return best, bestVer != nil +} + +func versionInts(s string) []int { + parts := strings.Split(s, "-") + out := make([]int, len(parts)) + for i, p := range parts { + out[i], _ = strconv.Atoi(p) + } + return out +} + +func versionLess(a, b []int) bool { + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] != b[i] { + return a[i] < b[i] + } + } + return len(a) < len(b) +} + +// applyAdditions inserts each new row after its sibling's line (or after the +// last row of the same vendor prefix, or at the end of the map), with a +// comment naming the source quote, then gofmt's the result. Existing rows and +// priceTableDate are not touched. +func applyAdditions(src []byte, t *table, r *addResult, today string) ([]byte, error) { + if len(t.Entries) == 0 { + return nil, fmt.Errorf("table has no rows") + } + byOffset := map[int][]addition{} + for _, a := range r.Added { + anchor := t.Entries[len(t.Entries)-1] + if a.Sibling != "" { + anchor, _ = t.entry(a.Sibling) + } else { + vendor, _, _ := strings.Cut(a.Key, "-") + for _, e := range t.Entries { + if strings.HasPrefix(e.Key, vendor+"-") { + anchor = e + } + } + } + at := strings.IndexByte(string(src[anchor.End:]), '\n') + if at < 0 { + return nil, fmt.Errorf("row %s has no line end", anchor.Key) + } + off := anchor.End + at + 1 + byOffset[off] = append(byOffset[off], a) + } + offsets := make([]int, 0, len(byOffset)) + for off := range byOffset { + offsets = append(offsets, off) + } + sort.Sort(sort.Reverse(sort.IntSlice(offsets))) + out := append([]byte(nil), src...) + for _, off := range offsets { + rows := byOffset[off] + sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) + var b strings.Builder + for _, a := range rows { + fmt.Fprintf(&b, "\t// %s: list price from the source's quote for %s (added %s by pricetable --add).\n", a.Key, a.SourceID, today) + fmt.Fprintf(&b, "\t%q: {InputPerMTok: %s, OutputPerMTok: %s, CacheReadPerMTok: %s, CacheWritePerMTok: %s},\n", + a.Key, formatPrice(a.Fields[fieldInput]), formatPrice(a.Fields[fieldOutput]), + formatPrice(a.Fields[fieldCacheRead]), formatPrice(a.Fields[fieldCacheWrite])) + } + out = append(out[:off], append([]byte(b.String()), out[off:]...)...) + } + formatted, err := format.Source(out) + if err != nil { + return nil, fmt.Errorf("gofmt: %w", err) + } + return formatted, nil +} + +// renderAddReport writes the markdown summary of an --add run. +func renderAddReport(r *addResult, source string) string { + var b strings.Builder + b.WriteString("## New price rows\n\n") + fmt.Fprintf(&b, "Source: `%s` (OpenRouter public model catalogue; per-token USD list prices). Every quote passed the same plausibility rules as a price refresh, compared with the newest existing row of the same model line.\n\n", source) + fmt.Fprintf(&b, "### Added (%d)\n\n", len(r.Added)) + if len(r.Added) == 0 { + b.WriteString("No row added.\n\n") + } else { + b.WriteString("All figures are USD per million tokens.\n\n") + b.WriteString("| model | source id | input | output | cache read | cache write | compared with |\n") + b.WriteString("|---|---|---|---|---|---|---|\n") + for _, a := range r.Added { + sib := "no earlier row of this line; ratio rules only" + if a.Sibling != "" { + sib = "`" + a.Sibling + "`" + } + fmt.Fprintf(&b, "| `%s` | `%s` | %s | %s | %s | %s | %s |\n", a.Key, a.SourceID, + formatPrice(a.Fields[fieldInput]), formatPrice(a.Fields[fieldOutput]), + formatPrice(a.Fields[fieldCacheRead]), formatPrice(a.Fields[fieldCacheWrite]), sib) + } + b.WriteString("\n") + } + if len(r.Held) > 0 { + fmt.Fprintf(&b, "### Held for review (%d)\n\n", len(r.Held)) + b.WriteString("The source's quote for these models failed a plausibility rule, so no row was added; the model falls back to its tier price until someone adds the row by hand from the vendor's price page.\n\n") + b.WriteString("| model | source id | input | output | cache read | cache write | rule |\n") + b.WriteString("|---|---|---|---|---|---|---|\n") + for _, s := range r.Held { + fmt.Fprintf(&b, "| `%s` | `%s` |", s.Key, s.SourceID) + for _, name := range priceFields { + fmt.Fprintf(&b, " %s |", formatPrice(s.Fields[name].New)) + } + fmt.Fprintf(&b, " %s |\n", strings.Join(s.Rules, "; ")) + } + b.WriteString("\n") + } + if len(r.Missing) > 0 { + fmt.Fprintf(&b, "### No quote (%d)\n\n", len(r.Missing)) + for _, u := range r.Missing { + fmt.Fprintf(&b, "- `%s` — %s\n", u.Key, u.Reason) + } + b.WriteString("\n") + } + if len(r.Present) > 0 { + fmt.Fprintf(&b, "### Already priced (%d)\n\n%s\n\n", len(r.Present), backtickList(r.Present)) + } + return b.String() +} + +// executeAdd runs --add: new rows only, existing rows and the date untouched. +func executeAdd(ctx context.Context, opts options, src []byte, t *table, today string) error { + var ids []string + for _, id := range strings.Split(opts.add, ",") { + if id = strings.TrimSpace(id); id != "" { + ids = append(ids, id) + } + } + raw, err := opts.fetch(ctx, opts.source) + if err != nil { + return fmt.Errorf("fetch %s: %w", opts.source, err) + } + catalogue, err := parseOpenRouter(raw) + if err != nil { + return fmt.Errorf("parse catalogue: %w", err) + } + result := planAdditions(t, catalogue, ids) + report := renderAddReport(result, opts.source) + if opts.report != "" { + if err := os.WriteFile(opts.report, []byte(report), 0o644); err != nil { + return fmt.Errorf("write report: %w", err) + } + } + fmt.Fprint(opts.stdout, report) + if len(result.Added) == 0 { + fmt.Fprintln(opts.stdout, "pricetable: no new rows") + return nil + } + out, err := applyAdditions(src, t, result, today) + if err != nil { + return fmt.Errorf("add rows: %w", err) + } + if opts.dryRun { + fmt.Fprintf(opts.stdout, "pricetable: dry run, %d new row(s) not written\n", len(result.Added)) + return nil + } + if err := os.WriteFile(opts.pricing, out, 0o644); err != nil { + return fmt.Errorf("write %s: %w", opts.pricing, err) + } + fmt.Fprintf(opts.stdout, "pricetable: added %d new row(s) to %s\n", len(result.Added), opts.pricing) + return nil +} diff --git a/src/internal/tools/pricetable/add_test.go b/src/internal/tools/pricetable/add_test.go new file mode 100644 index 0000000000..49f046efa8 --- /dev/null +++ b/src/internal/tools/pricetable/add_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// addTable is a miniature pricing.go for --add: two Fable rows, one Haiku +// and one Opus row as siblings, and a non-Claude row last so a Claude model +// with no sibling must still land among the Claude rows. +const addTable = `package tokens + +const priceTableDate = "2026-11-09" + +type ModelPrice struct { + InputPerMTok float64 + OutputPerMTok float64 + CacheReadPerMTok float64 + CacheWritePerMTok float64 +} + +var modelPrices = map[string]ModelPrice{ + // ---- Anthropic ---- + "claude-opus-5": {InputPerMTok: 5.00, OutputPerMTok: 25.00, CacheReadPerMTok: 0.50, CacheWritePerMTok: 6.25}, + "claude-fable-5": {InputPerMTok: 10.00, OutputPerMTok: 50.00, CacheReadPerMTok: 1.00, CacheWritePerMTok: 12.50}, + "claude-fable-5-1": {InputPerMTok: 10.00, OutputPerMTok: 50.00, CacheReadPerMTok: 0.25, CacheWritePerMTok: 12.50}, + "claude-haiku-4-5": {InputPerMTok: 1.00, OutputPerMTok: 5.00, CacheReadPerMTok: 0.10, CacheWritePerMTok: 1.25}, + + // ---- OpenAI ---- + "gpt-5-4": {InputPerMTok: 2.50, OutputPerMTok: 15.00, CacheReadPerMTok: 0.25, CacheWritePerMTok: 2.50}, +} +` + +const addIDs = "claude-fable-5-2,anthropic/claude-haiku-5.5,claude-opus-6,claude-sonnet-6,claude-nova-1,claude-fable-5-1,claude-fable-9" + +func addCatalogue(t *testing.T) *catalogue { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "openrouter-add.json")) + if err != nil { + t.Fatal(err) + } + c, err := parseOpenRouter(raw) + if err != nil { + t.Fatal(err) + } + return c +} + +func TestPlanAdditions(t *testing.T) { + tb, err := parseTable([]byte(addTable)) + if err != nil { + t.Fatal(err) + } + r := planAdditions(tb, addCatalogue(t), strings.Split(addIDs, ",")) + + var added []string + sib := map[string]string{} + for _, a := range r.Added { + added = append(added, a.Key) + sib[a.Key] = a.Sibling + } + if got, want := strings.Join(added, ","), "claude-fable-5-2,claude-haiku-5-5,claude-nova-1"; got != want { + t.Fatalf("added = %s, want %s", got, want) + } + for key, want := range map[string]string{"claude-fable-5-2": "claude-fable-5-1", "claude-haiku-5-5": "claude-haiku-4-5", "claude-nova-1": ""} { + if sib[key] != want { + t.Errorf("%s compared with %q, want %q", key, sib[key], want) + } + } + if got := strings.Join(r.Present, ","); got != "claude-fable-5-1" { + t.Errorf("present = %s, want claude-fable-5-1 (an existing row is never repriced by --add)", got) + } + if len(r.Missing) != 1 || r.Missing[0].Key != "claude-fable-9" { + t.Errorf("missing = %+v, want only claude-fable-9", r.Missing) + } + held := map[string]string{} + for _, s := range r.Held { + held[s.Key] = strings.Join(s.Rules, "; ") + } + if len(held) != 2 { + t.Fatalf("held = %v, want claude-opus-6 and claude-sonnet-6", held) + } + if !strings.Contains(held["claude-opus-6"], "InputPerMTok moves more than 5× (5.00 → 100.00)") { + t.Errorf("claude-opus-6 must be held against its sibling claude-opus-5: %s", held["claude-opus-6"]) + } + if !strings.Contains(held["claude-sonnet-6"], "source publishes no cache-read rate") { + t.Errorf("claude-sonnet-6 must be held for its missing cache-read rate: %s", held["claude-sonnet-6"]) + } +} + +func TestRunAddWritesRowsAndIsIdempotent(t *testing.T) { + pricing := filepath.Join(t.TempDir(), "pricing.go") + if err := os.WriteFile(pricing, []byte(addTable), 0o644); err != nil { + t.Fatal(err) + } + report := filepath.Join(t.TempDir(), "add.md") + args := []string{"--source", filepath.Join("testdata", "openrouter-add.json"), "--pricing", pricing, "--date", "2026-11-17", "--add", addIDs} + var stdout, stderr bytes.Buffer + if code := run(append(args, "--report", report), &stdout, &stderr, fixedNow); code != 0 { + t.Fatalf("exit %d: %s", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "added 3 new row(s)") { + t.Fatalf("stdout = %s", stdout.String()) + } + got, err := os.ReadFile(pricing) + if err != nil { + t.Fatal(err) + } + src := string(got) + tb, err := parseTable(got) + if err != nil { + t.Fatalf("rewritten table does not parse: %v\n%s", err, src) + } + if tb.Date != "2026-11-09" { + t.Errorf("priceTableDate = %s; --add must not move it", tb.Date) + } + row := func(key string) string { + e, ok := tb.entry(key) + if !ok { + t.Fatalf("%s missing from:\n%s", key, src) + } + return formatPrice(e.Fields[fieldInput].Value) + "/" + formatPrice(e.Fields[fieldOutput].Value) + "/" + + formatPrice(e.Fields[fieldCacheRead].Value) + "/" + formatPrice(e.Fields[fieldCacheWrite].Value) + } + for key, want := range map[string]string{ + "claude-fable-5-2": "10.00/50.00/1.00/12.50", + "claude-haiku-5-5": "1.00/5.00/0.10/1.25", + "claude-nova-1": "2.00/8.00/0.20/2.50", + "claude-fable-5-1": "10.00/50.00/0.25/12.50", + } { + if got := row(key); got != want { + t.Errorf("%s = %s, want %s", key, got, want) + } + } + for _, key := range []string{"claude-opus-6", "claude-sonnet-6", "claude-fable-9"} { + if _, ok := tb.entry(key); ok { + t.Errorf("%s must not be added", key) + } + } + // Placement: after the sibling's line, and a sibling-less Claude row stays + // among the Claude rows, before the OpenAI section. + if !(strings.Index(src, `"claude-fable-5-1"`) < strings.Index(src, `"claude-fable-5-2"`) && + strings.Index(src, `"claude-fable-5-2"`) < strings.Index(src, `"claude-haiku-4-5"`)) { + t.Errorf("claude-fable-5-2 not placed after claude-fable-5-1:\n%s", src) + } + if !(strings.Index(src, `"claude-nova-1"`) < strings.Index(src, "---- OpenAI ----")) { + t.Errorf("claude-nova-1 not placed among the Claude rows:\n%s", src) + } + if !strings.Contains(src, "// claude-fable-5-2: list price from the source's quote for anthropic/claude-fable-5.2 (added 2026-11-17 by pricetable --add).") { + t.Errorf("provenance comment missing:\n%s", src) + } + if md, err := os.ReadFile(report); err != nil || !strings.Contains(string(md), "### Added (3)") || !strings.Contains(string(md), "### Held for review (2)") { + t.Fatalf("report: err=%v body=%s", err, md) + } + + // Second run with the same ids: nothing to add, file byte-identical. + stdout.Reset() + if code := run(args, &stdout, &stderr, fixedNow); code != 0 { + t.Fatalf("second run exit %d: %s", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "pricetable: no new rows") { + t.Fatalf("second run stdout = %s", stdout.String()) + } + again, _ := os.ReadFile(pricing) + if !bytes.Equal(got, again) { + t.Fatal("second --add run modified pricing.go") + } +} + +func TestRunAddDryRunLeavesFile(t *testing.T) { + pricing := filepath.Join(t.TempDir(), "pricing.go") + if err := os.WriteFile(pricing, []byte(addTable), 0o644); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + code := run([]string{"--source", filepath.Join("testdata", "openrouter-add.json"), "--pricing", pricing, "--dry-run", "--add", "claude-fable-5-2"}, &stdout, &stderr, fixedNow) + if code != 0 { + t.Fatalf("exit %d: %s", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "dry run, 1 new row(s) not written") { + t.Fatalf("stdout = %s", stdout.String()) + } + got, _ := os.ReadFile(pricing) + if string(got) != addTable { + t.Fatal("dry run modified pricing.go") + } +} diff --git a/src/internal/tools/pricetable/main.go b/src/internal/tools/pricetable/main.go index bb6b21c989..ae09d51b0e 100644 --- a/src/internal/tools/pricetable/main.go +++ b/src/internal/tools/pricetable/main.go @@ -14,6 +14,14 @@ // go run ./internal/tools/pricetable # refresh in place // go run ./internal/tools/pricetable --dry-run # report only // go run ./internal/tools/pricetable --check-age 60 # CI staleness guard +// go run ./internal/tools/pricetable --add claude-fable-5-2,claude-haiku-5-5 +// # add rows the table lacks +// +// --add only inserts rows for the listed ids the table does not price yet, +// each from the source's quote and only when that quote passes the same +// plausibility rules a price move must pass; it never reprices an existing +// row or moves priceTableDate. .github/workflows/claude-model-freshness.yml +// uses it for the Claude models it adds to the dashboard's lists. // // The refresh is driven by .github/workflows/price-table-refresh.yml, which // turns a non-empty diff into a PR for a human to review; see @@ -56,6 +64,7 @@ type options struct { report string date string dryRun bool + add string checkAge int staleAfter int stdout io.Writer @@ -78,6 +87,7 @@ func run(args []string, stdout, stderr io.Writer, now func() time.Time) int { fs.StringVar(&opts.report, "report", "", "write the markdown report to this file (default: stdout only)") fs.StringVar(&opts.date, "date", "", "override today's date (YYYY-MM-DD) used for priceTableDate") fs.BoolVar(&opts.dryRun, "dry-run", false, "compute and report, but do not rewrite pricing.go") + fs.StringVar(&opts.add, "add", "", "comma-separated model ids to add as new rows when the table lacks them (existing rows are never repriced)") fs.IntVar(&opts.checkAge, "check-age", 0, "exit non-zero when priceTableDate is older than this many days (no fetch, no rewrite)") fs.IntVar(&opts.staleAfter, "stale-after", defaultStaleAfterDays, "bump priceTableDate even without price changes once it is older than this many days") if err := fs.Parse(args); err != nil { @@ -111,6 +121,9 @@ func execute(ctx context.Context, opts options) error { } today = opts.date } + if opts.add != "" { + return executeAdd(ctx, opts, src, table, today) + } raw, err := opts.fetch(ctx, opts.source) if err != nil { diff --git a/src/internal/tools/pricetable/reconcile.go b/src/internal/tools/pricetable/reconcile.go index 50b1aa353c..d0f7c94813 100644 --- a/src/internal/tools/pricetable/reconcile.go +++ b/src/internal/tools/pricetable/reconcile.go @@ -244,6 +244,13 @@ func implausible(e entry, ch change) []string { if len(ch.Fields) == 0 { return nil } + rules := moveRules(ch) + return append(rules, shapeRules(proposedFields(e, ch))...) +} + +// moveRules names the fields of ch that move more than maxMoveFactor in +// either direction, or to or from zero. +func moveRules(ch change) []string { var rules []string for _, name := range priceFields { fc, ok := ch.Fields[name] @@ -265,7 +272,14 @@ func implausible(e entry, ch change) []string { rules = append(rules, fmt.Sprintf("%s moves more than %d× (%s → %s)", name, maxMoveFactor, formatPrice(fc.Old), formatPrice(fc.New))) } } - p := proposedFields(e, ch) + return rules +} + +// shapeRules names the ratio rules a proposed row trips: output far above +// input, or a cache-read rate close to input. p maps every price field to its +// current (Old) and proposed (New) value. +func shapeRules(p map[string]fieldChange) []string { + var rules []string in, out, read := p[fieldInput].New, p[fieldOutput].New, p[fieldCacheRead].New // A zero proposed input would skip the ratio checks, so judge against the // current input instead. diff --git a/src/internal/tools/pricetable/table.go b/src/internal/tools/pricetable/table.go index 3632afb321..9febd6674a 100644 --- a/src/internal/tools/pricetable/table.go +++ b/src/internal/tools/pricetable/table.go @@ -35,6 +35,9 @@ type literal struct { type entry struct { Key string Fields map[string]literal + // End is the byte offset just past the row's ModelPrice literal; --add + // inserts new rows after the line it ends on. + End int } // table is the parsed price table: the date constant and every row. @@ -128,7 +131,7 @@ func parseEntries(cl *ast.CompositeLit, offset func(token.Pos) int) ([]entry, er if !ok { return nil, fmt.Errorf("%s: value is not a ModelPrice literal", key) } - e := entry{Key: key, Fields: make(map[string]literal, len(priceFields))} + e := entry{Key: key, Fields: make(map[string]literal, len(priceFields)), End: offset(kv.End())} for _, f := range val.Elts { fkv, ok := f.(*ast.KeyValueExpr) if !ok { diff --git a/src/internal/tools/pricetable/testdata/openrouter-add.json b/src/internal/tools/pricetable/testdata/openrouter-add.json new file mode 100644 index 0000000000..26a807c749 --- /dev/null +++ b/src/internal/tools/pricetable/testdata/openrouter-add.json @@ -0,0 +1,10 @@ +{ + "data": [ + {"id": "anthropic/claude-fable-5.2", "pricing": {"prompt": "0.00001", "completion": "0.00005", "input_cache_read": "0.000001", "input_cache_write": "0.0000125"}}, + {"id": "anthropic/claude-fable-5.1", "pricing": {"prompt": "0.00001", "completion": "0.00005", "input_cache_read": "0.000001", "input_cache_write": "0.0000125"}}, + {"id": "anthropic/claude-haiku-5.5", "pricing": {"prompt": "0.000001", "completion": "0.000005", "input_cache_read": "0.0000001", "input_cache_write": "0.00000125"}}, + {"id": "anthropic/claude-opus-6", "pricing": {"prompt": "0.0001", "completion": "0.0005", "input_cache_read": "0.00001", "input_cache_write": "0.000125"}}, + {"id": "anthropic/claude-sonnet-6", "pricing": {"prompt": "0.000003", "completion": "0.000015"}}, + {"id": "anthropic/claude-nova-1", "pricing": {"prompt": "0.000002", "completion": "0.000008", "input_cache_read": "0.0000002", "input_cache_write": "0.0000025"}} + ] +} diff --git a/src/scripts/test-check-cli-model-lists.sh b/src/scripts/test-check-cli-model-lists.sh index 788a8772e3..d0c725b9a2 100755 --- a/src/scripts/test-check-cli-model-lists.sh +++ b/src/scripts/test-check-cli-model-lists.sh @@ -54,6 +54,17 @@ sed -i 's/CODEX_VERSION=0.159.0/CODEX_VERSION=0.160.0/' "$TMP/Dockerfile" run env HIVE_MODEL_CHECK_STRICT=1 [ $RC -eq 1 ] && grep -q 'codex static list comment says 0.159.0 but src/Dockerfile pins 0.160.0' "$TMP/out" && pass "stale codex version comment is caught" || fail "codex comment" "$(cat "$TMP/out")" +# A line the weekly claude-model-freshness run appends (#18754): strict mode +# fails while the allowlist lacks the model and passes once it is listed. +printf 'claude claude-haiku-5-5 2.1.326\n' > "$TMP/reqs-new.txt" +fixtures 2.1.326 2.1.326 '"claude-sonnet-5-5",' +run env HIVE_MODEL_REQUIREMENTS="$TMP/reqs-new.txt" HIVE_MODEL_CHECK_STRICT=1 +[ $RC -eq 1 ] && grep -q 'missing claude-haiku-5-5, which Claude Code 2.1.326' "$TMP/out" && pass "strict mode fails on an appended requirement the allowlist lacks" || fail "appended requirement missing" "$(cat "$TMP/out")" +fixtures 2.1.326 2.1.326 '"claude-sonnet-5-5", + "claude-haiku-5-5",' +run env HIVE_MODEL_REQUIREMENTS="$TMP/reqs-new.txt" HIVE_MODEL_CHECK_STRICT=1 +[ $RC -eq 0 ] && grep -q consistent "$TMP/out" && pass "strict mode passes once the appended model is listed" || fail "appended requirement present" "$(cat "$TMP/out")" + echo echo "passed: $PASS failed: $FAIL" [ "$FAIL" -eq 0 ] From 2cadacb0383eb062580970bea6bdc06d159ff0c6 Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:51:03 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8C=B1=20Re-pin=20TRUSTED=5FSCRIPTS?= =?UTF-8?q?=5FTREE=20for=20the=20src/scripts=20test=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/scripts/test-check-cli-model-lists.sh changed, so the pinned tree in integration-governance.cjs moves to git rev-parse HEAD:src/scripts. Refs #18754 Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcWm7qFmFJp26r9LPdmi3t --- .github/scripts/integration-governance.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 2b3a06934f..bf26987da5 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '78d89054cc569554b1066b2eb2e322a7f38c285a'; +const TRUSTED_SCRIPTS_TREE = '05cb44dd8654da786a9b4414832df28daabaaaf4'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) =>