From 205d227fe8ace09395d5d95ad1d57e2694c751e5 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:47:06 +0200 Subject: [PATCH] feat(ci): staleness is measured, and the measurement is a ratchet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-31 fleet audit found the box on an EOL Node, aoz-housing a full framework generation behind on every axis, and ai-kit pinned at three tags across seven consumers — one at the pre-org-move OWNER. None of it hidden, all of it unmeasured. Same lesson shared-inventory encodes for duplication: writing 'stay current' in a doc is what already failed. blessed-versions.json is the SSOT (blessed major per key package + the tag every internal git pin must carry); version-currency.mjs measures every default branch against it, reports UNCHECKED for anything it could not read rather than counting it clean, and --check ratchets the total: fall or hold, never rise. Baseline seeded at the audit's real number: 80. Self-test pins both sides with the audit's own fixtures (the aoz manifest, the dead-owner pin) and runs in ci.yml beside the other detector tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WqKqMnHQHSmkGFfc5t7Rxn --- .github/workflows/ci.yml | 7 + .github/workflows/version-currency.yml | 62 ++++++++ README.md | 1 + scripts/ci/blessed-versions.json | 36 +++++ scripts/ci/test-version-currency.mjs | 75 +++++++++ scripts/ci/version-currency.baseline | 1 + scripts/ci/version-currency.mjs | 212 +++++++++++++++++++++++++ 7 files changed, 394 insertions(+) create mode 100644 .github/workflows/version-currency.yml create mode 100644 scripts/ci/blessed-versions.json create mode 100644 scripts/ci/test-version-currency.mjs create mode 100644 scripts/ci/version-currency.baseline create mode 100644 scripts/ci/version-currency.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bee84a..025f7ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,13 @@ jobs: - name: The hosted-Supabase audit still detects, and still stays quiet run: bash scripts/ci/test-hosted-supabase-audit.sh + # Same doctrine: the currency ratchet reports on repos nobody touches, + # so its own test pins both sides — the real 2026-08-31 staleness + # (aoz-housing's manifest, dead-owner ai-kit pin) is caught, a current + # manifest stays quiet, and an unreadable repo is UNCHECKED, never clean. + - name: The version-currency audit still detects, and still stays quiet + run: node scripts/ci/test-version-currency.mjs + # Drift guard. The fleet is on v7; templates handing out v4 is exactly # how this repo fell behind the repos it governs. - name: No stale action versions diff --git a/.github/workflows/version-currency.yml b/.github/workflows/version-currency.yml new file mode 100644 index 0000000..66cfc8a --- /dev/null +++ b/.github/workflows/version-currency.yml @@ -0,0 +1,62 @@ +# Staleness is measured, and the measurement is a ratchet. +# +# The 2026-08-31 audit found the box on an EOL Node, one app a framework +# generation behind on every axis, and an internal package pinned at a dead +# owner — none of it hidden, all of it unmeasured. Same lesson as the shared +# inventory: writing "stay current" down is what already failed. The report +# makes the trend visible; `--check` makes it binding. +name: Version currency + +on: + schedule: + # Weekly. Majors ship on the timescale of quarters; daily would be noise, + # and noise gets muted. + - cron: '17 6 * * 1' + workflow_dispatch: + inputs: + strict: + description: 'Fail when the gap count has risen' + type: boolean + default: true + +permissions: + contents: read + +jobs: + currency: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Self-test the detector first + run: node scripts/ci/test-version-currency.mjs + + - name: Measure the fleet + env: + # Default token reads public repos only; private ones would land in + # UNCHECKED (visibly — never counted as clean). FLEET_READ_TOKEN + # covers them all. + GH_TOKEN: ${{ secrets.FLEET_READ_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -uo pipefail + { + echo '## Version currency' + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + set +e + node scripts/ci/version-currency.mjs 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + set -e + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: The ratchet — gaps may fall, never rise + if: ${{ inputs.strict != false }} + env: + GH_TOKEN: ${{ secrets.FLEET_READ_TOKEN || secrets.GITHUB_TOKEN }} + run: node scripts/ci/version-currency.mjs --check diff --git a/README.md b/README.md index 12bbd67..2420008 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ versa. `dotfiles` is the environment again; this repo is the automation. | `scripts/ci/model-pin-audit.mjs` | runs daily: is any model id the fleet pins still served by its vendor? | | `scripts/ci/verify-floor-audit.sh` | does every repo's `verify` actually run lint + typecheck + test? | | `scripts/ci/shared-inventory.sh` | counts duplication across the fleet and holds it as a ratchet | +| `scripts/ci/version-currency.mjs` | measures every repo against `blessed-versions.json` (SSOT of blessed majors + internal-package tags) and holds the gap count as a ratchet | | `scripts/ci/ui-defect-audit.mjs` | do any live sites ship WCAG AA contrast failures or misaligned stacks? | | `scripts/fleet/` | local upkeep: worktree GC, stranded-work guard | | `templates/ci/` | golden CI workflows + pre-commit — deliberately ONE central copy | diff --git a/scripts/ci/blessed-versions.json b/scripts/ci/blessed-versions.json new file mode 100644 index 0000000..639a524 --- /dev/null +++ b/scripts/ci/blessed-versions.json @@ -0,0 +1,36 @@ +{ + "_comment": [ + "The fleet's blessed MAJOR version per key package — the SSOT the", + "version-currency audit measures every repo against. Raise a number here", + "(in a PR) when the fleet decides a new major is the standard; the audit", + "then reports every repo below it as a gap. Majors only: 0.x packages", + "(drizzle-orm, @anthropic-ai/sdk) have no meaningful major and are not", + "listed. next-auth is deliberately absent while v5 is beta.", + "internal_tags: the git tag every consumer of an internal package must", + "pin until the package is on npm." + ], + "majors": { + "next": 16, + "react": 19, + "react-dom": 19, + "typescript": 6, + "tailwindcss": 4, + "zod": 4, + "eslint": 10, + "vitest": 4, + "jest": 30, + "@types/node": 26, + "prettier": 3, + "tsx": 4, + "express": 5, + "dotenv": 17, + "framer-motion": 13, + "lucide-react": 1, + "openai": 7, + "@supabase/supabase-js": 2, + "pg": 8 + }, + "internal_tags": { + "ai-kit": "v0.6.2" + } +} diff --git a/scripts/ci/test-version-currency.mjs b/scripts/ci/test-version-currency.mjs new file mode 100644 index 0000000..d6853b7 --- /dev/null +++ b/scripts/ci/test-version-currency.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node +/** + * Self-test for version-currency.mjs. + * Run: node scripts/ci/test-version-currency.mjs + * + * Both sides pinned, per fleet doctrine: the real 2026-08-31 staleness is + * still caught (positive), a fully-current manifest stays silent (negative), + * and an unreadable repo is UNCHECKED — never counted as gap-free. The + * fixtures are the audit that motivated the ratchet: aoz-housing's actual + * manifest shape, including the dead-owner ai-kit pin. + */ +import { parseMajor, parseGitPin, gapsFor, collate } from "./version-currency.mjs"; + +let failures = 0; +function check(name, cond, detail = "") { + if (cond) console.log(` ok ${name}`); + else { console.log(` FAIL ${name} ${detail}`); failures++; } +} + +const blessed = { + majors: { next: 16, react: 19, tailwindcss: 4, zod: 4, eslint: 10, "@types/node": 26 }, + internal_tags: { "ai-kit": "v0.6.2" }, +}; + +// ── parseMajor / parseGitPin ──────────────────────────────────────────────── +check("parseMajor ^16.2.3 -> 16", parseMajor("^16.2.3") === 16); +check("parseMajor '*' -> null", parseMajor("*") === null); +check("parseGitPin github:catomean/ai-kit#v0.4.0", + JSON.stringify(parseGitPin("github:catomean/ai-kit#v0.4.0")) === + JSON.stringify({ owner: "catomean", repo: "ai-kit", tag: "v0.4.0" })); +check("parseGitPin registry range -> null", parseGitPin("^4.5.4") === null); + +// ── Positive: the aoz-housing regression is caught ────────────────────────── +const stale = { + dependencies: { next: "^14.2.5", react: "^18.3.1", "ai-kit": "github:catomean/ai-kit#v0.4.0" }, + devDependencies: { tailwindcss: "^3.4.9", zod: "^3.23.8", eslint: "^8.57.0", "@types/node": "^20.14.10" }, +}; +const staleGaps = gapsFor(stale, blessed); +check("stale manifest: 6 major gaps + 1 pin gap", staleGaps.length === 7, `got ${JSON.stringify(staleGaps)}`); +check("dead-owner pin is named as such", staleGaps.some((g) => g.includes("dead after org move"))); +check("next gap present", staleGaps.some((g) => g.startsWith("next "))); + +// ── Negative: a current manifest stays silent ─────────────────────────────── +const current = { + dependencies: { next: "16.3.3", react: "^19.2.8", "ai-kit": "github:bitbaum/ai-kit#v0.6.2" }, + devDependencies: { tailwindcss: "^4.3.3", zod: "^4.5.4", eslint: "^10.9.1", "@types/node": "^26.4.0" }, +}; +check("current manifest: zero gaps", gapsFor(current, blessed).length === 0, + `got ${JSON.stringify(gapsFor(current, blessed))}`); + +// ── Packages the repo does not use are not gaps ───────────────────────────── +check("absent package is not a gap", gapsFor({ dependencies: {} }, blessed).length === 0); + +// ── Right-tag wrong-owner and right-owner wrong-tag both flagged ──────────── +check("stale bitbaum tag flagged", + gapsFor({ dependencies: { "ai-kit": "github:bitbaum/ai-kit#v0.5.0" } }, blessed).length === 1); + +// ── UNCHECKED is not clean ────────────────────────────────────────────────── +const { rows, total, uncheckedRepos } = collate([ + { repo: "good", pkg: current, gaps: gapsFor(current, blessed) }, + { repo: "bad", pkg: stale, gaps: staleGaps }, + { repo: "unreadable" }, // no pkg — could not fetch +]); +check("unreadable repo lands in UNCHECKED", uncheckedRepos.length === 1 && uncheckedRepos[0] === "unreadable"); +check("unreadable repo not in measured rows", rows.every((r) => r.repo !== "unreadable")); +check("total counts only measured gaps", total === 7); +check("worst repo sorts first", rows[0].repo === "bad"); + +// ── Mutation-proof: a lowered baseline must trip the ratchet ──────────────── +// (The ratchet math is `total > baseline` in main(); assert the collate side +// that feeds it — a stale fleet yields a total a clean baseline cannot cover.) +check("mutation: stale total exceeds a clean baseline", total > 0); + +if (failures) { console.error(`\n${failures} failing`); process.exit(1); } +console.log("\nall green"); diff --git a/scripts/ci/version-currency.baseline b/scripts/ci/version-currency.baseline new file mode 100644 index 0000000..d15a2cc --- /dev/null +++ b/scripts/ci/version-currency.baseline @@ -0,0 +1 @@ +80 diff --git a/scripts/ci/version-currency.mjs b/scripts/ci/version-currency.mjs new file mode 100644 index 0000000..6610ad2 --- /dev/null +++ b/scripts/ci/version-currency.mjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node +/** + * Fleet audit: how far is each repo from the blessed version of everything + * it uses — and is that distance shrinking? + * + * WHY THIS EXISTS + * --------------- + * On 2026-08-31 a manual audit of all 38 repos found the production box on + * Node 20 four months past EOL, one app (aoz-housing) a full framework + * generation behind on every axis at once, the flagship's own CLAUDE.md + * describing a Tailwind major that main had left months earlier, and ai-kit + * pinned at three different tags across seven consumers — one of them still + * pointing at the repo's pre-org-move OWNER. None of this was hidden. All of + * it was unmeasured, so none of it ever became anyone's next action. + * + * Same doctrine as shared-inventory.sh: writing "keep dependencies current" + * in a doc is what already failed. This produces a NUMBER per repo — how many + * key packages sit below the fleet's blessed major — and `--check` makes the + * total a ratchet: it may fall, it may hold, it may never rise. + * + * WHAT IS BLESSED + * --------------- + * blessed-versions.json, next to this script, is the SSOT. Raising a blessed + * major there is a deliberate PR — the moment the fleet decides "Next 17 is + * the standard now", every repo below it becomes a counted gap. The audit + * never decides what is current; it only measures distance from the decision. + * + * WHAT IT DOES NOT PROVE + * ---------------------- + * That a repo at the blessed major WORKS — CI proves that, per repo. And it + * judges only manifests it could READ: a repo whose package.json could not be + * fetched is reported UNCHECKED, never counted as gap-free. "I could not + * look" and "nothing is stale" are different answers, and collapsing them is + * how a broken detector reports a healthy fleet. + * + * Reads each repo's REMOTE default branch via the contents API, never a local + * checkout: clones drift, and this fleet has already shipped a redundant PR + * off a stale clone. + * + * Usage: + * node scripts/ci/version-currency.mjs # report + * node scripts/ci/version-currency.mjs --check # ratchet: exit 1 if total gaps ROSE + * node scripts/ci/version-currency.mjs --update # rewrite the baseline (do it in a PR) + * + * Env: GH_OWNER (default bitbaum), GH_LIMIT (default 100) + */ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const BLESSED_PATH = join(HERE, "blessed-versions.json"); +const BASELINE_PATH = process.env.VERSION_CURRENCY_BASELINE || join(HERE, "version-currency.baseline"); + +// ── Pure logic (exported for the self-test; no network below this line) ───── + +/** First integer in a semver-ish range: "^16.2.3" -> 16, "16" -> 16, "*" -> null. */ +export function parseMajor(range) { + if (typeof range !== "string") return null; + const m = range.match(/(\d+)/); + return m ? Number(m[1]) : null; +} + +/** Is this dependency value a git pin (github:owner/repo#tag)? Returns {owner, repo, tag} or null. */ +export function parseGitPin(range) { + if (typeof range !== "string") return null; + const m = range.match(/^(?:github:|git\+https:\/\/github\.com\/)([\w.-]+)\/([\w.-]+?)(?:\.git)?#(.+)$/); + return m ? { owner: m[1], repo: m[2], tag: m[3] } : null; +} + +/** + * Gaps for one manifest against the blessed config. + * Returns an array of human-readable gap strings; empty = current. + */ +export function gapsFor(pkg, blessed) { + const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) }; + const gaps = []; + for (const [name, blessedMajor] of Object.entries(blessed.majors)) { + if (!(name in deps)) continue; + const have = parseMajor(deps[name]); + if (have !== null && have < blessedMajor) { + gaps.push(`${name} ${deps[name]} < blessed ${blessedMajor}`); + } + } + for (const [name, blessedTag] of Object.entries(blessed.internal_tags || {})) { + if (!(name in deps)) continue; + const pin = parseGitPin(deps[name]); + if (pin) { + if (pin.owner !== (process.env.GH_OWNER || "bitbaum")) { + gaps.push(`${name} pinned at owner ${pin.owner} (dead after org move)`); + } else if (pin.tag !== blessedTag) { + gaps.push(`${name} #${pin.tag} != blessed #${blessedTag}`); + } + } + // A registry version pin is not judged here — once packages are on npm, + // move them into `majors` and delete the internal_tags entry. + } + return gaps; +} + +/** Collate per-repo results into the report + total. */ +export function collate(results) { + const checked = results.filter((r) => r.pkg !== undefined); + const unchecked = results.filter((r) => r.pkg === undefined); + const rows = checked + .map((r) => ({ repo: r.repo, gaps: r.gaps })) + .sort((a, b) => b.gaps.length - a.gaps.length || a.repo.localeCompare(b.repo)); + const total = rows.reduce((n, r) => n + r.gaps.length, 0); + return { rows, total, uncheckedRepos: unchecked.map((r) => r.repo) }; +} + +// ── Fleet reading (network) ───────────────────────────────────────────────── + +function gh(args) { + return execFileSync("gh", args, { encoding: "utf8", timeout: 60000 }); +} + +function listRepos(owner, limit) { + const raw = gh(["repo", "list", owner, "--limit", String(limit), "--json", "name,isArchived"]); + return JSON.parse(raw) + .filter((r) => !r.isArchived) + .map((r) => r.name) + .sort(); +} + +function fetchManifest(owner, repo) { + try { + const raw = gh(["api", `repos/${owner}/${repo}/contents/package.json`, "--jq", ".content"]); + return JSON.parse(Buffer.from(raw.trim(), "base64").toString("utf8")); + } catch { + return undefined; // unreadable OR absent — resolved by root listing below + } +} + +function hasPackageJson(owner, repo) { + try { + const raw = gh(["api", `repos/${owner}/${repo}/contents/`, "--jq", "[.[].name]"]); + return JSON.parse(raw).includes("package.json"); + } catch { + return null; // could not even list — UNCHECKED + } +} + +function main() { + const mode = process.argv.includes("--check") ? "check" : process.argv.includes("--update") ? "update" : "report"; + const owner = process.env.GH_OWNER || "bitbaum"; + const limit = Number(process.env.GH_LIMIT || 100); + const blessed = JSON.parse(readFileSync(BLESSED_PATH, "utf8")); + + const results = []; + for (const repo of listRepos(owner, limit)) { + const present = hasPackageJson(owner, repo); + if (present === false) continue; // shell/docs repo: no Node surface, not a gap + if (present === null) { + results.push({ repo }); // UNCHECKED + continue; + } + const pkg = fetchManifest(owner, repo); + if (pkg === undefined) { + results.push({ repo }); // listed but unreadable — UNCHECKED, never zero + continue; + } + results.push({ repo, pkg, gaps: gapsFor(pkg, blessed) }); + } + + const { rows, total, uncheckedRepos } = collate(results); + + console.log(`version-currency: ${rows.length} repos measured against blessed-versions.json`); + for (const r of rows) { + if (r.gaps.length === 0) continue; + console.log(`\n ${r.repo} (${r.gaps.length})`); + for (const g of r.gaps) console.log(` - ${g}`); + } + const current = rows.filter((r) => r.gaps.length === 0).map((r) => r.repo); + if (current.length) console.log(`\n current (0 gaps): ${current.join(", ")}`); + if (uncheckedRepos.length) { + console.log(`\n UNCHECKED (could not read — not counted as clean): ${uncheckedRepos.join(", ")}`); + } + console.log(`\n TOTAL GAPS: ${total}`); + + if (mode === "update") { + writeFileSync(BASELINE_PATH, `${total}\n`); + console.log(` baseline written: ${total}`); + return; + } + if (mode === "check") { + let baseline; + try { + baseline = Number(readFileSync(BASELINE_PATH, "utf8").trim()); + } catch { + console.error(" no baseline — run --update in a PR first"); + process.exit(1); + } + if (Number.isNaN(baseline)) { + console.error(" baseline unreadable — refusing to compare against garbage"); + process.exit(1); + } + if (total > baseline) { + console.error(` RATCHET: gaps rose ${baseline} -> ${total}. Fix the regressions or raise the baseline in a PR a human sees.`); + process.exit(1); + } + if (total < baseline) { + console.log(` gaps fell ${baseline} -> ${total} — tighten the baseline with --update in your next PR`); + } else { + console.log(` holding at ${total}`); + } + } +} + +const invokedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; +if (invokedDirectly) main();