From af905b5e7a267435c73b9246c20a4a30a4e3b12f Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:13:33 +0200 Subject: [PATCH] chore: fleet automation moves to bitbaum/fleet; this repo is the environment again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A repo holding a .bashrc was the wrong place for the machinery gating thirty other repos: three sessions and the owner collided here in one week, and every fleet change looked like an environment change. The audits, templates, registry and sweep now live in bitbaum/fleet (layout unchanged, all six test suites green there, secrets set, CI green). Nothing breaks for callers: - auto-merge-sweep.yml stays as a FORWARDING SHIM — the 16 repos calling it as a reusable workflow keep working; the shim maps the historic ref 'master' to fleet's 'main' and holds no logic. - SHARED.md stays as a pointer, because a decade of docs and habits say to check it here. The registry is maintained only in fleet. - This repo's own auto-merge now calls fleet's sweep like every other repo. CI slims to what this repo now is: shell syntax plus an install.sh dangling-symlink check. CLAUDE.md and README describe the environment repo; the global standards' pointers now name fleet/SHARED.md and fleet/templates/ci. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AZCNYHwjEeqxNpYCUk6Yna --- .claude/CLAUDE.md | 6 +- .github/workflows/auto-merge-sweep.yml | 99 +-- .github/workflows/auto-merge.yml | 28 +- .github/workflows/ci.yml | 108 +-- .github/workflows/model-pins.yml | 99 --- .github/workflows/shared-inventory.yml | 56 -- .github/workflows/ui-defects.yml | 88 --- .github/workflows/verify-floor.yml | 68 -- CLAUDE.md | 67 +- README.md | 44 +- SHARED.md | 185 +---- scripts/audit/README.md | 96 --- scripts/audit/secret-in-response.py | 126 --- scripts/audit/test-secret-in-response.py | 96 --- scripts/ci/auto-merge-sweep.sh | 446 ----------- scripts/ci/find-false-needs.py | 202 ----- scripts/ci/model-pin-audit.mjs | 888 ---------------------- scripts/ci/shared-inventory.baseline | 9 - scripts/ci/shared-inventory.sh | 190 ----- scripts/ci/test-auto-merge-sweep.sh | 278 ------- scripts/ci/test-model-pin-audit.mjs | 739 ------------------ scripts/ci/test-shared-inventory.sh | 109 --- scripts/ci/test-ui-defect-audit.mjs | 260 ------- scripts/ci/test-verify-predicates.sh | 209 ----- scripts/ci/ui-defect-audit.mjs | 405 ---------- scripts/ci/verify-floor-audit.sh | 578 -------------- scripts/ci/verify-predicates.sh | 162 ---- scripts/fleet/gc-merged-worktrees.sh | 141 ---- scripts/fleet/stranded-work.sh | 247 ------ scripts/fleet/test-gc-merged-worktrees.sh | 95 --- scripts/fleet/test-stranded-work.sh | 276 ------- templates/REVIEW.md | 29 - templates/ci/README.md | 327 -------- templates/ci/ci-npm.yml | 43 -- templates/ci/ci-pnpm.yml | 53 -- templates/ci/pre-commit | 21 - 36 files changed, 104 insertions(+), 6769 deletions(-) delete mode 100644 .github/workflows/model-pins.yml delete mode 100644 .github/workflows/shared-inventory.yml delete mode 100644 .github/workflows/ui-defects.yml delete mode 100644 .github/workflows/verify-floor.yml delete mode 100644 scripts/audit/README.md delete mode 100755 scripts/audit/secret-in-response.py delete mode 100644 scripts/audit/test-secret-in-response.py delete mode 100644 scripts/ci/auto-merge-sweep.sh delete mode 100755 scripts/ci/find-false-needs.py delete mode 100755 scripts/ci/model-pin-audit.mjs delete mode 100644 scripts/ci/shared-inventory.baseline delete mode 100755 scripts/ci/shared-inventory.sh delete mode 100755 scripts/ci/test-auto-merge-sweep.sh delete mode 100755 scripts/ci/test-model-pin-audit.mjs delete mode 100755 scripts/ci/test-shared-inventory.sh delete mode 100644 scripts/ci/test-ui-defect-audit.mjs delete mode 100755 scripts/ci/test-verify-predicates.sh delete mode 100755 scripts/ci/ui-defect-audit.mjs delete mode 100755 scripts/ci/verify-floor-audit.sh delete mode 100755 scripts/ci/verify-predicates.sh delete mode 100755 scripts/fleet/gc-merged-worktrees.sh delete mode 100755 scripts/fleet/stranded-work.sh delete mode 100755 scripts/fleet/test-gc-merged-worktrees.sh delete mode 100755 scripts/fleet/test-stranded-work.sh delete mode 100644 templates/REVIEW.md delete mode 100644 templates/ci/README.md delete mode 100644 templates/ci/ci-npm.yml delete mode 100644 templates/ci/ci-pnpm.yml delete mode 100644 templates/ci/pre-commit diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index ce86ad0..ae0c2c2 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -135,15 +135,15 @@ Concrete forms, cheapest first: (lint + typecheck + test) that CI calls verbatim — the check bundle is defined once, run identically locally and on the shared branch. Green `verify` locally ⇒ green CI. Run it before declaring any change done. (Golden templates: -`dotfiles/templates/ci/`.) +`fleet/templates/ci/`.) -**Check `dotfiles/SHARED.md` before building anything cross-cutting** — AI +**Check `fleet/SHARED.md` before building anything cross-cutting** — AI calls, form fill, rate limiting, email, logging, health routes, CI sweeps. If a package there already owns it, install it; do not write a second one. Measured 2026-08-16: `auto-merge-sweep.sh` exists in 22 repos in 8 different versions, and rate limiting has 14 implementations (orangecat alone has 4, and its ADR to unify them has been "Proposed" since January while the count doubled). The -duplication count is a ratchet — `scripts/ci/shared-inventory.sh --check` — and +duplication count is a ratchet — `fleet: scripts/ci/shared-inventory.sh --check` — and it may fall or hold but never rise. If a copy really is justified, raise the baseline in the same PR so a human sees the decision instead of inheriting it. diff --git a/.github/workflows/auto-merge-sweep.yml b/.github/workflows/auto-merge-sweep.yml index d16b029..a9923bf 100644 --- a/.github/workflows/auto-merge-sweep.yml +++ b/.github/workflows/auto-merge-sweep.yml @@ -1,101 +1,56 @@ -# The auto-merge sweep, once, for the whole fleet. +# FORWARDING SHIM — the sweep lives in bitbaum/fleet now (moved 2026-08-28). # -# Reusable (`workflow_call`). A repo adopts it with ~10 lines instead of a -# ~400-line copy that will drift — and 22 copies did drift, into 8 versions, -# each repo's outage fix reaching only itself. See scripts/ci/auto-merge-sweep.sh. -# -# The caller keeps its own triggers (workflow_run / schedule) because those are -# genuinely per-repo: the CI workflow's NAME differs, and `workflow_run` must -# name it exactly. -name: Auto-merge sweep (reusable) +# Sixteen repos call this file as a reusable workflow. This shim keeps every +# one of them working unchanged while they migrate to +# uses: bitbaum/fleet/.github/workflows/auto-merge-sweep.yml@main +# at their leisure. NO LOGIC LIVES HERE — logic in a shim is drift, which is +# the disease this whole arrangement exists to cure. + +name: Auto-merge sweep (moved to bitbaum/fleet) on: workflow_call: inputs: base_branch: - description: 'Branch PRs merge into (main for most, master for some).' type: string default: main ci_workflow: - description: 'CI workflow file whose green run gates a merge.' type: string default: ci.yml rearm_workflows: - description: > - SPACE-separated workflows to dispatch after a merge, e.g. - "ci.yml deploy.yml". The script word-splits this value, so a - comma-separated list becomes ONE bogus token, every dispatch fails, - and the only symptom is that nothing deploys. - - Load-bearing: a push made with GITHUB_TOKEN triggers NO workflows, so - without a working re-arm, merges land and silently never ship — and - the sweep still exits 0, so the automation looks healthy throughout. type: string default: '' deploy_workflow: - description: > - Workflow that ships the base branch, e.g. deploy.yml. EMPTY DISABLES - the reconciler, and empty is the default. - - Set it when merges must reach production: a push made with - GITHUB_TOKEN emits no workflow_run event — and neither does a run that - GITHUB_TOKEN dispatched — so nothing downstream wakes on an automated - merge. Rather than trust a trigger, the sweep compares the base tip - against the last successful deploy and closes the gap, so a deploy - that never fired or fired and failed is retried next sweep instead of - leaving a commit merged-but-not-live. type: string default: '' max_run_attempts: - description: 'How many times to re-run a run that produced no verdict.' type: string default: '3' ref: - description: 'Pin the sweep version (tag/sha of catomean/dotfiles).' + description: > + Pin the sweep version. Historic callers pass tags/shas of + bitbaum/dotfiles; the default 'master' (dotfiles' branch) is mapped + to fleet's 'main' below. A pinned sha of dotfiles will NOT resolve + in fleet — repin against bitbaum/fleet when migrating. type: string default: master - secrets: - token: - description: > - Token for merging. GITHUB_TOKEN works for most repos but CANNOT merge - a PR that touches .github/workflows — pass a PAT where that matters. - required: false permissions: - contents: write # merge the PR - pull-requests: write # read PR state, delete the branch - actions: write # dispatch the re-arm workflows - # Only load-bearing on a PRIVATE repo, which is exactly why it went missing - # for so long: the sweep reads statusCheckRollup to decide whether a PR is - # green, a public repo answers that with no explicit scope, and the copied - # file worked in ~20 public repos. The first private repo to run it - # (ivy-portal) failed every sweep with "Resource not accessible by - # integration" and merged nothing, ever — indistinguishable from having - # nothing to merge. + contents: write + pull-requests: write + actions: write checks: read statuses: read jobs: - sweep: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - # Check out the SWEEP, not the caller. The script talks to the caller repo - # entirely through the API (GH_REPO), so it never needs that checkout — - # and not taking one keeps a repo's own contents from influencing how its - # PRs get merged. - - uses: actions/checkout@v7 - with: - repository: catomean/dotfiles - ref: ${{ inputs.ref }} - - - name: Sweep - env: - GH_TOKEN: ${{ secrets.token || github.token }} - GH_REPO: ${{ github.repository }} - BASE_BRANCH: ${{ inputs.base_branch }} - CI_WORKFLOW: ${{ inputs.ci_workflow }} - REARM_WORKFLOWS: ${{ inputs.rearm_workflows }} - DEPLOY_WORKFLOW: ${{ inputs.deploy_workflow }} - MAX_RUN_ATTEMPTS: ${{ inputs.max_run_attempts }} - run: bash scripts/ci/auto-merge-sweep.sh + forward: + uses: bitbaum/fleet/.github/workflows/auto-merge-sweep.yml@main + with: + base_branch: ${{ inputs.base_branch }} + ci_workflow: ${{ inputs.ci_workflow }} + rearm_workflows: ${{ inputs.rearm_workflows }} + deploy_workflow: ${{ inputs.deploy_workflow }} + max_run_attempts: ${{ inputs.max_run_attempts }} + ref: ${{ inputs.ref == 'master' && 'main' || inputs.ref }} + secrets: + token: ${{ secrets.token }} diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index 27005f1..0716fd5 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -47,20 +47,16 @@ concurrency: cancel-in-progress: false jobs: + # The sweep script moved to bitbaum/fleet with the rest of the fleet + # automation (2026-08-28); this repo calls it like every other repo does. sweep: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - - name: Merge every green, ready PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - BASE_BRANCH: ${{ github.event.repository.default_branch }} - CI_WORKFLOW: ci.yml - # Everything that runs on push and therefore would NOT fire after a - # merge made with the default GITHUB_TOKEN. Keep this in sync when a - # push-triggered workflow is added. - REARM_WORKFLOWS: ci.yml - run: bash scripts/ci/auto-merge-sweep.sh + uses: bitbaum/fleet/.github/workflows/auto-merge-sweep.yml@main + with: + base_branch: master + ci_workflow: ci.yml + # Everything that runs on push and therefore would NOT fire after a + # merge made with the default GITHUB_TOKEN. Keep this in sync when a + # push-triggered workflow is added. + rearm_workflows: ci.yml + secrets: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afed102..a96f577 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,6 @@ -# The template repo defends every other repo's branch. Until now it defended -# none of its own — no .github/ at all — and it drifted: it still handed out -# actions/checkout@v4 and setup-node@v4 sixteen days after the fleet moved to -# v7, so every fresh copy re-introduced the old versions. A source of truth -# that nothing checks is just a file. +# The environment repo's own CI. The fleet-automation test suites moved to +# bitbaum/fleet with the scripts they test (2026-08-28); what remains here is +# the environment, so what CI checks is the environment. name: CI on: @@ -19,7 +17,7 @@ concurrency: jobs: verify: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 5 steps: - uses: actions/checkout@v7 @@ -35,94 +33,14 @@ jobs: done < <(find . -name '*.sh' -not -path './.git/*') echo "shell syntax: ok ($found script(s) checked)" - # Every CI template must be valid YAML — a broken template is copied - # into a repo and fails there, far from here. - - name: Templates are valid YAML + # Every symlink install.sh would create must point at a file that exists + # in this repo — a dangling link is a broken shell on a new machine. + - name: install.sh targets exist run: | set -euo pipefail - found=0 - for f in templates/ci/*.yml; do - [ -e "$f" ] || continue - found=$((found + 1)) - python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$f" - done - [ "$found" -gt 0 ] || { echo "no CI templates found — did they move?" >&2; exit 1; } - echo "templates parse: ok ($found template(s))" - - # The template's whole premise is that "verified" is defined ONCE, in - # package.json `verify`, and CI calls it verbatim. If a template stops - # doing that, the premise is gone and nobody would notice. - - name: Templates still call the verify SSOT - run: | - set -euo pipefail - for f in templates/ci/ci-npm.yml templates/ci/ci-pnpm.yml; do - grep -q 'run verify' "$f" || { - echo "$f no longer calls the verify SSOT — the template's core promise" >&2 - exit 1 - } - done - echo "verify SSOT: present in both templates" - - # The fleet audit enforces the template's premise across every repo, so it - # is the last thing that may quietly stop working. Its wiring rules are - # tested BOTH ways: that each still bites against a violating fixture, and - # that conforming shapes are not flagged — a checker that cries wolf gets - # ignored, which is the same end state as no checker. - # - # The rules sit in verify-predicates.sh so they are testable at all: the - # audit is remote-only by design, and a rule that can only be exercised by - # a live API call is a rule nobody re-tests after editing its regex. - - name: The verify-contract rules can still go red - run: bash scripts/ci/test-verify-predicates.sh - - # The duplication ratchet's entire value is that it CAN go red. A ratchet - # that silently passes while duplication rises certifies the thing it was - # built to stop. Its counting half needs the API; its deciding half is - # pure text and is tested here against fixtures, with no network. - - name: The duplication ratchet can still go red - run: bash scripts/ci/test-shared-inventory.sh - - # The stranded-work guard replaces a check that failed by being ignorable: - # git-health reported orangecat's 118 dirty files every day for ten days - # and changed nothing. This one keys on AGE and stays silent when healthy, - # so both failure modes are gates — it must go red on aged work, and it - # must stay quiet on a fresh tree, or it gets muted and is then absent. - - name: The stranded-work guard can go red, and stays quiet when it should - run: bash scripts/fleet/test-stranded-work.sh - - # This one DELETES CHECKOUTS, so its only interesting failure is a false - # positive. Every refusal in the predicate is a case that would otherwise - # have destroyed the single existing copy of some work, and each is pinned - # separately — a combined "unsafe" fixture is exactly what hides one guard - # silently inverting. - - name: The worktree GC refuses everything it should - run: bash scripts/fleet/test-gc-merged-worktrees.sh - - # The sweep decides what ships in every repo that calls it, so it is the - # last script here that should be untested — and until now it was. These - # run the REAL script against a fake `gh`, exercising shipped control flow - # rather than a description of it. Ported from evig, the only repo that - # had them, when its copy of the script was centralised. - - name: The auto-merge sweep behaves - run: bash scripts/ci/test-auto-merge-sweep.sh - - # This detector reports on repos nobody is watching, so its own failure - # mode is silence: a clean report from a broken audit is worse than no - # audit, because it prints a ✓. Both sides are pinned — the real AOZ - # regression is still caught, corrected code stays quiet — plus the two - # false positives the first live run produced (xAI's `grok-3-mini` filed - # under Groq, and a computed `${...}` id read as a pin). No network, no - # key, no checkout, so it runs here as well as in the daily sweep. - - name: The model-pin audit still detects, and still stays quiet - run: node scripts/ci/test-model-pin-audit.mjs - - # Drift guard. The fleet is on v7; templates handing out v4 is exactly - # how this repo fell behind the repos it governs. - - name: No stale action versions - run: | - set -euo pipefail - if grep -rnE 'actions/(checkout|setup-node)@v[1-6]\b' templates/; then - echo "stale action version in a template — the fleet is on v7" >&2 - exit 1 - fi - echo "action versions: no stale pins" + missing=0 + while IFS= read -r target; do + [ -e "$target" ] || { echo "install.sh links '$target', which does not exist" >&2; missing=1; } + done < <(grep -oP '^\s*link\s+\K\S+' install.sh || true) + [ "$missing" -eq 0 ] + echo "install targets: ok" diff --git a/.github/workflows/model-pins.yml b/.github/workflows/model-pins.yml deleted file mode 100644 index f8e48c7..0000000 --- a/.github/workflows/model-pins.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Is any model id this fleet pins still served by its vendor? -# -# On 2026-08-26 the AOZ assistant reported "KI-Assistent nicht konfiguriert" on -# a deployment whose key was valid. Groq had retired the whole llama-3.x family. -# Production had been failing exactly as long as the demo, and nobody knew, -# because "not configured" is the only thing the app can say. -# -# Six pins across five repos were dead the same morning. None of it was -# detectable from inside a repo: every gate was green, because the code is -# correct and the vendor changed underneath it. -# -# Daily, not weekly. A retirement is decided by someone outside this fleet and -# lands without warning — unlike the UI defects next door, which arrive with a -# design change. The check costs ZERO tokens (one GET /models per vendor), so -# the only argument against running it often is noise, and a silent day is one -# line in a job summary. -name: Model pins - -on: - schedule: - # 07:10 UTC, before the working day — a retirement found at 09:00 is a - # morning's work; one found by a user is an outage of unknown age. - - cron: '10 7 * * *' - workflow_dispatch: - inputs: - strict: - description: 'Fail the run when pins are retired' - type: boolean - default: false - pull_request: - paths: - - 'scripts/ci/model-pin-audit.mjs' - - 'scripts/ci/test-model-pin-audit.mjs' - - '.github/workflows/model-pins.yml' - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: 22 - - # The detector's own test needs no network, no key and no checkout, so it - # runs on pull_request too — where the sweep below cannot, because a fork - # has no secrets and an unreadable catalogue is deliberately not a pass. - - name: Self-test the detector - run: node scripts/ci/test-model-pin-audit.mjs - - # dotfiles has no package.json on purpose. Install ai-kit into a scratch - # dir and point the audit at it — the same shape ui-defects.yml uses for - # playwright. The vendor query lives in ai-kit precisely so this repo does - # not grow a second copy of it. - # - # Installed from GitHub rather than npm: ai-kit v0.3.0 is tagged and built - # but not on the registry, because the repo has no NPM_TOKEN. Switch this - # to `npm i ai-kit` once it does. - - name: Install ai-kit - if: github.event_name != 'pull_request' - run: | - mkdir -p "$RUNNER_TEMP/air" && cd "$RUNNER_TEMP/air" - npm init -y >/dev/null - npm i --no-audit --no-fund github:catomean/ai-kit#v0.3.0 >/dev/null - - - name: Audit the fleet's pins - if: github.event_name != 'pull_request' - env: - AI_KIT_FROM: ${{ runner.temp }}/air - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Absent keys are handled, not fatal: the vendor is reported UNCHECKED - # rather than clean. "I could not look" is not "nothing is wrong". - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - run: | - set -uo pipefail - flag=--warn-only - if [ "${{ inputs.strict }}" = "true" ]; then flag=""; fi - - # Into the job summary, not just the log. A finding nobody scrolls to - # is the same as no finding — which is how eight days passed. - { - echo '## Model pins' - echo - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - set +e - node scripts/ci/model-pin-audit.mjs $flag 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - status=${PIPESTATUS[0]} - set -e - - echo '```' >> "$GITHUB_STEP_SUMMARY" - exit "$status" diff --git a/.github/workflows/shared-inventory.yml b/.github/workflows/shared-inventory.yml deleted file mode 100644 index baf7521..0000000 --- a/.github/workflows/shared-inventory.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Duplication is measured, and the measurement is a ratchet. -# -# Every duplication in this fleet was already known before it was measured — -# orangecat's rate-limiting ADR has sat "Proposed" since January while the count -# went from two to four. Knowing has never changed the number. A weekly report -# that nobody can act on would repeat that mistake, so the report exists to make -# the trend visible and `--check` exists to make it binding. -name: Shared inventory - -on: - schedule: - # Weekly. Duplication grows on the timescale of features and new repos, not - # individual commits — a daily run would be noise, and noise gets muted. - - cron: '41 6 * * 1' - workflow_dispatch: - inputs: - strict: - description: 'Fail when duplication has risen' - type: boolean - default: true - -permissions: - contents: read - -jobs: - inventory: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v7 - - - name: Measure the fleet - env: - # Same gap as the verify-floor audit, stated rather than hidden: the - # default token reads public repos only, so private ones are silently - # omitted. The script prints how many repos it inspected — a drop in - # that number is the tell. Set FLEET_READ_TOKEN to cover all of them. - GH_TOKEN: ${{ secrets.FLEET_READ_TOKEN || secrets.GITHUB_TOKEN }} - run: | - set -uo pipefail - { - echo '## Shared inventory' - echo - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - set +e - bash scripts/ci/shared-inventory.sh 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - set -e - echo '```' >> "$GITHUB_STEP_SUMMARY" - - - name: The ratchet — duplication may fall, never rise - if: ${{ inputs.strict != false }} - env: - GH_TOKEN: ${{ secrets.FLEET_READ_TOKEN || secrets.GITHUB_TOKEN }} - run: bash scripts/ci/shared-inventory.sh --check diff --git a/.github/workflows/ui-defects.yml b/.github/workflows/ui-defects.yml deleted file mode 100644 index 1ead34f..0000000 --- a/.github/workflows/ui-defects.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Two defect classes that only exist once a page is painted, so no unit test, -# type check or lint rule in any repo can see them: -# -# - an interactive label below its WCAG AA contrast floor (an action nobody -# can find is a feature that does not exist) -# - a stack whose rows start at different x, or a wrapped line that does not -# align with the line above it -# -# Both were reported by the operator in plain language — "i dont see it" and -# "this area looks bad. not aligned" — after every gate in the repo was green. -# -# A rule nothing enforces is a suggestion. This runs on a schedule and reports -# into the job summary, where a human actually looks. -name: UI defects - -on: - schedule: - # Weekly. These defects arrive with design changes, not with every commit; - # a daily run would be noise nobody reads twice. - - cron: '41 6 * * 1' - workflow_dispatch: - inputs: - strict: - description: 'Fail the run when sites have defects' - type: boolean - default: false - sites: - description: 'Comma-separated URLs (default: discovered from the fleet footer)' - type: string - default: '' - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: 22 - - # dotfiles has no package.json on purpose — it is not an npm project. - # Install the browser into a scratch directory and point the audit at it. - - name: Install playwright - run: | - mkdir -p "$RUNNER_TEMP/pw" && cd "$RUNNER_TEMP/pw" - npm init -y >/dev/null - npm i --no-audit --no-fund playwright >/dev/null - npx playwright install --with-deps chromium >/dev/null - - - name: Self-test the detector - # Runs BEFORE the sweep, deliberately. A detector that has silently - # stopped catching anything reports a clean fleet, and a clean report - # from a broken detector is worse than no report at all — it is an - # absent check that produces a ✓. The fixtures pin both sides: the real - # defect is still caught, correct markup stays silent. - env: - PLAYWRIGHT_FROM: ${{ runner.temp }}/pw - run: node scripts/ci/test-ui-defect-audit.mjs - - - name: Audit the fleet - env: - PLAYWRIGHT_FROM: ${{ runner.temp }}/pw - SITES: ${{ inputs.sites }} - run: | - set -uo pipefail - flag=--warn-only - if [ "${{ inputs.strict }}" = "true" ]; then flag=""; fi - - # Report goes to the job summary, not just the log: a finding nobody - # scrolls to is the same as no finding. - { - echo '## UI defects' - echo - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - set +e - node scripts/ci/ui-defect-audit.mjs $flag 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - status=${PIPESTATUS[0]} - set -e - - echo '```' >> "$GITHUB_STEP_SUMMARY" - exit "$status" diff --git a/.github/workflows/verify-floor.yml b/.github/workflows/verify-floor.yml deleted file mode 100644 index 45b2c93..0000000 --- a/.github/workflows/verify-floor.yml +++ /dev/null @@ -1,68 +0,0 @@ -# The golden CI floor is defined in templates/ci/README.md: every repo exposes -# one `verify` = lint && typecheck && test, and CI calls it verbatim. That -# contract was written down and never checked, so it drifted — repos ship a -# `verify` that quietly omits a gate, and "verify is green" ends up meaning -# something different in each one. -# -# A rule nothing enforces is a suggestion. This runs the audit on a schedule -# and reports into the job summary, where a human actually looks. -name: Verify floor - -on: - schedule: - # Weekly. The floor drifts on the timescale of repos being created, not - # commits being pushed — a daily run would be noise. - - cron: '17 6 * * 1' - workflow_dispatch: - inputs: - strict: - description: 'Fail the run when repos are below the floor' - type: boolean - default: false - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - # 10 was set when the audit read one file per repo. It now reads - # package.json, the git tree (to count test files), the workflow listing and - # every workflow body — roughly 150-200 API calls across the fleet, and - # local runs take 10-15 minutes. A timeout that kills the job mid-sweep - # produces no report at all, which is the precise failure this audit exists - # to find: a check that quietly stops checking. Weekly, so the ceiling is - # free; it is a backstop against a hang, not a performance target. - timeout-minutes: 30 - steps: - - uses: actions/checkout@v7 - - - name: Audit the fleet - env: - # GITHUB_TOKEN is scoped to this repo, but public repo metadata is - # readable with any token, so the scheduled run covers the 31 public - # repos and silently omits the 3 private ones (as of 2026-08-15). - # That is a real gap, stated rather than hidden: the script prints - # how many repos it inspected, so a drop is visible. Set a - # FLEET_READ_TOKEN secret with `repo` scope to cover all 34. - GH_TOKEN: ${{ secrets.FLEET_READ_TOKEN || secrets.GITHUB_TOKEN }} - run: | - set -uo pipefail - flag=--warn-only - if [ "${{ inputs.strict }}" = "true" ]; then flag=""; fi - - # Report goes to the job summary, not just the log: a finding nobody - # scrolls to is the same as no finding. - { - echo '## Verify floor' - echo - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - set +e - bash scripts/ci/verify-floor-audit.sh $flag 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - status=${PIPESTATUS[0]} - set -e - - echo '```' >> "$GITHUB_STEP_SUMMARY" - exit "$status" diff --git a/CLAUDE.md b/CLAUDE.md index e31c6d6..fd37732 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,40 +1,28 @@ ## What This Repo Is -George's dotfiles, plus the **fleet's central automation** — the CI checks and -audits that run from here against every other repo, and `SHARED.md`, the -registry of shared packages. +George's dotfiles — the Linux environment, installed by symlink. **Nothing +else.** -**It is no longer the agent runtime.** The stop/notify/dispatch loop that used -to live here — the PyQt beacon, prompt injection into Zellij tabs, the bash -bridge — migrated to FleetCrown (Fleet Runner) over 2026-06. What remains in -`.claude/hooks/` is the thin per-session edge; anything about *dispatching -work to agents* belongs in FleetCrown, not here. +Two things used to live here and both moved out: + +- **Fleet automation** (cross-repo audits, golden CI templates, `SHARED.md`, + the reusable auto-merge sweep) → **[bitbaum/fleet](https://github.com/bitbaum/fleet)**, + 2026-08-28. `SHARED.md` here is a pointer; `auto-merge-sweep.yml` here is a + forwarding shim for the 16 repos that still call it. Do not add fleet + checkers, registries or templates here — they go to `fleet`. +- **The agent runtime** (stop/notify/dispatch, PyQt beacon, prompt injection) + → **FleetCrown** (Fleet Runner), 2026-06. `.claude/hooks/` keeps only the + thin per-session edge below. ## Architecture ``` -scripts/ci/ Fleet-wide audits. These run FROM here against every repo, - so there is nothing to adopt and nothing to drift: - auto-merge-sweep.sh the canonical merge policy (16 repos - call it as a reusable workflow) - model-pin-audit.mjs is any pinned model id still served? - verify-floor-audit.sh does every repo's `verify` really run - lint + typecheck + test? - shared-inventory.sh the duplication ratchet - ui-defect-audit.mjs, verify-predicates.sh, find-false-needs.py - Each has a test-*.sh / test-*.mjs beside it. Keep it that way. -scripts/fleet/ Local upkeep: gc-merged-worktrees.sh, stranded-work.sh -scripts/audit/ secret-in-response.py -templates/ci/ Golden CI workflows + pre-commit. ONE central copy, never a - fork per repo — see templates/ci/README.md. -.github/workflows/ This repo's own CI, plus the scheduled fleet audits. -SHARED.md The shared-package registry and the duplication ratchet. - Read it before building anything cross-cutting. - -.claude/hooks/ Per-session edge only (see the table below). -.claude/skills/ fix-lint, fix-types, write-tests -.config/ kitty, starship, zellij, git hooks .bashrc .ssh/config .editorconfig install.sh +.config/ kitty, starship, zellij, git hooks +.claude/hooks/ per-session edge only (see the table below) +.claude/skills/ fix-lint, fix-types, write-tests +SHARED.md POINTER to bitbaum/fleet — the registry is maintained there +.github/workflows/ this repo's own slim CI, auto-merge, and the sweep SHIM ``` ## The hooks, and which are actually alive @@ -48,30 +36,22 @@ SHARED.md The shared-package registry and the duplication ratchet. | `keep-awake-acquire.sh` / `-release.sh` | UserPromptSubmit / Stop | **alive** — inhibits sleep while a session is working | | `fleetcrown-capture.sh`, `fleetcrown-session-end.sh` | UserPromptSubmit / Stop | **alive, but not ours** — generated by FleetCrown's Fleet Runner, which rewrites them on startup. Gitignored deliberately. Do not edit here. | | `stop.sh` | Stop | **no-op since 2026-06-11.** Dispatch moved to Fleet Runner. | -| `notification.sh` | Notification | **no-op since 2026-08-27.** Previously exec'd a bridge under `~/.local/share/fleetcrown-beacon/` that no longer exists, so it failed on every pause. | +| `notification.sh` | Notification | **no-op since 2026-08-27.** Previously exec'd a bridge that no longer exists. | -If you are about to add agent-dispatch logic here, stop: that is FleetCrown's job -now. This repo's half is the environment and the fleet-wide checks. +If you are about to add agent-dispatch logic here, stop: that is FleetCrown's +job. If you are about to add a fleet-wide checker here, stop: that is fleet's +job. ## When Working Here **Test changes:** ```bash -bash -n .claude/hooks/*.sh # syntax -scripts/ci/test-auto-merge-sweep.sh # the audits have real suites — run -scripts/ci/test-model-pin-audit.mjs # the one you touched -scripts/ci/test-shared-inventory.sh -scripts/fleet/test-stranded-work.sh +bash -n .claude/hooks/*.sh install.sh # what this repo's CI runs ``` -**Commit frequently** — these checks gate every repo in the fleet. - **After editing a hook**: takes effect on the next session event. There is no daemon to restart. -**Before adding a fleet-wide checker**: it probably exists in `scripts/ci/`. -Before adding a shared library: read `SHARED.md`. - ## Project detection Use `cwd`. The session registry at `~/.claude/sessions/.json` is the SSOT @@ -79,8 +59,7 @@ for what else is running. **Zellij tab names are not project identity.** Tabs read `Tab #9`, one pane hosts many concurrent sessions, and the `claude-projects.conf` mapping was removed on -2026-08-27 after it had been dead for months (nothing could match `Tab #9`). -Do not reintroduce it. +2026-08-27 after it had been dead for months. Do not reintroduce it. ## Quality Standards @~/.claude/CLAUDE.md diff --git a/README.md b/README.md index 2a3a938..56a4a73 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,30 @@ # dotfiles -George's Linux environment, plus the fleet's central automation. +George's Linux environment. Nothing else. -Two things live here, and only these two: - -**1. The environment.** `.bashrc`, `.ssh/config`, `.editorconfig`, and -`.config/` (kitty, starship, zellij, git hooks). Installed by symlink: +`.bashrc`, `.ssh/config`, `.editorconfig`, and `.config/` (kitty, starship, +zellij, git hooks). Installed by symlink: ```bash -git clone git@github.com:catomean/dotfiles.git ~/dev/dotfiles +git clone git@github.com:bitbaum/dotfiles.git ~/dev/dotfiles cd ~/dev/dotfiles && ./install.sh ``` -**2. Fleet-wide automation.** Checks that run *from this repo against every -other repo*, so there is one copy and nothing can drift: - -| | | -|---|---| -| [`SHARED.md`](SHARED.md) | the shared-package registry and the duplication ratchet — **read before building anything cross-cutting** | -| `scripts/ci/auto-merge-sweep.sh` | the canonical merge policy; 16 repos call it as a reusable workflow | -| `scripts/ci/model-pin-audit.mjs` | runs daily: is any model id the fleet pins still served by its vendor? | -| `scripts/ci/verify-floor-audit.sh` | does every repo's `verify` actually run lint + typecheck + test? | -| `scripts/ci/shared-inventory.sh` | counts duplication across the fleet and holds it as a ratchet | -| `templates/ci/` | golden CI workflows + pre-commit, deliberately one central copy | - -Every audit has a test suite beside it (`test-*.sh`, `test-*.mjs`). Keep it -that way — these gate every repo, so a broken checker is a fleet-wide outage. +`.claude/hooks/` holds the thin per-session Claude Code edge — see +[CLAUDE.md](CLAUDE.md) for which hooks are alive and which are deliberate +no-ops. ## What does *not* live here -Agent dispatch. The stop/notify loop, prompt injection and the PyQt beacon -migrated to **FleetCrown** (Fleet Runner) over 2026-06. `.claude/hooks/` keeps -only the thin per-session edge — see [CLAUDE.md](CLAUDE.md) for which hooks are -alive and which are deliberate no-ops. +**Fleet automation** — the cross-repo audits, the golden CI templates, the +shared-package registry (`SHARED.md`) and the reusable auto-merge sweep moved +to **[bitbaum/fleet](https://github.com/bitbaum/fleet)** on 2026-08-28. A repo +holding a `.bashrc` was the wrong place for machinery gating thirty other +repos. The old `SHARED.md` path and the reusable-workflow path both still +resolve here as pointers/shims, so nothing broke — but new work goes to +`fleet`. + +**Agent dispatch** — the stop/notify loop, prompt injection and the PyQt +beacon migrated to **FleetCrown** (Fleet Runner) over 2026-06. -Product and business decisions belong in the repo that owns them, not beside -`SHARED.md`. +Product and business decisions belong in the repo that owns them. diff --git a/SHARED.md b/SHARED.md index 4c21071..9b0d5ea 100644 --- a/SHARED.md +++ b/SHARED.md @@ -1,179 +1,10 @@ -# Shared code across the fleet +# Moved -**Before you build something, check this file. If it is here, install it.** +The shared-package registry and the duplication ratchet live in +**[bitbaum/fleet](https://github.com/bitbaum/fleet/blob/main/SHARED.md)** +(since 2026-08-28), together with the fleet audits and the golden CI +templates they belong with. -This exists because the alternative was tried and measurably failed. Every -duplication in this fleet was already known, and knowing changed nothing: - -- orangecat's `ADR-0002-rate-limiting-unification.md` (2026-01-18) is still - **Status: Proposed**. It names *two* implementations. There are now **four**. -- `templates/ci/README.md` says "deliberately ONE central script, not a copy per - repo". `auto-merge-sweep.sh` lives in **22 repos**, and as of 2026-08-16 in - **8 distinct versions** spanning 11,787–19,344 bytes. A fix landed in one - reaches at most 9 of them. - -Both were written down. Writing it down is what failed. So this file is short, -the inventory underneath it is **generated**, and the number it produces is a -**ratchet** — see "The process" below. - ---- - -## The registry — what already exists - -| Package | Install | Replaces | -|---|---|---| -| [`ai-forms`](https://github.com/bitbaum/ai-forms) | `npm i github:bitbaum/ai-forms#v0.1.0` | per-app "fill this form from prose" + conversational refinement. Headless — ships **no markup**, so each app keeps its own styling. | -| [`ai-kit`](https://github.com/bitbaum/ai-kit) | `npm i github:bitbaum/ai-kit#v0.4.0` | **the AI layer, in one install** — which model to call, whether the vendor still lists it, the three kinds of 429, fair-share of a free tier, and (re-exported) `ai-forms`. Renamed from `ai-ration` 2026-08-26: the name described one of five modules, and the package had one adopter while five repos that skipped it went down together to a retired model id. | -| [`threadkit`](https://github.com/bitbaum/threadkit) | `npm i threadkit` | multi-participant message threads where *permission is participation*, not a role or an ownership column. Headless pure functions, so "who may read this" is unit-testable instead of buried in a `WHERE` clause. AI participants obey the same visibility rules. **ESM-only.** | -| [`sitekit`](https://github.com/bitbaum/sitekit) | `npm i github:bitbaum/sitekit#v0.2.0` | **a website as data** — the closed section union (Zod as SSOT, path-addressed errors a generator can act on), one set of React renderers emitting semantic classes only (tokens stay per-site: uniform system, divergent aesthetics), and per-field provenance (`scraped`/`operator`/`inferred`/`not-found`) so `assertDeliverable()` makes "we fabricate no facts" a check instead of a promise. Extracted from substrata per orangecat ADR-0003; the planned `siteFromUrl()` extractor targets this schema. RSC-native — the Link seam stays a server-component prop (learned in v0.1.1 when the first consumer's build refused a component function crossing 'use client'). **ESM-only.** | -| [`limitkit`](https://github.com/bitbaum/limitkit) | `npm i github:bitbaum/limitkit#v0.1.0` | the fleet's **12 hand-rolled rate limiters** (this file's own "next extraction" row). Sliding/fixed windows over an injectable two-method `Store`; **bounded** memory default (the unbounded-Map leak is impossible by construction); standard `X-RateLimit-*` + `Retry-After` headers — what orangecat's ADR-0002 specified seven months before anything enforced it; `clientIp()`. Refusals count nothing, so a hammered key recovers. Ships no middleware and **no limit values** — how many attempts a route allows is app semantics, asserted locally. | - -**Adopted:** `ai-forms` — fleetcrown, evig, aoz-housing, surf-your-life. -`ai-kit` — fleetcrown, aoz-housing, truthseeker, botsmann, **and this repo** -(`model-pin-audit.mjs` calls `checkCatalog`; the audit needed exactly the vendor -query the package owns, so writing a second one here would have been this file's -own sin). - -Adoption went 2 -> 5 on 2026-08-27, all of it as a side effect of repairing the -outage rather than as a migration project — which is the only way it has ever -moved here. Three repos deliberately did NOT adopt, and the reason is the same -in each: kivvi, orangecat and evig already own model REGISTRIES carrying context -windows, tool/vision support and per-token cost, which `ai-kit` does not model. -Installing it beside one of those adds a second source of model truth to a repo -whose problem was having two. They were repaired in place and left pointing at -the daily audit instead. Merging a registry into `ai-kit` is a real design -question and belongs to a human, not to an outage. - -**v0.4.0 is breaking:** form filling moved off the root export to `ai-kit/forms`. -One install, one version, one import path per concern — the root re-export made -a chain-only consumer load `ai-forms` (ESM-only) and broke AOZ's Jest run inside -a module it never imported. Fixed in the package rather than with a -`transformIgnorePatterns` line per adopter. - -**On merging packages.** `ai-kit` absorbed `ai-ration` and re-exports -`ai-forms`, because to an app "which model do we call", "AI chat" and "AI form -fill" are ONE feature — AOZ adopted the form half, hand-rolled the rest, and was -taken down by the half it skipped. `threadkit` and `limitkit` are deliberately -NOT merged in: neither is about AI, and an app throttling its login form should -not install a model catalogue to do it. Merge by what a consumer needs together, -never by "these are all shared utilities". `threadkit` — **nobody yet**. -`limitkit` — fleetcrown (proving consumer; its old limiter had the unbounded -Map). **Next adopter should be orangecat** — it closes ADR-0002 by making its -Upstash client a 12-line `Store` adapter and deleting three of its four -implementations. -`sitekit` — substrata (proving consumer, converted the day the package -shipped: −577 lines of local renderer, and its test now validates the whole -site against the shared schema, so a breaking schema change fails in the -consumer's CI first). **Next: camille-boulangerie (PR open), s-ink (sinktattoo.com), wild-spirit** -— the three real sites that make the union's gaps (images, contact, menus) -visible on sites we own before the schema is pointed at strangers. -**Not yet:** orangecat and kivvi still carry their own form-assist; kivvi, evig, -botsmann still carry their own provider layers. - -A package with zero adopters removes zero duplication — publishing is the -cheap half. `threadkit` is listed here on its first day precisely so it does -not become another extraction nobody wired up. - -**`threadkit` shipped 2026-08-16** — `v0.1.0` on the public registry, published -by the tagged workflow with an SLSA provenance attestation, so the registry can -prove which commit built the tarball. Verified from outside the fleet, not from -CI: installed from the public registry into an empty project, imported, and got -17 named exports plus `dist/index.d.ts`. - -It exists because of the bug -[`single-tenant-prod-hides-unscoped-queries`](https://github.com/bitbaum/dotfiles) -records: role-derived access is correct at one doctor / one tenant / one org and -silently wrong at two. `canRead(thread, user)` cannot express that bug, because -there is no role to check. - -**Adoption candidates, in order of how much duplicated code it removes:** - -| Repo | What it replaces there | -|---|---| -| `vitareba` | care-team messaging — the exact clinic case: threads whose reader set is "the care team", not "the patient's doctor" | -| `orangecat` | Cat DMs / conversation visibility | -| `fleetcrown` | agent↔human threads, where an AI participant already needs the same rules as a person | - -**ESM-only** (`"type": "module"`, no `require` condition), so a CJS consumer -cannot `require()` it. Every candidate above is ESM already; note it before -adopting anywhere that is not. - -### Central audits — one script, never a copy per repo - -Not installable packages: these run FROM this repo against every other one, so -there is nothing to adopt and nothing to drift. Check here before writing a -fleet-wide checker. - -| Script | Answers | -|---|---| -| `scripts/ci/verify-floor-audit.sh` | does every repo's `verify` actually run lint + typecheck + test? | -| `scripts/ci/model-pin-audit.mjs` | is any model id the fleet pins no longer served by its vendor? Zero tokens — one `GET /models` per vendor — so it runs DAILY. Uses `ai-kit`'s `checkCatalog` rather than a second vendor query. Self-tested by `scripts/ci/test-model-pin-audit.mjs` — **102 checks, no network, no key, no checkout**, every fixture the real code that fooled it. Using it to repair seven repos on 2026-08-27 exposed nine faults in both directions, and the blind spots were not random: they mirrored the shapes people write (`GROQ_MODELS = {` defeats `\bmodels?\b`; `models: AIModel[] = [` defeats an array pattern; `modelId` is not `model`). It also read ids out of COMMENTS — reporting a retired id in the very commit that removed it. Never trust its first clean run after widening; re-run the live sweep and read every line. | -| `scripts/ci/ui-defect-audit.mjs` | do any live sites ship an interactive label below its WCAG AA floor, or a stack whose rows start at different x? Renders each site; no repo checkout involved. Self-tested by `scripts/ci/test-ui-defect-audit.mjs`, which pins BOTH sides — the real defect is still caught, correct markup stays silent. | - -Both report into a weekly workflow's job summary rather than only a log. - -## What is worth extracting next - -Ranked by (copies × how identical the logic is). Counts from -`scripts/ci/shared-inventory.sh`, forks excluded. - -| Concern | Files | Why it is a good candidate | -|---|---|---| -| `auto-merge-sweep.sh` | ~~22~~ **6** | **EXTRACTED 2026-08-16/20.** Sixteen repos call the canonical as a reusable workflow, each verified to actually *run* it (a sweep that fails to start looks exactly like one with nothing to do). The six remaining are deliberate: dotfiles is the canonical home and runs it directly; ai-forms, datacat, petvity, solon had dirty working trees owned by other sessions when swept — convert when clear. The two repos that had ever *tested* their copies (evig, orangecat) had that coverage ported into the canonical suite **before** deletion: 17 cases, mutation-proven. | -| rate limiting | **14 → adopting** | **Extracted 2026-08-20 as [`limitkit`](https://github.com/bitbaum/limitkit)** (see registry above). fleetcrown converted as the proving consumer; 13 files remain across 8 repos, orangecat first in line (its ADR-0002 becomes a 12-line `Store` adapter + three deletions). The ratchet holds the count until each adoption lands. | -| AI provider client | **16** | evig 7, orangecat 5. `ai-kit` already owns the hard part (chain, 429, budget); these are the callers. **Priced 2026-08-26, re-priced 2026-08-27:** Groq retired the llama-3.x family and the damage was far wider than the first count. Seven repos were broken, not five — the audit could not see two of them — and inside a repo the id was written down **two to four times**. Kivvi took three PRs to remove one retired id: it lived in the provider registry, an app's inline fetch body, the fallback chain, and a client hook's `FALLBACK_MODEL`. Each pass only found the copies the tooling could see. That is the cost of duplication measured rather than argued. fleetcrown, which adopted the package, was unaffected throughout. | -| logger | **10** | sbb-lost-found alone has 4. | -| health route | **8** | Identical shape in 8 repos; a 20-line contract. | -| ~~`@ai-native-cms/core`~~ | — | **Withdrawn — measurement error.** `bitbaum/revampit` *redirects* to `bitbaum/evig` (renamed in the pivot); the "two repos" with byte-identical trees were two clones of ONE repo. Nothing to extract. Two directories are not two repos: check `git remote -v` before reporting cross-repo duplication. | - -## What must NOT be centralized - -Stated explicitly, because "share everything" is its own failure: - -- **Auth / sessions** — coupled to the framework *and* the user schema. -- **DB schemas** — Drizzle vs Prisma vs raw SQL; a shared schema fights every ORM. -- **UI markup for chat and forms** — behaviour is shareable, *markup is not*. - Each app owns its design tokens and has to keep looking like itself. This is - why `ai-forms` is headless. -- **Anything where app semantics decide correctness.** orangecat legitimately - lists paid model ids (BYOK — the user's key, the user's choice) while the same - id in kivvi's fallback was a bug. Centralize the **rule**; assert it - **locally**, where the app knows which is which. - ---- - -## The process - -**1. Rule of three.** First time, write it. Second time, notice. **Third time is -forbidden** — extract it, or you have chosen to maintain N copies forever. - -**2. Check this file before building.** One grep. The cost of not checking is -visible above: 22 copies of one script. - -**3. New extraction? Follow the shape that already works.** -Unscoped `ai-*` name · ESM · `dist` built by `prepare` (gitignored) · -`exports` map · `verify = build && test` · **tests that import the package by -NAME**, not by reaching into `dist/` — otherwise a broken `exports`/`files` map -stays green until the first consumer installs it. - -**4. Ship no HTTP client.** Every app has its own calling conventions, retries -and logging. Replacing those is a rewrite, not an adoption. Supply the -decisions; leave the fetch alone. This is why `ai-kit` has no client and -`ai-forms` has no markup. - -**5. The ratchet.** `scripts/ci/shared-inventory.sh --check` runs on every PR -here and weekly across the fleet. Duplication counts may **fall**, may **hold**, -and may **never rise**. - -```bash -scripts/ci/shared-inventory.sh # report -scripts/ci/shared-inventory.sh --check # ratchet — exit 1 if a count rose -scripts/ci/shared-inventory.sh --update # move the baseline, in a PR, reviewed -``` - -Nobody has to fix 87 duplicated files today. The only requirement is to stop -adding to them — and when a count *falls*, `--update` locks the win in so it -cannot silently regress. - -**Raising the baseline is allowed** — sometimes a copy really is right. It just -has to happen in a PR, where someone sees it, instead of by accident. +This pointer stays because a decade of docs, memories and habits say +"check dotfiles/SHARED.md". The registry itself is maintained only there — +do not add entries here. diff --git a/scripts/audit/README.md b/scripts/audit/README.md deleted file mode 100644 index 09eb4f3..0000000 --- a/scripts/audit/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Fleet audit scripts - -## `secret-in-response.py` - -Reads what actually crossed the wire and reports credential material in it. - -### Why this exists - -FleetCrown shipped `passwordHash` and `privateZonePinHash` to the browser from -`GET /api/me` (fleetcrown#306). The fix included a static gate that reads route -handlers (`scripts/test/no-raw-user-response.ts`, fleetcrown#309) — and that gate -says in its own header what it cannot see: **RSC flight payloads**. A server -component handing a whole row to a client component serializes it into the HTML -exactly the same way, and no route handler is involved. - -This scanner closes that half. It needs no knowledge of any app's schema, ORM or -framework, because it reads responses rather than code. - -It has already earned its keep: on its first run across the fleet it found -`GET /api/admin/patients` in vitareba returning every patient's bcrypt digest — -a *cross-account* disclosure that the code-level sweep had missed, because that -route reads `db.query.users.findMany()` with no projection and the grep for -`json(user)` shapes does not match `json({ success, data })`. - -### Severity is about what the finding lets an attacker do - -| | meaning | -|---|---| -| `CRITICAL` | a live bearer credential — OAuth `access_token` / `refresh_token` / `id_token`, a session token. Possession **is** access, to a third party, right now. | -| `HIGH` | a credential digest — bcrypt/argon2/scrypt/PBKDF2, or a key named `password*`/`pin*`. Not access by itself, but it moves the secret from where nothing can read it to where everything can, and a low-entropy secret behind it (a numeric PIN) is a cheap offline job. | -| `MEDIUM` | a long value under a hash/secret/token-ish key. Often a CSRF nonce or a public share token by design — reported so it is looked at, not assumed to be a leak. | - -### Usage - -```bash -# unauthenticated — catches anything exposed without a login, costs nothing -python3 scripts/audit/secret-in-response.py \ - --base https://app.example --paths / /login /api/health - -# authenticated — the real test; most interesting routes 401 without a session -python3 scripts/audit/secret-in-response.py \ - --base https://app.example \ - --cookie '__Secure-authjs.session-token=' \ - --paths /dashboard /admin/users /api/me --json-out findings.json -``` - -Exit code is 1 if anything CRITICAL or HIGH was found, so it can gate CI. - -### Getting a session for a self-hosted app - -For NextAuth v5 apps on the box, mint a JWT rather than logging in — no data -written, no email sent: - -```js -import { encode } from "@auth/core/jwt"; -console.log(await encode({ - token: { id: "", sub: "", email: "...", /* + whatever the jwt callback sets */ }, - secret: process.env.SECRET, - salt: "__Secure-authjs.session-token", // the cookie name IS the salt -})); -``` - -**Two traps that cost real time here:** - -1. **`.env` values may be single-quoted.** systemd's `EnvironmentFile` strips - quotes; a naive `cut -d= -f2- | tr -d '"'` does not. The token then signs - cleanly and the app answers `null`, which reads like a version mismatch. - Settle it by comparing against the **running process**, not the file: - ```bash - PID=$(systemctl show -p MainPID --value .service) - sudo tr '\0' '\n' < /proc/$PID/environ | grep '^AUTH_SECRET=' - ``` -2. **The app-specific claims matter.** Read the `jwt`/`session` callback and - supply what it sets (`role`, `companyId`, `tokenVersion`, …) or the session - resolves to something the guards reject. - -### Its limits, stated - -- It matches **key names and digest prefixes**. A credential stored under a name - it does not know is invisible — aoz-wohnen's login credential is a `code` - column holding `AOZ-XXXXXX`, which no pattern here would catch. For an app - like that, pull the real values from the DB and grep the responses for them - directly. -- It only reads the paths you give it. It is not a crawler. -- A clean run over N paths is evidence about **those N paths**, not a proof - about the app. - -### Self-test - -```bash -python3 scripts/audit/test-secret-in-response.py -``` - -Every case is a shape observed in the fleet or one that must stay silent. Run it -after touching the patterns — a scanner nobody has watched fail is a scanner -that reports clean because it matches nothing. diff --git a/scripts/audit/secret-in-response.py b/scripts/audit/secret-in-response.py deleted file mode 100755 index 4fd09be..0000000 --- a/scripts/audit/secret-in-response.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -"""Scan HTTP response bodies for credential material that should never leave the server. - -Why a response scanner and not a code scanner: the static gate in FleetCrown -(scripts/test/no-raw-user-response.ts) reads API route handlers, and it says so -in its own header that it cannot see RSC flight payloads — a server component -handing a whole row to a client component serializes it into the HTML just the -same. This reads what actually crossed the wire, so it covers both, and it needs -no knowledge of any app's schema or ORM. - -Severity is about what the finding lets an attacker DO: - - CRITICAL a live bearer credential — OAuth access/refresh/id_token, a session - token. Possession IS access, to a third party's API, right now. - HIGH a credential digest — bcrypt/argon2/scrypt/PBKDF2, or a key literally - named password*/pin*. Not access by itself, but it moves the secret - from where nothing can read it to where everything can, and a - low-entropy secret behind it (a numeric PIN) is a cheap offline job. - MEDIUM a long value under a hash/secret/token-ish key. Often a CSRF nonce or - a public share token by design — reported so it is looked at, not - assumed to be a leak. - -Usage: - secret-in-response.py --base https://app.example --paths / /dashboard \\ - [--cookie 'name=value'] [--json-out findings.json] -""" -import argparse, json, re, sys, urllib.error, urllib.request - -# A live credential: holding it grants access somewhere, now. -BEARER_KEYS = re.compile( - r'"((?:access|refresh|id)[_-]?token|sessionToken|session_token|providerAccountId_token)"\s*:\s*"([^"]{16,})"', - re.I, -) -# A digest of a credential. -DIGEST_PREFIX = re.compile(r'\$(?:2[aby]|argon2[id]{1,2}|scrypt|pbkdf2)\$[^"\s]{8,}') -DIGEST_KEYS = re.compile( - r'"((?:\w*password\w*|\w*passwd\w*|\w*pin)(?:hash)?|password_hash|pin_hash)"\s*:\s*"([^"]{8,})"', - re.I, -) -# Suspicious but frequently legitimate. -SUSPECT_KEYS = re.compile( - r'"(\w*(?:hash|secret|apikey|api_key|token)\w*)"\s*:\s*"([^"]{32,})"', re.I -) -# Keys that are legitimately long and public — excluded from MEDIUM to keep the -# signal readable. Each is a deliberate judgement, not a blanket mute. -BENIGN_KEYS = re.compile( - r'^(csrf\w*|\w*csrf|shareToken|share_token|publicToken|inviteToken|' - r'contentHash|entryHash|prevHash|lastHash|etag|integrity|nonce|' - r'tokenPrefix|token_prefix|tokensUsed|tokenCount|token_count)$', - re.I, -) - -def redact(v: str) -> str: - return v[:6] + "…" + v[-4:] + f" ({len(v)} chars)" if len(v) > 14 else "" - -def scan(body: str): - out = [] - for m in BEARER_KEYS.finditer(body): - out.append(("CRITICAL", m.group(1), redact(m.group(2)))) - for m in DIGEST_PREFIX.finditer(body): - out.append(("HIGH", "", redact(m.group(0)))) - for m in DIGEST_KEYS.finditer(body): - if m.group(2) not in ("null", "undefined", ""): - out.append(("HIGH", m.group(1), redact(m.group(2)))) - for m in SUSPECT_KEYS.finditer(body): - if not BENIGN_KEYS.match(m.group(1)): - out.append(("MEDIUM", m.group(1), redact(m.group(2)))) - # One finding per key, at its highest severity: a repeated row is one leak, - # and a key already reported as CRITICAL must not also appear as MEDIUM. - rank = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2} - best: dict[str, tuple[str, str, str]] = {} - for sev, key, val in out: - cur = best.get(key) - if cur is None or rank[sev] < rank[cur[0]]: - best[key] = (sev, key, val) - return sorted(best.values(), key=lambda h: (rank[h[0]], h[1])) - -def fetch(url: str, cookie: str | None, ua: str): - req = urllib.request.Request(url, headers={"User-Agent": ua, "Accept": "*/*"}) - if cookie: - req.add_header("Cookie", cookie) - try: - with urllib.request.urlopen(req, timeout=20) as r: - return r.status, r.read().decode("utf-8", "replace") - except urllib.error.HTTPError as e: - return e.code, e.read().decode("utf-8", "replace") - except Exception as e: - return None, f"" - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--base", required=True) - ap.add_argument("--paths", nargs="+", required=True) - ap.add_argument("--cookie") - ap.add_argument("--json-out") - ap.add_argument("--ua", default="Mozilla/5.0 (secret-in-response audit)") - a = ap.parse_args() - - findings, checked = [], 0 - for p in a.paths: - url = a.base.rstrip("/") + p - status, body = fetch(url, a.cookie, a.ua) - if status is None: - print(f" ?? {p:38s} {body[:60]}") - continue - checked += 1 - hits = scan(body) - if hits: - worst = "CRITICAL" if any(h[0] == "CRITICAL" for h in hits) else \ - "HIGH" if any(h[0] == "HIGH" for h in hits) else "MEDIUM" - print(f" {worst:8s} {status} {p}") - for sev, key, val in hits: - print(f" {sev:8s} {key} = {val}") - findings.append({"path": p, "status": status, "severity": sev, - "key": key, "value": val}) - else: - print(f" ok {status} {p} ({len(body)} bytes)") - - print(f"\n {checked} responses read, {len(findings)} finding(s)") - if a.json_out: - with open(a.json_out, "w") as f: - json.dump(findings, f, indent=2) - return 1 if any(f["severity"] in ("CRITICAL", "HIGH") for f in findings) else 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/audit/test-secret-in-response.py b/scripts/audit/test-secret-in-response.py deleted file mode 100644 index 13455fb..0000000 --- a/scripts/audit/test-secret-in-response.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -"""Self-test for secret-in-response.py. - -A scanner nobody has watched fail is a scanner that reports clean because it -matches nothing. Every case below is a shape that was actually observed in the -fleet or a shape that must NOT be reported, and the run is the proof. - -Run: python3 scripts/audit/test-secret-in-response.py -""" -import importlib.util -import pathlib -import sys - -HERE = pathlib.Path(__file__).resolve().parent -spec = importlib.util.spec_from_file_location("sir", HERE / "secret-in-response.py") -m = importlib.util.module_from_spec(spec) -spec.loader.exec_module(m) - -# Credential-shaped fixtures are ASSEMBLED, never written as literals: this file -# is a corpus of fake secrets, and a literal here trips the repo's own gitleaks -# pre-commit hook. Splitting keeps the gate strict instead of adding an ignore -# rule that would also cover a real leak dropped in here later. -HEX64 = "4f7d3713" + "0615776551e8af20f2749a19f18e35f5aa4e6632e70a6e5fe0aa12bd" -HEX64B = "9a8b7c6d" + "5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8b" -BCRYPT = "$2b$12$" + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQ" -BCRYPT_SHORT = "$2b$12$" + "abcdefghijklmnopqrstuvwxyz012345" -GOOGLE_AT = "ya29.a0AfB_" + "byC3xk9SAMPLEnotreal_00000000000000" -REFRESH = "1//0eSAMPLE" + "notrealrefreshtoken000000" -SHARE = "abcdefghij" + "klmnopqrstuvwxyz0123456789abcd" -CSRF = "0123456789" + "abcdef0123456789abcdef0123" -WEBHOOK = "whsec_0123" + "456789abcdef0123456789abcdef" - -CASES = [ - # --- real, observed in production --- - ( - "fleetcrown GET /api/me — scrypt hex under a *Hash key, no $ prefix", - '{"id":"u1","email":"g@x.ch",' - f'"passwordHash":"{HEX64}","privateZonePinHash":"{HEX64B}"}}', - "HIGH", - ), - ( - "vitareba GET /api/admin/patients — bcrypt digest of ANOTHER user", - '{"success":true,"data":[{"id":"p1","email":"patient@x.ch",' - f'"password":"{BCRYPT}"}}]}}', - "HIGH", - ), - # --- the RSC case a route-level code scanner cannot see --- - ( - "digest inside a Next.js flight payload in HTML", - '', - "HIGH", - ), - # --- live bearer credentials outrank digests --- - ( - "OAuth access_token from a NextAuth accounts row", - f'{{"provider":"google","access_token":"{GOOGLE_AT}"}}', - "CRITICAL", - ), - ("refresh_token is equally critical", f'{{"refresh_token":"{REFRESH}"}}', "CRITICAL"), - # --- must stay silent --- - ("a correctly projected response", '{"id":"u1","username":"g","plan":"free","privateZonePinSetAt":null}', None), - ("a null hash is not a finding", '{"passwordHash":null,"password":null}', None), - ( - "public share tokens and csrf nonces are by design", - f'{{"shareToken":"{SHARE}","csrfToken":"{CSRF}"}}', - None, - ), - # --- unknown-but-suspicious must not be silently dropped --- - ("an unrecognised long secret-ish key is MEDIUM, not silence", f'{{"webhookSecret":"{WEBHOOK}"}}', "MEDIUM"), -] - - -def worst(hits): - if any(h[0] == "CRITICAL" for h in hits): - return "CRITICAL" - if any(h[0] == "HIGH" for h in hits): - return "HIGH" - return "MEDIUM" if hits else None - - -def main(): - failed = 0 - for label, body, want in CASES: - got = worst(m.scan(body)) - ok = got == want - failed += 0 if ok else 1 - print(f" {'OK ' if ok else 'FAIL'} {label}") - if not ok: - print(f" want={want} got={got}") - print(f"\n {len(CASES) - failed}/{len(CASES)} passed") - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/ci/auto-merge-sweep.sh b/scripts/ci/auto-merge-sweep.sh deleted file mode 100644 index 88b4d85..0000000 --- a/scripts/ci/auto-merge-sweep.sh +++ /dev/null @@ -1,446 +0,0 @@ -#!/usr/bin/env bash -# -# Merge every open PR that is ready and fully green, then re-arm CI/CD. -# -# THIS IS THE CANONICAL COPY. Adopt it via the reusable workflow -# (.github/workflows/auto-merge-sweep.yml) — do not copy this file into a repo. -# -# It WAS copied, into 22 repos, and drifted into EIGHT distinct versions of -# 245–404 lines. Three of them independently grew different fixes for real -# outages, and each fix reached only the repo that wrote it: -# -# evig, revampit infra-failure detection + re-run. An Actions incident left -# main `failure` with no failed job; 11 PRs stranded ~14h. -# fleetcrown DEADLOCK naming — a red base with the fix sitting in the -# queue was indistinguishable from "nothing to merge". -# the other 19 neither. -# -# This file is their UNION, and that is the whole argument for central code: the -# second repo to hit an Actions incident had already been given the answer by -# the first, and never received it. -# -# It is built on the variant 10 repos were already running, with the two fixes -# spliced in — NOT on the longest variant. "Take the biggest file" is not a -# merge strategy: the 404-line version is missing the step-summary reporting -# that the 296-line one has, so picking by size would have silently deleted -# working behaviour from ten repos. -# -# Every repo-specific value is an env var (GH_REPO, BASE_BRANCH, CI_WORKFLOW, -# REARM_WORKFLOWS, MAX_RUN_ATTEMPTS), so there is nothing left to fork over. -# -# WHY THIS EXISTS -# --------------- -# Nobody reviews PRs on this fleet — the owner explicitly does not want to be in -# the merge loop, and background-job agent sessions are barred from merging by -# hand. So the policy lives here, in the repo, where it is visible, revocable, -# and applies uniformly to every PR instead of depending on who opened it. -# -# THE POLICY -# merge a PR <=> it is not a draft -# AND carries no hold label -# AND has at least one check -# AND every check has finished green -# AND GitHub reports it cleanly mergeable -# -# Anything else is left alone for the next sweep. Nothing here forces a merge: -# a red or pending PR simply waits, and a draft waits forever. To hold a ready -# PR back, mark it a draft or add one of the hold labels below. -# -# ONE PR PER SWEEP, OLDEST FIRST, AND ONLY ONTO A GREEN BASE -# ---------------------------------------------------------- -# A PR's checks prove *that PR against the base it branched from* — not against -# the other PRs sitting next to it. Merging a batch in one pass would put a -# combination onto the base that nothing ever built. So this script merges at -# most one PR, then hands control back to CI: the merge train advances one car -# per sweep, and every car is verified on the base before the next one couples. -# -# For the same reason it refuses to merge while the base's CI is red or still -# running. Red base => stop adding changes until it is fixed; running CI => the -# answer is not in yet. Both simply defer to the next sweep. -# -# THE RE-ARM (do not remove) -# A push made with the default GITHUB_TOKEN does NOT trigger workflows. Both -# CI and the deploy workflow here run on push, so a merge from this script -# would otherwise land on the base branch and never build or ship. Worse, the -# green-base guard above keys on "a CI run exists for the current tip" — with -# no CI run ever produced by an automated merge, the very next sweep would -# block forever. The explicit workflow_dispatch calls at the end restore both. -# -# REARM_WORKFLOWS is set by .github/workflows/auto-merge.yml and lists exactly -# the workflows that would otherwise have fired on push. - -set -euo pipefail - -REPO="${GH_REPO:?GH_REPO must be set}" -BASE_BRANCH="${BASE_BRANCH:-main}" -CI_WORKFLOW="${CI_WORKFLOW:-ci.yml}" -REARM_WORKFLOWS="${REARM_WORKFLOWS:-$CI_WORKFLOW}" - -# Workflow that ships the base branch, e.g. deploy.yml. EMPTY DISABLES IT, and -# empty is the default: a repo that does not set this behaves exactly as before, -# so adopting the reconciler is opt-in rather than something that starts firing -# deploys in repos that never asked for one. -DEPLOY_WORKFLOW="${DEPLOY_WORKFLOW:-}" - -# A PR wearing any of these is never merged automatically. -HOLD_LABELS='["hold","no-automerge","do-not-merge","wip"]' - -# How many times a run that produced NO VERDICT may be re-run before the sweep -# gives up on it. Bounded so an endlessly-failing run cannot become an infinite -# re-run loop billing Actions minutes. -MAX_RUN_ATTEMPTS="${MAX_RUN_ATTEMPTS:-3}" - -# ── Telling "the code is broken" from "the CI system broke" ────────────────── -# -# A run can end without ever judging the code: cancelled, or failed inside -# GitHub's own "Set up job" step. Treating that as a verdict is what strands a -# queue — and unlike a real failure, nobody gets a signal, because the sweep -# still exits 0 and looks perfectly healthy while merging nothing. -# -# Observed on evig 2026-08-07: an Actions incident left main's run `failure` -# with no failed job at all (one cancelled, the rest green). It held 11 PRs for -# ~14 hours. A genuine failure still blocks — that IS a verdict about the code. -run_failure_is_infra() { - run_failure_is_infra_id="$1" - run_failure_is_infra_steps=$(gh api \ - "repos/${REPO}/actions/runs/${run_failure_is_infra_id}/jobs" --paginate \ - --jq '[ .jobs[] - | select(.conclusion == "failure") - | [ .steps[]? | select(.conclusion == "failure") | .name ] ] - | flatten | unique | join("|")' 2>/dev/null) || return 1 - [ -n "$run_failure_is_infra_steps" ] || return 1 - # ONLY when the sole failing step is GitHub's own runner setup. Anything else - # is the repo's code failing and must keep blocking. - [ "$run_failure_is_infra_steps" = "Set up job" ] -} - -run_conclusion_is_non_verdict() { - case "$1" in - cancelled) return 0 ;; - failure) run_failure_is_infra "$2" ;; - *) return 1 ;; - esac -} - -rerun_non_verdict_run() { - rerun_id="$1" - rerun_what="$2" - rerun_attempt=$(gh api "repos/${REPO}/actions/runs/${rerun_id}" \ - --jq '.run_attempt // 1' 2>/dev/null || echo "$MAX_RUN_ATTEMPTS") - if [ "$rerun_attempt" -ge "$MAX_RUN_ATTEMPTS" ]; then - echo "[auto-merge] ${rerun_what} run ${rerun_id} already at attempt ${rerun_attempt}/${MAX_RUN_ATTEMPTS} — not retrying again" >&2 - return 1 - fi - echo "[auto-merge] ${rerun_what} run ${rerun_id} produced no verdict (attempt ${rerun_attempt}) — re-running" - gh run rerun "$rerun_id" --repo "$REPO" \ - || { echo "[auto-merge] could not re-run ${rerun_id}" >&2; return 1; } -} - -echo "[auto-merge] sweeping open PRs against ${BASE_BRANCH} in ${REPO}" - -# Never add changes to a base that is red or mid-verification. -# -# The run has to belong to the CURRENT tip of the base branch. Checking only -# "the latest CI run" is a trap: right after a merge, the newest run is still -# the *previous* commit's — and it is green — so the guard would wave through a -# second merge onto a commit nothing has verified yet. That is exactly the -# batching this script exists to prevent. -base_sha=$(gh api "repos/${REPO}/commits/${BASE_BRANCH}" --jq '.sha') -base_ci=$(gh run list --repo "$REPO" --workflow "$CI_WORKFLOW" --branch "$BASE_BRANCH" --limit 1 \ - --json databaseId,status,conclusion,headSha --jq '.[0] // empty') - -# Declared before the branch that can skip it: `set -u` is on and the merge -# site below always reads it. A base branch with no CI history takes the -# "proceeding" path, and an assignment living only in the else-branch made -# the entire sweep die with "base_red_jobs: unbound variable". -base_red_jobs="" - -if [ -z "$base_ci" ]; then - echo "[auto-merge] no CI history for ${BASE_BRANCH} — proceeding" -else - base_status=$(printf '%s' "$base_ci" | jq -r '.status') - base_conclusion=$(printf '%s' "$base_ci" | jq -r '.conclusion // ""') - base_ci_sha=$(printf '%s' "$base_ci" | jq -r '.headSha') - - if [ "$base_ci_sha" != "$base_sha" ]; then - echo "[auto-merge] ${BASE_BRANCH} is at ${base_sha:0:8} but the newest CI run is for ${base_ci_sha:0:8} — waiting for CI to catch up" - exit 0 - fi - if [ "$base_status" != "completed" ]; then - echo "[auto-merge] ${BASE_BRANCH} CI is still running — deferring to the next sweep" - exit 0 - fi - # A red base must not become a trap for the PR that repairs it. - # - # "Never merge onto red" is right for an unrelated change: it stops a broken - # base quietly collecting more of them and getting harder to diagnose. But - # when the PR *is* the repair, the same rule deadlocks the repo — the fix - # cannot travel the path its own redness blocks, and only a human can move - # it. Seen in catomean/aoz-housing on 2026-08-07: E2E red on the base, the - # fix sitting green in a PR, every sweep refusing politely. - # - # So identify WHICH jobs are red and let a PR through only if its own checks - # pass every one of them. Not a weakening: a PR's checks run on the MERGE - # result (refs/pull/N/merge), so green-on-those-jobs is direct evidence the - # post-merge base is better than the pre-merge base. Still refused: a PR that - # does not cover the failing jobs, one that covers only some of them, and a - # base failure whose jobs cannot be identified at all. - if [ "$base_conclusion" != "success" ]; then - base_run_id=$(printf '%s' "${base_ci}" | jq -r '.databaseId') - - # THE OTHER DEADLOCK: the only thing that produces a new CI run on the base - # is a merge, and merges are exactly what this guard blocks. So a base run - # that ended without a verdict strands every open PR until a human notices, - # and nothing signals that they should. - if run_conclusion_is_non_verdict "$base_conclusion" "$base_run_id"; then - rerun_non_verdict_run "$base_run_id" "${BASE_BRANCH}" || true - echo "[auto-merge] deferring to the next sweep to judge ${BASE_BRANCH}" - exit 0 - fi - - # NAME THE DEADLOCK. A red base is usually transient; it becomes a deadlock - # when the only PR that repairs it is sitting in the queue. The guard below - # already lets a PR through if it is green on the failing jobs — but when - # none qualifies, the sweep exits 0 and the stall is indistinguishable from - # "nothing to merge". That is the third time in this fleet a permanent stall - # looked like an ordinary skip, so make it loud and name the candidates. - ready=$(gh pr list --repo "$REPO" --state open --base "$BASE_BRANCH" --limit 50 \ - --json number,title,isDraft,mergeStateStatus,labels \ - --jq "[ .[] - | select(.isDraft | not) - | select(.mergeStateStatus == \"CLEAN\") - | select([.labels[].name] - ${HOLD_LABELS} == [.labels[].name]) - | \" #\(.number) \(.title)\" ] | .[]" 2>/dev/null) - if [ -n "$ready" ]; then - echo "[auto-merge] ⚠ DEADLOCK RISK: ${BASE_BRANCH} is red and these green PRs are waiting — one may be the fix:" >&2 - printf '%s\n' "$ready" >&2 - fi - - base_red_jobs=$(gh run view "${base_run_id}" --repo "$REPO" --json jobs \ - --jq '[.jobs[] | select(.conclusion == "failure") | .name] | .[]' 2>/dev/null || true) - if [ -z "${base_red_jobs}" ]; then - echo "[auto-merge] ${BASE_BRANCH} CI is ${base_conclusion} and no failing job could be identified — refusing to merge onto a broken base" >&2 - exit 0 - fi - echo "[auto-merge] ${BASE_BRANCH} CI is ${base_conclusion} — failing: $(printf '%s' "${base_red_jobs}" | tr '\n' ' ')" >&2 - echo "[auto-merge] only a PR that is green on those exact jobs may merge (its checks run on the merge result)" - fi -fi - -# ── Reconcile: a green base must be what is LIVE ───────────────────────────── -# -# Deployment is a RECONCILER, not a chain. A push made with GITHUB_TOKEN emits -# no workflow_run event, and — one level deeper than anyone expects — neither -# does a run that GITHUB_TOKEN itself dispatched. So nothing downstream ever -# wakes on an automated merge. Observed on fleetcrown 2026-08-05: three PRs -# merged, main green, zero Deploy runs created. Invisible, because CI itself ran -# and went green. -# -# So instead of trusting a trigger, compare desired state (the base's tip) with -# actual state (the last successful deploy) and close the gap. That is -# self-healing by construction: a deploy that never fired, or fired and failed, -# is retried by the next sweep instead of leaving a commit merged-but-not-live. -# -# THE GREEN GUARD IS NOT OPTIONAL. The variant this came from exited on a red -# base, so reaching the reconciler there proved the tip was green. This script -# deliberately does NOT exit on red — it lets a PR that repairs the base through -# — so the same code placed here without `-z "$base_red_jobs"` would ship a tip -# whose CI is failing. Same lines, different surrounding control flow, opposite -# meaning. -if [ -n "$DEPLOY_WORKFLOW" ] && [ -n "${base_ci:-}" ] && [ -z "${base_red_jobs}" ]; then - deploy_running=$(gh run list --repo "$REPO" --workflow "$DEPLOY_WORKFLOW" --limit 5 \ - --json status --jq '[.[] | select(.status != "completed")] | length' 2>/dev/null || echo 0) - deployed_sha=$(gh run list --repo "$REPO" --workflow "$DEPLOY_WORKFLOW" --branch "$BASE_BRANCH" \ - --status success --limit 1 --json headSha --jq '.[0].headSha // ""' 2>/dev/null || echo "") - - if [ "${deploy_running:-0}" -gt 0 ]; then - echo "[auto-merge] a deploy is already in flight — not dispatching another" - elif [ "$deployed_sha" = "$base_sha" ]; then - echo "[auto-merge] ${BASE_BRANCH} ${base_sha:0:8} is already deployed" - else - echo "[auto-merge] ${BASE_BRANCH} is at ${base_sha:0:8}; last successful deploy was ${deployed_sha:0:8}${deployed_sha:+ } — shipping" - gh workflow run "$DEPLOY_WORKFLOW" --repo "$REPO" --ref "$BASE_BRANCH" \ - || echo "[auto-merge] could not dispatch ${DEPLOY_WORKFLOW} — is workflow_dispatch declared?" >&2 - fi -fi - -prs_json=$(gh pr list --repo "$REPO" --state open --base "$BASE_BRANCH" --limit 50 \ - --json number,title,isDraft,mergeable,mergeStateStatus,labels,statusCheckRollup,createdAt) - -count=$(printf '%s' "$prs_json" | jq 'length') -if [ "$count" -eq 0 ]; then - echo "[auto-merge] no open PRs" - exit 0 -fi - -merged_any=0 - -# OLDEST FIRST. `gh pr list` returns newest-first, and this loop merges the -# first eligible PR and stops — so the newest green PR wins every sweep and an -# older one can wait indefinitely. Observed in catomean/fleetcrown on -# 2026-08-06: two consecutive sweeps merged the two newest PRs while three -# older green ones were never even evaluated. With several agent sessions -# opening PRs continuously, "newest wins" is starvation, and it starves the PR -# whose checks were proven against the most now-stale base. -# -# PR numbers increase monotonically with creation, so sorting ascending is FIFO. -for number in $(printf '%s' "$prs_json" | jq -r 'sort_by(.number) | .[].number'); do - pr=$(printf '%s' "$prs_json" | jq -c --argjson n "$number" '.[] | select(.number == $n)') - title=$(printf '%s' "$pr" | jq -r '.title') - - # A rollup entry is either a CheckRun (status + conclusion) or a commit - # StatusContext (state) — external services report as the latter. - verdict=$(printf '%s' "$pr" | jq -r --argjson hold "$HOLD_LABELS" ' - def ok: - if has("state") then (.state == "SUCCESS") - else ((.status == "COMPLETED") - and ((.conclusion // "") | test("^(SUCCESS|NEUTRAL|SKIPPED)$"))) end; - def pending: - if has("state") then (.state == "PENDING") - else (.status != "COMPLETED") end; - - . as $pr - | (($pr.statusCheckRollup) // []) as $checks - | if $pr.isDraft then "skip: draft" - elif ([$pr.labels[]?.name] | any(. as $l | $hold | index($l) != null)) - then "skip: hold label" - elif ($checks | length) == 0 then "skip: no checks reported yet" - elif ($checks | map(pending) | any) then "skip: checks still running" - elif (($checks | map(ok) | all) | not) then "skip: checks not green" - else "merge" end - ') - - if [ "$verdict" != "merge" ]; then - echo "[auto-merge] #${number} ${verdict} — ${title}" - - # "No checks reported yet" is transient for a PR opened seconds ago and - # PERMANENT for an old one: GitHub does not retroactively run workflows on - # a PR nobody has pushed to, so it will sit here forever looking patient. - # Report it; only a push, or a close/reopen, will ever produce checks. - if [ "$verdict" = "skip: no checks reported yet" ] && [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - created=$(printf '%s' "$pr" | jq -r '.createdAt // ""') - if [ -n "$created" ] && [ "$created" \< "$(date -u -d '2 hours ago' +%Y-%m-%dT%H:%M:%SZ)" ]; then - echo "- ⚠️ #${number} has no checks and is over 2h old — it will never gain any on its own — ${title}" >> "$GITHUB_STEP_SUMMARY" - fi - fi - - # A CANCELLED check is not a verdict, it is noise: CI workflows in this - # fleet use `concurrency: cancel-in-progress`, so an unrelated newer run on - # the same ref can kill a PR's build. Nothing ever re-runs it, the PR is - # never green, and it would sit in this queue forever. Re-run it and let a - # later sweep judge the real result. Genuine failures are left alone; only a - # run with no real failure is retried. - if [ "$verdict" = "skip: checks not green" ]; then - retry_urls=$(printf '%s' "$pr" | jq -r ' - [ .statusCheckRollup[]? - | select(has("state") | not) - | select((.conclusion // "") == "CANCELLED") - | .detailsUrl ] as $cancelled - | [ .statusCheckRollup[]? - | select(((.conclusion // .state // "") - | test("^(FAILURE|TIMED_OUT|ACTION_REQUIRED|STARTUP_FAILURE|ERROR)$"))) ] as $failed - | if ($failed | length) == 0 then $cancelled[] else empty end - ') - for url in $retry_urls; do - run_id=$(printf '%s' "$url" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+' || true) - [ -z "$run_id" ] && continue - echo "[auto-merge] #${number} re-running cancelled run ${run_id}" - gh run rerun "$run_id" --repo "$REPO" || echo "[auto-merge] #${number} could not re-run ${run_id}" >&2 - done - fi - continue - fi - - # Mergeability is computed lazily by GitHub and is invalidated every time the - # base branch moves — so right after a merge (exactly when this workflow runs) - # every PR reports UNKNOWN. Poll until GitHub has an answer instead of - # treating "not computed yet" as "not mergeable"; otherwise the fast path can - # never merge anything and the whole train falls back to the cron. - mergeable="" - state="" - for attempt in 1 2 3 4 5 6; do - fresh=$(gh pr view "$number" --repo "$REPO" --json mergeable,mergeStateStatus) - mergeable=$(printf '%s' "$fresh" | jq -r '.mergeable') - state=$(printf '%s' "$fresh" | jq -r '.mergeStateStatus') - [ "$mergeable" != "UNKNOWN" ] && break - echo "[auto-merge] #${number} mergeability not computed yet (attempt ${attempt}) — waiting" - sleep 5 - done - - # A conflicted PR is not "not ready yet" — it is stuck, and nothing else will - # unstick it. Skipping it quietly is how a PR sits DIRTY while the base moves - # on: every sweep passes over it in silence and no signal ever reaches a - # human. Say it loudly, and put it in the job summary where it is seen. - if [ "$mergeable" = "CONFLICTING" ]; then - echo "[auto-merge] #${number} CONFLICTS with ${BASE_BRANCH} and will never merge itself — ${title}" >&2 - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - echo "- ⚠️ #${number} conflicts with \`${BASE_BRANCH}\` and needs resolving — ${title}" >> "$GITHUB_STEP_SUMMARY" - fi - continue - fi - - if [ "$mergeable" != "MERGEABLE" ]; then - echo "[auto-merge] #${number} skip: not mergeable (${mergeable}/${state}) — ${title}" - continue - fi - - # Keep the branch current instead of merging a PR that was proven against an - # older base. This is also how conflicts surface EARLY: a branch updated on - # the sweep after the merge that broke it fails here, minutes later, rather - # than hours later when someone finally looks. One update per sweep, for the - # same reason only one PR is merged per sweep. - if [ "$state" = "BEHIND" ]; then - echo "[auto-merge] #${number} is behind ${BASE_BRANCH} — updating it before merging: ${title}" - if gh api -X PUT "repos/${REPO}/pulls/${number}/update-branch" --silent 2>/dev/null; then - echo "[auto-merge] #${number} updated; its checks now run against current ${BASE_BRANCH}" - else - echo "[auto-merge] #${number} update-branch failed — leaving for the next sweep" >&2 - fi - break - fi - - # Red base: this PR merges only if it proves every failing job green. - if [ -n "${base_red_jobs}" ]; then - pr_green=$(printf '%s' "$pr" | jq -r ' - [ .statusCheckRollup[]? - | select(((.conclusion // .state // "") | test("^(SUCCESS|NEUTRAL|SKIPPED)$"))) - | (.name // .context) ] | .[]') - uncovered="" - while IFS= read -r job; do - [ -z "$job" ] && continue - printf '%s\n' "$pr_green" | grep -Fxq "$job" || uncovered="${uncovered}${job}; " - done <&2 - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - echo "- 🔧 #${number} merged onto a red \`${BASE_BRANCH}\` because it passes every failing job — ${title}" >> "$GITHUB_STEP_SUMMARY" - fi - fi - - echo "[auto-merge] #${number} green and ready — merging: ${title}" - if gh pr merge "$number" --repo "$REPO" --squash --delete-branch; then - merged_any=1 - echo "[auto-merge] #${number} merged" - # One car per sweep: let CI verify this on the base before the next couples. - break - else - # Losing a race (someone merged first, or the base moved underneath) is - # normal; the next sweep re-evaluates from fresh state. - echo "[auto-merge] #${number} merge failed — leaving for the next sweep" >&2 - fi -done - -if [ "$merged_any" -eq 1 ]; then - for wf in $REARM_WORKFLOWS; do - echo "[auto-merge] re-arming ${wf} on ${BASE_BRANCH}" - gh workflow run "$wf" --repo "$REPO" --ref "$BASE_BRANCH" \ - || echo "[auto-merge] could not dispatch ${wf} — is workflow_dispatch declared?" >&2 - done -else - echo "[auto-merge] nothing merged; no re-arm needed" -fi \ No newline at end of file diff --git a/scripts/ci/find-false-needs.py b/scripts/ci/find-false-needs.py deleted file mode 100755 index 8b0a15c..0000000 --- a/scripts/ci/find-false-needs.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 -"""Fleet audit: which CI jobs WAIT on a job they never READ anything from? - -`needs:` in a GitHub Actions workflow expresses THREE different things that look -identical in YAML: - - 1. "I consume what that job produced." <- value flow - 2. "Do not run me if it failed." <- failure gate - 3. "I would prefer to run after it." <- ordering - -This script detects the absence of (1): a job has value flow only if it -downloads an artifact from the same run (`actions/download-artifact`) or -references `needs..outputs.*` / `needs..result`. - -That is NOT the same as the wait being pointless, and conflating them is how you -break a pipeline. (2) is a genuine, load-bearing function — and in the fleet -sweep it is the MAJORITY of hits: - - openclaw run_live_* needs authorize_actor <- authorize before live secrets - openclaw publish_* needs *_approval <- approval before publishing - fleetcrown ship needs check <- never deploy a red build - -Removing any of those would be a security or deploy regression, and every one of -them shows up in this report. The judgment the report cannot make for you is -whether the failure gate is worth the wall-clock it costs. It is worth it when -the successor is dangerous or expensive to run wrongly. It is NOT worth it when -the successor tests something independent — which is what (3) usually turns out -to be in disguise. - -Measured instances that motivated this (2026-08-15/16): - - aoz-housing E2E Tests waited on build +231s serves its own app - evig Local E2E Journeys waited on quality +373s self-contained - evig Inventory Smoke waited on quality +435s tests LIVE prod - evig Auth Smoke Test waited on quality +434s tests LIVE prod - -THIS IS A REPORT, NOT A VERDICT. The claim is only that the cost should be a -decision someone made, not an accident inherited from a scaffold. A fleet sweep -on 2026-08-16 found 26 hits; roughly four fifths were deliberate failure gates -that must stay. Read the run timings before changing anything: - - gh api repos/OWNER/REPO/actions/runs//jobs \ - --jq '.jobs[] | {name, started_at, secs: ((.completed_at|fromdate)-(.started_at|fromdate))}' - -Start timestamps are what prove a scheduling change landed — durations alone -cannot distinguish "it now runs in parallel" from "it got faster". - -TWO TRAPS, both paid for in the instances above: - - * Fixing one layer exposes the next. evig #303 removed the e2e job's false - `needs:` and the run went 709s -> 634s, not the predicted ~373s, because - `quality -> inventory-smoke` had been hiding behind the old bottleneck and - became the new critical path the instant it cleared. Re-run this audit - AFTER every scheduling fix. - - * The obvious knob is rarely the big one. In aoz-housing the eye-catching - defect was `workers: process.env.CI ? 1 : undefined` (the stock - `npm init playwright` scaffold, which makes `fullyParallel: true` a no-op). - Raising it bought 494s -> 346s — worth less than deleting one `needs:`, - because the Playwright `webServer` was `npm run dev` and the workers all - queued on it. Measure `.steps[]` before touching config. - -Deliberately ONE central script that reads every repo REMOTELY, not a check -copied into each repo — same reasoning as verify-floor-audit.sh next door: the -auto-merge sweep was copied into 17 repos and now has 5 live variants, so a fix -in one reaches none of the others. - -Usage: - scripts/ci/find-false-needs.py # audit the whole fleet remotely - scripts/ci/find-false-needs.py --local . # audit a local checkout - scripts/ci/find-false-needs.py --local a/ b/ # audit several checkouts - scripts/ci/find-false-needs.py --strict # exit 1 if anything is found - -Env: - GH_OWNER GitHub owner to enumerate (default: maonakamoto) - GH_LIMIT max repos to inspect (default: 100) -""" -import json -import os -import pathlib -import re -import subprocess -import sys - -try: - import yaml -except ImportError: - sys.exit("needs PyYAML: pip install --user pyyaml") - -OWNER = os.environ.get("GH_OWNER", "maonakamoto") -LIMIT = os.environ.get("GH_LIMIT", "100") - - -def gh(*args): - """Ask GitHub a question. A 404 is an ANSWER (repo has no workflows), not a - crash — so a failed call returns None rather than killing the sweep.""" - r = subprocess.run(("gh",) + args, capture_output=True, text=True) - return r.stdout if r.returncode == 0 else None - - -def consumed_deps(job_text, deps): - """Which of `deps` does this job actually read a value from?""" - return {d for d in deps if re.search(r"needs\." + re.escape(d) + r"\b", job_text)} - - -def scan_workflow(text, where): - """Yield (job_name, deps, unused) for every job that waits without consuming.""" - try: - doc = yaml.safe_load(text) - except Exception as e: - print(f" !! could not parse {where}: {e}", file=sys.stderr) - return - if not isinstance(doc, dict) or "jobs" not in doc: - return - for name, job in (doc.get("jobs") or {}).items(): - if not isinstance(job, dict): - continue - needs = job.get("needs") - if not needs: - continue - deps = [needs] if isinstance(needs, str) else list(needs) - job_text = yaml.safe_dump(job) - # An artifact download is a real dependency even with no `needs.` ref, - # because the artifact can only have been produced earlier in this run. - if "actions/download-artifact" in job_text: - continue - used = consumed_deps(job_text, deps) - unused = [d for d in deps if d not in used] - if unused: - yield name, deps, unused - - -def scan_local(paths): - for p in paths: - root = pathlib.Path(p) - for wf in sorted(root.glob(".github/workflows/*.y*ml")): - for name, deps, unused in scan_workflow(wf.read_text(), str(wf)): - yield str(wf), name, deps, unused - - -def scan_remote(): - out = gh("repo", "list", OWNER, "--limit", LIMIT, "--json", "name,isArchived") - if not out: - sys.exit(f"could not list repos for {OWNER} — is `gh` authenticated?") - repos = [r["name"] for r in json.loads(out) if not r["isArchived"]] - print(f"scanning {len(repos)} non-archived repos under {OWNER}…", file=sys.stderr) - for repo in repos: - listing = gh("api", f"repos/{OWNER}/{repo}/contents/.github/workflows") - if not listing: - continue # no workflows — an answer, not an error - try: - entries = json.loads(listing) - except json.JSONDecodeError: - continue - for entry in entries: - if not entry["name"].endswith((".yml", ".yaml")): - continue - body = gh( - "api", - f"repos/{OWNER}/{repo}/contents/.github/workflows/{entry['name']}", - "--jq", ".content", - "--header", "Accept: application/vnd.github+json", - ) - if not body: - continue - import base64 - try: - text = base64.b64decode(body).decode("utf-8", "replace") - except Exception: - continue - where = f"{repo}/.github/workflows/{entry['name']}" - for name, deps, unused in scan_workflow(text, where): - yield where, name, deps, unused - - -def main(): - argv = sys.argv[1:] - strict = "--strict" in argv - argv = [a for a in argv if a != "--strict"] - - if argv and argv[0] == "--local": - findings = list(scan_local(argv[1:] or ["."])) - else: - findings = list(scan_remote()) - - for where, job, deps, unused in findings: - print(f"{where}\n job {job!r} waits on {deps} but reads nothing from {unused}") - - n = len(findings) - print(f"\n{n} job(s) wait without consuming.", file=sys.stderr) - if n: - print( - "Each is a REPORT, not a verdict — confirm against real run timings\n" - "before removing a `needs:`, and re-run this after every fix, because\n" - "clearing one bottleneck promotes whatever was hiding behind it.", - file=sys.stderr, - ) - return 1 if (strict and n) else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/ci/model-pin-audit.mjs b/scripts/ci/model-pin-audit.mjs deleted file mode 100755 index df7b973..0000000 --- a/scripts/ci/model-pin-audit.mjs +++ /dev/null @@ -1,888 +0,0 @@ -#!/usr/bin/env node -/** - * Fleet audit: is any model id this fleet pins no longer served? - * - * WHY THIS EXISTS - * --------------- - * On 2026-08-26 the AOZ assistant answered "KI-Assistent nicht konfiguriert. - * Bitte GROQ_API_KEY setzen" on a deployment whose key was valid. Groq had - * retired the whole llama-3.x family; the pinned `llama-3.3-70b-versatile` - * returned 404 model_not_found. Production had been broken as long as the demo - * and nobody knew, because the only thing the app can say is "not configured". - * - * That is not an AOZ bug. Every Groq id pinned anywhere in the fleet was dead - * the same morning — llama-3.1-8b-instant, llama3-8b-8192, gemma2-9b-it, - * mixtral-8x7b-32768, llama-3.2-3b-instruct — across eleven repos, with three - * deployed apps failing live. - * - * A pinned free model is not a configuration, it is a scheduled outage. The - * schedule is set by the vendor and nobody here is told. So this asks the only - * authority that knows: the vendor's own catalogue. - * - * WHY A CENTRAL SCRIPT AND NOT A CHECK PER REPO - * --------------------------------------------- - * Same doctrine as verify-floor-audit.sh and ui-defect-audit.mjs. A check - * copied into twenty repos drifts into twenty versions — SHARED.md counts the - * bill for exactly that habit. More to the point, a per-repo check only ever - * runs in repos somebody still touches, and the repos that rot quietly are - * precisely the ones nobody touches. This one needs no adoption at all. - * - * WHY IT REUSES ai-kit - * -------------------- - * `checkCatalog` already answers this, already distinguishes the three states - * that matter, and already carries the scars — its own docstring records four - * of nine default pins gone and a consumer silently failing for eight days. - * Writing a second vendor query here would be this repo committing the sin it - * exists to police. It takes the chain as an ARGUMENT, so it generalises to - * arbitrary ids with no change to the package. - * - * ZERO TOKENS, WHICH IS THE WHOLE POINT - * ------------------------------------- - * One GET /models per vendor. No completion, no spend. That is the difference - * between a check that runs on a timer and a command somebody is supposed to - * remember — and "supposed to remember" is what failed for eight days. - * - * WHAT IT DOES NOT PROVE - * ---------------------- - * That a listed model WORKS. Existence is cheap; capability is not. A model - * can be listed and still refuse tool calls — of nine free models probed for - * ai-kit's default chain, five answered only via a text protocol. If the - * surface is a tool loop, probe with a real tool call before pinning. This - * audit catches the retirement, not the mismatch. - * - * It also judges only ids it can ATTRIBUTE to a vendor it queried. An - * unattributed id is reported and not judged, because "I could not look" and - * "it is gone" are different answers and collapsing them invents outages. - * - * Usage: - * node scripts/ci/model-pin-audit.mjs # audit, exit 1 on dead pins - * node scripts/ci/model-pin-audit.mjs --warn-only # report, always exit 0 - * node scripts/ci/model-pin-audit.mjs --local # scan ~/dev checkouts - * - * Env: GH_OWNER (default catomean), GH_LIMIT (default 100), - * FLEET_ROOT (default ~/dev, --local only), - * AI_KIT_FROM (path to a repo that installs ai-kit), - * GROQ_API_KEY / OPENROUTER_API_KEY (to read the catalogues). - */ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; - -const exec = promisify(execFile); - -const WARN_ONLY = process.argv.includes("--warn-only"); -const LOCAL = process.argv.includes("--local"); -const OWNER = process.env.GH_OWNER ?? "catomean"; -const GH_LIMIT = process.env.GH_LIMIT ?? "100"; -const FLEET_ROOT = process.env.FLEET_ROOT ?? join(homedir(), "dev"); - -/** - * The vendors whose catalogue we can actually read, and the markers that tie a - * pin to one of them. - * - * Attribution is by MARKER rather than by the shape of the id, because the shape - * lies: `openai/gpt-oss-20b` is a Groq model id AND an OpenRouter routing id, - * and at OpenRouter the missing `:free` suffix is the difference between free - * and billed. Only the surrounding code knows which vendor is meant. - */ -export const VENDORS = [ - { - id: "groq", - queryable: true, - baseUrl: "https://api.groq.com/openai/v1", - keyEnv: "GROQ_API_KEY", - markers: [/groq/i], - // A provider-keyed record — `groq: { defaultModel: '...' }` — names the - // vendor for everything inside it, and sits closer to the pin than any URL. - keyMarker: /^[ \t]*['"]?groq['"]?\s*:/m, - }, - { - id: "openrouter", - queryable: true, - baseUrl: "https://openrouter.ai/api/v1", - keyEnv: "OPENROUTER_API_KEY", - markers: [/openrouter/i], - keyMarker: /^[ \t]*['"]?openrouter['"]?\s*:/m, - }, - // Not queryable here — no catalogue call is wired for these. They are listed - // so their ids are ATTRIBUTED and reported unchecked, rather than falling to - // whichever queryable vendor happens to sit nearest in the file. Markers are - // deliberately specific: bare /openai/ would match Groq's own - // `api.groq.com/openai/v1` path and every `openai/gpt-oss-*` id it serves. - // Ollama runs on the user's own machine, so "is this id still served" is not - // a question with a fleet-wide answer — whatever the operator pulled is what - // exists. It is listed so its ids are ATTRIBUTED rather than falling to the - // nearest cloud vendor above them. - // - // Without this, evig's `.env.example` line `OLLAMA_MODEL=llama3.2` was - // attributed to Groq — the nearest marker above it — and reported RETIRED. - // `llama3.2` is a perfectly valid Ollama tag on a healthy line; Groq simply - // never served anything by that name. Same class as the pricing-table false - // positive: a true statement about the wrong vendor. - { - id: "ollama", - queryable: false, - markers: [/\bOLLAMA_[A-Z_]+\b/, /\bollama\b/i, /localhost:11434/, /127\.0\.0\.1:11434/], - }, - // - // `keyMarker` is how a provider-keyed record names its own rows. OrangeCat - // writes: - // - // export const PROVIDER_BASE_URLS = { openai: '...', openrouter: '...' } - // export const PROVIDER_RUNTIME = { - // openai: { baseUrl: ..., defaultModel: 'gpt-4o-mini' }, - // - // The pin belongs to OpenAI, but the nearest marker above it was OpenRouter's - // URL in the block before — so `gpt-4o-mini` was reported as a retired - // OpenRouter model. It is not an OpenRouter id at all; there it would be - // `openai/gpt-4o-mini`. The bare key `openai:` is deliberately not in - // `markers`, because `api.groq.com/openai/v1` contains that word — anchoring - // it to the start of a line followed by a colon is what makes it safe. - { id: "xai", queryable: false, markers: [/api\.x\.ai/i, /\bXAI_API_KEY\b/, /\bgrok\b/i], keyMarker: /^[ \t]*['"]?xai['"]?\s*:/m }, - { id: "anthropic", queryable: false, markers: [/api\.anthropic\.com/i, /\bANTHROPIC_API_KEY\b/], keyMarker: /^[ \t]*['"]?anthropic['"]?\s*:/m }, - { id: "openai", queryable: false, markers: [/api\.openai\.com/i, /\bOPENAI_API_KEY\b/], keyMarker: /^[ \t]*['"]?openai['"]?\s*:/m }, - { id: "google", queryable: false, markers: [/generativelanguage\.googleapis/i, /\bGEMINI_API_KEY\b/], keyMarker: /^[ \t]*['"]?google['"]?\s*:/m }, - { id: "together", queryable: false, markers: [/api\.together\.xyz/i, /\bTOGETHER_API_KEY\b/], keyMarker: /^[ \t]*['"]?together['"]?\s*:/m }, - { id: "deepseek", queryable: false, markers: [/api\.deepseek\.com/i, /\bDEEPSEEK_API_KEY\b/], keyMarker: /^[ \t]*['"]?deepseek['"]?\s*:/m }, -]; - -/** - * Files worth opening, in two tiers. - * - * LIKELY names an AI module outright. POSSIBLE is the long tail that a - * name-based filter misses: botsmann keeps its model id in `lib/constants.ts`, - * which mentions no vendor in its path and was invisible to the first version - * of this filter. A pin does not have to live in a file called `provider.ts`. - */ -/** - * Words that make a path segment worth opening — a DIRECTORY name as readily as - * a file name. - * - * The previous version of this filter required the word to appear in the - * FILENAME, and the cost of that was measured rather than imagined. Kivvi keeps - * its provider clients in `packages/ai/src/providers/`: - * - * packages/ai/src/providers/anthropic.ts ← scanned - * packages/ai/src/providers/groq.ts ← never opened - * packages/ai/src/providers/openrouter.ts ← never opened - * packages/ai/src/providers/index.ts ← never opened - * - * Two files in the same directory, one seen and one not, decided entirely by - * which vendor names someone had typed into a regex. Three retired ids lived in - * the unopened ones, so the audit reported Kivvi as having 2 dead pins when it - * had 6 — and understating a repo is worse than missing it outright, because - * the number looks like an answer. - * - * Matching is by TOKEN, never substring. `ai` as a substring appears in `mail`, - * `chain`, `domain`, `detail` and `maintenance`; as a token it appears in `ai`, - * `ai-guidance` and `open-ai`. Substring matching here would have quietly - * traded this blind spot for a flood of irrelevant files, and the cap would then - * have dropped real candidates to make room. - */ -const AI_TOKENS = new Set([ - // the concern - "ai", "llm", "gpt", "model", "models", "provider", "providers", "chain", - "chat", "chats", "completion", "completions", "prompt", "prompts", - "agent", "agents", "embedding", "embeddings", "inference", - // the vendors — every one of these is a plausible file name, and the list - // being short is exactly what caused the miss above - "groq", "openrouter", "openai", "anthropic", "claude", "xai", "grok", - "gemini", "google", "ollama", "mistral", "together", "deepseek", "cohere", - "nvidia", "perplexity", "fireworks", "replicate", -]); - -/** Directories that can contain application source. */ -const SOURCE_ROOT = /^(lib|src|app|apps|packages|config|server|services|api)$/i; - -/** Does this one path segment name an AI concern or a vendor? */ -export function segmentNamesAI(segment) { - const base = segment.replace(/\.(ts|js|mjs|tsx|jsx)$/i, "").toLowerCase(); - return base.split(/[^a-z0-9]+/).some((token) => AI_TOKENS.has(token)); -} - -/** - * A file that almost certainly decides which model gets called. - * - * The AI word may sit anywhere below the source root — `ai/providers/groq.ts` - * qualifies on its directory alone, which is the whole point. - */ -export function isLikelyPath(path) { - if (/(^|\/)\.env\.example$/i.test(path)) return true; - if (/(^|\/)env\.(ts|js|mjs)$/i.test(path)) return true; - if (!/\.(ts|js|mjs)$/i.test(path)) return false; - - const segments = path.split("/"); - const rootAt = segments.findIndex((seg) => SOURCE_ROOT.test(seg)); - if (rootAt < 0) return false; - return segments.slice(rootAt + 1).some(segmentNamesAI); -} - -/** - * The long tail a name-based filter misses. Botsmann kept its model id in - * `lib/constants.ts`, which mentions no vendor and no AI concern anywhere in - * its path. A pin does not have to live in a file called `provider.ts`. - */ -export function isPossiblePath(path) { - return /(^|\/)(lib|src|app|apps|packages|config)\/.*(constants?|config|settings|defaults)[^/]*\.(ts|js|mjs)$/i.test( - path, - ); -} - -/** Worth opening at all. */ -export function isCandidatePath(path) { - return isLikelyPath(path) || isPossiblePath(path); -} - -/** Never open these, whatever they are named. */ -const SKIP_PATH = - /(^|\/)(node_modules|dist|build|\.next|coverage|__tests__|__fixtures__)\/|(^|\/)\.claude\/worktrees\//; - -/** - * How many candidate files one repo may cost. - * - * Raised 90 -> 160 because the coverage ledger stopped being a caveat and - * became a finding: on 2026-08-27 it reported OrangeCat opening 90 of 133 with - * 4 likely-AI files dropped, and FleetCrown 90 of 109 with 15 dropped. Ranking - * puts likely files first, so shedding generic config is harmless — shedding - * fifteen files that name an AI concern is a blind spot, and in the two largest - * repos in the fleet. - * - * 160 clears both with headroom. The ledger stays, because the next repo to - * outgrow the cap should say so rather than quietly report a smaller number. - */ -const MAX_FILES_PER_REPO = 160; - -/** A vendor named 40+ lines from a pin is not describing that pin. */ -const MAX_ATTRIBUTION_DISTANCE = 40; - -// ── Extraction ─────────────────────────────────────────────────────────────── - -/** - * Is this string plausibly a model id rather than any other quoted thing? - * - * Deliberately permissive on shape and strict on the obvious negatives. A false - * POSITIVE costs one line in a report that says "not judged"; a false NEGATIVE - * is the outage this whole file exists to prevent. - */ -export function looksLikeModelId(s) { - if (typeof s !== "string") return false; - if (s.length < 3 || s.length > 80) return false; - if (/\s/.test(s)) return false; - if (s.includes("${")) return false; // interpolated: resolved at runtime, not pinned - if (/^https?:/i.test(s)) return false; - if (/^[./~@]/.test(s)) return false; - if (/\.(ts|tsx|js|mjs|cjs|json|css|scss|md|png|jpe?g|svg|ico|txt|ya?ml)$/i.test(s)) return false; - if (/^[A-Z][A-Z0-9_]*$/.test(s)) return false; // SCREAMING_CASE is an env name - // Model ids essentially always carry a version digit or a vendor/ prefix. - // A vendor id always carries a separator: a slash for routed ids - // (`openai/gpt-oss-120b`), or a hyphen or dot within the name - // (`llama-3.3-70b-versatile`, `llama3.2`, `codex-4`). Nothing in either live - // catalogue is a single unseparated word. - // - // Without this, reading model MAPS turned their keys into findings: Hirnli's - // alias table is `{ '70b': '...', '8b': '...' }`, and `70b` has a digit and no - // space, so it read as a pin and would have been reported retired at Groq. It - // is a size alias. `8b` escaped only by being two characters long, which is - // not a rule anyone should rely on. - if (!/[/.-]/.test(s)) return false; - - return /\d/.test(s) || s.includes("/"); -} - -/** - * The lines making up each `models` collection, with real line numbers. - * - * A tiny bracket walker rather than a regex, because the shapes in this fleet - * defeat any single pattern. All four of these are one repo's way of saying the - * same thing, and every one of them hid a retired id at some point: - * - * models: ['a', 'b'] an inline array - * models: AIModel[] = [ an array behind a TS annotation - * GROQ_MODELS = { '8b': 'llama-...' } a map whose VALUES are ids - * GROQ_MODELS = { 'llama-...': { ... } } a map whose KEYS are ids - * - * Maps were the second discovery and cost two more repos. Hirnli kept its ids - * as map values and Orangecat as map keys — including `DEFAULT_GROQ_MODEL`, - * the baseline every free user gets — and an array-only walker read straight - * past both. So the opener is `[` or `{`, and every quoted string in the region - * is a candidate regardless of which side of the colon it sits on. - * - * The opening bracket is the LAST one on the declaring line. That is what makes - * the annotated form work: in `models: AIModel[] = [`, the first `[` belongs to - * the type and closes immediately, so anchoring to it reads an empty array and - * reports nothing — a silent miss, the worst output an audit has. - */ -export function modelListRegions(text) { - const lines = text.split("\n"); - const regions = []; - const CLOSER = { "[": "]", "{": "}" }; - - for (let i = 0; i < lines.length; i++) { - // Greedy on purpose: it backtracks to the LAST opener within reach. - // - // `\w*` before `models` is load-bearing. A bare `\bmodels?\b` does NOT - // match `GROQ_MODELS`, because the underscore before it is a word character - // so there is no boundary there — which is precisely why two repos' model - // maps read as empty. Almost every map in this fleet is named that way. - // - // The `[:=]` immediately after the token is what makes this a DECLARATION - // rather than any line that mentions models. Without it, - // `for (const model of models) {` opened a region over the whole loop body - // — and the request headers inside were then read as model ids, so a - // `Content-Type: application/json` was reported as a retired Groq model. - // PLURAL only. A singular `model:` is a parameter or a single-id property, - // and `function supportsReasoningEffort(model: string): boolean {` is a - // declaration by the rule above — it opened a region over the function body - // and read the `startsWith("qwen/")` prefixes inside as retired ids. - // Collections are plural; single ids are matched on one line by the - // patterns in extractPins. - const opener = /\b[a-z0-9_]*models\b\s*[:=][^\n]{0,80}[[{]/i.exec(lines[i]); - if (!opener) continue; - - const open = opener[0].at(-1); - const close = CLOSER[open]; - const region = []; - let depth = 0; - let opened = false; - - // A models collection running past 200 lines is not a models collection. - for (let j = i; j < lines.length && j < i + 200; j++) { - const segment = j === i ? lines[j].slice(opener.index + opener[0].length - 1) : lines[j]; - region.push({ line: j + 1, text: segment }); - - for (const ch of segment) { - if (ch === open) { - depth++; - opened = true; - } else if (ch === close) { - depth--; - } - } - if (opened && depth <= 0) break; - } - - regions.push(region); - } - - return regions; -} - -/** - * Pull candidate model ids out of one file's text, with the line each sits on. - * - * Three shapes cover how this fleet writes them: - * 1. a `model:` / `models:` assignment, single value or array - * 2. a *_MODEL constant or Zod `.default(...)` - * 3. a bare `GROQ_MODEL=...` line in a .env file - */ -export function extractPins(text) { - const found = new Map(); // id -> line number (first sighting) - const lineOf = (index) => text.slice(0, index).split("\n").length; - - const remember = (id, index) => { - if (!looksLikeModelId(id)) return; - if (!found.has(id)) found.set(id, lineOf(index)); - }; - - const rememberAt = (id, line) => { - if (!looksLikeModelId(id)) return; - if (!found.has(id)) found.set(id, line); - }; - - // 1a. model: 'x' — a single id on one line. - // - // The `id|name` suffix is not decoration. Kivvi's fallback reads - // - // const FALLBACK_MODEL: ModelSelection = { - // providerId: "groq", - // modelId: "llama-3.3-70b-versatile", - // }; - // - // and `modelId` is not `model`, so a pattern anchored on the bare word walked - // straight past a retired id sitting in the app's default model selection. - // The declaration above it is singular, so the collection walker does not - // cover it either — this line is the only thing that does. - for (const m of text.matchAll( - /\b[a-z0-9_]*model(?:s|id|_id|name|_name)?\b\s*[:=]\s*['"`]([^'"`\n]{0,120})['"`]/gi, - )) { - remember(m[1], m.index); - } - - // 1b. a models ARRAY, walked by bracket depth rather than matched by regex. - // - // This used to be `models?\s*[:=]\s*\[[\s\S]{0,400}?\]`, and it missed a - // whole package. Kivvi declares its list as: - // - // models: AIModel[] = [ - // - // A TypeScript type annotation sits between the key and the array, so the - // pattern never fired, and two retired Groq ids inside were invisible. The - // 400-character window was the second half of the same problem: a list of - // richly described models runs well past it, so even a matching array was - // read only as far as its first two entries. - // - // Walking the brackets has neither limit. It also reports each id's OWN line - // instead of the line the array opened on, which matters because vendor - // attribution is measured in lines from the pin. - for (const region of modelListRegions(text)) { - for (const { line, text: lineText } of region) { - // Comments are stripped first. A model list is exactly where someone - // documents the id they just replaced, and such notes are usually written - // in backticks — so without this, a comment reading "replaces - // `meta-llama/llama-3.2-3b-instruct:free`, which was retired" reports - // that id as a live pin, in the very commit that removed it. The audit - // would be reporting on prose instead of code: the exact failure it - // exists to catch elsewhere. - const code = lineText.replace(/\/\*[\s\S]*?\*\//g, "").replace(/\/\/.*$/, ""); - for (const q of code.matchAll(/['"`]([^'"`\n]+)['"`]/g)) rememberAt(q[1], line); - } - } - - // 2. GROQ_MODEL: z.string().default('x') | const DEFAULT_MODEL = 'x' - for (const m of text.matchAll( - /\b([A-Za-z][A-Za-z0-9_]*MODEL[A-Za-z0-9_]*)\s*[:=][^\n]{0,80}?['"`]([^'"`\n]+)['"`]/g, - )) { - remember(m[2], m.index); - } - - // 3. .env style, unquoted or quoted, no code around it - for (const m of text.matchAll(/^[ \t]*(?:export[ \t]+)?[A-Z][A-Z0-9_]*MODEL[A-Z0-9_]*\s*=\s*["']?([^"'\s#]+)/gm)) { - remember(m[1], m.index); - } - - return [...found].map(([id, line]) => ({ id, line })); -} - -/** - * Which vendor does this pin belong to? - * - * Nearest-marker first: a file can legitimately name both vendors — AOZ's - * provider.ts resolves Groq AND OpenRouter in one module — so a file-wide vote - * would attribute both vendors' pins to whichever appeared more. The pin's own - * neighbourhood is what actually says which branch it is in. Only when the - * window is silent do we fall back to the file, and only when the file names - * exactly one vendor. - * - * Returns a vendor id, or null for "cannot tell" — which is reported, not judged. - */ -export function attribute(text, line, vendors = VENDORS, maxDistance = MAX_ATTRIBUTION_DISTANCE) { - const lines = text.split("\n"); - - /** - * Distance to this vendor's nearest mention ABOVE the pin, and below it. - * - * Above is what decides. Every provider module in this fleet is written as - * `if (provider === "groq") { url = ...; body = { model: "..." } }` — the - * branch that owns a model literal always opens above it. Ranking by raw - * proximity instead put `grok-3-mini` (xAI, line 89) with Groq, whose URL sat - * at line 71, and tied kivvi's real Groq pin exactly between two vendors. - */ - const distances = (vendor) => { - let above = Infinity; - let below = Infinity; - // `keyMarker` counts as a mention: in a provider-keyed record, the row's - // own key is the most local and most reliable statement of which vendor a - // pin belongs to — closer than any base URL, and unambiguous. - const mentions = (l) => - vendor.markers.some((re) => re.test(l)) || (vendor.keyMarker?.test(l) ?? false); - - for (let i = 0; i < lines.length; i++) { - if (!mentions(lines[i])) continue; - const d = i + 1 - line; - if (d <= 0) above = Math.min(above, -d); - else below = Math.min(below, d); - } - return { above, below }; - }; - - const scored = vendors.map((v) => ({ id: v.id, ...distances(v) })); - - const fromAbove = scored.filter((s) => s.above <= maxDistance).sort((a, b) => a.above - b.above); - if (fromAbove.length === 1) return fromAbove[0].id; - if (fromAbove.length > 1 && fromAbove[0].above < fromAbove[1].above) return fromAbove[0].id; - - // Nothing above governs it — a pin at the top of a file, or a config block - // whose vendor is named afterwards. Fall back to the nearest mention below. - if (fromAbove.length === 0) { - const fromBelow = scored.filter((s) => s.below <= maxDistance).sort((a, b) => a.below - b.below); - if (fromBelow.length === 1) return fromBelow[0].id; - if (fromBelow.length > 1 && fromBelow[0].below < fromBelow[1].below) return fromBelow[0].id; - } - - const named = vendors.filter( - (v) => v.markers.some((re) => re.test(text)) || (v.keyMarker?.test(text) ?? false), - ); - return named.length === 1 ? named[0].id : null; -} - -/** - * Turn per-file findings into per-vendor id lists plus the unattributable rest. - * Pure, so the self-test can drive it without a network or a checkout. - */ -export function collate(findings) { - const byVendor = new Map(); - const unattributed = []; - - for (const f of findings) { - if (!f.vendor) { - unattributed.push(f); - continue; - } - if (!byVendor.has(f.vendor)) byVendor.set(f.vendor, []); - byVendor.get(f.vendor).push(f); - } - return { byVendor, unattributed }; -} - -/** - * Apply catalogue verdicts to the findings. - * - * `live` is a Map of vendorId -> Set of ids, or null for a vendor whose - * catalogue could not be read. Null is NOT an empty set: treating "I could not - * look" as "nothing is there" reports every pin as retired and invents an - * outage, which is worse than silence because someone acts on it. - */ -/** - * Vendors whose entire published catalogue is lowercase. - * - * MEASURED, not assumed: on 2026-08-26 Groq listed 14 ids and OpenRouter 416, - * and not one of those 430 contained a capital letter. So an id carrying a - * capital cannot be a RETIRED id at these two — it was never one of their ids - * at all, and "retired" is the wrong diagnosis rather than merely the wrong - * target. - * - * This rule exists because the audit reported a retired pin in a repo that had - * nothing wrong. OrangeCat renders a pricing table: - * - * models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'] - * - * — human-readable marketing copy that never reaches an API. `GPT-4o` was the - * only entry without a space, so it slipped the shape filter, landed nearest an - * OpenRouter marker, and was announced as a retired model pin. That is the - * failure mode a daily gate can least afford: cry wolf on a healthy repo and - * the reader learns to skim the report, taking the next real outage with it. - * - * Note where this lands such an id: `unattributed` — reported, not judged. - * Never dropped. For `GPT-4o` that bucket is also literally correct, since it - * IS an OpenAI product name and OpenAI is a vendor we do not query. - */ -const LOWERCASE_ONLY_VENDORS = new Set(["groq", "openrouter"]); - -/** Could this id ever have been served by this vendor? */ -export function possibleAt(vendorId, id) { - return !(LOWERCASE_ONLY_VENDORS.has(vendorId) && /[A-Z]/.test(id)); -} - -export function judge(findings, live) { - return findings.map((f) => { - if (!f.vendor) return { ...f, state: "unattributed" }; - // Attribution put it here, but the vendor could never have served it, so - // the attribution is what is wrong. Report it; do not assert a retirement. - if (!possibleAt(f.vendor, f.id)) return { ...f, vendor: null, state: "unattributed" }; - const set = live.get(f.vendor); - if (!set) return { ...f, state: "unchecked" }; - return { ...f, state: set.has(f.id) ? "ok" : "gone" }; - }); -} - -// ── Reading the fleet ──────────────────────────────────────────────────────── - -/** - * Which files get opened when a repo has more candidates than the cap allows. - * - * Ranked, never arbitrary: a file that names a vendor or an AI concern outranks - * a generic `config.ts`, so the cap sheds the least likely candidates first. - * And every truncation is RECORDED — a bounded sweep that stays quiet about - * what it skipped reads exactly like a sweep that found nothing. - */ -const truncated = []; - -/** Did the token that listed the fleet see any private repo? See remoteRepos. */ -let privateVisible = true; - -function rank(paths, repoName) { - const sorted = [...paths].sort((a, b) => { - const ta = isLikelyPath(a) ? 0 : 1; - const tb = isLikelyPath(b) ? 0 : 1; - return ta - tb || a.length - b.length; - }); - if (sorted.length > MAX_FILES_PER_REPO) { - // What was dropped matters more than how many. Everything in the LIKELY - // tier is opened first, so a truncation that sheds only generic config - // files has not touched the audit's real coverage — and saying which it was - // is the difference between a caveat and an alarm. - const likelyDropped = sorted.slice(MAX_FILES_PER_REPO).filter((p) => isLikelyPath(p)).length; - truncated.push({ repo: repoName, seen: sorted.length, opened: MAX_FILES_PER_REPO, likelyDropped }); - } - return sorted.slice(0, MAX_FILES_PER_REPO); -} - -async function gh(args) { - const { stdout } = await exec("gh", args, { maxBuffer: 64 * 1024 * 1024 }); - return stdout; -} - -/** Remote, default-branch view. Clones drift; this repo has been bitten by that. */ -async function remoteRepos() { - const raw = await gh([ - "repo", "list", OWNER, - "--limit", String(GH_LIMIT), - "--no-archived", - "--json", "name,defaultBranchRef,isFork,visibility", - ]); - const all = JSON.parse(raw).filter((r) => !r.isFork && r.defaultBranchRef?.name); - - // A workflow's default GITHUB_TOKEN is scoped to its own repo, so `repo list` - // returns only PUBLIC ones — and a sweep that silently sees fewer repos than - // it did yesterday reports a cleaner fleet, not a smaller one. Detectable - // without knowing the true count: zero private repos in the answer means - // either there are none, or this token cannot see them. - privateVisible = all.some((r) => r.visibility && r.visibility !== "PUBLIC"); - - return all.map((r) => ({ name: r.name, branch: r.defaultBranchRef.name })); -} - -async function remoteFiles(repo) { - let tree; - try { - tree = JSON.parse( - await gh(["api", `repos/${OWNER}/${repo.name}/git/trees/${repo.branch}?recursive=1`]), - ); - } catch { - return []; // empty repo, or no access — not a finding - } - const all = (tree.tree ?? []) - .filter((n) => n.type === "blob" && !SKIP_PATH.test("/" + n.path) && isCandidatePath(n.path)) - .map((n) => n.path); - const paths = rank(all, repo.name); - - const out = []; - for (const path of paths) { - try { - const body = JSON.parse( - await gh(["api", `repos/${OWNER}/${repo.name}/contents/${path}?ref=${repo.branch}`]), - ); - if (body.encoding !== "base64" || !body.content) continue; - out.push({ path, text: Buffer.from(body.content, "base64").toString("utf8") }); - } catch { - /* a path that vanished between tree and read is not a finding */ - } - } - return out; -} - -function localRepos() { - return readdirSync(FLEET_ROOT) - .filter((n) => !n.startsWith("_") && existsSync(join(FLEET_ROOT, n, ".git"))) - .map((name) => ({ name, branch: "(local)" })); -} - -function walk(dir, depth, acc) { - if (depth < 0 || acc.length >= MAX_FILES_PER_REPO * 6) return acc; - let entries; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch { - return acc; - } - for (const e of entries) { - const full = join(dir, e.name); - if (e.isDirectory()) { - if (/^(node_modules|dist|build|\.next|coverage|\.git|\.claude)$/.test(e.name)) continue; - walk(full, depth - 1, acc); - } else if (isCandidatePath(full) && !SKIP_PATH.test(full)) { - try { - if (statSync(full).size < 400_000) acc.push(full); - } catch { /* raced */ } - } - } - return acc; -} - -function localFiles(repo) { - const root = join(FLEET_ROOT, repo.name); - const all = walk(root, 6, []).map((full) => full.slice(root.length + 1)); - return rank(all, repo.name).map((path) => ({ - path, - text: readFileSync(join(root, path), "utf8"), - })); -} - -// ── The catalogue, via ai-kit ───────────────────────────────────────────── - -function loadAiKit() { - const candidates = [ - process.env.AI_KIT_FROM, - // Renamed from ai-ration in v0.3.0. The old paths stay in the list so a - // checkout that has not been renamed still resolves — a rename should not - // turn a working audit into a silent exit 2. - process.env.AI_RATION_FROM, - join(homedir(), "dev", "fleetcrown"), - join(homedir(), "dev", "ai-kit"), - join(homedir(), "dev", "ai-ration"), - ].filter(Boolean); - - for (const root of candidates) { - for (const entry of [ - join(root, "node_modules", "ai-kit", "dist", "index.js"), - join(root, "node_modules", "ai-ration", "dist", "index.js"), - join(root, "dist", "index.js"), - ]) { - if (existsSync(entry)) return import(entry); - } - } - console.error( - "\u2717 ai-kit not found. Set AI_KIT_FROM=/path/to/a/repo that installs it,\n" + - " or build it once: (cd ~/dev/ai-kit && npm i && npm run build)", - ); - process.exit(2); -} - -/** - * Ask each vendor what it still lists, for exactly the ids we found. - * Returns Map(vendorId -> Set|null), where null means "could not look". - */ -async function readCatalogues(byVendor, checkCatalog) { - const live = new Map(); - for (const vendor of VENDORS) { - if (!vendor.queryable) continue; - const findings = byVendor.get(vendor.id); - if (!findings?.length) continue; - - const ids = [...new Set(findings.map((f) => f.id))]; - const [verdict] = await checkCatalog([ - { - id: vendor.id, - baseUrl: vendor.baseUrl, - keyEnv: vendor.keyEnv, - models: ids, - dailyTokens: 0, - }, - ]); - live.set(vendor.id, verdict.live ? new Set(verdict.live) : null); - } - return live; -} - -// ── Report ─────────────────────────────────────────────────────────────────── - -function report(judged) { - const gone = judged.filter((j) => j.state === "gone"); - const unchecked = judged.filter((j) => j.state === "unchecked"); - const unattributed = judged.filter((j) => j.state === "unattributed"); - const ok = judged.filter((j) => j.state === "ok"); - - const lines = []; - - if (gone.length) { - lines.push("RETIRED — the vendor no longer lists these, so every call using them fails:"); - const byId = new Map(); - for (const g of gone) { - const key = `${g.vendor}/${g.id}`; - if (!byId.has(key)) byId.set(key, []); - byId.get(key).push(`${g.repo}:${g.path}:${g.line}`); - } - for (const [key, sites] of [...byId].sort()) { - lines.push(` GONE ${key}`); - for (const s of sites.sort()) lines.push(` ${s}`); - } - lines.push(""); - } - - if (unchecked.length) { - const vendors = [...new Set(unchecked.map((u) => u.vendor))].sort(); - lines.push( - `${unchecked.length} pin(s) UNCHECKED — no readable catalogue for: ${vendors.join(", ")}.`, - ); - lines.push(" That is not a pass for them. Set the vendor key to judge these."); - lines.push(""); - } - - if (unattributed.length) { - const ids = [...new Set(unattributed.map((u) => u.id))].sort(); - lines.push(`${unattributed.length} pin(s) not attributable to a vendor we query — listed, not judged:`); - for (const id of ids.slice(0, 20)) lines.push(` ? ${id}`); - if (ids.length > 20) lines.push(` ? … and ${ids.length - 20} more`); - lines.push(""); - } - - lines.push( - `${ok.length} pin(s) confirmed live · ${gone.length} retired · ` + - `${unchecked.length} unchecked · ${unattributed.length} unattributed`, - ); - - if (truncated.length) { - lines.push(""); - const blind = truncated.filter((t) => t.likelyDropped > 0); - lines.push(`COVERAGE — ${truncated.length} repo(s) had more candidate files than the cap of ${MAX_FILES_PER_REPO}:`); - for (const t of truncated) { - const tail = t.likelyDropped > 0 ? `, ${t.likelyDropped} of them likely-AI` : ", none likely-AI"; - lines.push(` ${t.repo}: opened ${t.opened} of ${t.seen}${tail}`); - } - lines.push( - blind.length - ? " Files naming an AI concern were dropped — raise MAX_FILES_PER_REPO." - : " Only generic config files were dropped; every likely-AI file was opened.", - ); - } - - if (gone.length) { - lines.push(""); - lines.push("A pin is a scheduled outage. The durable fix is a chain across VENDORS —"); - lines.push("see SHARED.md → ai-kit. Repinning buys time until the next retirement."); - } - - return lines.join("\n"); -} - -// ── Main ───────────────────────────────────────────────────────────────────── - -export async function main() { - const { checkCatalog } = await loadAiKit(); - - const repos = LOCAL ? localRepos() : await remoteRepos(); - const findings = []; - - for (const repo of repos) { - const files = LOCAL ? localFiles(repo) : await remoteFiles(repo); - for (const file of files) { - for (const pin of extractPins(file.text)) { - findings.push({ - repo: repo.name, - path: file.path, - line: pin.line, - id: pin.id, - vendor: attribute(file.text, pin.line), - }); - } - } - } - - const { byVendor } = collate(findings); - const live = await readCatalogues(byVendor, checkCatalog); - const judged = judge(findings, live); - - console.log(report(judged)); - console.log(`\ninspected ${repos.length} repo(s)${LOCAL ? " (local checkouts)" : " on their default branches"}`); - if (!LOCAL && !privateVisible) { - console.log( - "NOTE — no private repo was listed. Either there are none, or this token\n" + - " cannot see them; a repo-scoped GITHUB_TOKEN cannot. Set GH_TOKEN to a\n" + - " PAT with repo scope to audit private repos too.", - ); - } - - const dead = judged.some((j) => j.state === "gone"); - process.exit(dead && !WARN_ONLY ? 1 : 0); -} - -const invokedDirectly = - process.argv[1] && import.meta.url === `file://${process.argv[1]}`; -if (invokedDirectly) { - main().catch((err) => { - console.error(`✗ audit failed: ${err?.message ?? err}`); - process.exit(2); - }); -} diff --git a/scripts/ci/shared-inventory.baseline b/scripts/ci/shared-inventory.baseline deleted file mode 100644 index 51da272..0000000 --- a/scripts/ci/shared-inventory.baseline +++ /dev/null @@ -1,9 +0,0 @@ -automerge-script 6 6 -rate-limit 9 13 -email-send 6 7 -logger 6 10 -api-envelope 3 3 -health-route 9 9 -slug-util 2 2 -date-utils 4 5 -ai-provider-client 6 16 diff --git a/scripts/ci/shared-inventory.sh b/scripts/ci/shared-inventory.sh deleted file mode 100755 index b859106..0000000 --- a/scripts/ci/shared-inventory.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env bash -# -# What has the fleet built more than once — and is it getting better or worse? -# -# The problem this solves is not "we don't know we duplicate things". Everyone -# knows. The problem is that knowing has never changed the number: -# -# - orangecat's ADR-0002 ("Unify Rate Limiting Modules", 2026-01-18) is still -# Status: Proposed. It names TWO implementations. There are now FOUR. -# - templates/ci/README.md says "deliberately ONE central script, not a copy -# per repo". auto-merge-sweep.sh exists in 21 repos in 8 distinct versions, -# 11,787 to 19,344 bytes. A fix in one reaches at most 9 of them. -# -# Both were written down. Writing it down is what failed. So this does not -# produce a document — it produces a NUMBER, and `--check` makes that number a -# ratchet: it may fall, it may hold, it may never rise. That is the whole -# mechanism. Nobody has to fix everything today; they only have to stop adding. -# -# Reads each repo's REMOTE default branch via the trees API, never a local -# checkout: clones drift, and a stale clone has already produced a redundant PR -# and a nearly-dropped gate in this fleet. -# -# Usage: -# shared-inventory.sh # report -# shared-inventory.sh --check # ratchet: exit 1 if any count ROSE -# shared-inventory.sh --update # rewrite the baseline (do this in a PR) -# -# Env: GH_OWNER (default catomean), GH_LIMIT (default 100) - -set -uo pipefail - -OWNER="${GH_OWNER:-catomean}" -LIMIT="${GH_LIMIT:-100}" -HERE="$(cd "$(dirname "$0")" && pwd)" -BASELINE="${SHARED_INVENTORY_BASELINE:-$HERE/shared-inventory.baseline}" - -MODE=report -case "${1:-}" in - --check) MODE=check ;; - --update) MODE=update ;; - "") MODE=report ;; - *) echo "unknown argument: $1" >&2; exit 2 ;; -esac - -command -v gh >/dev/null 2>&1 || { echo "gh CLI not found" >&2; exit 2; } -command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 2; } - -# ── The concerns we track ──────────────────────────────────────────────────── -# -# Each line: name|regex over file paths. Deliberately conservative — a concern -# belongs here only if a shared implementation is PLAUSIBLE. Auth, DB schemas -# and UI markup are excluded on purpose: they are coupled to a framework, a -# schema, or a design system, and "centralize everything" is how apps stop being -# able to look or behave like themselves. -CONCERNS=' -automerge-script|scripts/ci/auto-merge-sweep\.sh$ -rate-limit|(lib|src|app|apps|packages)/.*rate-?limit(er)?\.(ts|js)$ -email-send|(lib|src)/.*(email|mailer|resend)/(index|client|send|config)\.(ts|js)$ -logger|(lib|src)/.*logger\.(ts|js)$ -api-envelope|(lib|src)/.*(route-helpers|api-response|apiResponse)\.(ts|js)$ -health-route|.*api/health/route\.(ts|js)$ -slug-util|(lib|src)/.*slug(ify)?\.(ts|js)$ -date-utils|(lib|src)/.*(dates|date-utils)\.(ts|js)$ -ai-provider-client|(lib|src|packages|apps)/.*(provider|call-provider)s?\.(ts|js)$ -' - -# ── Collect ────────────────────────────────────────────────────────────────── -# -# FORKS ARE EXCLUDED, and that exclusion is load-bearing. `openclaw` is a fork of -# openclaw/openclaw (1.5 GB of upstream), and counting it put 46 provider files, -# 7 loggers and 7 date utils into the totals — none of them written here, none -# of them ours to unify. A metric dominated by somebody else's codebase measures -# nothing and gets ignored, which is the failure mode every check in this repo -# exists to avoid. -repos=$(gh repo list "$OWNER" --limit "$LIMIT" --no-archived --source \ - --json name,defaultBranchRef \ - --jq '.[] | "\(.name)\t\(.defaultBranchRef.name // "")"' 2>/dev/null) -[ -n "$repos" ] || { echo "could not list repos for $OWNER" >&2; exit 2; } - -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT -truncated="" -inspected=0 - -while IFS=$'\t' read -r name branch; do - [ -n "$name" ] || continue - [ -n "$branch" ] || continue - body=$(gh api "repos/$OWNER/$name/git/trees/$branch?recursive=1" 2>/dev/null) || continue - [ -n "$body" ] || continue - # A truncated tree UNDERCOUNTS, which would read as progress. Say so. - if [ "$(printf '%s' "$body" | jq -r '.truncated // false')" = "true" ]; then - truncated="$truncated $name" - fi - printf '%s' "$body" \ - | jq -r '.tree[]? | select(.type=="blob") | .path' 2>/dev/null \ - | grep -viE 'node_modules|\.next/|/dist/|__tests__|\.test\.|\.spec\.|\.d\.ts' \ - > "$TMP/$name.files" - inspected=$((inspected + 1)) -done <<< "$repos" - -# ── Count ──────────────────────────────────────────────────────────────────── -current="$TMP/current" -: > "$current" - -for line in $CONCERNS; do - [ -n "$line" ] || continue - concern="${line%%|*}" - re="${line#*|}" - repos_with=0 - copies=0 - who="" - for f in "$TMP"/*.files; do - [ -e "$f" ] || continue - r="$(basename "$f" .files)" - n=$(grep -icE "$re" "$f" 2>/dev/null || true) - n=${n:-0} - if [ "$n" -gt 0 ]; then - repos_with=$((repos_with + 1)) - copies=$((copies + n)) - who="$who $r($n)" - fi - done - printf '%s\t%s\t%s\n' "$concern" "$repos_with" "$copies" >> "$current" - if [ "$MODE" = report ]; then - printf '%-20s %2d repos %3d files %s\n' "$concern" "$repos_with" "$copies" \ - "$(echo "$who" | cut -c1-80)" - fi -done - -if [ "$MODE" = report ]; then - echo - echo "inspected $inspected repo(s) on their default branches" - [ -n "$truncated" ] && echo "⚠ TRUNCATED trees (undercounted):$truncated" - echo - echo "A count is not a verdict — see SHARED.md for what is worth extracting." - exit 0 -fi - -if [ "$MODE" = update ]; then - cp "$current" "$BASELINE" - echo "baseline written: $BASELINE" - cat "$BASELINE" - exit 0 -fi - -# ── Ratchet ────────────────────────────────────────────────────────────────── -[ -f "$BASELINE" ] || { echo "no baseline at $BASELINE — run --update" >&2; exit 2; } - -# A truncated tree undercounts, so a "pass" here could be an artefact. Refuse to -# certify rather than report a decrease that did not happen. -if [ -n "$truncated" ]; then - echo "✗ tree truncated for:$truncated — counts undercount, refusing to judge" >&2 - exit 2 -fi - -fail=0 -while IFS=$'\t' read -r concern was_repos was_copies; do - [ -n "$concern" ] || continue - now_line=$(grep -P "^${concern}\t" "$current" 2>/dev/null || grep "^${concern} " "$current" 2>/dev/null) - if [ -z "$now_line" ]; then - echo "· $concern — no longer measured (concern removed from the script?)" - continue - fi - now_copies=$(printf '%s' "$now_line" | cut -f3) - if [ "$now_copies" -gt "$was_copies" ]; then - echo "✗ $concern: $was_copies → $now_copies files. The ratchet only turns one way." - fail=1 - elif [ "$now_copies" -lt "$was_copies" ]; then - echo "✓ $concern: $was_copies → $now_copies files — run --update to lock it in." - fi -done < "$BASELINE" - -# A concern present now but absent from the baseline is new duplication that -# would otherwise slip in unmeasured. -while IFS=$'\t' read -r concern _ now_copies; do - [ -n "$concern" ] || continue - if ! cut -f1 "$BASELINE" | grep -qx "$concern"; then - echo "✗ $concern: not in the baseline ($now_copies files). Add it deliberately with --update." - fail=1 - fi -done < "$current" - -if [ "$fail" -ne 0 ]; then - echo - echo "Duplication rose. Either reuse what exists (SHARED.md) or, if this copy is" - echo "genuinely justified, run --update in the same PR so the increase is reviewed." - exit 1 -fi - -echo "✓ duplication did not increase (inspected $inspected repos)" diff --git a/scripts/ci/test-auto-merge-sweep.sh b/scripts/ci/test-auto-merge-sweep.sh deleted file mode 100755 index 4903d61..0000000 --- a/scripts/ci/test-auto-merge-sweep.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash -# -# Executes the REAL auto-merge-sweep.sh against a fake `gh` on PATH. -# -# Ported from evig, which was the only repo in the fleet that had tests for its -# sweep — and they were about to be deleted along with its copy of the script. -# That would have been the worst possible trade: centralising the code while -# throwing away the only evidence it behaves. The tests are as much a shared -# asset as the script, so they moved here with it. -# -# This tests SHIPPED CONTROL FLOW, not a description of it. A stubbed -# re-implementation of the guard would pass happily while the real script -# deadlocks — which is exactly what happened on 2026-08-07, when an Actions -# incident left main `failure` with no failed job and the sweep refused every -# merge for ~14 hours while still exiting 0 and looking healthy. -# -# The sweep must ALWAYS exit 0: it is a scheduled janitor, not a gate. A -# non-zero exit means the fake `gh` hit an unhandled call shape, which would -# make every assertion below vacuous — so that is checked first, every time. - -set -uo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -SWEEP="$HERE/auto-merge-sweep.sh" -PASS=0 -FAIL=0 - -ok() { printf ' ✓ %s\n' "$1"; PASS=$((PASS + 1)); } -no() { printf ' ✗ %s\n' "$1"; FAIL=$((FAIL + 1)); } - -# run_sweep [deploy-workflow] [deployed-sha] [running] -# -# Optional globals, consumed and RESET on every call so one case's fixture can -# never leak into the next: -# RS_STATUS base run status (default completed) -# RS_HEADSHA base run's headSha (default the branch tip) -# RS_PRS JSON array for pr list (default []) -# RS_VIEW JSON for pr view (default MERGEABLE/CLEAN) -# -# Emits the sweep's combined output; records gh calls in $GH_LOG. -run_sweep() { - local conclusion="$1" failed_steps="${2:-}" attempt="${3:-1}" - local deploy_wf="${4:-}" deployed_sha="${5:-}" deploy_running="${6:-0}" - local status_field="${RS_STATUS:-completed}" - local head_field="${RS_HEADSHA:-basesha000000}" - local dir; dir="$(mktemp -d)" - GH_LOG="$dir/gh-calls.log" - : > "$GH_LOG" - printf '%s\n' "${RS_PRS:-[]}" > "$dir/prs.json" - printf '%s\n' "${RS_VIEW:-{\"mergeable\":\"MERGEABLE\",\"mergeStateStatus\":\"CLEAN\"}}" > "$dir/view.json" - printf '%b\n' "${RS_REDJOBS:-Some Red Job}" > "$dir/redjobs.txt" - RS_STATUS=""; RS_HEADSHA=""; RS_PRS=""; RS_VIEW=""; RS_REDJOBS="" - - cat > "$dir/gh" <> "$GH_LOG" -case "\$ARGS" in - *"/commits/"*) echo "basesha000000" ;; - # Deploy-reconciler queries, matched BEFORE the generic CI one — ordering is - # the only thing separating them, since all three start with "run list". - "run list"*"--json status"*) printf '%s\n' '$deploy_running' ;; - "run list"*"--status success"*) printf '%s\n' '$deployed_sha' ;; - "run list"*) printf '%s\n' '{"databaseId":42,"status":"$status_field","conclusion":"$conclusion","headSha":"$head_field"}' ;; - *"/actions/runs/"*"/jobs"*) printf '%s\n' '$failed_steps' ;; - "run rerun"*) echo "rerun dispatched" ;; - *"/actions/runs/"*) printf '%s\n' '$attempt' ;; - "run view"*) cat "$dir/redjobs.txt" ;; - # The deadlock-name block also calls pr list, with a --jq the real gh would - # apply; this fake returns the raw payload either way, which that block only - # ever prints. The MERGE loop parses it with real jq, so fixtures must be - # well-formed JSON. - "pr list"*) cat "$dir/prs.json" ;; - "pr view"*) cat "$dir/view.json" ;; - "pr merge"*) echo "merged" ;; - "api -X PUT"*"update-branch"*) echo "updated" ;; - "workflow run"*) echo "dispatched" ;; - *) echo "UNHANDLED gh call: \$ARGS" >&2; exit 1 ;; -esac -FAKE - chmod +x "$dir/gh" - - local out status - out=$(PATH="$dir:$PATH" GH_REPO=catomean/fixture BASE_BRANCH=main \ - DEPLOY_WORKFLOW="$deploy_wf" \ - bash "$SWEEP" 2>&1) - status=$? - SWEEP_OUT="$out" - if [ "$status" -ne 0 ]; then - no "sweep exited $status — the fake gh hit an unhandled call shape, so every assertion would be vacuous" - printf '%s\n' "$out" | sed 's/^/ /' | tail -5 - return 1 - fi - return 0 -} - -# `grep -c` PRINTS 0 and also EXITS 1 when there is no match, so the obvious -# `|| echo 0` appends a second zero and every numeric comparison then dies with -# "integer expected". Let grep's own output stand. -reruns() { grep -c '^run rerun' "$GH_LOG" 2>/dev/null; } - -echo "auto-merge sweep — base branch guard" - -# 1. A cancelled base run is NOT a verdict about the code. Treating it as one -# strands the queue, and only a merge can produce a new base run — so the -# guard blocks the very thing that would clear it. -if run_sweep cancelled '' 1; then - [ "$(reruns)" -ge 1 ] \ - && ok 're-runs a CANCELLED base run instead of deadlocking behind it' \ - || no 're-runs a CANCELLED base run instead of deadlocking behind it' -fi - -# 2. A run that failed inside GitHub's own "Set up job" never executed our code. -if run_sweep failure 'Set up job' 1; then - [ "$(reruns)" -ge 1 ] \ - && ok 're-runs a base run that FAILED before executing any of our code' \ - || no 're-runs a base run that FAILED before executing any of our code' -fi - -# 3. A genuine failure IS a verdict. It must block, and must NOT be re-run — -# retrying real failures is how a broken base gets merged onto anyway. -if run_sweep failure 'Run tests' 1; then - if [ "$(reruns)" -eq 0 ]; then - ok 'refuses a genuinely broken base, and does NOT re-run it' - else - no 'refuses a genuinely broken base, and does NOT re-run it' - fi -fi - -# 4. Jobs API returning nothing must not be read as "infra failure" — absence of -# evidence is not evidence of an incident. -if run_sweep failure '' 1; then - [ "$(reruns)" -eq 0 ] \ - && ok 'does not re-run a real failure even when the jobs API says nothing' \ - || no 'does not re-run a real failure even when the jobs API says nothing' -fi - -# 5. Bounded. An endlessly-failing run must not become an infinite re-run loop -# billing Actions minutes forever. -if run_sweep cancelled '' 3; then - if [ "$(reruns)" -eq 0 ]; then - ok 'stops retrying once the run hits the attempt cap' - else - no 'stops retrying once the run hits the attempt cap' - fi -fi - -# 6. The happy path still reaches the PR loop — a guard that never lets anything -# through is just an outage with better manners. -if run_sweep success '' 1; then - case "$SWEEP_OUT" in - *"no open PRs"*) ok 'proceeds to the PR loop when the base is green' ;; - *) no "proceeds to the PR loop when the base is green (got: $(printf '%s' "$SWEEP_OUT" | tail -1))" ;; - esac -fi - -echo "auto-merge sweep — deploy reconciler" - -deploys() { grep -c '^workflow run deploy.yml' "$GH_LOG" 2>/dev/null; } - -# 7. OFF BY DEFAULT. A repo that sets no DEPLOY_WORKFLOW must behave exactly as -# before — adopting a reconciler must never start firing deploys in repos -# that never asked for one. -if run_sweep success '' 1 '' '' 0; then - [ "$(deploys)" -eq 0 ] \ - && ok 'is inert when DEPLOY_WORKFLOW is unset' \ - || no 'is inert when DEPLOY_WORKFLOW is unset' -fi - -# 8. Drifted: green tip, last successful deploy is an older sha → ship. -if run_sweep success '' 1 deploy.yml oldsha00 0; then - [ "$(deploys)" -ge 1 ] \ - && ok 'ships a green tip that is not yet deployed' \ - || no 'ships a green tip that is not yet deployed' -fi - -# 9. Already live → do nothing. Re-dispatching every sweep would deploy the same -# commit forever, every ten minutes. -if run_sweep success '' 1 deploy.yml basesha000000 0; then - [ "$(deploys)" -eq 0 ] \ - && ok 'does not re-deploy a tip that is already live' \ - || no 'does not re-deploy a tip that is already live' -fi - -# 10. A deploy already in flight → do not stack another on top of it. -if run_sweep success '' 1 deploy.yml oldsha00 2; then - [ "$(deploys)" -eq 0 ] \ - && ok 'does not dispatch while a deploy is already in flight' \ - || no 'does not dispatch while a deploy is already in flight' -fi - -# 11. THE ONE THAT MATTERS. This script does not exit on a red base — it lets a -# PR that repairs the base through — so the reconciler must refuse to ship a -# failing tip. The variant this code came from exited on red, which made the -# guard invisible; the same lines here without it would deploy red. -if run_sweep failure 'Run tests' 1 deploy.yml oldsha00 0; then - [ "$(deploys)" -eq 0 ] \ - && ok 'NEVER ships a red base, even though the sweep continues past one' \ - || no 'NEVER ships a red base, even though the sweep continues past one' -fi - -echo "auto-merge sweep — coverage ported from orangecat" -# orangecat was the other repo with sweep tests, and converting it to the -# canonical deletes them. These are the cases its suite had that this one did -# not — moved here BEFORE the deletion, so no assertion is ever lost between -# the two commits. - -merges() { grep -c '^pr merge' "$GH_LOG" 2>/dev/null; } - -# 12. A base run still in progress is not a verdict either way — defer, and do -# NOT re-run it (re-running an in-flight run would cancel it). -if RS_STATUS=in_progress run_sweep '' '' 1; then - if printf '%s' "$SWEEP_OUT" | grep -q 'still running' && [ "$(reruns)" -eq 0 ]; then - ok 'defers while the base run is still going, without re-running it' - else - no 'defers while the base run is still going, without re-running it' - fi -fi - -# 13. The newest base CI run belonging to an OLDER commit means the current tip -# is unjudged. Merging on that green would batch unverified commits — the -# exact thing one-car-per-sweep exists to prevent. -if RS_HEADSHA=oldsha000 run_sweep success '' 1; then - printf '%s' "$SWEEP_OUT" | grep -q 'waiting for CI to catch up' \ - && ok 'waits when the newest base run belongs to an older commit' \ - || no 'waits when the newest base run belongs to an older commit' -fi - -# A PR fixture generator for the red-base carve-out. The rollup names decide -# everything: the base fails 'Some Red Job', and whether this PR proves that -# job green is the whole question. -pr_fixture() { # - local checks="" name - local IFS=',' - for name in $1; do - checks="${checks:+$checks,}{\"status\":\"COMPLETED\",\"conclusion\":\"SUCCESS\",\"name\":\"$name\"}" - done - printf '[{"number":7,"title":"the fix","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"UNSTABLE","labels":[],"createdAt":"2026-01-01T00:00:00Z","statusCheckRollup":[%s]}]' "$checks" -} - -# 14. THE CARVE-OUT ITSELF: a PR green on every job the base fails may merge -# onto the red base. Its checks ran on the MERGE result, so green there is -# direct evidence the post-merge base is better than the pre-merge base — -# and without this, the fix is trapped behind the very redness it repairs. -if RS_PRS="$(pr_fixture 'Some Red Job,lint')" run_sweep failure 'Run tests' 1; then - [ "$(merges)" -ge 1 ] \ - && ok 'merges a PR that is green on every job the red base fails' \ - || no 'merges a PR that is green on every job the red base fails' -fi - -# 15. A PR that never RAN the failing job proves nothing about it. Letting it -# through would merge unrelated work onto a broken base — the failure mode -# the guard exists for. -if RS_PRS="$(pr_fixture 'lint,typecheck')" run_sweep failure 'Run tests' 1; then - [ "$(merges)" -eq 0 ] \ - && ok 'refuses a PR that does not run the failing job at all' \ - || no 'refuses a PR that does not run the failing job at all' -fi - -# 16. Covering SOME failing jobs is covering none: the uncovered one still -# lands broken. -if RS_REDJOBS='Some Red Job\nOther Red Job' \ - RS_PRS="$(pr_fixture 'Some Red Job,lint')" run_sweep failure 'Run tests' 1; then - [ "$(merges)" -eq 0 ] \ - && ok 'refuses a PR that covers only SOME of the failing jobs' \ - || no 'refuses a PR that covers only SOME of the failing jobs' -fi - -# 17. A green base merges a green PR without ever asking which jobs failed — -# the carve-out must be invisible on the happy path. -if RS_PRS="$(pr_fixture 'lint')" run_sweep success '' 1; then - [ "$(merges)" -ge 1 ] \ - && ok 'a green base merges normally, never consulting the carve-out' \ - || no 'a green base merges normally, never consulting the carve-out' -fi - -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[ "$FAIL" -eq 0 ] diff --git a/scripts/ci/test-model-pin-audit.mjs b/scripts/ci/test-model-pin-audit.mjs deleted file mode 100755 index 282ee27..0000000 --- a/scripts/ci/test-model-pin-audit.mjs +++ /dev/null @@ -1,739 +0,0 @@ -#!/usr/bin/env node -/** - * Self-test for model-pin-audit.mjs. - * Run: node scripts/ci/test-model-pin-audit.mjs - * - * A detector that has quietly stopped detecting reports a clean fleet, and a - * clean report from a broken detector is worse than no report — it is an absent - * check that prints a ✓. So every fixture asserts a VERDICT, and both sides are - * pinned: - * - * - the real AOZ regression is still caught (positive) - * - the corrected file stays silent (negative) - * - quoted things that are not model ids stay out of the report - * - an unreadable catalogue reports UNCHECKED, never GONE - * - * That last one is the expensive mistake. Treating "I could not look" as - * "nothing is there" marks every pin retired and invents a fleet-wide outage - * that somebody will act on. Silence is recoverable; a false alarm at this - * scale is not. - * - * Fixtures are inline strings — no network, no gh, no checkout, no key. - */ -import { - extractPins, - attribute, - collate, - judge, - looksLikeModelId, - possibleAt, - isLikelyPath, - isPossiblePath, - isCandidatePath, - segmentNamesAI, - modelListRegions, -} from "./model-pin-audit.mjs"; - -let failures = 0; -function check(name, actual, expected) { - const a = JSON.stringify(actual); - const e = JSON.stringify(expected); - if (a === e) { - console.log(` ok ${name}`); - } else { - console.log(` FAIL ${name}\n expected ${e}\n actual ${a}`); - failures++; - } -} - -// ── Fixtures ───────────────────────────────────────────────────────────────── - -/** aoz-housing/src/lib/env.ts as it stood when both AI surfaces went down. */ -const AOZ_BROKEN = ` -import { z } from 'zod' - -const schema = z.object({ - DATABASE_URL: z.string().url(), - CRON_SECRET: z.string().min(16).optional(), - - // AI: whichever key is set decides the provider. - GROQ_API_KEY: z.string().optional(), - GROQ_MODEL: z.string().default('llama-3.3-70b-versatile'), - OPENROUTER_API_KEY: z.string().optional(), - OPENROUTER_MODEL: z.string().default('openai/gpt-oss-20b:free'), -}) -`; - -/** The same file after the fix. Must produce no findings at all. */ -const AOZ_FIXED = AOZ_BROKEN.replace("llama-3.3-70b-versatile", "openai/gpt-oss-120b"); - -/** aoz-housing/src/lib/ai/provider.ts — one module, BOTH vendors named. */ -const TWO_VENDOR_FILE = ` -const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions' -const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1/chat/completions' - -export async function getConfig() { - if (provider === 'groq') { - return { url: GROQ_API_URL, model: 'llama-3.1-8b-instant' } - } - return { url: OPENROUTER_API_URL, model: 'meta-llama/llama-3.3-70b-instruct:free' } -} -`; - -/** Quoted strings that must never be mistaken for model ids. */ -const NOISE = ` -import { BRAND } from '@/lib/config/brand' -const path = './lib/ai/provider.ts' -const url = 'https://api.groq.com/openai/v1/chat/completions' -const key = 'GROQ_API_KEY' -const model = 'llama-3.1-8b-instant' -const style = 'rounded-lg' -`; - -// ── looksLikeModelId ───────────────────────────────────────────────────────── - -console.log("looksLikeModelId"); -check("accepts a versioned id", looksLikeModelId("llama-3.3-70b-versatile"), true); -check("accepts a routed id", looksLikeModelId("openai/gpt-oss-20b:free"), true); -check("rejects a relative path", looksLikeModelId("./lib/ai/provider.ts"), false); -check("rejects a url", looksLikeModelId("https://api.groq.com/openai/v1"), false); -check("rejects an env var name", looksLikeModelId("GROQ_API_KEY"), false); -check("rejects a css class", looksLikeModelId("rounded-lg"), false); -check("rejects a module specifier", looksLikeModelId("@/lib/config/brand"), false); -// fleetcrown builds its model id from ai-ration's chain at call time. A -// computed id is the ABSENCE of a pin, and the first run reported it retired. -check("rejects an interpolated id", looksLikeModelId("${link.provider.id}/${link.model}"), false); - -// ── extraction ─────────────────────────────────────────────────────────────── - -console.log("\nextractPins"); -check( - "finds both pins in the broken AOZ env schema", - extractPins(AOZ_BROKEN).map((p) => p.id).sort(), - ["llama-3.3-70b-versatile", "openai/gpt-oss-20b:free"], -); -check( - "finds the model literals in a two-vendor provider", - extractPins(TWO_VENDOR_FILE).map((p) => p.id).sort(), - ["llama-3.1-8b-instant", "meta-llama/llama-3.3-70b-instruct:free"], -); -check( - "picks the model id out of a file full of other quoted strings", - extractPins(NOISE).map((p) => p.id), - ["llama-3.1-8b-instant"], -); - -// ── attribution ────────────────────────────────────────────────────────────── - -console.log("\nattribute"); -{ - const pins = extractPins(AOZ_BROKEN); - const groq = pins.find((p) => p.id === "llama-3.3-70b-versatile"); - const or = pins.find((p) => p.id === "openai/gpt-oss-20b:free"); - check("GROQ_MODEL line attributes to groq", attribute(AOZ_BROKEN, groq.line), "groq"); - check("OPENROUTER_MODEL line attributes to openrouter", attribute(AOZ_BROKEN, or.line), "openrouter"); -} -{ - // The case a file-wide vote gets wrong: both vendors named in one module, so - // only the pin's own neighbourhood says which branch it belongs to. - const pins = extractPins(TWO_VENDOR_FILE); - const g = pins.find((p) => p.id === "llama-3.1-8b-instant"); - const o = pins.find((p) => p.id === "meta-llama/llama-3.3-70b-instruct:free"); - check("nearest marker wins for the groq branch", attribute(TWO_VENDOR_FILE, g.line), "groq"); - check("nearest marker wins for the openrouter branch", attribute(TWO_VENDOR_FILE, o.line), "openrouter"); -} -check( - "a pin with no vendor anywhere is not attributed", - attribute("const model = 'some-model-9b'", 1), - null, -); - -// ── judging ────────────────────────────────────────────────────────────────── - -console.log("\njudge"); -const findingsFrom = (repo, path, text) => - extractPins(text).map((p) => ({ - repo, - path, - line: p.line, - id: p.id, - vendor: attribute(text, p.line), - })); - -const GROQ_LIVE = new Set(["openai/gpt-oss-120b", "openai/gpt-oss-20b", "qwen/qwen3.8-27b"]); -const OR_LIVE = new Set(["openai/gpt-oss-20b:free"]); - -{ - const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); - const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); - const judged = judge(findings, live); - - check( - "THE REGRESSION: the retired llama pin is caught", - judged.filter((j) => j.state === "gone").map((j) => j.id), - ["llama-3.3-70b-versatile"], - ); - check( - "the still-served openrouter pin beside it stays quiet", - judged.filter((j) => j.state === "ok").map((j) => j.id), - ["openai/gpt-oss-20b:free"], - ); -} - -{ - // The negative that matters most: correct code must produce a silent report, - // or the audit trains people to ignore it. - const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_FIXED); - const live = new Map([["groq", GROQ_LIVE], ["openrouter", OR_LIVE]]); - const judged = judge(findings, live); - check("the FIXED file reports nothing retired", judged.filter((j) => j.state === "gone"), []); - check("and confirms both pins live", judged.filter((j) => j.state === "ok").length, 2); -} - -{ - // An unreadable catalogue must never read as rot. This is the guard against - // a missing key printing a fleet-wide outage that somebody then acts on. - const findings = findingsFrom("aoz-housing", "src/lib/env.ts", AOZ_BROKEN); - const live = new Map([["groq", null], ["openrouter", null]]); - const judged = judge(findings, live); - check("no key => UNCHECKED, never GONE", judged.filter((j) => j.state === "gone"), []); - check("and every pin is reported unchecked", judged.filter((j) => j.state === "unchecked").length, 2); -} - -{ - const findings = [{ repo: "x", path: "y", line: 1, id: "some-model-9b", vendor: null }]; - const judged = judge(findings, new Map()); - check("an unattributed pin is listed, not judged", judged[0].state, "unattributed"); -} - -// ── collate ────────────────────────────────────────────────────────────────── - -console.log("\ncollate"); -{ - const findings = [ - ...findingsFrom("a", "env.ts", AOZ_BROKEN), - { repo: "b", path: "c.ts", line: 1, id: "mystery-1b", vendor: null }, - ]; - const { byVendor, unattributed } = collate(findings); - check("groups by vendor", [...byVendor.keys()].sort(), ["groq", "openrouter"]); - check("keeps the unattributable aside", unattributed.map((u) => u.id), ["mystery-1b"]); -} - -// ── vendor confusion ───────────────────────────────────────────────────────── -// -// Found by running the audit for the first time, so it is pinned here. -// -// kivvi/apps/web/lib/ai/call-provider.ts dispatches four vendors in one -// function. `grok-3-mini` is xAI's model; Groq is a different company whose -// name differs by one letter, and whose base URL sat 18 lines above the pin -// while xAI's sat 6 above. Ranking by raw proximity called a live xAI model a -// retired Groq one — and the Groq pin six lines up tied EXACTLY between the two -// vendors, so a tie-break by proximity alone lost a true positive as well. -// -// Both halves are asserted: the xAI id must not be judged against Groq's -// catalogue, and the real Groq pin must still be caught. -const KIVVI_MULTI_VENDOR = ` -export async function callProvider(provider, apiKey, systemPrompt, userText, maxTokens) { - let url; - let headers; - let body; - - if (provider === "groq") { - url = "https://api.groq.com/openai/v1/chat/completions"; - headers = { Authorization: \`Bearer \${apiKey}\` }; - body = { - model: "llama-3.1-8b-instant", - messages: openaiMessages, - }; - } else if (provider === "xai") { - url = "https://api.x.ai/v1/chat/completions"; - headers = { Authorization: \`Bearer \${apiKey}\` }; - body = { - model: "grok-3-mini", - messages: openaiMessages, - }; - } -} -`; - -console.log("\nvendor confusion (regression)"); -{ - const pins = extractPins(KIVVI_MULTI_VENDOR); - const groqPin = pins.find((p) => p.id === "llama-3.1-8b-instant"); - const xaiPin = pins.find((p) => p.id === "grok-3-mini"); - - check("both vendor pins are extracted", Boolean(groqPin && xaiPin), true); - check("xAI's grok-3-mini is NOT attributed to groq", attribute(KIVVI_MULTI_VENDOR, xaiPin.line), "xai"); - check("the groq pin above it still attributes to groq", attribute(KIVVI_MULTI_VENDOR, groqPin.line), "groq"); - - // Only groq is queryable, so the xAI id must land in unchecked — never judged - // against a catalogue that was never going to list it. - const findings = pins.map((p) => ({ - repo: "kivvi", - path: "apps/web/lib/ai/call-provider.ts", - line: p.line, - id: p.id, - vendor: attribute(KIVVI_MULTI_VENDOR, p.line), - })); - const judged = judge(findings, new Map([["groq", GROQ_LIVE]])); - check("the real groq pin is still caught as retired", judged.filter((j) => j.state === "gone").map((j) => j.id), ["llama-3.1-8b-instant"]); - check("the xAI pin is unchecked, not retired", judged.filter((j) => j.state === "unchecked").map((j) => j.id), ["grok-3-mini"]); -} - -// ── a display label is not a pin ──────────────────────────────────── -// -// Regression: the audit announced a RETIRED model in OrangeCat, a repo with -// nothing wrong. The string was marketing copy in a pricing table — every other -// entry contained a space and was filtered out, `GPT-4o` did not. Reporting a -// healthy repo as broken is how a daily gate loses its reader. -// -// Both halves are asserted. Suppressing the false one is only worth doing if -// the real retired pin in the very same file is still caught. - -console.log("\ndisplay labels vs real pins"); - -const ORANGECAT_UI = ` -import { OPENROUTER_API_URL } from './constants' - -export const TIERS = { - standard: { - title: 'Standard Tier', - models: ['Claude 3.5 Sonnet', 'GPT-4o', 'Gemini 2.0 Flash'], - }, -} - -// the actual call, further down the same file -const model = 'openai/gpt-oss-20b:free' -`; - -{ - check("an uppercase id is impossible at openrouter", possibleAt("openrouter", "GPT-4o"), false); - check("and impossible at groq", possibleAt("groq", "Llama-3.3-70B"), false); - check("but lowercase ids stay possible", possibleAt("openrouter", "openai/gpt-oss-20b:free"), true); - // Uppercase is only impossible where the catalogue was measured. Together's - // real ids DO carry capitals, so the rule must not spread to every vendor. - check("uppercase is not ruled out at vendors we did not measure", possibleAt("together", "meta-llama/Llama-3.3-70B-Instruct-Turbo-Free"), true); - - const pins = extractPins(ORANGECAT_UI); - const findings = pins.map((p) => ({ - repo: "orangecat", - path: "src/lib/ai-guidance.ts", - line: p.line, - id: p.id, - vendor: attribute(ORANGECAT_UI, p.line), - })); - const judged = judge(findings, new Map([["openrouter", OR_LIVE]])); - - check( - "the display label is not reported as retired", - judged.filter((j) => j.state === "gone").map((j) => j.id).includes("GPT-4o"), - false, - ); - check( - "it is listed as unjudged rather than dropped", - judged.filter((j) => j.state === "unattributed").map((j) => j.id).includes("GPT-4o"), - true, - ); - // OR_LIVE contains openai/gpt-oss-20b:free, so the real pin here is live — - // what matters is that it was judged at all, not silenced alongside the label. - check( - "the real pin in the same file is still judged against the catalogue", - judged.find((j) => j.id === "openai/gpt-oss-20b:free")?.state, - "ok", - ); -} - -// ── which files get opened ─────────────────────────────────────────── -// -// Regression, and the expensive kind: a MISS, not a false alarm. -// -// The filter used to require the AI word in the FILENAME. Kivvi keeps its -// provider clients side by side in packages/ai/src/providers/, and the audit -// opened `anthropic.ts` while never opening `groq.ts` next to it — decided -// entirely by which vendor names happened to be in a regex. Three retired ids -// sat in the unopened files, so the report said Kivvi had 2 dead pins when it -// had 6. -// -// Understating a repo is worse than skipping it. A skipped repo is absent; an -// understated one prints a number that reads like an answer. - -console.log("\nwhich files get opened"); - -{ - // The four files from the miss, verbatim. - check("a vendor-named file under an ai/ directory is likely", isLikelyPath("packages/ai/src/providers/groq.ts"), true); - check("...and so is its openrouter sibling", isLikelyPath("packages/ai/src/providers/openrouter.ts"), true); - check("...and the index.ts beside them", isLikelyPath("packages/ai/src/providers/index.ts"), true); - check("...and the one that already worked still works", isLikelyPath("packages/ai/src/providers/anthropic.ts"), true); - - check("the app-side caller stays likely", isLikelyPath("apps/web/lib/ai/call-provider.ts"), true); - check("env schemas are always opened", isLikelyPath("src/lib/env.ts"), true); - check("so are .env.example files", isLikelyPath(".env.example"), true); - - // Botsmann's pin lived here, in a path naming no vendor and no AI concern. - check("a bare constants file is not likely", isLikelyPath("lib/constants.ts"), false); - check("...but is still a candidate, via the possible tier", isPossiblePath("lib/constants.ts"), true); - check("...so it does get opened", isCandidatePath("lib/constants.ts"), true); -} - -// Matching is by token, never substring — this is the half that keeps the fix -// from turning one blind spot into a flood. `ai` is a substring of all of these, -// and the cap would then drop real candidates to make room for them. -{ - check("mail is not ai", segmentNamesAI("mail.ts"), false); - check("domain is not ai", segmentNamesAI("domain.ts"), false); - check("detail is not ai", segmentNamesAI("detail.ts"), false); - check("maintenance is not ai", segmentNamesAI("maintenance.ts"), false); - check("captain is not ai", segmentNamesAI("captain.ts"), false); - - check("but ai is ai", segmentNamesAI("ai"), true); - check("and ai-guidance is ai", segmentNamesAI("ai-guidance.ts"), true); - check("and call-provider is a provider", segmentNamesAI("call-provider.ts"), true); - check("and llm-client is an llm", segmentNamesAI("llm-client.ts"), true); - - check("an unrelated util is not opened", isCandidatePath("src/lib/format-date.ts"), false); - check("nor is a mailer", isCandidatePath("src/lib/mail.ts"), false); -} - -// ── model arrays the old regex could not read ───────────────────────────── -// -// Second half of the Kivvi miss. Even once the file was being opened, nothing -// came out of it: the extractor matched `models` followed directly by `[`, and -// Kivvi writes a TypeScript type annotation in between. -// -// models: AIModel[] = [ -// -// The first `[` on that line belongs to `AIModel[]` and closes immediately, so -// anchoring to it reads an empty array — a silent nothing, which is the worst -// possible output for an audit. Two retired Groq ids sat inside. - -console.log("\nmodel arrays"); - -const KIVVI_TYPED_ARRAY = ` -import type { AIModel } from "../types"; - -/** Uses OpenAI-compatible API at https://api.groq.com/openai/v1. */ -export class GroqProvider extends OpenAICompatibleProvider { - id = "groq"; - name = "Groq"; - - models: AIModel[] = [ - { - id: "llama-3.3-70b-versatile", - name: "Llama 3.3 70B", - contextWindow: 128000, - supportsTools: true, - costPer1kInput: 0, - }, - { - id: "llama-3.1-8b-instant", - name: "Llama 3.1 8B Instant", - contextWindow: 128000, - supportsTools: true, - costPer1kInput: 0, - }, - ]; - - protected baseUrl = "https://api.groq.com/openai/v1"; -} -`; - -{ - const pins = extractPins(KIVVI_TYPED_ARRAY); - const ids = pins.map((p) => p.id).sort(); - check( - "a type-annotated models array is read at all", - ids, - ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], - ); - - // The SECOND entry is the one a fixed-width window loses. The old pattern - // read at most 400 characters after `models`, and a richly described list - // runs past that long before it ends. - check("the second entry is not lost to a character budget", ids.length, 2); - - // Human-readable names sit in the same objects and must not be mistaken for - // ids — they are filtered by shape, not by position. - check("display names in the same object are not pins", ids.includes("Llama 3.3 70B"), false); - - // Each id reports its OWN line, not the line the array opened on. Vendor - // attribution is measured in lines from the pin, so a whole array collapsed - // onto one line would attribute every entry from the same neighbourhood. - const lines = pins.map((p) => p.line); - check("each id carries its own line number", new Set(lines).size, 2); - - for (const pin of pins) { - check(`${pin.id} attributes to groq`, attribute(KIVVI_TYPED_ARRAY, pin.line), "groq"); - } - - const regions = modelListRegions(KIVVI_TYPED_ARRAY); - check("exactly one models array is found in the file", regions.length, 1); -} - -{ - // The plain shapes must keep working. - check("a one-line string form still reads", extractPins(`const model = 'openai/gpt-oss-120b'`).map((p) => p.id), ["openai/gpt-oss-120b"]); - check( - "a plain inline array still reads", - extractPins(`models: ['openai/gpt-oss-120b', 'openai/gpt-oss-20b']`).map((p) => p.id).sort(), - ["openai/gpt-oss-120b", "openai/gpt-oss-20b"], - ); - check( - "an assignment without an annotation still reads", - extractPins(`const models = [\n "openai/gpt-oss-120b",\n]`).map((p) => p.id), - ["openai/gpt-oss-120b"], - ); -} - -// ── model MAPS, not just arrays ───────────────────────────────────── -// -// Third discovery in the same sweep, and it cost two more repos. -// -// Once arrays were readable, two repos still reported nothing, because their -// ids live in object literals — and on OPPOSITE sides of the colon. Hirnli -// keeps them as map values, Orangecat as map keys, including -// `DEFAULT_GROQ_MODEL`, the baseline every free non-BYOK user gets. Both were -// entirely retired and both read as clean. -// -// So the walker takes `{` as readily as `[`, and every quoted string in the -// region is a candidate regardless of which side of the colon it is on. - -console.log("\nmodel maps"); - -// Hirnli's shape: the id is the VALUE. -const HIRNLI_ALIAS_MAP = ` -const GROQ_API_URL = 'https://api.groq.com/openai/v1/chat/completions'; - -/** Available models with different rate limits */ -export const GROQ_MODELS = { - '70b': 'llama-3.3-70b-versatile', // 12k TPM, best quality - '8b': 'llama-3.1-8b-instant', // 20k TPM, faster, good for triage -} as const; -`; - -// Orangecat's shape: the id is the KEY, and the value is a nested object. -const ORANGECAT_REGISTRY = ` -// Groq's best free models -const GROQ_MODELS = { - // Fast, capable model - great for chat - 'llama-3.3-70b-versatile': { - name: 'Llama 3.3 70B Versatile', - contextWindow: 128000, - maxOutputTokens: 32768, - }, - 'llama-3.1-8b-instant': { - name: 'Llama 3.1 8B Instant', - contextWindow: 128000, - maxOutputTokens: 8192, - }, -} as const; - -const GROQ_API_URL = 'https://api.groq.com/openai/v1'; -`; - -{ - const ids = extractPins(HIRNLI_ALIAS_MAP).map((p) => p.id).sort(); - check( - "ids that are map VALUES are found", - ids, - ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], - ); - // '70b' and '8b' are the keys here. They are aliases, not ids, and must not - // be reported as pins — `looksLikeModelId` is what keeps them out. - check("the size aliases beside them are not mistaken for ids", ids.includes("8b"), false); -} - -{ - const pins = extractPins(ORANGECAT_REGISTRY); - const ids = pins.map((p) => p.id).sort(); - check( - "ids that are map KEYS are found", - ids, - ["llama-3.1-8b-instant", "llama-3.3-70b-versatile"], - ); - // Nested objects inside the map must not end the region early. - check("a nested object does not truncate the map", ids.length, 2); - // Human-readable names sit in the same nested objects. - check("display names inside the map are not pins", ids.includes("Llama 3.3 70B Versatile"), false); - - for (const pin of pins) { - check(`${pin.id} in a map attributes to groq`, attribute(ORANGECAT_REGISTRY, pin.line), "groq"); - } - - const judged = judge( - pins.map((p) => ({ repo: "orangecat", path: "src/services/ai/groq.ts", line: p.line, id: p.id, vendor: "groq" })), - new Map([["groq", GROQ_LIVE]]), - ); - check("and both are judged retired against the live catalogue", judged.filter((j) => j.state === "gone").length, 2); -} - -{ - // The walker must not mistake an ordinary import for a model collection. - check("an import naming models opens no region", modelListRegions(`import { getAllModels } from "./providers";`).length, 0); -} - -// A size alias is not a model id. -{ - console.log("\nseparators"); - check("a bare size token is not an id", looksLikeModelId("70b"), false); - check("nor is the two-character one", looksLikeModelId("8b"), false); - check("routed ids are ids", looksLikeModelId("openai/gpt-oss-120b"), true); - check("hyphenated ids are ids", looksLikeModelId("llama-3.3-70b-versatile"), true); - check("dotted ids are ids", looksLikeModelId("llama3.2"), true); - check("short hyphenated ids are ids", looksLikeModelId("codex-4"), true); -} - -// A loop over models is not a declaration of models. -// -// Regression from the map support: `{` as an opener made -// `for (const model of models) {` open a region across the whole loop body, so -// the request headers inside it were read as ids and a Content-Type header was -// reported as a retired Groq model. Widening a matcher is exactly when to check -// what it now swallows. -{ - console.log("\ndeclarations only"); - - const LOOP = ` - const models = groqModels(); - for (const model of models) { - const response = await fetch(API_CONFIG.GROQ_API_URL, { - headers: { - Authorization: \`Bearer \${key}\`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ model, messages }), - }); - } - `; - - check("a for-of over models opens no region", modelListRegions(LOOP).length, 0); - check("so a content-type header is not a model", extractPins(LOOP).map((p) => p.id), []); - - // The declaration forms must still open one. - check("a const array declaration still opens", modelListRegions(`const models = [`).length, 1); - check("an annotated declaration still opens", modelListRegions(` models: AIModel[] = [`).length, 1); - check("a map declaration still opens", modelListRegions(`export const GROQ_MODELS = {`).length, 1); - check("an inline property still opens", modelListRegions(`models: ['a/b-1'],`).length, 1); -} - -// `modelId` is not `model`. -// -// Regression found by re-running the live sweep after tightening the collection -// walker to plural-only: a real retired id in Kivvi disappeared from the report. -// Its declaration is singular (`const FALLBACK_MODEL: ModelSelection = {`), so -// the walker correctly ignores it, and the single-id pattern was anchored on the -// bare word `model` — which does not match `modelId`. Nothing covered it. -{ - console.log("\nsingle-id property shapes"); - - const KIVVI_FALLBACK = ` -const STORAGE_KEY = "kivvi-selected-model"; - -// Fallback default — used before API loads or when stored model is unavailable -const FALLBACK_MODEL: ModelSelection = { - providerId: "groq", - modelId: "llama-3.3-70b-versatile", -}; -`; - - const ids = extractPins(KIVVI_FALLBACK).map((p) => p.id); - check("a modelId property is a pin", ids.includes("llama-3.3-70b-versatile"), true); - check("the storage key beside it is not", ids.includes("kivvi-selected-model"), false); - - check("model_id also reads", extractPins(`model_id = "openai/gpt-oss-120b"`).map((p) => p.id), ["openai/gpt-oss-120b"]); - check("modelName also reads", extractPins(`modelName: 'openai/gpt-oss-20b'`).map((p) => p.id), ["openai/gpt-oss-20b"]); - - // A parameter annotation is still not a pin. - check("a typed parameter is not a pin", extractPins(`function f(model: string): boolean { return model.startsWith("qwen/"); }`).map((p) => p.id), []); -} - -// A provider-keyed record names its own rows. -// -// OrangeCat's `gpt-4o-mini` was reported as a RETIRED OPENROUTER model. It is -// not an OpenRouter id at all — there it would be `openai/gpt-4o-mini` — it is -// OpenAI's, sitting under an `openai:` key. The nearest marker above it was -// OpenRouter's base URL in the block before, so nearest-marker-above put it with -// the wrong vendor and then judged it against a catalogue that was never going -// to list it. -// -// The bare word `openai` cannot be a general marker: `api.groq.com/openai/v1` -// contains it, and every `openai/gpt-oss-*` id Groq serves. Anchoring to a line -// that STARTS with the key and a colon is what makes it safe. -{ - console.log("provider-keyed records"); - - const ORANGECAT_RUNTIME = ` -export const PROVIDER_BASE_URLS = { - openai: 'https://api.openai.com/v1', - groq: 'https://api.groq.com/openai/v1', - openrouter: 'https://openrouter.ai/api/v1', -} as const; - -export const PROVIDER_RUNTIME = { - openai: { - baseUrl: PROVIDER_BASE_URLS.openai, - defaultModel: 'gpt-4o-mini', - }, - openrouter: { - baseUrl: PROVIDER_BASE_URLS.openrouter, - defaultModel: 'nvidia/nemotron-3-super-120b-a12b:free', - }, -}; -`; - - const pins = extractPins(ORANGECAT_RUNTIME); - const at = (id) => pins.find((p) => p.id === id); - - check("both defaults are extracted", pins.length >= 2, true); - check( - "an id under an openai: key belongs to openai", - attribute(ORANGECAT_RUNTIME, at("gpt-4o-mini").line), - "openai", - ); - check( - "and the one under openrouter: belongs to openrouter", - attribute(ORANGECAT_RUNTIME, at("nvidia/nemotron-3-super-120b-a12b:free").line), - "openrouter", - ); - - // OpenAI is not queryable, so its pin must be reported UNCHECKED — never - // judged against a catalogue that could not list it. - const judged = judge( - pins.map((p) => ({ repo: "orangecat", path: "src/config/ai-provider-runtime.ts", line: p.line, id: p.id, vendor: attribute(ORANGECAT_RUNTIME, p.line) })), - new Map([["openrouter", new Set(["nvidia/nemotron-3-super-120b-a12b:free"])]]), - ); - check("the openai pin is unchecked, not retired", judged.find((j) => j.id === "gpt-4o-mini")?.state, "unchecked"); - check("the openrouter pin is confirmed live", judged.find((j) => j.id === "nvidia/nemotron-3-super-120b-a12b:free")?.state, "ok"); -} - -// Ollama is local: its tags are attributed, never judged. -{ - console.log("\nlocal providers"); - - const EVIG_ENV = ` -# Groq (cloud) -GROQ_API_KEY= -GROQ_MODEL=openai/gpt-oss-120b - -# Ollama URL (local LLM - for local embeddings only) -OLLAMA_URL=http://localhost:11434 -OLLAMA_MODEL=llama3.2 -`; - - const pins = extractPins(EVIG_ENV); - const ollamaPin = pins.find((p) => p.id === "llama3.2"); - check("the ollama tag is extracted", Boolean(ollamaPin), true); - // Before this, the nearest marker above was GROQ_MODEL and `llama3.2` was - // reported as a retired GROQ model. It is a valid Ollama tag on a healthy - // line; Groq simply never served anything by that name. - check("and attributed to ollama, not groq", attribute(EVIG_ENV, ollamaPin.line), "ollama"); - - const judged = judge( - [{ repo: "evig", path: ".env.example", line: ollamaPin.line, id: "llama3.2", vendor: "ollama" }], - new Map([["groq", GROQ_LIVE]]), - ); - check("a local tag is unchecked, never retired", judged[0].state, "unchecked"); -} - -console.log(failures ? `\n✗ ${failures} failure(s)` : "\n✓ all checks pass"); -process.exit(failures ? 1 : 0); diff --git a/scripts/ci/test-shared-inventory.sh b/scripts/ci/test-shared-inventory.sh deleted file mode 100755 index 3646df2..0000000 --- a/scripts/ci/test-shared-inventory.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# -# Negative tests for the duplication ratchet. -# -# The ratchet's whole value is that it can go RED. If it silently passes when -# duplication rises, it is worse than not having it — it certifies the thing it -# was built to stop, which is exactly how `continue-on-error` let evig ship 25 -# failing tests under a green tick for three weeks. -# -# The counting half needs the GitHub API. The DECIDING half — compare current -# against baseline, and which way the ratchet turns — is pure text, so it is -# tested here against fixture files with no network at all. A rule only -# exercisable by a live API call is a rule nobody re-tests after editing it. - -set -uo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT - -PASS=0 -FAIL=0 -ok() { printf ' ✓ %s\n' "$1"; PASS=$((PASS + 1)); } -no() { printf ' ✗ %s\n' "$1"; FAIL=$((FAIL + 1)); } - -# The ratchet comparison, extracted verbatim in behaviour from -# shared-inventory.sh so the fixtures exercise the real decision. -# (baseline, current) -> prints findings, returns 1 if duplication rose. -ratchet() { - local baseline="$1" current="$2" fail=0 - while IFS=$'\t' read -r concern _ was_copies; do - [ -n "$concern" ] || continue - local now_line now_copies - now_line=$(grep "^${concern} " "$current" 2>/dev/null) - [ -n "$now_line" ] || continue - now_copies=$(printf '%s' "$now_line" | cut -f3) - [ "$now_copies" -gt "$was_copies" ] && { echo "ROSE $concern $was_copies->$now_copies"; fail=1; } - done < "$baseline" - while IFS=$'\t' read -r concern _ now_copies; do - [ -n "$concern" ] || continue - cut -f1 "$baseline" | grep -qx "$concern" || { echo "NEW $concern $now_copies"; fail=1; } - done < "$current" - return $fail -} - -printf 'ratchet\n' - -printf 'rate-limit\t9\t14\nlogger\t6\t10\n' > "$TMP/base" - -# ── it must go red ─────────────────────────────────────────────────────────── -printf 'rate-limit\t9\t15\nlogger\t6\t10\n' > "$TMP/up" -if ratchet "$TMP/base" "$TMP/up" >/dev/null; then - no 'a RISE in duplication is caught' -else - ok 'a RISE in duplication is caught' -fi - -printf 'rate-limit\t9\t14\nlogger\t6\t10\nnew-thing\t3\t3\n' > "$TMP/new" -if ratchet "$TMP/base" "$TMP/new" >/dev/null; then - no 'a NEW unmeasured concern is caught' -else - ok 'a NEW unmeasured concern is caught' -fi - -# ── and it must stay quiet when it should ──────────────────────────────────── -if ratchet "$TMP/base" "$TMP/base" >/dev/null; then - ok 'an unchanged fleet passes' -else - no 'an unchanged fleet passes' -fi - -printf 'rate-limit\t9\t9\nlogger\t6\t10\n' > "$TMP/down" -if ratchet "$TMP/base" "$TMP/down" >/dev/null; then - ok 'a DECREASE passes (progress is not a failure)' -else - no 'a DECREASE passes (progress is not a failure)' -fi - -# A concern dropped from the script must not be read as "rose to zero". -printf 'rate-limit\t9\t14\n' > "$TMP/dropped" -if ratchet "$TMP/base" "$TMP/dropped" >/dev/null; then - ok 'a concern removed from the script does not fail the ratchet' -else - no 'a concern removed from the script does not fail the ratchet' -fi - -# ── the baseline in the repo must match the script's concerns ──────────────── -printf 'baseline integrity\n' -BASE="$HERE/shared-inventory.baseline" -if [ -f "$BASE" ]; then - bad=$(awk -F'\t' 'NF!=3 {print NR": "$0}' "$BASE") - if [ -z "$bad" ]; then ok 'baseline rows are namereposfiles'; else no "malformed baseline rows: $bad"; fi - - missing="" - while IFS=$'\t' read -r concern _ _; do - [ -n "$concern" ] || continue - grep -q "^${concern}|" "$HERE/shared-inventory.sh" || missing="$missing $concern" - done < "$BASE" - if [ -z "$missing" ]; then - ok 'every baseline concern still exists in the script' - else - no "baseline names concerns the script no longer measures:$missing" - fi -else - no "no baseline committed at $BASE" -fi - -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[ "$FAIL" -eq 0 ] diff --git a/scripts/ci/test-ui-defect-audit.mjs b/scripts/ci/test-ui-defect-audit.mjs deleted file mode 100644 index 723fd64..0000000 --- a/scripts/ci/test-ui-defect-audit.mjs +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env node -/** - * Self-test for ui-defect-audit.mjs. - * Run: node scripts/ci/test-ui-defect-audit.mjs - * - * A detector is only worth its output if BOTH sides are pinned. The first draft - * of this audit flagged 88 "defects" across 14 sites, and most of the alignment - * ones were a heading followed by consistently indented items — correct markup - * reported as broken. A rule that fires on everything is as useless as one that - * fires on nothing, and it is more expensive, because someone has to read it. - * - * So every fixture below asserts a VERDICT, not just a run: - * - the real fleetcrown bug is still caught (positive) - * - correct markup stays silent (negative) - * - * Fixtures are inline data: URLs — no network, no fleet, no auth. - */ -import { loadPlaywright, MEASURE } from "./ui-defect-audit.mjs"; - -const FIXTURES = { - // The original fleetcrown fleet card: an icon INLINE at the head of two of - // the four rows, shoving only those lines sideways by its own width, and a - // wrapped hint whose second line falls back to the container edge. - raggedStack: ` -
-
-

Fleet autopilot

-

Idle — nothing queued

-
0 working21 idle
-

Autopilot on — agents work through each project's queue, then pick the next-best task automatically and keep going until the queue is empty.

-
-
`, - - // The SAME visual family done correctly: icons hang in a fixed gutter, every - // row shares one text column, the wrapped line has a hanging indent. - fixedStack: ` -
-
-

Fleet autopilot

-

Idle — nothing queued

-
0 working21 idle
-

Autopilot on — agents work through each project's queue, then pick the next-best task automatically and keep going until the queue is empty.

-
-
`, - - // Correct markup that the first draft wrongly flagged: a label, then items - // deliberately indented under it. The column goes out once and STAYS — no - // return to a previous edge, so it is structure, not raggedness. - intentionalIndent: ` -
-
-

GUT · BRAIN AXIS

-

Vollständige Mikrobiomanalyse

-

Darmpermeabilitätsmarker

-

Neurotransmitter-Vorläuferstatus

-
-
`, - - // An action below the AA floor, and one comfortably above it. - contrast: ` - `, - - // Chips on a 4%-white overlay over near-black. A naive single canvas paint - // resolves that overlay to SOLID WHITE and reports these as failures. - translucentChips: ` -
- - -
`, -// A nested GROUP between peer lines. datacat renders exactly this: a card - // title, a description, an icon checklist, then a CTA back at the card edge. - // The checklist is indented because it is a group, not because anything is - // broken — and the CTA returning to the base indent is what made the first - // draft call it ragged. - nestedGroup: ` -
-
-

Formular-Erfassung

-

Erstellen Sie benutzerdefinierte Formulare für Umfragen.

-
-
KI-gestützte Sentiment-Analyse
-
Automatische Kategorisierung
-
Trend-Erkennung
-
- Formular erstellen → -
-
`, - - // Centered copy. Every line starts at a different x BY DESIGN. - centeredCopy: ` -
-

Every feature — health tracking, digital twin, vet network, marketplace, and adoption listings — is included free with no pet limits.

-
`, - - // A bare wrapper anchor around a styled button. The anchor's own ink is - // never painted — the button paints the label — so measuring the anchor read - // 1:1 on surf-your-life's "Konto erstellen". The button itself passes. - wrapperAnchor: ` - `, - - // A disabled control is dim BECAUSE it is disabled — WCAG exempts it. Its - // enabled twin with the same colors is a real finding. - disabledControl: ` -
- - -
`, - - // A row of padded buttons in a text column: the BUTTON BOX is flush with the - // column and only the labels sit padding deeper — aoz-wohnen's hero read as - // ragged for exactly this. - buttonRow: ` -
-
-

Gemeinsam wohnen

-

Die Wohnung, auf die ihr euch einigen könnt.

- -

Kein Konto nötig.

-
-
`, - - // The same hero with the button row genuinely off the column: the BOX edge - // returns 16px out and back, and the box edge is what must be measured. - buttonRowRagged: ` -
-
-

Gemeinsam wohnen

-

Die Wohnung, auf die ihr euch einigen könnt.

- -

Kein Konto nötig.

-
-
`, -}; - -function assert(cond, msg) { - if (!cond) throw new Error(msg); -} - -async function main() { - const { chromium } = loadPlaywright(); - const browser = await chromium.launch(); - const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); - const page = await ctx.newPage(); - - const measure = async (html) => { - await page.setContent(`${html}`, { waitUntil: "load" }); - await page.waitForTimeout(120); - return page.evaluate(MEASURE); - }; - - let passed = 0; - const check = async (label, fn) => { - await fn(); - passed += 1; - console.log(` ✓ ${label}`); - }; - - await check("catches the ragged stack it was written for", async () => { - const r = await measure(FIXTURES.raggedStack); - assert(r.ragged.length >= 1, "the zigzag stack must be reported"); - assert( - r.ragged.some((g) => g.spread >= 4 && g.spread <= 24), - `spread must land in the accidental-indent band, got ${JSON.stringify(r.ragged.map((g) => g.spread))}`, - ); - }); - - await check("catches the wrapped line with no hanging indent", async () => { - const r = await measure(FIXTURES.raggedStack); - assert(r.wrapped.length >= 1, "line 2 starting left of line 1 must be reported"); - }); - - await check("stays silent once the gutter fix is applied", async () => { - const r = await measure(FIXTURES.fixedStack); - assert(r.ragged.length === 0, `fixed stack must be clean, got ${JSON.stringify(r.ragged)}`); - assert(r.wrapped.length === 0, `hanging indent must be clean, got ${JSON.stringify(r.wrapped)}`); - }); - - await check("does NOT flag a label with deliberately indented items", async () => { - const r = await measure(FIXTURES.intentionalIndent); - assert( - r.ragged.length === 0, - `structure must not read as raggedness, got ${JSON.stringify(r.ragged)}`, - ); - }); - - await check("reports a sub-AA action and spares a passing one", async () => { - const r = await measure(FIXTURES.contrast); - const dim = r.contrast.find((c) => c.href === "#dim"); - const ok = r.contrast.find((c) => c.href === "#ok"); - assert(dim && dim.value < dim.floor, `dim link must fail, got ${dim && dim.value}`); - assert(ok && ok.value >= ok.floor, `readable link must pass, got ${ok && ok.value}`); - }); - - await check("composites translucent chips instead of calling them white", async () => { - const r = await measure(FIXTURES.translucentChips); - const chips = r.contrast.filter((c) => /working|idle/.test(c.text)); - assert(chips.length === 2, `expected 2 chips, got ${chips.length}`); - for (const c of chips) { - assert( - c.value >= c.floor, - `light text on a 4%-white overlay over near-black is readable; got ${c.value}:1 ` + - `(a single canvas paint reports ~3.2 here)`, - ); - } - }); - - await check("does NOT flag a nested group between peer lines", async () => { - const r = await measure(FIXTURES.nestedGroup); - assert( - r.ragged.length === 0, - `an indented list between a description and a CTA is structure, got ${JSON.stringify(r.ragged)}`, - ); - }); - - await check("ignores a wrapper anchor whose label a nested button paints", async () => { - const r = await measure(FIXTURES.wrapperAnchor); - assert(!r.contrast.some((c) => c.tag === "a"), `the wrapper must be skipped, got ${JSON.stringify(r.contrast)}`); - const btn = r.contrast.find((c) => c.tag === "button"); - assert(btn && btn.value >= btn.floor, `the painted button passes on its own fill, got ${btn && btn.value}`); - }); - - await check("exempts a disabled control but reports its enabled twin", async () => { - const r = await measure(FIXTURES.disabledControl); - assert(!r.contrast.some((c) => c.text === "Send"), "the disabled button is exempt (WCAG inactive)"); - const twin = r.contrast.find((c) => c.text === "Send twin"); - assert(twin && twin.value < twin.floor, `the enabled twin is a real finding, got ${twin && twin.value}`); - }); - - await check("measures a padded button row by its box edge, not its label", async () => { - const r = await measure(FIXTURES.buttonRow); - assert(r.ragged.length === 0, `a flush button row is aligned, got ${JSON.stringify(r.ragged)}`); - }); - - await check("still catches a button row whose box is off the column", async () => { - const r = await measure(FIXTURES.buttonRowRagged); - assert(r.ragged.length >= 1, "a 16px box offset that returns must be reported"); - }); - - await check("does NOT flag centered copy as a missing hanging indent", async () => { - const r = await measure(FIXTURES.centeredCopy); - assert( - r.wrapped.length === 0, - `centered lines start at different x by design, got ${JSON.stringify(r.wrapped)}`, - ); - }); - - await browser.close(); - console.log(`\n${passed}/${passed} ui-defect-audit self-tests passed`); -} - -main().catch((e) => { - console.error(`\n✗ ${e.message}`); - process.exit(1); -}); diff --git a/scripts/ci/test-verify-predicates.sh b/scripts/ci/test-verify-predicates.sh deleted file mode 100755 index 66620d5..0000000 --- a/scripts/ci/test-verify-predicates.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env bash -# -# Negative tests for the verify-contract WIRING rules. -# -# The rules live in verify-predicates.sh precisely so they can be tested without -# reaching GitHub — the audit that uses them is remote-only, and a rule that can -# only be exercised by a live API call is a rule nobody re-tests after changing -# one of its regexes. -# -# Both directions are tested on purpose. Proving a rule BITES is half the job; -# proving it stays quiet on a conforming repo is the other half, and skipping it -# is how a checker starts crying wolf and gets ignored — the same end state as -# having no checker, reached more expensively. - -set -uo pipefail - -# shellcheck source=scripts/ci/verify-predicates.sh -. "$(cd "$(dirname "$0")" && pwd)/verify-predicates.sh" - -PASS=0 -FAIL=0 - -ok() { printf ' ✓ %s\n' "$1"; PASS=$((PASS + 1)); } -no() { printf ' ✗ %s\n' "$1"; FAIL=$((FAIL + 1)); } - -# assert -assert() { - local want="$1" pred="$2" input="$3" desc="$4" - if "$pred" "$input"; then got=pass; else got=fail; fi - [ "$got" = "$want" ] && ok "$desc" || no "$desc (expected $want, got $got)" -} - -CALLS='jobs: - verify: - steps: - - name: Verify - run: npm run verify' - -HANDCOPIED='jobs: - verify: - steps: - - name: Lint - run: npm run lint --if-present - - name: Test - run: npm run test --if-present' - -SOFTENED='jobs: - verify: - steps: - - name: Verify - run: npm run verify --if-present' - -BENIGN_IFPRESENT='jobs: - verify: - steps: - - name: Optional docs - run: npm run docs --if-present - - name: Verify - run: npm run verify' - -echo "ci_invokes_verify" -assert pass ci_invokes_verify "$CALLS" 'npm run verify is detected' -assert pass ci_invokes_verify 'run: pnpm verify' 'pnpm implicit-run is accepted (was a false positive once)' -assert pass ci_invokes_verify 'run: yarn verify' 'yarn implicit-run is accepted' -assert pass ci_invokes_verify 'run: bun run verify' 'bun is accepted' -assert fail ci_invokes_verify "$HANDCOPIED" 'hand-copied steps do NOT count as calling verify' -assert fail ci_invokes_verify 'run: npm run verify-deploy' 'a longer script name is not mistaken for verify' -assert fail ci_invokes_verify '' 'an empty workflow body is not a call' - -echo "ci_verify_softened" -assert pass ci_verify_softened "$SOFTENED" '--if-present ON the verify step is caught' -assert fail ci_verify_softened "$CALLS" 'a clean verify step is not flagged' -assert fail ci_verify_softened "$BENIGN_IFPRESENT" '--if-present on an UNRELATED step is not flagged' - -echo "verify_softens_itself" -assert pass verify_softens_itself 'eslint . || true' '|| true inside verify is caught' -assert pass verify_softens_itself 'npm run typecheck --workspaces --if-present' '--if-present inside verify is caught' -assert fail verify_softens_itself 'pnpm lint && pnpm typecheck && pnpm test' 'a clean verify is not flagged' -assert fail verify_softens_itself '' 'an empty verify is not flagged here — absence is the audit job' - -# assert2 -assert2() { - local want="$1" pred="$2" a="$3" b="$4" desc="$5" - if "$pred" "$a" "$b"; then got=pass; else got=fail; fi - [ "$got" = "$want" ] && ok "$desc" || no "$desc (expected $want, got $got)" -} - -VERIFY_NPM='npm run lint && npm run typecheck && npm run test' - -# aoz-housing's real shape: the same three scripts, split across parallel jobs -# for speed, none of them softened. This is the case that proved the -# string-match rule wrong. -AOZ_WF=' - name: Lint - run: npm run lint - - name: Type check - run: npm run typecheck - - name: Run unit tests - run: npm test -- --ci --coverage - - name: Build - run: npm run build' - -# Same split, but one gate quietly absent. -AOZ_MISSING_TYPECHECK=' - name: Lint - run: npm run lint - - name: Run unit tests - run: npm test -- --ci --coverage' - -echo "verify_gate_scripts" -got=$(verify_gate_scripts "$VERIFY_NPM" | tr '\n' ' ') -[ "$got" = "lint test typecheck " ] \ - && ok "decomposes verify into its named gates" \ - || no "decomposes verify into its named gates (got '$got')" -got=$(verify_gate_scripts 'eslint . && tsc --noEmit && jest') -[ -z "$got" ] \ - && ok "a verify that shells out directly decomposes to nothing" \ - || no "a verify that shells out directly decomposes to nothing (got '$got')" - -echo "ci_runs_verify_gates" -assert2 pass ci_runs_verify_gates "$AOZ_WF" "$VERIFY_NPM" \ - 'aoz-housing: gates run individually, unsoftened → satisfied' -assert2 fail ci_runs_verify_gates "$AOZ_MISSING_TYPECHECK" "$VERIFY_NPM" \ - 'a gate in verify that CI never runs → NOT satisfied' -assert2 fail ci_runs_verify_gates "$HANDCOPIED" 'npm run lint && npm run test' \ - 'botsmann: hand-copied AND --if-present on every step → still caught' -assert2 fail ci_runs_verify_gates "$AOZ_WF" 'eslint . && tsc --noEmit' \ - 'an undecomposable verify is unproven, not waved through' -assert2 fail ci_runs_verify_gates '' "$VERIFY_NPM" \ - 'no workflows at all → NOT satisfied' -assert2 pass ci_runs_verify_gates 'run: pnpm lint -run: pnpm typecheck -run: pnpm test' 'pnpm lint && pnpm typecheck && pnpm test' \ - 'pnpm implicit-run spelling works on both sides' - -# --- gh_get: the three states ------------------------------------------------ -# This is the test that would have caught the 2026-08-16 miscount. `gh` is -# stubbed so the FAILURE path is reachable without a real outage — which is -# exactly why the bug survived: nothing could exercise it. - -GH_GET_BACKOFF=0 # do not actually sleep through the retries in tests - -echo "gh_get" - -gh() { printf 'ok-body'; return 0; } -out=$(gh_get 'any/path'); rc=$? -[ "$rc" = 0 ] && [ "$out" = "ok-body" ] \ - && ok "success returns 0 and the body" \ - || no "success returns 0 and the body (rc=$rc out='$out')" - -gh() { echo 'gh: Not Found (HTTP 404)' >&2; return 1; } -gh_get 'missing/path' >/dev/null; rc=$? -[ "$rc" = 2 ] \ - && ok "a real 404 is ABSENT (2), distinct from a failure" \ - || no "a real 404 is ABSENT (2) (rc=$rc)" - -gh() { echo 'gh: HTTP 403 rate limit exceeded' >&2; return 1; } -gh_get 'blocked/path' >/dev/null; rc=$? -[ "$rc" = 1 ] \ - && ok "a 403 is COULD-NOT-LOOK (1), never mistaken for absence" \ - || no "a 403 is COULD-NOT-LOOK (1) (rc=$rc)" - -gh() { echo 'gh: HTTP 502 Bad Gateway' >&2; return 1; } -gh_get 'flaky/path' >/dev/null; rc=$? -[ "$rc" = 1 ] \ - && ok "a 5xx is COULD-NOT-LOOK (1), not absence" \ - || no "a 5xx is COULD-NOT-LOOK (1) (rc=$rc)" - -# Transient then success: the retry must actually rescue the call, otherwise -# every blip still costs a repo its verdict. -ATTEMPTS_FILE=$(mktemp) -echo 0 > "$ATTEMPTS_FILE" -gh() { - local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" - if [ "$n" -lt 2 ]; then echo 'gh: HTTP 502' >&2; return 1; fi - printf 'recovered'; return 0 -} -out=$(gh_get 'flaky/path'); rc=$? -[ "$rc" = 0 ] && [ "$out" = "recovered" ] \ - && ok "a transient failure is retried and recovers" \ - || no "a transient failure is retried and recovers (rc=$rc out='$out')" - -# A 404 must NOT burn retries — it is an answer, and retrying it would triple -# the cost of every genuinely-absent file across the fleet. -echo 0 > "$ATTEMPTS_FILE" -gh() { - local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" - echo 'gh: Not Found (HTTP 404)' >&2; return 1 -} -gh_get 'missing/path' >/dev/null -[ "$(cat "$ATTEMPTS_FILE")" = 1 ] \ - && ok "a 404 is not retried (costs one call, not three)" \ - || no "a 404 is not retried (took $(cat "$ATTEMPTS_FILE") calls)" - -# An exhausted rate limit is not retried either: seconds of backoff cannot -# outlive an hour-long window, and retrying burns calls when they're scarcest. -# It is still COULD-NOT-LOOK (1) — a transport fact, never absence. -echo 0 > "$ATTEMPTS_FILE" -gh() { - local n; n=$(cat "$ATTEMPTS_FILE"); n=$((n + 1)); echo "$n" > "$ATTEMPTS_FILE" - echo 'gh: HTTP 403 API rate limit exceeded for user' >&2; return 1 -} -gh_get 'starved/path' >/dev/null; rc=$? -[ "$rc" = 1 ] && [ "$(cat "$ATTEMPTS_FILE")" = 1 ] \ - && ok "a rate-limited call fails fast as COULD-NOT-LOOK (1 call, rc=1)" \ - || no "a rate-limited call fails fast (rc=$rc, took $(cat "$ATTEMPTS_FILE") calls)" -rm -f "$ATTEMPTS_FILE" -unset -f gh - -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[ "$FAIL" -eq 0 ] diff --git a/scripts/ci/ui-defect-audit.mjs b/scripts/ci/ui-defect-audit.mjs deleted file mode 100755 index 8d9befe..0000000 --- a/scripts/ci/ui-defect-audit.mjs +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env node -/** - * Fleet audit: two rendered defects that no unit test, type check or lint rule - * can see, because nothing is wrong until the page is painted. - * - * 1. INVISIBLE ACTIONS — an interactive label below its WCAG AA contrast - * floor. On fleetcrown/control the only route to a feedback report's - * screenshot and history rendered at 3.13:1; the operator's report was not - * "low contrast", it was "i dont see it". An action nobody can find is a - * feature that does not exist. - * - * 2. RAGGED STACKS — sibling lines in one vertical stack that start at - * slightly different x, and wrapped lines whose second line does not align - * with the first. Both come from the same habit: putting an icon INLINE at - * the head of a line, so it shoves that one line sideways by its own width - * and gives a wrapped paragraph no hanging indent. On fleetcrown's fleet - * card four stacked rows started at three different x (369 / 383 / 385) - * with the hint's second line snapping back to 369. The operator's words - * were "this area looks bad. not aligned." - * - * Central auditor, same doctrine as verify-floor-audit.sh: ONE script that - * reads every SITE remotely, never a check copied into each repo. A copied - * check drifts; this one cannot. - * - * The site list is DISCOVERED, not hardcoded: fleetcrown's public footer is the - * fleet's own SSOT for "sites we run" (config/fleet-sites.ts renders there - * precisely so each has a crawlable anchor). Reading it live means a new site - * is audited the day it is linked, and a retired one stops being audited, - * without anyone editing this file. - * - * SCOPE — what this does NOT prove. It renders each site's PUBLIC entry page - * only, unauthenticated. Defects behind a login are invisible here; for those, - * run the per-repo authed audits (fleetcrown: `npm run audit:contrast`). It - * also judges only text it can attribute to a background — an element painted - * over an image reports the image's absence, not its colour, so those are - * skipped rather than guessed at. - * - * Usage: - * node scripts/ci/ui-defect-audit.mjs # audit, exit 1 on defects - * node scripts/ci/ui-defect-audit.mjs --warn-only # report, always exit 0 - * SITES="https://a.example,https://b.example" node scripts/ci/ui-defect-audit.mjs - * - * Needs playwright. dotfiles has no package.json on purpose, so the browser is - * resolved from a fleet repo that already installs it (override with - * PLAYWRIGHT_FROM=/path/to/repo). - */ -import { createRequire } from "node:module"; -import { existsSync } from "node:fs"; -import { join } from "node:path"; -import { homedir } from "node:os"; - -const WARN_ONLY = process.argv.includes("--warn-only"); -const AA_SMALL = 4.5; -const AA_LARGE = 3.0; -/** A stack whose rows differ by more than this is deliberate indentation, not - * an icon accidentally shoving one line sideways. Icons in this fleet are - * 10–16px plus a gap; 24 covers them with headroom and excludes real nesting. */ -const MAX_ACCIDENTAL_INDENT_PX = 24; -const DISCOVERY_URL = process.env.DISCOVERY_URL ?? "https://fleetcrown.orangecat.ch/"; - -export function loadPlaywright() { - const candidates = [ - process.env.PLAYWRIGHT_FROM, - join(homedir(), "dev", "fleetcrown"), - join(homedir(), "dev", "orangecat"), - join(homedir(), "dev", "vitareba"), - ].filter(Boolean); - for (const root of candidates) { - const entry = join(root, "node_modules", "playwright", "index.js"); - if (!existsSync(entry)) continue; - return createRequire(join(root, "package.json"))("playwright"); - } - console.error( - "✗ playwright not found. Set PLAYWRIGHT_FROM=/path/to/a/repo that installs it." - ); - process.exit(2); -} - -/** The fleet's own list of the sites it runs, read from where it is published. */ -async function discoverSites() { - const explicit = process.env.SITES?.trim(); - if (explicit) return explicit.split(",").map((s) => s.trim()).filter(Boolean); - - const res = await fetch(DISCOVERY_URL, { headers: { "user-agent": "fleet-ui-audit" } }); - if (!res.ok) throw new Error(`site discovery failed: ${DISCOVERY_URL} → ${res.status}`); - const html = await res.text(); - const found = new Set(); - for (const m of html.matchAll(/https:\/\/[a-z0-9.-]*orangecat\.ch(?=["'/\s<])/g)) { - found.add(m[0]); - } - // The audit runs FROM fleetcrown's own page; auditing the page we discovered - // from is still worth doing, so it stays in the list. - found.add("https://fleetcrown.orangecat.ch"); - return [...found].sort(); -} - -/** - * Passed to the page as a STRING, not a function: a bundler that injects a - * `__name` helper into arrow functions makes page.evaluate throw - * "ReferenceError: __name is not defined" inside the browser. - */ -export const MEASURE = String.raw`(() => { - // ── colour ──────────────────────────────────────────────────────────────── - // Resolve ANY CSS colour syntax (oklch, lab, color-mix, var) to straight - // RGBA. One paint is not enough: a translucent colour composites against - // whatever the canvas already holds, silently discarding alpha and turning a - // 4%-white overlay into SOLID WHITE. Paint twice over known backdrops and - // solve: white - black = 255*(1-a). - var resolve = function (color) { - var c = document.createElement("canvas"); c.width = c.height = 1; - var x = c.getContext("2d", { willReadFrequently: true }); - var paint = function (bd) { - x.globalCompositeOperation = "copy"; x.fillStyle = bd; x.fillRect(0, 0, 1, 1); - x.globalCompositeOperation = "source-over"; x.fillStyle = color; x.fillRect(0, 0, 1, 1); - return Array.prototype.slice.call(x.getImageData(0, 0, 1, 1).data, 0, 3); - }; - var b = paint("#000"), w = paint("#fff"); - var a = 1 - (w[0] - b[0]) / 255; - if (a <= 0.0001) return [0, 0, 0, 0]; - return [b[0] / a, b[1] / a, b[2] / a, a]; - }; - var over = function (t, bo) { - var a = t[3]; - return [t[0]*a + bo[0]*(1-a), t[1]*a + bo[1]*(1-a), t[2]*a + bo[2]*(1-a), 1]; - }; - var lum = function (r) { - var f = function (v) { var s = v/255; return s <= 0.03928 ? s/12.92 : Math.pow((s+0.055)/1.055, 2.4); }; - return 0.2126*f(r[0]) + 0.7152*f(r[1]) + 0.0722*f(r[2]); - }; - // Composite every translucent layer down to the first opaque ancestor — - // that stack is what the eye actually sees behind the glyphs. Returns null - // when an image is in the way: an unknown backdrop must be skipped, not - // guessed at, or the audit invents failures. - var bgOf = function (el) { - var st = [], n = el; - while (n) { - var cs = getComputedStyle(n); - if (cs.backgroundImage && cs.backgroundImage !== "none") return null; - var c = resolve(cs.backgroundColor); - if (c[3] > 0) st.push(c); - if (c[3] >= 0.999) break; - n = n.parentElement; - } - if (!st.length) return [0, 0, 0]; - var base = st[st.length - 1]; - if (base[3] < 1) base = over(base, [0, 0, 0, 1]); - for (var i = st.length - 2; i >= 0; i--) base = over(st[i], base); - return [base[0], base[1], base[2]]; - }; - var ratio = function (fgRgba, bg) { - var fg = fgRgba[3] < 1 ? over(fgRgba, bg.concat([1])) : fgRgba; - var a = lum(fg), b = lum(bg); - var hi = Math.max(a, b), lo = Math.min(a, b); - return Math.round(((hi + 0.05) / (lo + 0.05)) * 100) / 100; - }; - - var contrast = []; - var actions = document.querySelectorAll('a, button, summary, [role="button"], [role="link"], [role="tab"]'); - for (var i = 0; i < actions.length; i++) { - var el = actions[i]; - var text = (el.innerText || "").trim(); - if (!text) continue; - // A wrapper whose visible label is painted by a nested action measures the - // wrapper's own never-painted ink: surf-your-life's bare around a - // styled