From e9a89cecce034ad8167fadfb39dc2e460c9da89c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 10 Aug 2026 15:51:08 +0000 Subject: [PATCH] ci(shadcn): close the three declared alarm-channel gaps (#3586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #3497 built the alarm channel and disclosed three gaps it left open. This closes all three, and moves the classification out of YAML so the new branching is testable: - gap 2: `timeout-minutes: 20` on the job. The online step is 46 serial registry requests and `fetchUrl` sets no socket timeout, so a hang was bounded only by GitHub's 360-minute default. Sized from measurement: 49s longest of 30 observed runs, ~13min compound degraded worst case. - gap 3: the `analyze` step's exit code is captured instead of hidden behind `continue-on-error`, and an analyze crash enters the SAME issue channel as check failures. Its output is now captured with `2>&1`, so the crash the alarm reports is actually in the alarm's body. - gap 1: N consecutive unreachable runs escalate into that same channel (N=3). Cross-run state is carried by two marker steps whose names and conclusions the next run reads via the Actions API — the cheapest honest mechanism of the five costed in `readRegistryStreak`'s header. The single-run tolerance ruling is untouched for runs 1..N-1. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017Qqyix2QcnpUC9XeYVDzx3 --- .github/workflows/shadcn-check.yml | 244 +++---- scripts/__tests__/shadcn-check-report.test.ts | 394 +++++++++++ scripts/shadcn-check-report.mjs | 612 ++++++++++++++++++ 3 files changed, 1110 insertions(+), 140 deletions(-) create mode 100644 scripts/__tests__/shadcn-check-report.test.ts create mode 100644 scripts/shadcn-check-report.mjs diff --git a/.github/workflows/shadcn-check.yml b/.github/workflows/shadcn-check.yml index 7770772735..d1af1549db 100644 --- a/.github/workflows/shadcn-check.yml +++ b/.github/workflows/shadcn-check.yml @@ -12,15 +12,44 @@ on: # default token cannot do unless it is asked for. Declared explicitly so an # org-wide tightening of the default workflow permissions cannot silently turn # the only alarm channel this workflow has back into a no-op. +# +# `actions: read` is what lets a run see the PREVIOUS runs' marker steps, which +# is the cross-run state the consecutive-unreachability escalation is built on +# (objectui#3586; the alternatives and their costs are in +# `scripts/shadcn-check-report.mjs`, `readRegistryStreak`). permissions: contents: read issues: write + actions: read jobs: check-components: name: Check for Shadcn Component Updates runs-on: ubuntu-latest + # objectui#3586 gap ②. Without this the job inherits GitHub's 360-minute + # default, and the online step is 46 SERIAL registry requests through + # `fetchUrl` in `scripts/shadcn-sync.js`, which sets no socket timeout — so a + # black-holed connection is bounded by nothing but this line. Six hours of a + # runner, once a week, on a schedule nobody watches. + # + # The arithmetic, from measurement rather than taste: + # + # - all 30 runs in the API history completed in 19-49s wall clock; the + # longest was 49s (2026-04-06). The most recent (31374857502) took 34s. + # - inside it the online step took ~1.7s for all 46 components + # (09:30:22.71 -> 09:30:24.03), i.e. ~37ms per serial request, with + # `Registry: 0 cached, 46 fetched` and 0 errors. + # - degraded-but-alive worst case, at 10s per request: 46 x 10s = 7.7min, + # plus ~35s of checkout/install/upload overhead. + # - `pnpm install --frozen-lockfile` took 7s on a pnpm store cache hit, and + # all 30 observed runs hit it. A lockfile change misses; budget ~5min. + # + # 8.3min + 5min = ~13min of compound worst case. 20 gives it ~1.5x headroom, + # is ~24x the longest run ever observed, and turns a hang from 6 hours into + # 20 minutes. + timeout-minutes: 20 + steps: - name: Checkout code uses: actions/checkout@v7 @@ -42,16 +71,30 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # Left tolerant deliberately: `component-analysis.js` has exactly one - # non-zero exit (an unhandled crash in `main()`), so there is no verdict - # here to swallow — its output is advisory context for the report below. + # objectui#3586 gap ③. This step used to carry `continue-on-error: true`, + # on the reasoning that `component-analysis.js` has "no verdict here to + # swallow" — true of the DRIFT verdict, false of the step itself. Its one + # non-zero exit is an uncaught crash, and a crash on a weekly schedule + # turned the job red where nobody looks: the same silent-failure shape as + # the swallowed exit code, one step over. + # + # The exit code is captured explicitly instead and classified with the + # check step's, into the SAME issue channel — not a second one, and not + # behind `continue-on-error`, which is what made the first gap invisible. + # + # `2>&1` matters: the crash writes its stack to stderr, and the old + # `> analysis.txt` sent only stdout to the file the issue body quotes. An + # alarm that cannot show the error it is alarming about is half an alarm. - name: Analyze components (offline) id: analyze + shell: bash run: | echo "Running offline component analysis..." - pnpm shadcn:analyze > analysis.txt - cat analysis.txt - continue-on-error: true + set +e + pnpm shadcn:analyze 2>&1 | tee analysis.ansi.txt + status=${PIPESTATUS[0]} + set -e + echo "exit_code=$status" >> "$GITHUB_OUTPUT" # `pnpm shadcn:check` has carried a REAL exit code since #3455: it exits # non-zero for one reason only — a declared local patch that is missing @@ -64,7 +107,7 @@ jobs: # away wholesale — and because the reporting step was gated on `failure()`, # which a tolerated step never produces, the issue-creation path below had # never once run (objectstack#5805). The code is captured explicitly here - # instead, classified, and routed into that issue path: this workflow runs + # instead, and classified by the step after this one: this workflow runs # weekly on a schedule, and a red run on a page nobody opens is not an # alarm — an issue in the triage queue is. - name: Check component status (online) @@ -76,86 +119,50 @@ jobs: pnpm shadcn:check 2>&1 | tee check.ansi.txt status=${PIPESTATUS[0]} set -e - - # The script colours every line unconditionally (no TTY or NO_COLOR - # check), so the raw capture is dense with ANSI escapes. Strip them for - # the artifact and for the issue body. `\e` below is perl's own escape - # sequence — never write the byte itself into a repo file - # (objectstack#4890). - perl -pe 's/\e\[[0-9;]*[A-Za-z]//g' check.ansi.txt > check.txt - rm -f check.ansi.txt - - # How many components the registry could not serve. Both shapes the - # script produces for that: a rejected fetch, and a response whose - # file content is unusable (proxy error page, egress block, schema - # change). Counted for reporting only — see the tolerance rule below. - registry_errors=$(grep -cE 'Registry returned no usable file content|Error fetching from registry:' check.txt || true) - - # Three classes. Only the benign one is tolerated, so a failure mode - # nobody anticipated cannot fall through the same gap the swallowed - # exit code did: - # - # patch the patch gate's own verdict line is present. Upstream - # moved an anchor the next `--update` must re-apply, or a - # required edit vanished from the file on disk. ALARM. - # ok exit 0 and no such verdict. Includes an unreachable - # registry, which the script reports per component and still - # exits 0 — tolerated, per objectstack#5805. - # broken any other non-zero exit: the check could not run at all - # (fatal error, bad invocation, tooling). ALARM — a check - # that cannot report is not a passing check. - # - # The verdict line is tested BEFORE the exit code on purpose: the - # message is the evidence, the exit code is a policy that a later - # change to the script could revise without touching this workflow. - if grep -qF 'component(s) with declared local patch failures' check.txt; then - check_class=patch - elif [ "$status" -eq 0 ]; then - check_class=ok - else - check_class=broken - fi - - alarm=false - if [ "$check_class" != 'ok' ]; then - alarm=true - fi - - { - echo "exit_code=$status" - echo "class=$check_class" - echo "registry_errors=$registry_errors" - echo "alarm=$alarm" - } >> "$GITHUB_OUTPUT" - - { - echo "### Shadcn component check" - echo "" - echo "- \`pnpm shadcn:check\` exit code: \`$status\` (class: \`$check_class\`)" - echo "- components the registry could not serve: $registry_errors" - } >> "$GITHUB_STEP_SUMMARY" - - case "$check_class" in - patch) - echo "::error::Declared local patches are failing (exit $status). Opening/updating the tracking issue." - echo "- Verdict: a declared local patch failed. Tracking issue opened or updated." >> "$GITHUB_STEP_SUMMARY" - ;; - broken) - echo "::error::shadcn:check exited $status without a patch verdict — the check itself could not run. Opening/updating the tracking issue." - echo "- Verdict: the check could not run. Tracking issue opened or updated." >> "$GITHUB_STEP_SUMMARY" - ;; - ok) - if [ "$registry_errors" -gt 0 ]; then - # Tolerated, but never reported as a clean bill of health: with - # the registry unreachable the upstream-anchor half of the check - # did not execute, so this run proved nothing about upstream. - echo "::warning::$registry_errors component(s) could not be fetched from the registry, so the upstream-anchor check did not run. Tolerated by design — no issue opened." - echo "- Verdict: no patch failure, but the online half did not run (registry unreachable). Tolerated, no issue opened." >> "$GITHUB_STEP_SUMMARY" - else - echo "- Verdict: all declared local patches still apply to current upstream." >> "$GITHUB_STEP_SUMMARY" - fi - ;; - esac + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + + # Everything the two steps above produce is classified here, in + # `scripts/shadcn-check-report.mjs` — ANSI stripping, #3497's three check + # classes, the analyze class, the cross-run registry streak, the step + # summary, the annotations, and the issue body. It lives in a file because + # #3497's version lived in YAML and therefore could not be tested; it is + # covered by `scripts/__tests__/shadcn-check-report.test.ts`. + # + # It exits 0 for every CLASSIFIED outcome, alarms included: the alarm is + # the issue, not the job colour. It going red means the classifier itself + # crashed, which is the one failure this mechanism cannot route into its + # own channel — hence the unit tests. + - name: Classify this run + id: verdict + env: + ANALYZE_EXIT_CODE: ${{ steps.analyze.outputs.exit_code }} + CHECK_EXIT_CODE: ${{ steps.check.outputs.exit_code }} + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/shadcn-check-report.mjs + + # ── Cross-run state ───────────────────────────────────────────────────── + # These two steps do nothing in this run. Their names and conclusions ARE + # the state the NEXT run reads back through the Actions API, which is how + # "the registry has been unreachable N runs running" can be known at all + # without a database (objectui#3586 gap ①). + # + # Three-valued by construction, and that is the point: exactly one of them + # succeeds in a run that reached a verdict, and NEITHER appears in a run + # that died before it — so "we don't know" is distinguishable from + # "reachable", and only "unreachable" extends the streak. + # + # The names are a contract with `MARKER_STEPS` in + # `scripts/shadcn-check-report.mjs`; the pin test holds them equal, because + # a rename here would silently reset the streak forever. + - name: 'Cross-run marker: registry reachable' + if: steps.verdict.outputs.registry_state == 'reachable' + run: echo "Registry reachable in this run — the unreachable streak is broken here." + + - name: 'Cross-run marker: registry unreachable' + if: steps.verdict.outputs.registry_state == 'unreachable' + run: | + echo "Registry unreachable in this run (${{ steps.verdict.outputs.registry_errors }} component(s))." + echo "Consecutive unreachable runs: ${{ steps.verdict.outputs.registry_streak }}." - name: Upload analysis results uses: actions/upload-artifact@v7 @@ -171,64 +178,21 @@ jobs: # cannot deliver (missing permission, API outage), the job must go red, # because a silently broken alarm channel is the bug this workflow was # just fixed for. + # + # One channel for every reason — patch failure, a check that could not run, + # an analyze crash, a registry blind for `ESCALATION_THRESHOLD` runs, or a + # cross-run read that failed. Same labels, same de-duplication. The body is + # rendered by the classifier step into `alarm-issue.md`; this step only + # delivers it. - name: Report check failure as an issue - if: steps.check.outputs.alarm == 'true' + if: steps.verdict.outputs.alarm == 'true' uses: actions/github-script@v9 env: - CHECK_CLASS: ${{ steps.check.outputs.class }} - CHECK_EXIT: ${{ steps.check.outputs.exit_code }} - REGISTRY_ERRORS: ${{ steps.check.outputs.registry_errors }} + ISSUE_TITLE: ${{ steps.verdict.outputs.issue_title }} with: script: | const fs = require('fs'); - - const checkClass = process.env.CHECK_CLASS; - const isPatchFailure = checkClass === 'patch'; - - const title = isPatchFailure - ? 'Shadcn sync: declared local patches are failing' - : 'Shadcn sync: the weekly component check could not run'; - - let body = '## Shadcn Components Status Report\n\n'; - if (isPatchFailure) { - body += 'The weekly component sync check found a **declared local patch failure**: '; - body += 'either a required edit is missing from the file on disk, or upstream moved '; - body += 'the anchor it is applied to, so the next `pnpm shadcn:update` would refuse '; - body += 'to write rather than drop it. Details in the check output below.\n\n'; - } else { - body += 'The weekly component sync check **could not complete**: `pnpm shadcn:check` '; - body += 'exited `' + process.env.CHECK_EXIT + '` without reaching a patch verdict. '; - body += 'Until this is fixed the weekly upstream early-warning is not running.\n\n'; - } - body += '- Exit code: `' + process.env.CHECK_EXIT + '` (class: `' + checkClass + '`)\n'; - body += '- Components the registry could not serve: ' + process.env.REGISTRY_ERRORS + '\n'; - body += '- Run: ' + context.serverUrl + '/' + context.repo.owner + '/' + context.repo.repo + - '/actions/runs/' + context.runId + '\n\n'; - - if (fs.existsSync('analysis.txt')) { - const analysis = fs.readFileSync('analysis.txt', 'utf8'); - body += '### Offline Analysis\n\n'; - body += '```\n' + analysis.substring(0, 5000) + '\n```\n\n'; - } - - if (fs.existsSync('check.txt')) { - const check = fs.readFileSync('check.txt', 'utf8'); - body += '### Online Check Results\n\n'; - body += '```\n' + check.substring(0, 5000) + '\n```\n\n'; - } - - body += '### Next Steps\n\n'; - if (isPatchFailure) { - body += '1. Read the `DECLARED LOCAL PATCHES` section above — it names the patch id, its tracking issue and the reason\n'; - body += '2. Marker missing from disk: restore it with `pnpm shadcn:update `\n'; - body += '3. Anchor no longer found upstream: re-target `find`/`occurrences` in `scripts/shadcn-local-patches.mjs`\n'; - body += '4. See [SHADCN_SYNC.md](../blob/main/docs/SHADCN_SYNC.md) for detailed guide\n\n'; - } else { - body += '1. Open the workflow run linked above and read the failure\n'; - body += '2. Reproduce locally with `pnpm shadcn:check`\n'; - body += '3. See [SHADCN_SYNC.md](../blob/main/docs/SHADCN_SYNC.md) for detailed guide\n\n'; - } - body += '> This issue was automatically created by the Shadcn Components Check workflow.\n'; + const body = fs.readFileSync('alarm-issue.md', 'utf8'); // Check if there's already an open issue const issues = await github.rest.issues.listForRepo({ @@ -251,7 +215,7 @@ jobs: await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, - title: title, + title: process.env.ISSUE_TITLE, body: body, labels: ['maintenance', 'shadcn-sync', 'dependencies'], }); diff --git a/scripts/__tests__/shadcn-check-report.test.ts b/scripts/__tests__/shadcn-check-report.test.ts new file mode 100644 index 0000000000..a124213119 --- /dev/null +++ b/scripts/__tests__/shadcn-check-report.test.ts @@ -0,0 +1,394 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// Plain-JS CI helper. Its types are INFERRED from the .mjs source by +// `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here — see +// objectui#3494. +import { + ESCALATION_THRESHOLD, + MARKER_STEPS, + alarmTitle, + classifyAnalyze, + classifyCheck, + consecutiveUnreachable, + countRegistryErrors, + decideAlarm, + main, + readRegistryStreak, + registryStateFromJobs, + renderAlarmIssue, + stripAnsi, +} from '../shadcn-check-report.mjs'; + +/** + * objectui#3586 — the three gaps PR #3497 declared and left open. + * + * #3497 built the alarm channel and tested it by hand: five fixtures, run once, + * results written into the PR body. Nothing re-runs that between weekly cron + * fires, and the logic it covered lived in YAML where nothing could. This file + * is the standing version of that dry run, and it exists because this card adds + * two more classified inputs to the same decision. + * + * What each block pins: + * + * 1. #3497's three check classes, UNCHANGED. This card must not move the + * tolerance ruling, so the ruling is asserted here as a regression fence + * rather than described in a comment. + * 2. Gap ③ — an `analyze` crash reaches the SAME channel. The discriminating + * assertion is the reverse one: with the crash classified back to `ok`, the + * identical run produces NO alarm. That is the "silently red on an unwatched + * weekly job" state the card is closing, asserted from both sides. + * 3. Gap ① — consecutive unreachability. Both directions again: runs 1..N-1 + * stay tolerated exactly as #3497 ruled (no alarm, no issue), and run N + * escalates into the same channel. + * 4. Gap ② and the cross-run contract, read out of the workflow YAML. The + * marker step names are the one part of the mechanism a rename can break + * SILENTLY — a renamed marker reads back as `unknown`, resets the streak, + * and the escalation simply never fires again — so they are pinned against + * the module's own constants rather than re-spelled. + */ + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const workflowPath = path.join(repoRoot, '.github/workflows/shadcn-check.yml'); +const workflow = fs.readFileSync(workflowPath, 'utf8'); + +/** + * The workflow YAML with whole-line comments removed — the same precaution + * `docs-links-workflow.test.ts` documents. This file's header prose discusses + * `continue-on-error` and `timeout-minutes` at length, and a scan that counted + * those sentences would assert the opposite of the truth. + */ +const workflowCode = workflow + .split('\n') + .filter((line) => !/^\s*#/.test(line)) + .join('\n'); + +/** A realistic `shadcn:check` capture: 46 components, every fetch refused. */ +const unreachableOutput = Array.from( + { length: 46 }, + (_, i) => `✗ component-${i} Registry returned no usable file content`, +).join('\n'); + +const cleanOutput = ['✓ button Identical to upstream', 'Registry: 0 cached, 46 fetched'].join('\n'); + +const patchOutput = [ + cleanOutput, + 'DECLARED LOCAL PATCHES — FAILED', + '✗ 1 component(s) with declared local patch failures', +].join('\n'); + +describe('classifyCheck — #3497 three classes, unchanged', () => { + it('classifies a clean run as ok with no alarm', () => { + const verdict = classifyCheck({ output: cleanOutput, exitCode: 0 }); + expect(verdict).toEqual({ checkClass: 'ok', registryErrors: 0, registryState: 'reachable' }); + expect(decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: 0 }).alarm).toBe(false); + }); + + it('tolerates an unreachable registry: still ok, still exit 0, still no issue', () => { + const verdict = classifyCheck({ output: unreachableOutput, exitCode: 0 }); + expect(verdict.checkClass).toBe('ok'); + expect(verdict.registryErrors).toBe(46); + expect(verdict.registryState).toBe('unreachable'); + // The single-run ruling this card leaves untouched. + expect(decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: 1 }).alarm).toBe(false); + }); + + it('classifies a declared-patch failure as patch', () => { + expect(classifyCheck({ output: patchOutput, exitCode: 1 }).checkClass).toBe('patch'); + }); + + it('reads the verdict LINE before the exit code — the evidence outranks the policy', () => { + // #3497's ordering ruling: a script revision that stopped exiting non-zero + // for a patch failure must not silently downgrade the verdict to `ok`. + expect(classifyCheck({ output: patchOutput, exitCode: 0 }).checkClass).toBe('patch'); + }); + + it('classifies any other non-zero exit as broken', () => { + expect(classifyCheck({ output: 'Error loading manifest', exitCode: 1 }).checkClass).toBe('broken'); + expect(decideAlarm({ checkClass: 'broken', analyzeClass: 'ok', registryStreak: 0 }).alarm).toBe(true); + }); + + it('counts both registry-failure shapes the sync script produces', () => { + expect( + countRegistryErrors( + ['Error fetching from registry: HTTP 403', 'Registry returned no usable file content', 'fine'].join('\n'), + ), + ).toBe(2); + }); + + it('strips the ANSI the shadcn scripts emit unconditionally', () => { + const esc = String.fromCharCode(0x1b); + expect(stripAnsi(`${esc}[32m✓ button${esc}[0m`)).toBe('✓ button'); + }); +}); + +describe('gap ③ — an analyze crash lands in the same channel, not silently red', () => { + it('classifies a non-zero analyze exit as broken and alarms on it', () => { + expect(classifyAnalyze({ exitCode: 1 })).toBe('broken'); + const { alarm, reasons } = decideAlarm({ checkClass: 'ok', analyzeClass: 'broken', registryStreak: 0 }); + expect(alarm).toBe(true); + expect(reasons).toContain('analyze-broken'); + }); + + it('REVERSE: the identical run with the crash classified ok produces no alarm at all', () => { + // This is the pre-#3586 behaviour, stated as an assertion: `analyze` exits + // non-zero, the job goes red on a weekly schedule, and nothing reaches the + // triage queue. If a later change reverts the classification, this pair — + // not the positive test above — is what catches it. + expect(classifyAnalyze({ exitCode: 0 })).toBe('ok'); + expect(decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: 0 }).alarm).toBe(false); + }); + + it('reports the crash in the SAME issue body, with the labels-bearing title of the existing channel', () => { + const { title, body } = renderAlarmIssue({ + reasons: ['analyze-broken'], + analyzeClass: 'broken', + analyzeExit: 1, + analysisLog: 'Fatal error: Cannot read properties of undefined', + }); + // #3497's own "could not run" title — deliberately not a new one, because a + // new title on a de-duplicated channel is how a second channel starts. + expect(title).toBe('Shadcn sync: the weekly component check could not run'); + expect(body).toContain('The offline analysis step **crashed**'); + expect(body).toContain('Cannot read properties of undefined'); + }); + + it('still names the patch failure first when both fail — one issue, both facts', () => { + const { title, body } = renderAlarmIssue({ reasons: ['patch', 'analyze-broken'], checkExit: 1, analyzeExit: 1 }); + expect(title).toBe('Shadcn sync: declared local patches are failing'); + expect(body).toContain('declared local patch failure'); + expect(body).toContain('The offline analysis step **crashed**'); + }); +}); + +describe('gap ① — consecutive unreachability escalates, single runs still do not', () => { + const jobsWith = (stepName: string) => ({ jobs: [{ steps: [{ name: stepName, conclusion: 'success' }] }] }); + const unreachableRun = jobsWith(MARKER_STEPS.unreachable); + const reachableRun = jobsWith(MARKER_STEPS.reachable); + /** A run that died before the markers — neither name present. */ + const diedEarlyRun = { jobs: [{ steps: [{ name: 'Install dependencies', conclusion: 'failure' }] }] }; + + const stubApi = (previous: object[]) => { + const calls: string[] = []; + return { + calls, + listRuns: async (perPage: number) => { + calls.push(`runs:${perPage}`); + return { workflow_runs: previous.map((_, i) => ({ id: 100 + i })) }; + }, + listJobs: async (runId: number) => { + calls.push(`jobs:${runId}`); + return previous[runId - 100]; + }, + }; + }; + + it('reads three states out of a jobs payload, and "unknown" is not "reachable"', () => { + expect(registryStateFromJobs(unreachableRun.jobs)).toBe('unreachable'); + expect(registryStateFromJobs(reachableRun.jobs)).toBe('reachable'); + expect(registryStateFromJobs(diedEarlyRun.jobs)).toBe('unknown'); + expect(registryStateFromJobs([])).toBe('unknown'); + }); + + it('does not touch the API at all when this run reached the registry', async () => { + const api = stubApi([unreachableRun, unreachableRun]); + const result = await readRegistryStreak({ currentState: 'reachable', api }); + expect(result).toEqual({ streak: 0, readable: true, error: '' }); + expect(api.calls).toEqual([]); + }); + + it('run 1 of a new outage: streak 1, tolerated', async () => { + const api = stubApi([reachableRun, reachableRun]); + const { streak } = await readRegistryStreak({ currentState: 'unreachable', api }); + expect(streak).toBe(1); + expect(decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: streak }).alarm).toBe(false); + }); + + it('run 2: streak 2, STILL tolerated — the ruling holds up to N-1', async () => { + const api = stubApi([unreachableRun, reachableRun]); + const { streak } = await readRegistryStreak({ currentState: 'unreachable', api }); + expect(streak).toBe(2); + expect(streak).toBeLessThan(ESCALATION_THRESHOLD); + expect(decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: streak }).alarm).toBe(false); + }); + + it('run 3: streak reaches the threshold and escalates into the same channel', async () => { + const api = stubApi([unreachableRun, unreachableRun, unreachableRun]); + const { streak } = await readRegistryStreak({ currentState: 'unreachable', api }); + expect(streak).toBe(ESCALATION_THRESHOLD); + + const { alarm, reasons } = decideAlarm({ checkClass: 'ok', analyzeClass: 'ok', registryStreak: streak }); + expect(alarm).toBe(true); + expect(reasons).toEqual(['registry-blind']); + + const { title, body } = renderAlarmIssue({ reasons, registryStreak: streak, registryErrors: 46 }); + expect(title).toBe('Shadcn sync: the registry has been unreachable for 3 consecutive runs'); + expect(body).toContain('unreachable for **3 consecutive runs**'); + expect(body).toContain('proved nothing about upstream'); + }); + + it('stops walking at the threshold — never more than N-1 job reads', async () => { + const api = stubApi(Array.from({ length: 8 }, () => unreachableRun)); + await readRegistryStreak({ currentState: 'unreachable', api }); + expect(api.calls.filter((c) => c.startsWith('jobs:'))).toHaveLength(ESCALATION_THRESHOLD - 1); + }); + + it('a reachable run breaks the streak; an unknown one breaks it too', () => { + expect(consecutiveUnreachable(['unreachable', 'unreachable', 'reachable', 'unreachable'])).toBe(2); + expect(consecutiveUnreachable(['unreachable', 'unknown', 'unreachable'])).toBe(1); + expect(consecutiveUnreachable(['reachable'])).toBe(0); + }); + + it('a failed cross-run read alarms rather than silently disabling the escalation', async () => { + const api = { + listRuns: async () => { + throw new Error('HTTP 500'); + }, + listJobs: async () => ({ jobs: [] }), + }; + const { readable, error } = await readRegistryStreak({ currentState: 'unreachable', api }); + expect(readable).toBe(false); + expect(error).toContain('HTTP 500'); + + const { alarm, reasons } = decideAlarm({ + checkClass: 'ok', + analyzeClass: 'ok', + registryStreak: 1, + streakReadable: false, + }); + expect(alarm).toBe(true); + expect(reasons).toContain('streak-unreadable'); + expect(alarmTitle(reasons)).toBe('Shadcn sync: the weekly component check could not run'); + }); +}); + +describe('main() — the wiring the workflow actually depends on', () => { + const made: string[] = []; + + afterEach(() => { + for (const dir of made.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + async function run(env: Record, files: Record) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'shadcn-check-report-')); + made.push(dir); + for (const [name, content] of Object.entries(files)) fs.writeFileSync(path.join(dir, name), content); + + const saved = { ...process.env }; + Object.assign(process.env, { + GITHUB_OUTPUT: path.join(dir, 'outputs.txt'), + GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md'), + CHECK_RAW_LOG: path.join(dir, 'check.ansi.txt'), + ANALYZE_RAW_LOG: path.join(dir, 'analysis.ansi.txt'), + CHECK_LOG: path.join(dir, 'check.txt'), + ANALYZE_LOG: path.join(dir, 'analysis.txt'), + ALARM_BODY_FILE: path.join(dir, 'alarm-issue.md'), + ...env, + }); + + try { + const result = await main({ api: { listRuns: async () => ({ workflow_runs: [] }), listJobs: async () => ({ jobs: [] }) } }); + const outputs = Object.fromEntries( + fs + .readFileSync(path.join(dir, 'outputs.txt'), 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)]), + ); + const alarmBody = fs.existsSync(path.join(dir, 'alarm-issue.md')) + ? fs.readFileSync(path.join(dir, 'alarm-issue.md'), 'utf8') + : null; + return { result, outputs, alarmBody, dir }; + } finally { + process.env = saved; + } + } + + it('a clean run: no alarm, no issue body, stripped logs written for the artifact', async () => { + const esc = String.fromCharCode(0x1b); + const { outputs, alarmBody, dir } = await run( + { CHECK_EXIT_CODE: '0', ANALYZE_EXIT_CODE: '0' }, + { 'check.ansi.txt': `${esc}[32m${cleanOutput}${esc}[0m`, 'analysis.ansi.txt': 'Summary' }, + ); + expect(outputs.alarm).toBe('false'); + expect(outputs.check_class).toBe('ok'); + expect(outputs.analyze_class).toBe('ok'); + expect(outputs.registry_state).toBe('reachable'); + expect(alarmBody).toBeNull(); + // The artifact must carry the readable copy, not the escape-dense capture. + expect(fs.readFileSync(path.join(dir, 'check.txt'), 'utf8')).not.toContain(esc); + }); + + it('an analyze crash alone drives the alarm outputs end to end', async () => { + const { outputs, alarmBody } = await run( + { CHECK_EXIT_CODE: '0', ANALYZE_EXIT_CODE: '1' }, + { 'check.ansi.txt': cleanOutput, 'analysis.ansi.txt': 'Fatal error: boom' }, + ); + expect(outputs.alarm).toBe('true'); + expect(outputs.alarm_reasons).toBe('analyze-broken'); + expect(outputs.issue_title).toBe('Shadcn sync: the weekly component check could not run'); + expect(alarmBody).toContain('boom'); + }); + + it('an unreachable registry on run 1 stays tolerated but is recorded for the next run', async () => { + const { outputs, alarmBody } = await run( + { CHECK_EXIT_CODE: '0', ANALYZE_EXIT_CODE: '0' }, + { 'check.ansi.txt': unreachableOutput, 'analysis.ansi.txt': 'Summary' }, + ); + expect(outputs.alarm).toBe('false'); + expect(outputs.registry_state).toBe('unreachable'); + expect(outputs.registry_streak).toBe('1'); + expect(alarmBody).toBeNull(); + }); +}); + +describe('shadcn-check.yml — the gaps, read off the workflow itself', () => { + it('gap ②: the job declares a timeout instead of inheriting the 360-minute default', () => { + const timeout = workflowCode.match(/^\s*timeout-minutes:\s*(\d+)\s*$/m); + expect(timeout, 'a hung 46-request serial check must not be able to burn six hours').not.toBeNull(); + const minutes = Number(timeout?.[1]); + // Generous over the measured worst case (49s observed, ~13min compound + // degraded) and far under the default it replaces. + expect(minutes).toBeGreaterThan(13); + expect(minutes).toBeLessThan(60); + }); + + it('gap ③: the analyze step no longer hides its exit code behind continue-on-error', () => { + expect(workflowCode).toContain('pnpm shadcn:analyze'); + expect(workflowCode).not.toContain('continue-on-error'); + // Captured explicitly, the way #3497 captured the check step's. + expect(workflowCode).toMatch(/pnpm shadcn:analyze 2>&1 \| tee/); + }); + + it('gap ①: the marker step names match the module constants exactly', () => { + // Pinned against the constants, never re-spelled: a rename on one side only + // makes every future run read `unknown`, which resets the streak, which + // means the escalation never fires again — and nothing else would notice. + expect(workflowCode).toContain(`- name: 'Cross-run marker: registry reachable'`); + expect(workflowCode).toContain(`- name: 'Cross-run marker: registry unreachable'`); + expect(MARKER_STEPS.reachable).toBe('Cross-run marker: registry reachable'); + expect(MARKER_STEPS.unreachable).toBe('Cross-run marker: registry unreachable'); + expect(workflowCode).toMatch(/if: steps\.verdict\.outputs\.registry_state == 'reachable'/); + expect(workflowCode).toMatch(/if: steps\.verdict\.outputs\.registry_state == 'unreachable'/); + }); + + it('gap ①: the cross-run read has the permission it needs', () => { + expect(workflowCode).toMatch(/^\s*actions:\s*read\s*$/m); + expect(workflowCode).toMatch(/^\s*issues:\s*write\s*$/m); + }); + + it('the classification runs from the tested module, not from a shell block', () => { + expect(workflowCode).toContain('node scripts/shadcn-check-report.mjs'); + expect(fs.existsSync(path.join(repoRoot, 'scripts/shadcn-check-report.mjs'))).toBe(true); + }); + + it('one channel: the alarm step is still the single issue writer, still not tolerated', () => { + const labels = workflowCode.match(/labels: \['maintenance', 'shadcn-sync', 'dependencies'\]/g); + expect(labels).toHaveLength(1); + expect(workflowCode.match(/github\.rest\.issues\.create\(/g)).toHaveLength(1); + expect(workflowCode).toContain(`if: steps.verdict.outputs.alarm == 'true'`); + }); +}); diff --git a/scripts/shadcn-check-report.mjs b/scripts/shadcn-check-report.mjs new file mode 100644 index 0000000000..0013b1efbe --- /dev/null +++ b/scripts/shadcn-check-report.mjs @@ -0,0 +1,612 @@ +#!/usr/bin/env node +/** + * Classifies one run of `.github/workflows/shadcn-check.yml` and renders the + * alarm issue body it may need to open. + * + * ## Why this is a file and not an inline `script:` block + * + * PR #3497 built the alarm channel this workflow has: the `check` step's exit + * code stopped being swallowed, was sorted into three classes, and the two + * classes worth waking someone for were routed into a tracking issue — because + * "this workflow runs weekly on a schedule, and a red run on a page nobody + * opens is not an alarm, an issue in the triage queue is". + * + * All of that logic lived in YAML, so none of it could be tested; #3497 checked + * it by hand-running five fixtures once and wrote the result into its PR body. + * objectui#3586 adds a fourth classified input (the `analyze` step's exit code) + * and a fifth (cross-run registry reachability), which is more branching than a + * shell block that nothing executes between weekly runs should carry. The + * classification is therefore extracted here and covered by + * `scripts/__tests__/shadcn-check-report.test.ts`, the same way + * `scripts/render-budget-comment.mjs` was extracted for + * `.github/workflows/performance-budget.yml`. + * + * ## The classification, and the three gaps objectui#3586 closes + * + * `check` keeps #3497's vocabulary exactly — the tolerance ruling is unchanged: + * + * patch the patch gate's own verdict line is present. ALARM. + * ok exit 0 and no such verdict. Includes an unreachable registry, + * which the script reports per component and still exits 0. + * broken any other non-zero exit — the check could not run at all. ALARM. + * + * Extended, not replaced: + * + * analyze `ok` | `broken`. `component-analysis.js` has exactly one non-zero + * exit — an uncaught crash in `main()`. #3497 left the step + * `continue-on-error: true` because there was "no verdict here to + * swallow"; that is true of the *drift* verdict and false of the + * step itself, so a crash turned the job red on an unwatched weekly + * schedule and reached nobody. `broken` now enters the SAME issue + * channel — same labels, same de-duplication, no second channel. + * + * registry `reachable` | `unreachable`, plus how many consecutive runs have + * been `unreachable`. A single unreachable run stays tolerated + * exactly as #3497 ruled (exit 0 + `::warning::` + summary, no + * issue). What was missing is that the tolerance never expired: + * with the registry unreachable the upstream-anchor half of the + * check does not execute, so N consecutive unreachable runs are N + * consecutive weeks of proving nothing, and that never escalated. + * At `ESCALATION_THRESHOLD` it does. + * + * The predicate for `unreachable` is deliberately the SAME one #3497 already + * warns on (`registry_errors > 0`), not a new threshold. One tolerated + * condition, one definition: the escalation says "we have now tolerated this N + * runs running", and a single component whose URL moved for good is a real + * finding by week three, not noise. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +/** + * How many consecutive unreachable runs escalate into the issue channel. + * + * The cron is weekly, so N is measured in weeks of blindness: + * + * N=2 (14 days) fires on any fortnight-long upstream or egress hiccup — a CDN + * change or a runner-egress policy edit plausibly spans two Mondays, and + * alarming on it is the noise #3497's single-run tolerance was ruled to + * avoid. + * N=3 (21 days) is well past "transient" for a registry that answered all 46 + * requests in ~1.7s as recently as run 31374857502, and still inside + * every retention window the mechanism depends on (see + * `readRegistryStreak`). + * N=4 (28 days) is a month of an early-warning system warning about nothing, + * and would sit outside the 30-day artifact retention had the artifact + * mechanism been the one chosen. + * + * Three. Changing it is a one-line edit here; the workflow reads it from this + * module rather than repeating it. + */ +export const ESCALATION_THRESHOLD = 3; + +/** + * The two step names that carry this run's registry reachability forward to the + * next run. + * + * This is a CONTRACT BETWEEN RUNS, and the only one in the mechanism that a + * rename can break silently: a run whose marker steps are named something else + * reads back as `unknown`, which resets the streak, which means the escalation + * quietly never fires — the exact silent-failure shape this card exists to + * close. `scripts/__tests__/shadcn-check-report.test.ts` pins the workflow's + * step names against these constants so the rename goes red instead. + */ +export const MARKER_STEPS = Object.freeze({ + reachable: 'Cross-run marker: registry reachable', + unreachable: 'Cross-run marker: registry unreachable', +}); + +/** How much of each captured log is quoted into the issue body. #3497's cap. */ +const LOG_EXCERPT_LIMIT = 5000; + +/** + * The two shapes `scripts/shadcn-sync.js` produces when the registry cannot be + * served: a rejected fetch, and a 2xx whose file content is unusable (proxy + * error page, egress block, schema change). Same pair #3497 counted. + */ +const REGISTRY_ERROR_LINE = /Registry returned no usable file content|Error fetching from registry:/; + +/** The patch gate's own verdict line — the evidence, tested before the exit code. */ +const PATCH_VERDICT_LINE = 'component(s) with declared local patch failures'; + +/** + * `ESC` is written as an escape sequence, never as the byte itself + * (objectstack#4890, and `scripts/check-control-bytes.mjs`). Both shadcn + * scripts colour every line unconditionally, with no TTY or NO_COLOR check, so + * the raw capture is dense with escapes and an issue body quoting it verbatim + * is unreadable. + */ +const ANSI_SEQUENCE = /\u001b\[[0-9;]*[A-Za-z]/g; + +/** @param {string} text */ +export function stripAnsi(text) { + return text.replace(ANSI_SEQUENCE, ''); +} + +/** @param {string} output ANSI-stripped `pnpm shadcn:check` output */ +export function countRegistryErrors(output) { + return output.split('\n').filter((line) => REGISTRY_ERROR_LINE.test(line)).length; +} + +/** + * #3497's three classes, unchanged, plus the registry state derived from the + * same output. + * + * The verdict line is tested BEFORE the exit code on purpose, and that ordering + * is #3497's ruling, quoted: "the message is the evidence, the exit code is a + * policy that a later change to the script could revise without touching this + * workflow". + * + * @param {{ output?: string, exitCode?: number|string }} input + */ +export function classifyCheck({ output = '', exitCode = 0 } = {}) { + const status = Number(exitCode); + const registryErrors = countRegistryErrors(output); + + let checkClass; + if (output.includes(PATCH_VERDICT_LINE)) { + checkClass = 'patch'; + } else if (status === 0) { + checkClass = 'ok'; + } else { + checkClass = 'broken'; + } + + return { + checkClass, + registryErrors, + registryState: registryErrors > 0 ? 'unreachable' : 'reachable', + }; +} + +/** + * `component-analysis.js` is offline and reads only local files; its single + * non-zero exit is the `main().catch()` at the bottom of the file. So there are + * exactly two states, and `broken` alarms. + * + * @param {{ exitCode?: number|string }} input + */ +export function classifyAnalyze({ exitCode = 0 } = {}) { + return Number(exitCode) === 0 ? 'ok' : 'broken'; +} + +/** + * Read one previous run's registry state out of its `/actions/runs/{id}/jobs` + * payload. + * + * Three-valued on purpose. `unknown` is not a synonym for `reachable`: it means + * the run never reached a verdict (it died at checkout or install), or it ran a + * version of the workflow older than the markers. Both break the streak, which + * under-alarms rather than over-alarms — the honest direction for a mechanism + * whose whole job is to eventually open an issue. + * + * Every job is searched rather than the one named in the workflow: the job name + * is not part of this contract, and matching on it would add a second string a + * rename could silently break. + * + * @param {{ steps?: Array<{ name?: string, conclusion?: string }> }[]} jobs + */ +export function registryStateFromJobs(jobs = []) { + for (const job of jobs) { + for (const step of job?.steps ?? []) { + if (step?.conclusion !== 'success') continue; + if (step.name === MARKER_STEPS.unreachable) return 'unreachable'; + if (step.name === MARKER_STEPS.reachable) return 'reachable'; + } + } + return 'unknown'; +} + +/** + * Length of the current unreachable streak, current run included. + * + * @param {string[]} states newest-first, current run at index 0 + */ +export function consecutiveUnreachable(states = []) { + let streak = 0; + for (const state of states) { + if (state !== 'unreachable') break; + streak += 1; + } + return streak; +} + +/** + * The alarm decision. Every reason routes to ONE channel (see + * `renderAlarmIssue`); this function only says whether to open it and why. + * + * `streak-unreadable` alarms, and that is #3497's rule applied to the new + * moving part: "a check that cannot report is not a passing check". If the + * cross-run read breaks for good, the escalation can never fire again, and a + * silently dead alarm channel is the whole bug family this workflow keeps + * closing. It costs a false alarm on a hard GitHub API outage, which is rare, + * loud and self-explaining — the cheap direction to be wrong in. + * + * @param {{ checkClass: string, analyzeClass: string, registryStreak: number, + * streakReadable?: boolean, threshold?: number }} input + */ +export function decideAlarm({ + checkClass, + analyzeClass, + registryStreak = 0, + streakReadable = true, + threshold = ESCALATION_THRESHOLD, +}) { + const reasons = []; + if (checkClass === 'patch') reasons.push('patch'); + if (checkClass === 'broken') reasons.push('check-broken'); + if (analyzeClass === 'broken') reasons.push('analyze-broken'); + if (registryStreak >= threshold) reasons.push('registry-blind'); + if (!streakReadable) reasons.push('streak-unreadable'); + return { alarm: reasons.length > 0, reasons }; +} + +/** + * The issue title. Only ever used when there is no open `shadcn-sync` issue — + * an existing one is commented on instead, which is why the title is chosen + * from the most actionable reason rather than trying to name all of them. + * + * @param {string[]} reasons + * @param {number} streak + */ +export function alarmTitle(reasons, streak = 0) { + if (reasons.includes('patch')) return 'Shadcn sync: declared local patches are failing'; + if (reasons.includes('check-broken') || reasons.includes('analyze-broken') || reasons.includes('streak-unreadable')) { + return 'Shadcn sync: the weekly component check could not run'; + } + if (reasons.includes('registry-blind')) { + return `Shadcn sync: the registry has been unreachable for ${streak} consecutive runs`; + } + return 'Shadcn sync: the weekly component check needs attention'; +} + +const excerpt = (text) => text.slice(0, LOG_EXCERPT_LIMIT); + +/** + * Renders the issue body. One body for every reason: the channel is shared, so + * a reader who opens it for a patch failure and finds the registry has also + * been blind for three weeks learns both facts at once. + * + * @param {object} input + */ +export function renderAlarmIssue({ + reasons = [], + checkClass = 'ok', + checkExit = 0, + analyzeClass = 'ok', + analyzeExit = 0, + registryErrors = 0, + registryStreak = 0, + streakReadable = true, + streakError = '', + threshold = ESCALATION_THRESHOLD, + runUrl = '', + analysisLog = '', + checkLog = '', +} = {}) { + const title = alarmTitle(reasons, registryStreak); + let body = '## Shadcn Components Status Report\n\n'; + + if (reasons.includes('patch')) { + body += 'The weekly component sync check found a **declared local patch failure**: '; + body += 'either a required edit is missing from the file on disk, or upstream moved '; + body += 'the anchor it is applied to, so the next `pnpm shadcn:update` would refuse '; + body += 'to write rather than drop it. Details in the check output below.\n\n'; + } + if (reasons.includes('check-broken')) { + body += 'The weekly component sync check **could not complete**: `pnpm shadcn:check` '; + body += 'exited `' + checkExit + '` without reaching a patch verdict. '; + body += 'Until this is fixed the weekly upstream early-warning is not running.\n\n'; + } + if (reasons.includes('analyze-broken')) { + body += 'The offline analysis step **crashed**: `pnpm shadcn:analyze` exited `' + analyzeExit + '`. '; + body += 'Its only non-zero exit is an uncaught error, so this is a defect in '; + body += '`scripts/component-analysis.js` or in what it reads — the crash output is '; + body += 'quoted below.\n\n'; + } + if (reasons.includes('registry-blind')) { + body += 'The registry has been unreachable for **' + registryStreak + ' consecutive runs** '; + body += '(threshold: ' + threshold + '). Each run on its own was tolerated by design and exited 0, '; + body += 'but with the registry unreachable the upstream-anchor half of the check never '; + body += 'executes: those runs proved nothing about upstream, and the tolerance has now '; + body += 'been extended long enough to be worth a look.\n\n'; + } + if (reasons.includes('streak-unreadable')) { + body += 'The cross-run registry state **could not be read**' + (streakError ? ' (' + streakError + ')' : '') + '. '; + body += 'Consecutive-unreachability can no longer escalate until this is fixed, so it is '; + body += 'reported here rather than tolerated.\n\n'; + } + + body += '- `pnpm shadcn:check` exit code: `' + checkExit + '` (class: `' + checkClass + '`)\n'; + body += '- `pnpm shadcn:analyze` exit code: `' + analyzeExit + '` (class: `' + analyzeClass + '`)\n'; + body += '- Components the registry could not serve: ' + registryErrors + '\n'; + body += '- Consecutive runs with an unreachable registry: ' + + (streakReadable ? registryStreak + ' (escalates at ' + threshold + ')' : 'unknown — cross-run read failed') + '\n'; + if (runUrl) body += '- Run: ' + runUrl + '\n'; + body += '\n'; + + if (analysisLog) { + body += '### Offline Analysis\n\n'; + body += '```\n' + excerpt(analysisLog) + '\n```\n\n'; + } + if (checkLog) { + body += '### Online Check Results\n\n'; + body += '```\n' + excerpt(checkLog) + '\n```\n\n'; + } + + body += '### Next Steps\n\n'; + if (reasons.includes('patch')) { + body += '1. Read the `DECLARED LOCAL PATCHES` section above — it names the patch id, its tracking issue and the reason\n'; + body += '2. Marker missing from disk: restore it with `pnpm shadcn:update `\n'; + body += '3. Anchor no longer found upstream: re-target `find`/`occurrences` in `scripts/shadcn-local-patches.mjs`\n'; + } else if (reasons.includes('registry-blind') && reasons.length === 1) { + body += '1. Check whether the registry URLs in `packages/components/shadcn-components.json` still resolve\n'; + body += '2. Reproduce locally with `pnpm shadcn:check --no-cache` — a local success points at runner egress\n'; + body += '3. If upstream moved for good, re-point the sources; the check has proven nothing for ' + + registryStreak + ' runs\n'; + } else { + body += '1. Open the workflow run linked above and read the failure\n'; + body += '2. Reproduce locally with `pnpm shadcn:analyze` / `pnpm shadcn:check`\n'; + } + body += '4. See [SHADCN_SYNC.md](../blob/main/docs/SHADCN_SYNC.md) for detailed guide\n\n'; + body += '> This issue was automatically created by the Shadcn Components Check workflow.\n'; + + return { title, body }; +} + +// ── Cross-run state ───────────────────────────────────────────────────────── + +/** + * The Actions REST surface this mechanism needs, injectable so the tests never + * touch the network. + * + * The workflow file name is DERIVED from `GITHUB_WORKFLOW_REF` + * (`owner/repo/.github/workflows/shadcn-check.yml@refs/heads/main`) rather than + * spelled out: a hard-coded copy is a second source of truth that a file rename + * would break silently, and silence is the failure mode being closed. + */ +export function createActionsApi({ + token = process.env.GITHUB_TOKEN ?? '', + apiUrl = process.env.GITHUB_API_URL ?? 'https://api.github.com', + repository = process.env.GITHUB_REPOSITORY ?? '', + workflowRef = process.env.GITHUB_WORKFLOW_REF ?? '', + fetchImpl = globalThis.fetch, +} = {}) { + const workflowFile = path.basename((workflowRef.split('@')[0] ?? '').trim()); + if (!workflowFile) throw new Error('GITHUB_WORKFLOW_REF did not yield a workflow file name'); + if (!repository) throw new Error('GITHUB_REPOSITORY is not set'); + + const get = async (route) => { + const res = await fetchImpl(`${apiUrl}${route}`, { + headers: { + accept: 'application/vnd.github+json', + 'x-github-api-version': '2022-11-28', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + }); + if (!res.ok) throw new Error(`GET ${route} -> HTTP ${res.status}`); + return res.json(); + }; + + return { + listRuns: (perPage) => + get(`/repos/${repository}/actions/workflows/${workflowFile}/runs?status=completed&per_page=${perPage}`), + listJobs: (runId) => get(`/repos/${repository}/actions/runs/${runId}/jobs?per_page=100`), + }; +} + +/** + * How many consecutive runs — this one included — found the registry + * unreachable. + * + * ## Why the Actions API and not the alternatives (objectui#3586, measured) + * + * The card named two candidates and asked for the cheaper honest one. Both were + * costed, and two more were found on the way: + * + * previous run's `conclusion` 1 call, and structurally BLIND. #3497's + * tolerance ruling makes an unreachable run exit 0, so its conclusion is + * `success` — byte-identical to a clean run (run 31374857502: `success`, + * 0 registry errors). It cannot express the distinction at any price. + * + * previous runs' STEP conclusions 1 + (N-1) calls = 3 at N=3, `actions: + * read`, no new artifacts, and the state lives as long as the run itself + * (90 days by default, ~13 weekly runs). Emitting it costs two no-op steps. + * CHOSEN. + * + * previous runs' artifacts the existing `shadcn-analysis` artifact already + * carries `check.txt`, so nothing new is emitted — but reading it is list + + * download + unzip per run (1 + 2(N-1) calls plus transfers), and retention + * is 30 days ≈ 4 weekly runs, which leaves no headroom above N=3. + * + * previous runs' annotations 1 + 2(N-1) = 5 calls at N=3, and the state is + * the `::warning::` PROSE, re-parsed a week later. + * + * comments on the alarm issue 1-2 calls with the already-granted + * `issues: write`, and disqualified on design before cost: a tolerated + * unreachable run has no issue to comment on (`label:shadcn-sync` has + * matched 0 issues, ever). Recording there means either opening the issue on + * run 1 — which IS the alarm, contradicting the single-run tolerance ruling + * this card leaves untouched — or a private state issue, i.e. the second + * channel the card forbids. + * + * `actions/cache` ~0 API cost, and disqualified: GitHub evicts entries 7 + * days after last access and the cron is exactly 7 days, so the state sits + * on the eviction boundary. A silently reset streak is an escalation that + * never fires — the failure shape being closed, rebuilt inside the fix. + * + * At most `threshold - 1` job reads happen: the walk stops at the first run + * that was reachable or unknown, and never needs to look past the threshold. + * + * @param {{ currentState: string, threshold?: number, currentRunId?: string|number, api?: object }} input + */ +export async function readRegistryStreak({ + currentState, + threshold = ESCALATION_THRESHOLD, + currentRunId = process.env.GITHUB_RUN_ID ?? '', + api, +}) { + if (currentState !== 'unreachable') return { streak: 0, readable: true, error: '' }; + + const states = ['unreachable']; + try { + const client = api ?? createActionsApi(); + const { workflow_runs: runs = [] } = await client.listRuns(threshold + 2); + const previous = runs.filter((run) => String(run.id) !== String(currentRunId)); + + for (const run of previous) { + if (states.length >= threshold) break; + const { jobs = [] } = await client.listJobs(run.id); + const state = registryStateFromJobs(jobs); + states.push(state); + if (state !== 'unreachable') break; + } + } catch (error) { + return { streak: consecutiveUnreachable(states), readable: false, error: String(error?.message ?? error) }; + } + + return { streak: consecutiveUnreachable(states), readable: true, error: '' }; +} + +// ── Workflow entry point ──────────────────────────────────────────────────── + +const readIfPresent = (file) => { + try { + return fs.readFileSync(file, 'utf8'); + } catch { + return ''; + } +}; + +const appendTo = (envVar, text) => { + const target = process.env[envVar]; + if (target) fs.appendFileSync(target, text.endsWith('\n') ? text : `${text}\n`); +}; + +/** + * Classifies the run, records the verdict where the workflow can read it, and + * writes the alarm body when there is one. + * + * Exits 0 in every classified case, including every alarm: the alarm is the + * issue, not the job colour — #3497's ruling, and the reason gap ③ existed at + * all. The one thing that still turns the job red is this file itself throwing, + * which is the irreducible bottom of the mechanism and the reason the logic was + * moved somewhere it can be unit tested. + */ +export async function main({ api } = {}) { + const checkExit = Number(process.env.CHECK_EXIT_CODE ?? 0); + const analyzeExit = Number(process.env.ANALYZE_EXIT_CODE ?? 0); + + const checkLog = stripAnsi(readIfPresent(process.env.CHECK_RAW_LOG ?? 'check.ansi.txt')); + const analysisLog = stripAnsi(readIfPresent(process.env.ANALYZE_RAW_LOG ?? 'analysis.ansi.txt')); + fs.writeFileSync(process.env.CHECK_LOG ?? 'check.txt', checkLog); + fs.writeFileSync(process.env.ANALYZE_LOG ?? 'analysis.txt', analysisLog); + + const { checkClass, registryErrors, registryState } = classifyCheck({ output: checkLog, exitCode: checkExit }); + const analyzeClass = classifyAnalyze({ exitCode: analyzeExit }); + const { streak, readable, error } = await readRegistryStreak({ currentState: registryState, api }); + const { alarm, reasons } = decideAlarm({ + checkClass, + analyzeClass, + registryStreak: streak, + streakReadable: readable, + }); + + const runUrl = process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : ''; + + const { title, body } = renderAlarmIssue({ + reasons, + checkClass, + checkExit, + analyzeClass, + analyzeExit, + registryErrors, + registryStreak: streak, + streakReadable: readable, + streakError: error, + runUrl, + analysisLog, + checkLog, + }); + + if (alarm) fs.writeFileSync(process.env.ALARM_BODY_FILE ?? 'alarm-issue.md', body); + + appendTo( + 'GITHUB_OUTPUT', + [ + `check_exit_code=${checkExit}`, + `check_class=${checkClass}`, + `analyze_exit_code=${analyzeExit}`, + `analyze_class=${analyzeClass}`, + `registry_errors=${registryErrors}`, + `registry_state=${registryState}`, + `registry_streak=${streak}`, + `streak_readable=${readable}`, + `alarm=${alarm}`, + `alarm_reasons=${reasons.join(',')}`, + `issue_title=${title}`, + ].join('\n'), + ); + + const summary = [ + '### Shadcn component check', + '', + `- \`pnpm shadcn:analyze\` exit code: \`${analyzeExit}\` (class: \`${analyzeClass}\`)`, + `- \`pnpm shadcn:check\` exit code: \`${checkExit}\` (class: \`${checkClass}\`)`, + `- components the registry could not serve: ${registryErrors}`, + ]; + + if (analyzeClass === 'broken') { + console.log(`::error::pnpm shadcn:analyze crashed (exit ${analyzeExit}). Opening/updating the tracking issue.`); + summary.push('- Verdict: the offline analysis step crashed. Tracking issue opened or updated.'); + } + if (checkClass === 'patch') { + console.log(`::error::Declared local patches are failing (exit ${checkExit}). Opening/updating the tracking issue.`); + summary.push('- Verdict: a declared local patch failed. Tracking issue opened or updated.'); + } else if (checkClass === 'broken') { + console.log( + `::error::shadcn:check exited ${checkExit} without a patch verdict — the check itself could not run. Opening/updating the tracking issue.`, + ); + summary.push('- Verdict: the check could not run. Tracking issue opened or updated.'); + } else if (registryErrors > 0) { + if (reasons.includes('registry-blind')) { + console.log( + `::error::The registry has been unreachable for ${streak} consecutive runs (threshold ${ESCALATION_THRESHOLD}) — the upstream-anchor check has not run in that time. Opening/updating the tracking issue.`, + ); + summary.push( + `- Verdict: unreachable for ${streak} consecutive runs — tolerance exhausted. Tracking issue opened or updated.`, + ); + } else { + // Tolerated, but never reported as a clean bill of health: with the + // registry unreachable the upstream-anchor half of the check did not + // execute, so this run proved nothing about upstream. #3497's ruling for + // runs 1..N-1, unchanged — only the counter is new. + console.log( + `::warning::${registryErrors} component(s) could not be fetched from the registry, so the upstream-anchor check did not run. Tolerated by design — no issue opened (consecutive unreachable runs: ${streak} of ${ESCALATION_THRESHOLD}).`, + ); + summary.push( + `- Verdict: no patch failure, but the online half did not run (registry unreachable ${streak} run(s) running, escalates at ${ESCALATION_THRESHOLD}). Tolerated, no issue opened.`, + ); + } + } else if (analyzeClass === 'ok') { + summary.push('- Verdict: all declared local patches still apply to current upstream.'); + } + + if (!readable) { + console.log(`::error::The cross-run registry state could not be read (${error}). Opening/updating the tracking issue.`); + summary.push('- Verdict: the cross-run state read failed — escalation is blind. Tracking issue opened or updated.'); + } + + appendTo('GITHUB_STEP_SUMMARY', summary.join('\n')); + return { alarm, reasons, checkClass, analyzeClass, registryState, streak }; +} + +const invokedDirectly = + process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url; +if (invokedDirectly) { + await main(); +}