feat(ci): automatically rerun expired Renovate quarantine checks

.github/workflows/renovate-quarantine-recheck-test.yml

@@ -0,0 +1,26 @@
+name: Renovate expired-quarantine recheck tests
+
+on:
+  pull_request:
+    paths:
+      - "scripts/renovate-quarantine-recheck*.mjs"
+      - ".github/workflows/renovate-quarantine-recheck-test.yml"
+  push:
+    branches: [main]
+    paths:
+      - "scripts/renovate-quarantine-recheck*.mjs"
+      - ".github/workflows/renovate-quarantine-recheck-test.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Test release-age parser, check selection, and GitHub API orchestration
+        run: node --test scripts/renovate-quarantine-recheck*.test.mjs

deploy/renovate-quarantine-recheck/README.md

@@ -0,0 +1,49 @@
+# Central Renovate quarantine recheck
+
+One hourly cold-storage systemd timer, using the existing scoped GitHub App
+identity through /usr/local/bin/gh and github-app-git-credential.
+
+This is not a release-age bypass. The script only re-runs an Actions job
+after its failure log proves every new crates.io version has aged at least
+72 hours, plus a 15-minute margin. The actual CI gate runs again; normal
+Renovate automerge alone handles successful PRs.
+
+- Discover all nonarchived repositories in the GitHub App installation.
+- Inspect only open Renovate PRs at the exact current head.
+- Select completed failed GitHub Actions quarantine or strict-Rust jobs.
+- Require that the sole failed step be the Cargo quarantine.
+- Refuse missing timestamps, unexpected reasons, other failed jobs,
+  queued/in-progress checks, changed PR heads, or repeat run attempts.
+- Revalidate the PR head, job and check state immediately before mutation.
+- Rerun only the specific failed job, not whole workflow runs.
+- JSON-lines audit output includes crate versions and eligibility dates.
+- No daemon, database, persistent state, cache or build artifacts.
+- Default read-only mode; --apply is required, --max caps one sweep.
+
+## Installation on cold-storage
+
+Run the following one time as the existing authorized operator. The source
+is the merged ci-workflows main branch; the live checkout is under /srv.
+
+    sudo install -d -o sentinelx -g sentinelx -m 0755 /srv/ci-workflows
+    sudo -u sentinelx git clone --depth=1 https://github.com/codeandsolder/ci-workflows.git /srv/ci-workflows
+    sudo install -m 0644 deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service /etc/systemd/system/
+    sudo install -m 0644 deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer /etc/systemd/system/
+    sudo systemctl daemon-reload
+    sudo systemctl enable --now renovate-quarantine-recheck.timer
+
+If /srv/ci-workflows already exists, pull --ff-only rather than re-cloning.
+The service updates its checkout before every run; a failed pull blocks retry.
+It uses the existing GitHub App helper, not an unscoped PAT.
+The privileged ExecStartPre invokes the existing root-owned token-minting oneshot, so the actual fleet scanner can run under NoNewPrivileges=true with a fresh scoped credential.
+
+## Verification and operations
+
+    sudo -u sentinelx /usr/bin/node /srv/ci-workflows/scripts/renovate-quarantine-recheck.mjs --repo=codeandsolder/sccache-deduplicated
+    sudo systemctl start renovate-quarantine-recheck.service
+    sudo journalctl -u renovate-quarantine-recheck.service -n 100 --no-pager
+    systemctl list-timers renovate-quarantine-recheck.timer
+
+    node --test scripts/renovate-quarantine-recheck*.test.mjs
+
+Do not deploy this timer on each runner; this is one fleet controller.

deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Recheck expired Cargo quarantine checks for Renovate
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=sentinelx
+Group=sentinelx
+WorkingDirectory=/srv/ci-workflows
+# Mint a scoped installation token via its existing root-owned systemd oneshot.
+ExecStartPre=+/usr/bin/systemctl start github-app-token.service
+ExecStartPre=/usr/bin/git -C /srv/ci-workflows pull --ff-only --quiet origin main
+ExecStart=/usr/bin/node /srv/ci-workflows/scripts/renovate-quarantine-recheck.mjs --apply --max=10 --grace-minutes=15
+TimeoutStartSec=20min
+Nice=10
+IOSchedulingClass=idle
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ReadWritePaths=/srv/ci-workflows
+ProtectHome=true
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectKernelLogs=true
+ProtectControlGroups=true
+LockPersonality=true
+RestrictRealtime=true

deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Hourly check for expired Renovate Cargo quarantines
+
+[Timer]
+OnCalendar=hourly
+RandomizedDelaySec=5min
+AccuracySec=1min
+Persistent=true
+Unit=renovate-quarantine-recheck.service
+
+[Install]
+WantedBy=timers.target

scripts/renovate-quarantine-recheck.integration.test.mjs

@@ -0,0 +1,44 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, chmodSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+test('fake GitHub API: dry run is passive, apply reruns only eligible quarantine job', () => {
+  const root = mkdtempSync(join(tmpdir(), 'renovate-recheck-'));
+  try {
+    const bin = join(root, 'bin');
+    mkdirSync(bin);
+    const now = new Date(Date.now() - 4 * 86_400_000).toISOString();
+    const stub = join(bin, 'gh');
+    writeFileSync(stub, [
+      '#!/usr/bin/env node',
+      "const fs=require('node:fs');",
+      "const args=process.argv.slice(2);",
+      "const path=args.at(-1);",
+      "fs.appendFileSync(process.env.FAKE_GH_CALLS,args.join(' ')+'\\n');",
+      "const json=x=>process.stdout.write(JSON.stringify(x));",
+      "if(path==='repos/demo/repo/pulls?state=open&per_page=100&page=1') json([{ number:1, state:'open', user:{login:'renovate[bot]'}, head:{sha:'abc'} }]);",
+      "else if(path==='repos/demo/repo/commits/abc/check-runs?per_page=100&page=1') json({check_runs:[{ id:123, name:'strict-rust / Strict Rust + Python policy', app:{slug:'github-actions'}, status:'completed', conclusion:'failure' }]});",
+      "else if(path==='repos/demo/repo/actions/jobs/123') json({id:123,run_id:444,status:'completed',conclusion:'failure',head_sha:'abc',run_attempt:1,steps:[{name:'Quarantine newly introduced crates.io versions across all workspaces',conclusion:'failure'}]});",
+      "else if(path==='repos/demo/repo/actions/jobs/123/logs') process.stdout.write('Cargo release-age quarantine is still active:\\n- abc 1.0.0: published "+now+", younger than 72h\\n');",
+      "else if(path==='repos/demo/repo/pulls/1') json({state:'open',head:{sha:'abc'}});",
+      "else if(path==='repos/demo/repo/actions/jobs/123/rerun' && args.includes('POST')) process.exit(0);",
+      "else {console.error('unexpected API '+args.join(' '));process.exit(42)}",
+    ].join('\n')+'\n');
+    chmodSync(stub, 0o755);
+    const script = new URL('./renovate-quarantine-recheck.mjs', import.meta.url);
+    const env = {...process.env, PATH: bin+':'+process.env.PATH, FAKE_GH_CALLS:join(root,'calls')};
+    const dry = spawnSync(process.execPath, [script.pathname, '--repo=demo/repo'], {env,encoding:'utf8'});
+    assert.equal(dry.status,0,dry.stderr || dry.stdout);
+    assert.match(dry.stdout, /"type":"dry-run"/);
+    assert.equal(readFileSync(env.FAKE_GH_CALLS,'utf8').includes('POST'),false);
+    const live = spawnSync(process.execPath, [script.pathname, '--repo=demo/repo','--apply'], {env,encoding:'utf8'});
+    assert.equal(live.status,0,live.stderr || live.stdout);
+    assert.match(live.stdout, /"type":"rerun"/);
+    assert.equal(readFileSync(env.FAKE_GH_CALLS,'utf8').split('POST').length-1,1);
+  } finally {
+    rmSync(root,{recursive:true,force:true});
+  }
+});

scripts/renovate-quarantine-recheck.mjs

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+// Rerun a completed Renovate CI job only when its *sole* failure is an expired
+// Cargo publication-age hold. GitHub Actions performs the real quarantine again.
+import { execFileSync } from 'node:child_process';
+
+export const BOT_LOGINS = new Set(['renovate[bot]', 'renovate']);
+const AGE_HEADER = 'Cargo release-age quarantine is still active:';
+const PUBLICATION = /^-\s+([a-zA-Z0-9_-]+)\s+(\S+): published (\S+), younger than (\d+)h$/;
+const QUARANTINE_STEP = /quarantine|release.age/i;
+const AGE_CHECK_NAME = /^(?:quarantine|strict.rust(?:\s*\/|$))/i;
+const ANSI = /\x1b\[[0-9;]*m/g;
+const DEFAULT_GRACE_MS = 15 * 60_000;
+
+export function parseQuarantineHold(log) {
+  const lines = log.replace(ANSI, '').split(/\r?\n/);
+  let markerCount = 0;
+  const blocks = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (!lines[i].includes(AGE_HEADER)) continue;
+    markerCount++;
+    const entries = [];
+    for (let j = i + 1; j < lines.length; j++) {
+      // CI log timestamps prefix the actual stderr messages.
+      const body = lines[j].replace(/^\d{4}-\d\d-\d\dT\S+?\s+/, '').trim();
+      if (!body.startsWith('- ')) break;
+      const m = PUBLICATION.exec(body);
+      if (!m) return { eligible: false, reason: 'unknown-quarantine-reason' };
+      const published = Date.parse(m[3]);
+      const hours = Number(m[4]);
+      if (!Number.isFinite(published) || !Number.isSafeInteger(hours) || hours < 72) {
+        return { eligible: false, reason: 'invalid-age-metadata' };
+      }
+      entries.push({ name: m[1], version: m[2], publishedAt: m[3],
+        eligibleAt: published + hours * 3_600_000, hours });
+    }
+    if (entries.length === 0) return { eligible: false, reason: 'no-crates' };
+    blocks.push(...entries);
+  }
+  if (markerCount === 0) return { eligible: false, reason: 'not-a-quarantine-hold' };
+  const eligibleAt = Math.max(...blocks.map(e => e.eligibleAt));
+  if (!Number.isFinite(eligibleAt)) return { eligible: false, reason: 'invalid-eligibility' };
+  return { eligible: true, blockers: blocks, eligibleAt };
+}
+
+export function classifyJob(job, prHead) {
+  if (job.status !== 'completed' || job.conclusion !== 'failure' || job.head_sha !== prHead) {
+    return 'stale-or-not-failed';
+  }
+  // Prevent a failing rerun of a permanently broken PR from looping hourly.
+  if (job.run_attempt !== 1) return 'already-retried';
+  const failures = (job.steps || []).filter(s => s.conclusion === 'failure');
+  if (!failures.length || failures.some(s => !QUARANTINE_STEP.test(s.name))) {
+    return 'other-job-failure';
+  }
+  return null;
+}
+
+export function decide(hold, now, graceMs = DEFAULT_GRACE_MS) {
+  if (!hold.eligible) return hold.reason;
+  return now >= hold.eligibleAt + graceMs ? null : 'release-too-young';
+}
+
+export function eligibleChecks(checks) {
+  const runs = checks.filter(c => c.app?.slug === 'github-actions');
+  if (runs.some(c => c.status !== 'completed')) return null;
+  const failures = runs.filter(c => c.conclusion === 'failure');
+  if (!failures.length || failures.some(c => !AGE_CHECK_NAME.test(c.name))) return null;
+  return failures;
+}
+
+function api(path, options = {}) {
+  const args = ['api', path];
+  if (options.method) args.splice(1, 0, '-X', options.method);
+  const raw = execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024,
+    timeout: 45_000, stdio: ['ignore', 'pipe', 'pipe'] });
+  return options.raw ? raw : options.method ? true : JSON.parse(raw);
+}
+
+function pages(endpoint) {
+  const all = [];
+  for (let page = 1; ; page++) {
+    if (page > 100) throw Error('unexpected API pagination depth: ' + endpoint);
+    const payload = api(endpoint + (endpoint.includes('?') ? '&' : '?') +
+      'per_page=100&page=' + page);
+    const values = Array.isArray(payload) ? payload :
+      payload.repositories ?? payload.check_runs ?? [];
+    all.push(...values);
+    if (values.length < 100) return all;
+  }
+}
+
+function emit(type, fields = {}) {
+  process.stdout.write(JSON.stringify({ type, ...fields }) + '\n');
+}
+
+function optionsOf(args) {
+  const options = { apply: false, repo: null, max: 10, graceMs: DEFAULT_GRACE_MS };
+  for (const arg of args) {
+    if (arg === '--apply') options.apply = true;
+    else if (arg.startsWith('--repo=')) options.repo = arg.slice(7);
+    else if (arg.startsWith('--max=')) options.max = Number(arg.slice(6));
+    else if (arg.startsWith('--grace-minutes=')) options.graceMs = Number(arg.slice(16)) * 60_000;
+    else if (arg === '--help') {
+      console.log('Usage: node scripts/renovate-quarantine-recheck.mjs [--apply] [--repo=owner/name] [--max=10] [--grace-minutes=15]');
+      process.exit(0);
+    } else throw Error('unknown argument ' + arg);
+  }
+  if (!Number.isSafeInteger(options.max) || options.max < 1 || options.max > 100 ||
+      !Number.isFinite(options.graceMs) || options.graceMs < 0 ||
+      (options.repo && !/^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/.test(options.repo))) {
+    throw Error('invalid options');
+  }
+  return options;
+}
+
+async function main() {
+  const opts = optionsOf(process.argv.slice(2));
+  const now = Date.now();
+  const repos = opts.repo
+    ? [{ full_name: opts.repo, archived: false }]
+    : pages('installation/repositories').filter(r => !r.archived && !r.disabled);
+  let actions = 0, candidates = 0, failures = 0;
+  for (const repo of repos) {
+    const full = repo.full_name;
+    let prs;
+    try {
+      prs = pages('repos/' + full + '/pulls?state=open');
+    } catch (err) {
+      failures++;
+      emit('api-error', { repo: full, stage: 'pulls', message: err.message.slice(0, 250) });
+      continue;
+    }
+    for (const pr of prs.filter(p => BOT_LOGINS.has(p.user?.login))) {
+      if (actions >= opts.max) break;
+      const head = pr.head.sha;
+      let checks;
+      try {
+        checks = pages('repos/' + full + '/commits/' + head + '/check-runs');
+      } catch (err) {
+        failures++;
+        emit('api-error', { repo: full, pr: pr.number, stage: 'checks',
+          message: err.message.slice(0, 250) });
+        continue;
+      }
+      const ageChecks = eligibleChecks(checks);
+      if (!ageChecks) continue;
+      for (const check of ageChecks) {
+        if (actions >= opts.max) break;
+        const jobId = check.id;
+        try {
+          const job = api('repos/' + full + '/actions/jobs/' + jobId);
+          const jobReason = classifyJob(job, head);
+          if (jobReason) {
+            emit('skip', { repo: full, pr: pr.number, jobId, reason: jobReason });
+            continue;
+          }
+          const hold = parseQuarantineHold(api('repos/' + full + '/actions/jobs/' +
+            jobId + '/logs', { raw: true }));
+          const reason = decide(hold, now, opts.graceMs);
+          candidates++;
+          if (reason) {
+            emit('skip', { repo: full, pr: pr.number, jobId, reason,
+              eligibleAt: hold.eligibleAt ? new Date(hold.eligibleAt + opts.graceMs).toISOString() : null });
+            continue;
+          }
+          // Revalidate the head and the exact job immediately before any mutation.
+          const live = api('repos/' + full + '/pulls/' + pr.number);
+          if (live.state !== 'open' || live.head.sha !== head ||
+              classifyJob(api('repos/' + full + '/actions/jobs/' + jobId), head)) {
+            emit('skip', { repo: full, pr: pr.number, jobId, reason: 'changed-during-scan' });
+            continue;
+          }
+          const currentChecks = pages('repos/' + full + '/commits/' + head + '/check-runs');
+          if (!(eligibleChecks(currentChecks) || []).some(c => c.id === jobId)) {
+            emit('skip', { repo: full, pr: pr.number, jobId, reason: 'check-state-changed' });
+            continue;
+          }
+          const payload = { repo: full, pr: pr.number, jobId, head, runId: job.run_id,
+            blockingCrates: hold.blockers.map(b => b.name + ' ' + b.version),
+            eligibleAt: new Date(hold.eligibleAt + opts.graceMs).toISOString() };
+          if (!opts.apply) emit('dry-run', payload);
+          else {
+            api('repos/' + full + '/actions/jobs/' + jobId + '/rerun', { method: 'POST' });
+            emit('rerun', payload);
+          }
+          actions++;
+        } catch (err) {
+          failures++;
+          emit('api-error', { repo: full, pr: pr.number, jobId, stage: 'job',
+            message: err.message.slice(0, 250) });
+        }
+      }
+    }
+    if (actions >= opts.max) break;
+  }
+  emit('summary', { repos: repos.length, candidates, actions, failures, apply: opts.apply });
+  if (failures) process.exitCode = 1;
+}
+
+if (import.meta.url === 'file://' + process.argv[1]) {
+  main().catch(err => { emit('fatal', { message: err.message.slice(0, 300) }); process.exitCode = 1; });
+}