diff --git a/.github/workflows/renovate-quarantine-recheck-test.yml b/.github/workflows/renovate-quarantine-recheck-test.yml new file mode 100644 index 0000000..2708936 --- /dev/null +++ b/.github/workflows/renovate-quarantine-recheck-test.yml @@ -0,0 +1,26 @@ +name: Renovate expired-quarantine recheck tests + +on: + pull_request: + paths: + - "scripts/renovate-quarantine-recheck*.mjs" + - ".github/workflows/renovate-quarantine-recheck-test.yml" + push: + branches: [main] + paths: + - "scripts/renovate-quarantine-recheck*.mjs" + - ".github/workflows/renovate-quarantine-recheck-test.yml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Test release-age parser, check selection, and GitHub API orchestration + run: node --test scripts/renovate-quarantine-recheck*.test.mjs diff --git a/deploy/renovate-quarantine-recheck/README.md b/deploy/renovate-quarantine-recheck/README.md new file mode 100644 index 0000000..ad8d1a6 --- /dev/null +++ b/deploy/renovate-quarantine-recheck/README.md @@ -0,0 +1,49 @@ +# Central Renovate quarantine recheck + +One hourly cold-storage systemd timer, using the existing scoped GitHub App +identity through /usr/local/bin/gh and github-app-git-credential. + +This is not a release-age bypass. The script only re-runs an Actions job +after its failure log proves every new crates.io version has aged at least +72 hours, plus a 15-minute margin. The actual CI gate runs again; normal +Renovate automerge alone handles successful PRs. + +- Discover all nonarchived repositories in the GitHub App installation. +- Inspect only open Renovate PRs at the exact current head. +- Select completed failed GitHub Actions quarantine or strict-Rust jobs. +- Require that the sole failed step be the Cargo quarantine. +- Refuse missing timestamps, unexpected reasons, other failed jobs, + queued/in-progress checks, changed PR heads, or repeat run attempts. +- Revalidate the PR head, job and check state immediately before mutation. +- Rerun only the specific failed job, not whole workflow runs. +- JSON-lines audit output includes crate versions and eligibility dates. +- No daemon, database, persistent state, cache or build artifacts. +- Default read-only mode; --apply is required, --max caps one sweep. + +## Installation on cold-storage + +Run the following one time as the existing authorized operator. The source +is the merged ci-workflows main branch; the live checkout is under /srv. + + sudo install -d -o sentinelx -g sentinelx -m 0755 /srv/ci-workflows + sudo -u sentinelx git clone --depth=1 https://github.com/codeandsolder/ci-workflows.git /srv/ci-workflows + sudo install -m 0644 deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service /etc/systemd/system/ + sudo install -m 0644 deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer /etc/systemd/system/ + sudo systemctl daemon-reload + sudo systemctl enable --now renovate-quarantine-recheck.timer + +If /srv/ci-workflows already exists, pull --ff-only rather than re-cloning. +The service updates its checkout before every run; a failed pull blocks retry. +It uses the existing GitHub App helper, not an unscoped PAT. +The privileged ExecStartPre invokes the existing root-owned token-minting oneshot, so the actual fleet scanner can run under NoNewPrivileges=true with a fresh scoped credential. + +## Verification and operations + + sudo -u sentinelx /usr/bin/node /srv/ci-workflows/scripts/renovate-quarantine-recheck.mjs --repo=codeandsolder/sccache-deduplicated + sudo systemctl start renovate-quarantine-recheck.service + sudo journalctl -u renovate-quarantine-recheck.service -n 100 --no-pager + systemctl list-timers renovate-quarantine-recheck.timer + + node --test scripts/renovate-quarantine-recheck*.test.mjs + +Do not deploy this timer on each runner; this is one fleet controller. diff --git a/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service b/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service new file mode 100644 index 0000000..c6878e2 --- /dev/null +++ b/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.service @@ -0,0 +1,28 @@ +[Unit] +Description=Recheck expired Cargo quarantine checks for Renovate +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=sentinelx +Group=sentinelx +WorkingDirectory=/srv/ci-workflows +# Mint a scoped installation token via its existing root-owned systemd oneshot. +ExecStartPre=+/usr/bin/systemctl start github-app-token.service +ExecStartPre=/usr/bin/git -C /srv/ci-workflows pull --ff-only --quiet origin main +ExecStart=/usr/bin/node /srv/ci-workflows/scripts/renovate-quarantine-recheck.mjs --apply --max=10 --grace-minutes=15 +TimeoutStartSec=20min +Nice=10 +IOSchedulingClass=idle +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ReadWritePaths=/srv/ci-workflows +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +LockPersonality=true +RestrictRealtime=true diff --git a/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer b/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer new file mode 100644 index 0000000..d98119c --- /dev/null +++ b/deploy/renovate-quarantine-recheck/renovate-quarantine-recheck.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Hourly check for expired Renovate Cargo quarantines + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=5min +AccuracySec=1min +Persistent=true +Unit=renovate-quarantine-recheck.service + +[Install] +WantedBy=timers.target diff --git a/scripts/renovate-quarantine-recheck.integration.test.mjs b/scripts/renovate-quarantine-recheck.integration.test.mjs new file mode 100644 index 0000000..fb77916 --- /dev/null +++ b/scripts/renovate-quarantine-recheck.integration.test.mjs @@ -0,0 +1,44 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, chmodSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +test('fake GitHub API: dry run is passive, apply reruns only eligible quarantine job', () => { + const root = mkdtempSync(join(tmpdir(), 'renovate-recheck-')); + try { + const bin = join(root, 'bin'); + mkdirSync(bin); + const now = new Date(Date.now() - 4 * 86_400_000).toISOString(); + const stub = join(bin, 'gh'); + writeFileSync(stub, [ + '#!/usr/bin/env node', + "const fs=require('node:fs');", + "const args=process.argv.slice(2);", + "const path=args.at(-1);", + "fs.appendFileSync(process.env.FAKE_GH_CALLS,args.join(' ')+'\\n');", + "const json=x=>process.stdout.write(JSON.stringify(x));", + "if(path==='repos/demo/repo/pulls?state=open&per_page=100&page=1') json([{ number:1, state:'open', user:{login:'renovate[bot]'}, head:{sha:'abc'} }]);", + "else if(path==='repos/demo/repo/commits/abc/check-runs?per_page=100&page=1') json({check_runs:[{ id:123, name:'strict-rust / Strict Rust + Python policy', app:{slug:'github-actions'}, status:'completed', conclusion:'failure' }]});", + "else if(path==='repos/demo/repo/actions/jobs/123') json({id:123,run_id:444,status:'completed',conclusion:'failure',head_sha:'abc',run_attempt:1,steps:[{name:'Quarantine newly introduced crates.io versions across all workspaces',conclusion:'failure'}]});", + "else if(path==='repos/demo/repo/actions/jobs/123/logs') process.stdout.write('Cargo release-age quarantine is still active:\\n- abc 1.0.0: published "+now+", younger than 72h\\n');", + "else if(path==='repos/demo/repo/pulls/1') json({state:'open',head:{sha:'abc'}});", + "else if(path==='repos/demo/repo/actions/jobs/123/rerun' && args.includes('POST')) process.exit(0);", + "else {console.error('unexpected API '+args.join(' '));process.exit(42)}", + ].join('\n')+'\n'); + chmodSync(stub, 0o755); + const script = new URL('./renovate-quarantine-recheck.mjs', import.meta.url); + const env = {...process.env, PATH: bin+':'+process.env.PATH, FAKE_GH_CALLS:join(root,'calls')}; + const dry = spawnSync(process.execPath, [script.pathname, '--repo=demo/repo'], {env,encoding:'utf8'}); + assert.equal(dry.status,0,dry.stderr || dry.stdout); + assert.match(dry.stdout, /"type":"dry-run"/); + assert.equal(readFileSync(env.FAKE_GH_CALLS,'utf8').includes('POST'),false); + const live = spawnSync(process.execPath, [script.pathname, '--repo=demo/repo','--apply'], {env,encoding:'utf8'}); + assert.equal(live.status,0,live.stderr || live.stdout); + assert.match(live.stdout, /"type":"rerun"/); + assert.equal(readFileSync(env.FAKE_GH_CALLS,'utf8').split('POST').length-1,1); + } finally { + rmSync(root,{recursive:true,force:true}); + } +}); diff --git a/scripts/renovate-quarantine-recheck.mjs b/scripts/renovate-quarantine-recheck.mjs new file mode 100644 index 0000000..a0c6a68 --- /dev/null +++ b/scripts/renovate-quarantine-recheck.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node +// Rerun a completed Renovate CI job only when its *sole* failure is an expired +// Cargo publication-age hold. GitHub Actions performs the real quarantine again. +import { execFileSync } from 'node:child_process'; + +export const BOT_LOGINS = new Set(['renovate[bot]', 'renovate']); +const AGE_HEADER = 'Cargo release-age quarantine is still active:'; +const PUBLICATION = /^-\s+([a-zA-Z0-9_-]+)\s+(\S+): published (\S+), younger than (\d+)h$/; +const QUARANTINE_STEP = /quarantine|release.age/i; +const AGE_CHECK_NAME = /^(?:quarantine|strict.rust(?:\s*\/|$))/i; +const ANSI = /\x1b\[[0-9;]*m/g; +const DEFAULT_GRACE_MS = 15 * 60_000; + +export function parseQuarantineHold(log) { + const lines = log.replace(ANSI, '').split(/\r?\n/); + let markerCount = 0; + const blocks = []; + for (let i = 0; i < lines.length; i++) { + if (!lines[i].includes(AGE_HEADER)) continue; + markerCount++; + const entries = []; + for (let j = i + 1; j < lines.length; j++) { + // CI log timestamps prefix the actual stderr messages. + const body = lines[j].replace(/^\d{4}-\d\d-\d\dT\S+?\s+/, '').trim(); + if (!body.startsWith('- ')) break; + const m = PUBLICATION.exec(body); + if (!m) return { eligible: false, reason: 'unknown-quarantine-reason' }; + const published = Date.parse(m[3]); + const hours = Number(m[4]); + if (!Number.isFinite(published) || !Number.isSafeInteger(hours) || hours < 72) { + return { eligible: false, reason: 'invalid-age-metadata' }; + } + entries.push({ name: m[1], version: m[2], publishedAt: m[3], + eligibleAt: published + hours * 3_600_000, hours }); + } + if (entries.length === 0) return { eligible: false, reason: 'no-crates' }; + blocks.push(...entries); + } + if (markerCount === 0) return { eligible: false, reason: 'not-a-quarantine-hold' }; + const eligibleAt = Math.max(...blocks.map(e => e.eligibleAt)); + if (!Number.isFinite(eligibleAt)) return { eligible: false, reason: 'invalid-eligibility' }; + return { eligible: true, blockers: blocks, eligibleAt }; +} + +export function classifyJob(job, prHead) { + if (job.status !== 'completed' || job.conclusion !== 'failure' || job.head_sha !== prHead) { + return 'stale-or-not-failed'; + } + // Prevent a failing rerun of a permanently broken PR from looping hourly. + if (job.run_attempt !== 1) return 'already-retried'; + const failures = (job.steps || []).filter(s => s.conclusion === 'failure'); + if (!failures.length || failures.some(s => !QUARANTINE_STEP.test(s.name))) { + return 'other-job-failure'; + } + return null; +} + +export function decide(hold, now, graceMs = DEFAULT_GRACE_MS) { + if (!hold.eligible) return hold.reason; + return now >= hold.eligibleAt + graceMs ? null : 'release-too-young'; +} + +export function eligibleChecks(checks) { + const runs = checks.filter(c => c.app?.slug === 'github-actions'); + if (runs.some(c => c.status !== 'completed')) return null; + const failures = runs.filter(c => c.conclusion === 'failure'); + if (!failures.length || failures.some(c => !AGE_CHECK_NAME.test(c.name))) return null; + return failures; +} + +function api(path, options = {}) { + const args = ['api', path]; + if (options.method) args.splice(1, 0, '-X', options.method); + const raw = execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, + timeout: 45_000, stdio: ['ignore', 'pipe', 'pipe'] }); + return options.raw ? raw : options.method ? true : JSON.parse(raw); +} + +function pages(endpoint) { + const all = []; + for (let page = 1; ; page++) { + if (page > 100) throw Error('unexpected API pagination depth: ' + endpoint); + const payload = api(endpoint + (endpoint.includes('?') ? '&' : '?') + + 'per_page=100&page=' + page); + const values = Array.isArray(payload) ? payload : + payload.repositories ?? payload.check_runs ?? []; + all.push(...values); + if (values.length < 100) return all; + } +} + +function emit(type, fields = {}) { + process.stdout.write(JSON.stringify({ type, ...fields }) + '\n'); +} + +function optionsOf(args) { + const options = { apply: false, repo: null, max: 10, graceMs: DEFAULT_GRACE_MS }; + for (const arg of args) { + if (arg === '--apply') options.apply = true; + else if (arg.startsWith('--repo=')) options.repo = arg.slice(7); + else if (arg.startsWith('--max=')) options.max = Number(arg.slice(6)); + else if (arg.startsWith('--grace-minutes=')) options.graceMs = Number(arg.slice(16)) * 60_000; + else if (arg === '--help') { + console.log('Usage: node scripts/renovate-quarantine-recheck.mjs [--apply] [--repo=owner/name] [--max=10] [--grace-minutes=15]'); + process.exit(0); + } else throw Error('unknown argument ' + arg); + } + if (!Number.isSafeInteger(options.max) || options.max < 1 || options.max > 100 || + !Number.isFinite(options.graceMs) || options.graceMs < 0 || + (options.repo && !/^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/.test(options.repo))) { + throw Error('invalid options'); + } + return options; +} + +async function main() { + const opts = optionsOf(process.argv.slice(2)); + const now = Date.now(); + const repos = opts.repo + ? [{ full_name: opts.repo, archived: false }] + : pages('installation/repositories').filter(r => !r.archived && !r.disabled); + let actions = 0, candidates = 0, failures = 0; + for (const repo of repos) { + const full = repo.full_name; + let prs; + try { + prs = pages('repos/' + full + '/pulls?state=open'); + } catch (err) { + failures++; + emit('api-error', { repo: full, stage: 'pulls', message: err.message.slice(0, 250) }); + continue; + } + for (const pr of prs.filter(p => BOT_LOGINS.has(p.user?.login))) { + if (actions >= opts.max) break; + const head = pr.head.sha; + let checks; + try { + checks = pages('repos/' + full + '/commits/' + head + '/check-runs'); + } catch (err) { + failures++; + emit('api-error', { repo: full, pr: pr.number, stage: 'checks', + message: err.message.slice(0, 250) }); + continue; + } + const ageChecks = eligibleChecks(checks); + if (!ageChecks) continue; + for (const check of ageChecks) { + if (actions >= opts.max) break; + const jobId = check.id; + try { + const job = api('repos/' + full + '/actions/jobs/' + jobId); + const jobReason = classifyJob(job, head); + if (jobReason) { + emit('skip', { repo: full, pr: pr.number, jobId, reason: jobReason }); + continue; + } + const hold = parseQuarantineHold(api('repos/' + full + '/actions/jobs/' + + jobId + '/logs', { raw: true })); + const reason = decide(hold, now, opts.graceMs); + candidates++; + if (reason) { + emit('skip', { repo: full, pr: pr.number, jobId, reason, + eligibleAt: hold.eligibleAt ? new Date(hold.eligibleAt + opts.graceMs).toISOString() : null }); + continue; + } + // Revalidate the head and the exact job immediately before any mutation. + const live = api('repos/' + full + '/pulls/' + pr.number); + if (live.state !== 'open' || live.head.sha !== head || + classifyJob(api('repos/' + full + '/actions/jobs/' + jobId), head)) { + emit('skip', { repo: full, pr: pr.number, jobId, reason: 'changed-during-scan' }); + continue; + } + const currentChecks = pages('repos/' + full + '/commits/' + head + '/check-runs'); + if (!(eligibleChecks(currentChecks) || []).some(c => c.id === jobId)) { + emit('skip', { repo: full, pr: pr.number, jobId, reason: 'check-state-changed' }); + continue; + } + const payload = { repo: full, pr: pr.number, jobId, head, runId: job.run_id, + blockingCrates: hold.blockers.map(b => b.name + ' ' + b.version), + eligibleAt: new Date(hold.eligibleAt + opts.graceMs).toISOString() }; + if (!opts.apply) emit('dry-run', payload); + else { + api('repos/' + full + '/actions/jobs/' + jobId + '/rerun', { method: 'POST' }); + emit('rerun', payload); + } + actions++; + } catch (err) { + failures++; + emit('api-error', { repo: full, pr: pr.number, jobId, stage: 'job', + message: err.message.slice(0, 250) }); + } + } + } + if (actions >= opts.max) break; + } + emit('summary', { repos: repos.length, candidates, actions, failures, apply: opts.apply }); + if (failures) process.exitCode = 1; +} + +if (import.meta.url === 'file://' + process.argv[1]) { + main().catch(err => { emit('fatal', { message: err.message.slice(0, 300) }); process.exitCode = 1; }); +} diff --git a/scripts/renovate-quarantine-recheck.test.mjs b/scripts/renovate-quarantine-recheck.test.mjs new file mode 100644 index 0000000..e5a7444 --- /dev/null +++ b/scripts/renovate-quarantine-recheck.test.mjs @@ -0,0 +1,43 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseQuarantineHold, classifyJob, decide, eligibleChecks } from './renovate-quarantine-recheck.mjs'; + +const log = [ + '2026-11-12T01:00:00Z Cargo release-age quarantine is still active:', + '2026-11-12T01:00:01Z - abc 1.2.3: published 2026-11-11T21:56:00+00:00, younger than 72h', + '2026-11-12T01:00:01Z - def 0.2.4: published 2026-11-11T21:57:08.123456+00:00, younger than 72h', + '2026-11-12T01:00:02Z ##[error]Process completed with exit code 1.', +].join('\n'); + +test('all crates must age out, including fractions and UTC offsets', () => { + const hold = parseQuarantineHold(log); + assert.equal(hold.eligible, true); + assert.equal(hold.blockers.length, 2); + assert.equal(decide(hold, Date.parse('2026-11-14T22:20:00Z')), null); + assert.equal(decide(hold, Date.parse('2026-11-14T21:57:08Z')), 'release-too-young'); +}); +test('missing metadata, malformed timestamps and unexpected reasons stay blocked', () => { + for (const replacement of ['- x 1.0: missing authoritative crates.io publication time', + '- x 1.0: crates.io lookup failed: timeout', + '- x 1.0: published invalid, younger than 72h', + '- x 1.0: published 2026-11-11T21:56:00+00:00, younger than 12h']) { + assert.equal(parseQuarantineHold(log.replace(/- abc 1\.2\.3:.+/, replacement)).eligible, false); + } + assert.equal(parseQuarantineHold('error: build failed').eligible, false); +}); +test('only one sole quarantine step, original attempt, matching commit qualifies', () => { + const job = { status: 'completed', conclusion: 'failure', head_sha: 'abc', run_attempt: 1, + steps: [{ name: 'Quarantine newly introduced crates.io versions across all workspaces', conclusion: 'failure' }] }; + assert.equal(classifyJob(job, 'abc'), null); + assert.equal(classifyJob({ ...job, run_attempt: 2 }, 'abc'), 'already-retried'); + assert.equal(classifyJob(job, 'def'), 'stale-or-not-failed'); + assert.equal(classifyJob({ ...job, steps: [...job.steps, { name: 'Build', conclusion: 'failure' }] }, 'abc'), 'other-job-failure'); +}); +test('do not retry when other checks are pending or failed', () => { + const age = { name: 'quarantine', app: { slug: 'github-actions' }, status: 'completed', conclusion: 'failure', id: 1 }; + assert.deepEqual(eligibleChecks([age]), [age]); + const strictAge = { ...age, id: 8, name: 'strict-rust / Strict Rust + Python policy' }; + assert.deepEqual(eligibleChecks([strictAge]), [strictAge]); + assert.equal(eligibleChecks([age, { ...age, id: 2, name: 'build' }]), null); + assert.equal(eligibleChecks([age, { ...age, id: 3, status: 'in_progress' }]), null); +});