diff --git a/.github/scripts/native-queue-policy.cjs b/.github/scripts/native-queue-policy.cjs index 9bd9b8df13..b90fc0f2da 100644 --- a/.github/scripts/native-queue-policy.cjs +++ b/.github/scripts/native-queue-policy.cjs @@ -58,6 +58,23 @@ class ProviderUnavailable extends Error { } } +// A provider that completed red on this exact head is that PR's own verdict +// (a missing sign-off, a missing changelog fragment, a failed verifier), never +// a broken admission workflow. Name it as such: four PRs each red on their own +// provider read like one shared incident and got filed as one (#18280). It is +// a hard failure, deliberately not PendingEvidence: waiting cannot change a +// completed verdict, and only a new push to the PR re-runs the providers. +const PROVIDER_VERDICT_NOTE = + 'per-PR provider verdict, not a shared CI incident: fix it on this PR; the next push re-evaluates'; + +class ProviderVerdict extends Error { + constructor(provider, conclusion, head) { + super(`${provider}: ${conclusion} on ${head} (${PROVIDER_VERDICT_NOTE})`); + this.provider = provider; + this.conclusion = conclusion; + } +} + function dcoEnumerationUnavailable(run) { // This text only classifies a diagnostic, never supplies passing evidence. // Call only after validating provider, head and completed status. @@ -134,7 +151,7 @@ function evidenceForHead(runs, head, checks) { // Existing PR path scoping legitimately skips technical jobs only. Policy // proofs, especially external DCO, must have actually succeeded. const scopedSkip = Object.hasOwn(BUILD_CHECKS, name) && run.conclusion === 'skipped'; - if (run.conclusion !== 'success' && !scopedSkip) throw new Error(`${name}: ${run.conclusion}`); + if (run.conclusion !== 'success' && !scopedSkip) throw new ProviderVerdict(name, run.conclusion, head); return { run, source }; }); } @@ -229,7 +246,7 @@ async function assertProwDCO(github, repo, head) { .sort((a, b) => b.id - a.id)[0]; if (!status) throw new PendingEvidence(`Missing prow ${PROW.context} on ${head}`); if (status.state === 'pending') throw new PendingEvidence(`prow ${PROW.context} is pending`); - if (status.state !== 'success') throw new Error(`prow ${PROW.context}: ${status.state}`); + if (status.state !== 'success') throw new ProviderVerdict(`prow ${PROW.context}`, status.state, head); } async function assertEvidence(github, repo, number, head, checks, pr) { @@ -427,7 +444,7 @@ module.exports = { ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS, PendingEvidence, membersForGroup, assertEligible, evidenceForHead, closedNoop, evaluate, isTransientAPIError, isCosmeticLabelFlip, replayLastVerdict, BLOCKED_LABEL, - ProviderUnavailable, FORWARD_MERGE, PROW, + ProviderUnavailable, ProviderVerdict, PROVIDER_VERDICT_NOTE, FORWARD_MERGE, PROW, DCO_ENUMERATION_SUMMARY_PREFIX, DCO_ENUMERATION_SUMMARY_FALLBACK, DCO_PROVIDER_UNAVAILABLE_TEXT, DCO_PROVIDER_RUNBOOK, }; diff --git a/.github/scripts/native-queue-policy.test.cjs b/.github/scripts/native-queue-policy.test.cjs index 8e7d626d63..ea58100a1c 100644 --- a/.github/scripts/native-queue-policy.test.cjs +++ b/.github/scripts/native-queue-policy.test.cjs @@ -5,7 +5,7 @@ const { ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS, PendingEvidence, membersForGroup, assertEligible, evidenceForHead, closedNoop, evaluate, isTransientAPIError, isCosmeticLabelFlip, replayLastVerdict, - ProviderUnavailable, FORWARD_MERGE, PROW, + ProviderUnavailable, ProviderVerdict, PROVIDER_VERDICT_NOTE, FORWARD_MERGE, PROW, DCO_ENUMERATION_SUMMARY_PREFIX, DCO_ENUMERATION_SUMMARY_FALLBACK, DCO_PROVIDER_UNAVAILABLE_TEXT, @@ -203,7 +203,30 @@ test('UNKNOWN reporting requires trusted exact-head completed DCO and the specif error => !(error instanceof ProviderUnavailable)); } assert.throws(() => evidenceForHead([{ ...dco, conclusion: 'failure' }], sha(11), { DCO: null }), - error => !(error instanceof ProviderUnavailable) && error.message === 'DCO: failure'); + error => !(error instanceof ProviderUnavailable) && error instanceof ProviderVerdict && + error.message.startsWith(`DCO: failure on ${sha(11)} `)); +}); + +test('a red provider is named as a per-PR verdict, not a shared CI incident (#18280)', async () => { + for (const [name, conclusion] of [ + ['DCO', 'action_required'], ['changelog-fragment-guard', 'failure'], + ['verify PR contents', 'cancelled'], ['Sign-off survives the squash', 'timed_out'], + ]) { + const f = fixture('pull_request_target'); + f.state.runs.get(sha(11)).find(r => r.name === name).conclusion = conclusion; + await assert.rejects(evaluate(f), error => { + assert.ok(error instanceof ProviderVerdict, `${name} must be a ProviderVerdict`); + assert.ok(!(error instanceof PendingEvidence), 'a completed verdict is not worth waiting on'); + assert.equal(error.provider, name); + assert.equal(error.conclusion, conclusion); + assert.ok(error.message.startsWith(`${name}: ${conclusion} on ${sha(11)} (`), error.message); + assert.ok(error.message.includes(PROVIDER_VERDICT_NOTE), error.message); + return true; + }); + } + // A path-scoped skip of a technical job is still not a verdict. + const runs = checks(sha(11), BUILD_CHECKS).map(r => ({ ...r, conclusion: 'skipped' })); + assert.doesNotThrow(() => evidenceForHead(runs, sha(11), BUILD_CHECKS)); }); // #17030: v6-topup.yml's PR whose probot DCO rejects a commit already on v5. @@ -277,19 +300,20 @@ test('another head branch, base, fork or author keeps requiring probot DCO', asy ]) { const f = forwardMergeFixture(); change(f.state.prs[0]); - await assert.rejects(evaluate(f), error => error.message === 'DCO: action_required'); + await assert.rejects(evaluate(f), error => error.message === `DCO: action_required on ${sha(11)} (${PROVIDER_VERDICT_NOTE})`); assert.equal(f.state.seen.length, 0); } const v5 = forwardMergeFixture(); v5.state.prs[0].base.ref = 'v5'; v5.context.payload.pull_request.base.ref = 'v5'; - await assert.rejects(evaluate(v5), error => error.message === 'DCO: action_required'); + await assert.rejects(evaluate(v5), error => error.message === `DCO: action_required on ${sha(11)} (${PROVIDER_VERDICT_NOTE})`); }); test('forward-merge path fails without a successful prow-created dco status', async () => { const failed = forwardMergeFixture(); failed.state.statuses.push({ id: 2, context: 'dco', state: 'failure', creator: { login: PROW.login, id: PROW.id } }); - await assert.rejects(evaluate(failed), /prow dco: failure.*DCO path: prow dco/); + await assert.rejects(evaluate(failed), error => error instanceof ProviderVerdict && + /prow dco: failure on .*per-PR provider verdict.*DCO path: prow dco/.test(error.message)); // Any workflow token can post a `dco` status; it never stands in for prow. const forged = forwardMergeFixture(); forged.state.statuses = [{ id: 3, context: 'dco', state: 'success', creator: { login: 'github-actions[bot]', id: 41898282 } }]; diff --git a/.github/scripts/test-native-queue-workflows.py b/.github/scripts/test-native-queue-workflows.py index 26a806b6f3..7ed06431dd 100644 --- a/.github/scripts/test-native-queue-workflows.py +++ b/.github/scripts/test-native-queue-workflows.py @@ -99,6 +99,18 @@ def test_cosmetic_label_flips_skip_the_evidence_wait(self): self.assertIn("steps.label-flip.outputs.cosmetic != 'true'", wait["if"]) self.assertLess(steps.index(gate), steps.index(wait)) + def test_red_provider_is_reported_not_thrown(self): + # A PR's own red provider must surface as a named per-PR verdict, not + # an "Unhandled error" stack trace that reads like a broken shared + # workflow and gets filed as an incident (#18280). + steps = workflow("native-queue-policy.yml")["jobs"]["policy"]["steps"] + wait = next(s for s in steps if s.get("name", "").startswith("Validate exact-head evidence")) + script = wait["with"]["script"] + self.assertIn("error instanceof ProviderVerdict", script) + verdict = script[script.index("error instanceof ProviderVerdict"):] + self.assertLess(verdict.index("core.setFailed(error.message)"), + verdict.index("instanceof PendingEvidence")) + def test_dependabot_runs_the_same_trusted_pr_verifier(self): verifier = workflow("pr-verifier.yml") events = verifier.get("on", verifier.get(True)) diff --git a/.github/workflows/native-queue-policy.yml b/.github/workflows/native-queue-policy.yml index 9469ed0203..d761e558c9 100644 --- a/.github/workflows/native-queue-policy.yml +++ b/.github/workflows/native-queue-policy.yml @@ -74,7 +74,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { evaluate, PendingEvidence, ProviderUnavailable } = require('./.github/scripts/native-queue-policy.cjs'); + const { evaluate, PendingEvidence, ProviderUnavailable, ProviderVerdict } = require('./.github/scripts/native-queue-policy.cjs'); const attempts = 90; const intervalMs = 15000; for (let attempt = 0; attempt < attempts; attempt++) { @@ -90,6 +90,17 @@ jobs: core.setFailed(error.message); return; } + if (error instanceof ProviderVerdict) { + // The PR's own provider is red (#18280): a per-PR verdict to + // fix on the PR, reported plainly rather than as an + // unhandled exception that reads like a broken workflow. + await core.summary + .addHeading('Admission blocked: ' + error.provider + ' is ' + error.conclusion + ' on this PR') + .addRaw(error.message) + .write(); + core.setFailed(error.message); + return; + } if (!(error instanceof PendingEvidence) || attempt === attempts - 1) throw error; core.info(error.message); await new Promise(resolve => setTimeout(resolve, intervalMs));