Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions .github/scripts/native-queue-policy.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,23 @@ class ProviderUnavailable extends Error {
}
}

// A provider that completed red on this exact head is that PR's own verdict
// (a missing sign-off, a missing changelog fragment, a failed verifier), never
// a broken admission workflow. Name it as such: four PRs each red on their own
// provider read like one shared incident and got filed as one (#18280). It is
// a hard failure, deliberately not PendingEvidence: waiting cannot change a
// completed verdict, and only a new push to the PR re-runs the providers.
const PROVIDER_VERDICT_NOTE =
'per-PR provider verdict, not a shared CI incident: fix it on this PR; the next push re-evaluates';

class ProviderVerdict extends Error {
constructor(provider, conclusion, head) {
super(`${provider}: ${conclusion} on ${head} (${PROVIDER_VERDICT_NOTE})`);
this.provider = provider;
this.conclusion = conclusion;
}
}

function dcoEnumerationUnavailable(run) {
// This text only classifies a diagnostic, never supplies passing evidence.
// Call only after validating provider, head and completed status.
Expand Down Expand Up @@ -134,7 +151,7 @@ function evidenceForHead(runs, head, checks) {
// Existing PR path scoping legitimately skips technical jobs only. Policy
// proofs, especially external DCO, must have actually succeeded.
const scopedSkip = Object.hasOwn(BUILD_CHECKS, name) && run.conclusion === 'skipped';
if (run.conclusion !== 'success' && !scopedSkip) throw new Error(`${name}: ${run.conclusion}`);
if (run.conclusion !== 'success' && !scopedSkip) throw new ProviderVerdict(name, run.conclusion, head);
return { run, source };
});
}
Expand Down Expand Up @@ -229,7 +246,7 @@ async function assertProwDCO(github, repo, head) {
.sort((a, b) => b.id - a.id)[0];
if (!status) throw new PendingEvidence(`Missing prow ${PROW.context} on ${head}`);
if (status.state === 'pending') throw new PendingEvidence(`prow ${PROW.context} is pending`);
if (status.state !== 'success') throw new Error(`prow ${PROW.context}: ${status.state}`);
if (status.state !== 'success') throw new ProviderVerdict(`prow ${PROW.context}`, status.state, head);
}

async function assertEvidence(github, repo, number, head, checks, pr) {
Expand Down Expand Up @@ -427,7 +444,7 @@ module.exports = {
ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS,
PendingEvidence, membersForGroup, assertEligible, evidenceForHead, closedNoop, evaluate,
isTransientAPIError, isCosmeticLabelFlip, replayLastVerdict, BLOCKED_LABEL,
ProviderUnavailable, FORWARD_MERGE, PROW,
ProviderUnavailable, ProviderVerdict, PROVIDER_VERDICT_NOTE, FORWARD_MERGE, PROW,
DCO_ENUMERATION_SUMMARY_PREFIX, DCO_ENUMERATION_SUMMARY_FALLBACK,
DCO_PROVIDER_UNAVAILABLE_TEXT, DCO_PROVIDER_RUNBOOK,
};
34 changes: 29 additions & 5 deletions .github/scripts/native-queue-policy.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const {
ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS,
PendingEvidence, membersForGroup, assertEligible, evidenceForHead, closedNoop, evaluate,
isTransientAPIError, isCosmeticLabelFlip, replayLastVerdict,
ProviderUnavailable, FORWARD_MERGE, PROW,
ProviderUnavailable, ProviderVerdict, PROVIDER_VERDICT_NOTE, FORWARD_MERGE, PROW,
DCO_ENUMERATION_SUMMARY_PREFIX,
DCO_ENUMERATION_SUMMARY_FALLBACK,
DCO_PROVIDER_UNAVAILABLE_TEXT,
Expand Down Expand Up @@ -203,7 +203,30 @@ test('UNKNOWN reporting requires trusted exact-head completed DCO and the specif
error => !(error instanceof ProviderUnavailable));
}
assert.throws(() => evidenceForHead([{ ...dco, conclusion: 'failure' }], sha(11), { DCO: null }),
error => !(error instanceof ProviderUnavailable) && error.message === 'DCO: failure');
error => !(error instanceof ProviderUnavailable) && error instanceof ProviderVerdict &&
error.message.startsWith(`DCO: failure on ${sha(11)} `));
});

test('a red provider is named as a per-PR verdict, not a shared CI incident (#18280)', async () => {
for (const [name, conclusion] of [
['DCO', 'action_required'], ['changelog-fragment-guard', 'failure'],
['verify PR contents', 'cancelled'], ['Sign-off survives the squash', 'timed_out'],
]) {
const f = fixture('pull_request_target');
f.state.runs.get(sha(11)).find(r => r.name === name).conclusion = conclusion;
await assert.rejects(evaluate(f), error => {
assert.ok(error instanceof ProviderVerdict, `${name} must be a ProviderVerdict`);
assert.ok(!(error instanceof PendingEvidence), 'a completed verdict is not worth waiting on');
assert.equal(error.provider, name);
assert.equal(error.conclusion, conclusion);
assert.ok(error.message.startsWith(`${name}: ${conclusion} on ${sha(11)} (`), error.message);
assert.ok(error.message.includes(PROVIDER_VERDICT_NOTE), error.message);
return true;
});
}
// A path-scoped skip of a technical job is still not a verdict.
const runs = checks(sha(11), BUILD_CHECKS).map(r => ({ ...r, conclusion: 'skipped' }));
assert.doesNotThrow(() => evidenceForHead(runs, sha(11), BUILD_CHECKS));
});

// #17030: v6-topup.yml's PR whose probot DCO rejects a commit already on v5.
Expand Down Expand Up @@ -277,19 +300,20 @@ test('another head branch, base, fork or author keeps requiring probot DCO', asy
]) {
const f = forwardMergeFixture();
change(f.state.prs[0]);
await assert.rejects(evaluate(f), error => error.message === 'DCO: action_required');
await assert.rejects(evaluate(f), error => error.message === `DCO: action_required on ${sha(11)} (${PROVIDER_VERDICT_NOTE})`);
assert.equal(f.state.seen.length, 0);
}
const v5 = forwardMergeFixture();
v5.state.prs[0].base.ref = 'v5';
v5.context.payload.pull_request.base.ref = 'v5';
await assert.rejects(evaluate(v5), error => error.message === 'DCO: action_required');
await assert.rejects(evaluate(v5), error => error.message === `DCO: action_required on ${sha(11)} (${PROVIDER_VERDICT_NOTE})`);
});

test('forward-merge path fails without a successful prow-created dco status', async () => {
const failed = forwardMergeFixture();
failed.state.statuses.push({ id: 2, context: 'dco', state: 'failure', creator: { login: PROW.login, id: PROW.id } });
await assert.rejects(evaluate(failed), /prow dco: failure.*DCO path: prow dco/);
await assert.rejects(evaluate(failed), error => error instanceof ProviderVerdict &&
/prow dco: failure on .*per-PR provider verdict.*DCO path: prow dco/.test(error.message));
// Any workflow token can post a `dco` status; it never stands in for prow.
const forged = forwardMergeFixture();
forged.state.statuses = [{ id: 3, context: 'dco', state: 'success', creator: { login: 'github-actions[bot]', id: 41898282 } }];
Expand Down
12 changes: 12 additions & 0 deletions .github/scripts/test-native-queue-workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,18 @@ def test_cosmetic_label_flips_skip_the_evidence_wait(self):
self.assertIn("steps.label-flip.outputs.cosmetic != 'true'", wait["if"])
self.assertLess(steps.index(gate), steps.index(wait))

def test_red_provider_is_reported_not_thrown(self):
# A PR's own red provider must surface as a named per-PR verdict, not
# an "Unhandled error" stack trace that reads like a broken shared
# workflow and gets filed as an incident (#18280).
steps = workflow("native-queue-policy.yml")["jobs"]["policy"]["steps"]
wait = next(s for s in steps if s.get("name", "").startswith("Validate exact-head evidence"))
script = wait["with"]["script"]
self.assertIn("error instanceof ProviderVerdict", script)
verdict = script[script.index("error instanceof ProviderVerdict"):]
self.assertLess(verdict.index("core.setFailed(error.message)"),
verdict.index("instanceof PendingEvidence"))

def test_dependabot_runs_the_same_trusted_pr_verifier(self):
verifier = workflow("pr-verifier.yml")
events = verifier.get("on", verifier.get(True))
Expand Down
13 changes: 12 additions & 1 deletion .github/workflows/native-queue-policy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const { evaluate, PendingEvidence, ProviderUnavailable } = require('./.github/scripts/native-queue-policy.cjs');
const { evaluate, PendingEvidence, ProviderUnavailable, ProviderVerdict } = require('./.github/scripts/native-queue-policy.cjs');
const attempts = 90;
const intervalMs = 15000;
for (let attempt = 0; attempt < attempts; attempt++) {
Expand All @@ -90,6 +90,17 @@ jobs:
core.setFailed(error.message);
return;
}
if (error instanceof ProviderVerdict) {
// The PR's own provider is red (#18280): a per-PR verdict to
// fix on the PR, reported plainly rather than as an
// unhandled exception that reads like a broken workflow.
await core.summary
.addHeading('Admission blocked: ' + error.provider + ' is ' + error.conclusion + ' on this PR')
.addRaw(error.message)
.write();
core.setFailed(error.message);
return;
}
if (!(error instanceof PendingEvidence) || attempt === attempts - 1) throw error;
core.info(error.message);
await new Promise(resolve => setTimeout(resolve, intervalMs));
Expand Down
Loading