Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 68 additions & 7 deletions .github/scripts/integration-governance.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,74 @@ async function snapshot(github, subject) {

// Readonly adaptation of verify/type.go, verify/issue.go and notes/common/prefix.go
// at SOURCE (SPDX-License-Identifier: Apache-2.0, Copyright 2020 The Kubernetes Authors).
// https://www.apache.org/licenses/LICENSE-2.0 . Unlike the
// advisory image wrapper, unavailable validation never becomes success.
// https://www.apache.org/licenses/LICENSE-2.0 . The same two verdicts back
// the required `PR Type` / `PR Issue` checks that pr-verifier.yml posts
// (.github/scripts/pr-verifier.cjs, #18764) and the integration `contents`
// policy below, where unavailable validation never becomes success.
// [type, String(), shortcodes and emoji in the order PRTypeFromTitle tries them]
const PR_TYPES = [
['feature', ':sparkles:', '✨'], ['bugfix', ':bug:', '🐛'], ['docs', ':book:', '📖'],
['infra', ':seedling:', '🌱'], ['breaking', ':warning:', '⚠'], ['release', ':rocket:', '🚀'],
['infra', ':running:', '🏃'],
];
const EMOJI = {feature: '✨', bugfix: '🐛', docs: '📖', infra: '🌱', breaking: '⚠', release: '🚀'};

// notes/common/prefix.go PRTypeFromTitle: [type or null, title without prefix].
function prTypeFromTitle(title) {
title = title.trim();
for (const [type, code, emoji] of PR_TYPES) {
if (title.startsWith(code) || title.startsWith(emoji)) {
title = title.startsWith(code) ? title.slice(code.length) : title;
title = title.startsWith(emoji) ? title.slice(emoji.length) : title;
title = title.startsWith('\uFE0F') ? title.slice(1) : title;
return [type, title.trim()];
}
}
return [null, title];
}

// Go's %#q: a backquoted string when it can be, else a quoted one.
const goQuote = text => (/[`\u0000-\u0008\u000A-\u001F\u007F\uFEFF]/.test(text) ? JSON.stringify(text) : `\`${text}\``);

// verify/type.go verifyPRType; output title from verify/main.go.
function prTypeVerdict(rawTitle) {
const title = String(rawTitle ?? '').replace(/^\W?WIP\W/iu, '').trim().replace(/^\[[\w.-]*\]/, '').trim();
const [type, finalTitle] = prTypeFromTitle(title);
if (!type) {
return {name: 'PR Type', conclusion: 'failure', title: 'PR Type in Title',
summary: 'No matching PR type indicator found in title.',
text: `I saw a title of ${goQuote(title)}, which doesn't seem to have any of the acceptable prefixes.

You need to have one of these as the prefix of your PR title:

- Breaking change: ⚠ (\`:warning:\`)
- Non-breaking feature: ✨ (\`:sparkles:\`)
- Patch fix: 🐛 (\`:bug:\`)
- Docs: 📖 (\`:book:\`)
- Release: 🚀 (\`:rocket:\`)
- Infra/Tests/Other: 🌱 (\`:seedling:\`)

More details can be found at [sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md](https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md).`};
}
return {name: 'PR Type', conclusion: 'success', title: 'PR Type in Title',
summary: `Found ${EMOJI[type]} PR (${type})`, text: `Final title:\n\n\t${finalTitle}\n`};
}

// verify/issue.go checkIssueInTitle; output title from verify/main.go.
function prIssueVerdict(rawTitle) {
const [, title] = prTypeFromTitle(String(rawTitle ?? ''));
if (/#[0-9]+\b/.test(title)) {
return {name: 'PR Issue', conclusion: 'failure', title: 'Issue/PR tag in PR title',
summary: 'Your PR has an Issue or PR number in the title.',
text: `The title should just be descriptive.
Issue numbers belong in the PR body as either \`Fixes #XYZ\` (if it closes the issue or PR), or something like \`Related to #XYZ\` (if it's just related).`};
}
return {name: 'PR Issue', conclusion: 'success', title: 'Issue/PR tag in PR title',
summary: 'Your PR title does not contain any Issue or PR tags', text: ''};
}

function verifyContents(title) {
const trimmed = title.replace(/^\W?WIP\W/iu, '').trim().replace(/^\[[\w.-]*\]/, '').trim();
const prefixes = ['✨', '🐛', '📖', '🌱', '⚠', '🚀', '🏃',
':sparkles:', ':bug:', ':book:', ':seedling:', ':warning:', ':rocket:', ':running:'];
if (!prefixes.some(prefix => trimmed.startsWith(prefix)) || /#[0-9]+\b/.test(title)) {
if (prTypeVerdict(title).conclusion !== 'success' || prIssueVerdict(title).conclusion !== 'success') {
throw new Error(`PR content policy rejected title; immutable source ${SOURCE}`);
}
}
Expand Down Expand Up @@ -155,4 +216,4 @@ async function verify(kind, subject, members, github) {
}
}

module.exports = {SOURCE, row, digest, snapshot, verifyContents, verify};
module.exports = {SOURCE, row, digest, snapshot, prTypeVerdict, prIssueVerdict, verifyContents, verify};
17 changes: 17 additions & 0 deletions .github/scripts/integration-governance.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,23 @@ test('pinned pure verifier actually validates the immutable upstream rules', ()
}
});

test('PR Type and PR Issue verdicts keep the upstream check text (#18764)', () => {
assert.deepEqual(policy.prTypeVerdict('WIP: [scope] ✨\uFE0F add thing'), {name: 'PR Type',
conclusion: 'success', title: 'PR Type in Title', summary: 'Found ✨ PR (feature)',
text: 'Final title:\n\n\tadd thing\n'});
assert.equal(policy.prTypeVerdict(':running: chore').summary, 'Found 🌱 PR (infra)');
const missing = policy.prTypeVerdict('📝 proposal');
assert.equal(missing.conclusion, 'failure');
assert.equal(missing.summary, 'No matching PR type indicator found in title.');
assert.ok(missing.text.startsWith("I saw a title of `📝 proposal`, which doesn't seem to have"));
assert.deepEqual(policy.prIssueVerdict('🐛 fix flake'), {name: 'PR Issue', conclusion: 'success',
title: 'Issue/PR tag in PR title', summary: 'Your PR title does not contain any Issue or PR tags', text: ''});
const tagged = policy.prIssueVerdict('🐛 fix #123');
assert.equal(tagged.conclusion, 'failure');
assert.equal(tagged.summary, 'Your PR has an Issue or PR number in the title.');
assert.equal(policy.prIssueVerdict('🐛 fix #12a').conclusion, 'success');
});

test('member snapshots never accept another head/repo/member or an unrelated merge', async () => {
assert.equal((await policy.snapshot(client(member()), subject())).length, 1);
const mutations = [
Expand Down
89 changes: 89 additions & 0 deletions .github/scripts/pr-verifier.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
'use strict';

// Records the required `PR Type` and `PR Issue` check runs for one PR head
// (#18764). Each check is ONE create call that already carries its final
// status, conclusion and output: no in_progress run is ever created, so no
// later update can fail and leave one hanging. The verdicts come from the
// in-repo port of the upstream verifier (integration-governance.cjs).
//
// A content verdict (a title without a type prefix, an issue number in the
// title) is reported by the check itself; the wrapper job stays green for it.
// Failing to RECORD a verdict is different: the job fails and says so, and a
// completed `failure` run carrying the reason is posted when GitHub allows it.

const {prTypeVerdict, prIssueVerdict} = require('./integration-governance.cjs');

const CHECK_NAMES = ['PR Type', 'PR Issue'];

const describe = error =>
[error?.status, error?.message || String(error)].filter(Boolean).join(' ');

async function create(github, owner, repo, headSha, verdict) {
const output = {title: verdict.title, summary: verdict.summary};
if (verdict.text) output.text = verdict.text;
const {data} = await github.rest.checks.create({
owner, repo, name: verdict.name, head_sha: headSha,
status: 'completed', conclusion: verdict.conclusion, output,
});
// The response is GitHub's own record of the run: anything other than the
// completed verdict on this head means the required context is not settled.
if (data?.name !== verdict.name || data?.head_sha !== headSha ||
data?.status !== 'completed' || data?.conclusion !== verdict.conclusion) {
throw new Error(`GitHub recorded ${JSON.stringify({name: data?.name, head_sha: data?.head_sha,
status: data?.status, conclusion: data?.conclusion})} instead of a completed ${verdict.conclusion}`);
}
return data;
}

async function record({github, core, owner, repo, pr}) {
const headSha = pr?.head?.sha;
if (!/^[0-9a-f]{40}$/.test(headSha || '') || typeof pr?.title !== 'string') {
core.setFailed('PR Verifier could not record its result: the event carries no PR head SHA or title (#18764).');
return {recorded: [], unrecorded: CHECK_NAMES.slice()};
}
const verdicts = [prTypeVerdict(pr.title), prIssueVerdict(pr.title)];
const recorded = [], unrecorded = [];
for (const verdict of verdicts) {
let error;
// One retry covers a transient API error; each attempt is a single,
// already-completed create, so a retry can never leave a run in progress.
for (let attempt = 1; attempt <= 2; attempt++) {
try {
const run = await create(github, owner, repo, headSha, verdict);
core.info(`${verdict.name}: ${verdict.conclusion} — ${verdict.summary} (check run ${run.id} on ${headSha})`);
recorded.push(verdict.name);
error = undefined;
break;
} catch (caught) {
error = caught;
core.info(`${verdict.name}: attempt ${attempt} to record the result failed: ${describe(caught)}`);
}
}
if (!error) {
if (verdict.conclusion !== 'success') {
// Advisory for the wrapper job: the required context carries it.
core.warning(`${verdict.name}: ${verdict.summary} The required "${verdict.name}" check reports this; the wrapper job stays green.`);
}
continue;
}
unrecorded.push(verdict.name);
const reason = `PR Verifier could not record its "${verdict.name}" result on ${headSha}: ${describe(error)}`;
core.error(reason);
try {
await github.rest.checks.create({
owner, repo, name: verdict.name, head_sha: headSha,
status: 'completed', conclusion: 'failure',
output: {title: verdict.title, summary: `${reason}. This is a verifier failure, not a verdict on the PR: re-run this PR Verifier run, or add or remove any label, to record a real result (#18764).`},
});
} catch (fallback) {
core.error(`${verdict.name}: the fallback failure run could not be posted either: ${describe(fallback)}`);
}
}
if (unrecorded.length) {
core.setFailed(`PR Verifier could not record its result for ${unrecorded.join(' and ')} on ${headSha}; ` +
'the required check(s) may be missing or show no verdict for this head (#18764).');
}
return {recorded, unrecorded};
}

module.exports = {CHECK_NAMES, record};
143 changes: 143 additions & 0 deletions .github/scripts/pr-verifier.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
'use strict';
// #18764: PR Verifier records PR Type / PR Issue with one completed create
// each, and fails the job only when a result cannot be recorded.
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {CHECK_NAMES, record} = require('./pr-verifier.cjs');

const headSha = 'a'.repeat(40);
const pr = title => ({number: 7, title, head: {sha: headSha}});

function fakeCore() {
const core = {failed: [], warnings: [], errors: [], infos: []};
core.setFailed = message => core.failed.push(message);
core.warning = message => core.warnings.push(message);
core.error = message => core.errors.push(message);
core.info = message => core.infos.push(message);
return core;
}

// failures: name -> number of create calls for that name that throw.
function fakeGithub({failures = {}, record: mangle} = {}) {
const calls = [];
const remaining = {...failures};
let id = 100;
const checks = {
create: async params => {
calls.push({method: 'create', ...params});
if (remaining[params.name] > 0) {
remaining[params.name]--;
throw Object.assign(new Error('Not Found'), {status: 404});
}
const data = {id: ++id, name: params.name, head_sha: params.head_sha,
status: params.status, conclusion: params.conclusion};
return {data: mangle ? mangle(data) : data};
},
};
for (const forbidden of ['update', 'get', 'listForRef', 'listForSuite']) {
checks[forbidden] = async () => { throw new Error(`checks.${forbidden} must not be called`); };
}
return {calls, github: {rest: {checks}, request: async () => { throw new Error('raw requests must not be made'); }}};
}

const run = (github, core, title) =>
record({github, core, owner: 'hivecommons', repo: 'hive', pr: pr(title)});

test('check names are the required contexts', () => {
assert.deepEqual(CHECK_NAMES, ['PR Type', 'PR Issue']);
});

test('a good title records two completed successes in one create each', async () => {
const {calls, github} = fakeGithub();
const core = fakeCore();
const result = await run(github, core, '🐛 fix: stuck checks');
assert.deepEqual(result, {recorded: ['PR Type', 'PR Issue'], unrecorded: []});
assert.equal(calls.length, 2);
assert.deepEqual(calls.map(c => c.name), ['PR Type', 'PR Issue']);
for (const call of calls) {
assert.equal(call.method, 'create');
assert.equal(call.head_sha, headSha);
assert.equal(call.status, 'completed');
assert.equal(call.conclusion, 'success');
}
assert.equal(calls[0].output.title, 'PR Type in Title');
assert.equal(calls[0].output.summary, 'Found 🐛 PR (bugfix)');
assert.equal(calls[1].output.title, 'Issue/PR tag in PR title');
assert.equal(calls[1].output.summary, 'Your PR title does not contain any Issue or PR tags');
assert.equal(calls[1].output.text, undefined);
assert.deepEqual(core.failed, []);
assert.deepEqual(core.warnings, []);
});

test('a bad title is a completed failure on the check, advisory for the job', async () => {
const {calls, github} = fakeGithub();
const core = fakeCore();
await run(github, core, '📝 proposal for #12');
assert.deepEqual(calls.map(c => [c.name, c.status, c.conclusion]),
[['PR Type', 'completed', 'failure'], ['PR Issue', 'completed', 'failure']]);
assert.equal(calls[0].output.summary, 'No matching PR type indicator found in title.');
assert.equal(calls[1].output.summary, 'Your PR has an Issue or PR number in the title.');
assert.deepEqual(core.failed, []);
assert.equal(core.warnings.length, 2);
});

test('one transient create error is retried with the real verdict', async () => {
const {calls, github} = fakeGithub({failures: {'PR Type': 1}});
const core = fakeCore();
const result = await run(github, core, '✨ feature');
assert.deepEqual(result.unrecorded, []);
assert.deepEqual(calls.map(c => [c.name, c.conclusion]),
[['PR Type', 'success'], ['PR Type', 'success'], ['PR Issue', 'success']]);
assert.deepEqual(core.failed, []);
});

test('a result that cannot be recorded fails the job and posts a completed failure', async () => {
const {calls, github} = fakeGithub({failures: {'PR Issue': 2}});
const core = fakeCore();
const result = await run(github, core, '✨ feature');
assert.deepEqual(result, {recorded: ['PR Type'], unrecorded: ['PR Issue']});
const fallback = calls.at(-1);
assert.equal(fallback.name, 'PR Issue');
assert.equal(fallback.status, 'completed');
assert.equal(fallback.conclusion, 'failure');
assert.match(fallback.output.summary, /could not record its "PR Issue" result/);
assert.match(fallback.output.summary, /404 Not Found/);
assert.equal(core.failed.length, 1);
assert.match(core.failed[0], /could not record its result for PR Issue/);
});

test('the job still fails when even the fallback cannot be posted', async () => {
const {github} = fakeGithub({failures: {'PR Type': 3, 'PR Issue': 3}});
const core = fakeCore();
const result = await run(github, core, '✨ feature');
assert.deepEqual(result.unrecorded, ['PR Type', 'PR Issue']);
assert.equal(core.failed.length, 1);
assert.match(core.failed[0], /PR Type and PR Issue/);
assert.ok(core.errors.some(e => /fallback failure run could not be posted/.test(e)));
});

test('a create GitHub answers with anything but the completed verdict is not a recorded result', async () => {
const {github} = fakeGithub({record: data => ({...data, status: 'in_progress', conclusion: null})});
const core = fakeCore();
const result = await run(github, core, '✨ feature');
assert.deepEqual(result.unrecorded, ['PR Type', 'PR Issue']);
assert.equal(core.failed.length, 1);
});

test('an event without a head SHA or title fails instead of passing silently', async () => {
const {calls, github} = fakeGithub();
const core = fakeCore();
await record({github, core, owner: 'o', repo: 'r', pr: {title: '✨ x', head: {}}});
await record({github, core, owner: 'o', repo: 'r', pr: undefined});
assert.equal(calls.length, 0);
assert.equal(core.failed.length, 2);
});

test('the verifier never updates or looks up a check run', () => {
const source = fs.readFileSync(path.join(__dirname, 'pr-verifier.cjs'), 'utf8');
for (const forbidden of [/checks\.update/, /listForRef/, /PATCH/, /in_progress/, /status: 'queued'/]) {
assert.doesNotMatch(source.replace(/^\s*\/\/.*$/gm, ''), forbidden);
}
});
32 changes: 30 additions & 2 deletions .github/scripts/test-native-queue-workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,35 @@ def test_dependabot_runs_the_same_trusted_pr_verifier(self):
self.assertEqual(job["name"], "verify PR contents")
verify_step = next(step for step in job["steps"]
if step.get("name") == "Verifier action")
self.assertIn("docker run --rm", verify_step["run"])
# #18764: one completed create per check from the in-repo verifier,
# no image and no create-then-update.
self.assertTrue(verify_step["uses"].startswith("actions/github-script@"))
self.assertIn(".github/scripts/pr-verifier.cjs", verify_step["with"]["script"])
self.assertNotIn("continue-on-error", verify_step)

def test_pr_verifier_writes_completed_checks_and_fails_when_it_cannot(self):
# #18764: the image created PR Type / PR Issue in_progress and then
# PATCHed them; a 404 on the PATCH left the required runs in_progress
# for ever while the continue-on-error wrapper stayed green.
verifier = workflow("pr-verifier.yml")
job = verifier["jobs"]["verify"]
self.assertNotIn("packages", verifier["permissions"])
self.assertEqual(verifier["permissions"]["checks"], "write")
for step in job["steps"]:
self.assertNotIn("continue-on-error", step, step.get("name"))
self.assertNotIn("docker", str(step.get("run", "")) + str(step.get("uses", "")))
self.assertNotIn("env", job)
checkout = next(s for s in job["steps"] if str(s.get("uses", "")).startswith("actions/checkout@"))
# pull_request_target: github.sha is the default-branch tip, never PR code.
self.assertEqual(checkout["with"]["ref"], "${{ github.sha }}")
self.assertFalse(checkout["with"]["persist-credentials"])
script = (ROOT / ".github/scripts/pr-verifier.cjs").read_text()
code = re.sub(r"(?m)^\s*//.*$", "", script)
self.assertIn("CHECK_NAMES = ['PR Type', 'PR Issue']", code)
self.assertIn("status: 'completed'", code)
for forbidden in ("checks.update", "listForRef", "PATCH", "in_progress"):
self.assertNotIn(forbidden, code)
self.assertIn("core.setFailed(", code)

def test_autofix_push_deterministically_redispatches_the_verifier(self):
# #16567: the autofix push landed after the last prow label and no
Expand All @@ -146,7 +174,7 @@ def test_autofix_push_deterministically_redispatches_the_verifier(self):
verify_step = next(s for s in steps if s.get("id") == "verifier")
self.assertIn("steps.dispatch_event.outputs.stale != 'true'", verify_step["if"])
self.assertIn("steps.dispatch_event.outputs.event_path", verify_step["env"]["VERIFIER_EVENT_PATH"])
self.assertIn('-v "$VERIFIER_EVENT_PATH:/github/event.json:ro"', verify_step["run"])
self.assertIn("process.env.VERIFIER_EVENT_PATH", verify_step["with"]["script"])

guard = workflow("changelog-fragment-guard.yml")
self.assertEqual(guard["permissions"]["actions"], "write")
Expand Down
Loading
Loading