diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index dced6e68ad..54ce3ade11 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -46,13 +46,74 @@ async function snapshot(github, subject) { // Readonly adaptation of verify/type.go, verify/issue.go and notes/common/prefix.go // at SOURCE (SPDX-License-Identifier: Apache-2.0, Copyright 2020 The Kubernetes Authors). -// https://www.apache.org/licenses/LICENSE-2.0 . Unlike the -// advisory image wrapper, unavailable validation never becomes success. +// https://www.apache.org/licenses/LICENSE-2.0 . The same two verdicts back +// the required `PR Type` / `PR Issue` checks that pr-verifier.yml posts +// (.github/scripts/pr-verifier.cjs, #18764) and the integration `contents` +// policy below, where unavailable validation never becomes success. +// [type, String(), shortcodes and emoji in the order PRTypeFromTitle tries them] +const PR_TYPES = [ + ['feature', ':sparkles:', '✨'], ['bugfix', ':bug:', 'πŸ›'], ['docs', ':book:', 'πŸ“–'], + ['infra', ':seedling:', '🌱'], ['breaking', ':warning:', '⚠'], ['release', ':rocket:', 'πŸš€'], + ['infra', ':running:', 'πŸƒ'], +]; +const EMOJI = {feature: '✨', bugfix: 'πŸ›', docs: 'πŸ“–', infra: '🌱', breaking: '⚠', release: 'πŸš€'}; + +// notes/common/prefix.go PRTypeFromTitle: [type or null, title without prefix]. +function prTypeFromTitle(title) { + title = title.trim(); + for (const [type, code, emoji] of PR_TYPES) { + if (title.startsWith(code) || title.startsWith(emoji)) { + title = title.startsWith(code) ? title.slice(code.length) : title; + title = title.startsWith(emoji) ? title.slice(emoji.length) : title; + title = title.startsWith('\uFE0F') ? title.slice(1) : title; + return [type, title.trim()]; + } + } + return [null, title]; +} + +// Go's %#q: a backquoted string when it can be, else a quoted one. +const goQuote = text => (/[`\u0000-\u0008\u000A-\u001F\u007F\uFEFF]/.test(text) ? JSON.stringify(text) : `\`${text}\``); + +// verify/type.go verifyPRType; output title from verify/main.go. +function prTypeVerdict(rawTitle) { + const title = String(rawTitle ?? '').replace(/^\W?WIP\W/iu, '').trim().replace(/^\[[\w.-]*\]/, '').trim(); + const [type, finalTitle] = prTypeFromTitle(title); + if (!type) { + return {name: 'PR Type', conclusion: 'failure', title: 'PR Type in Title', + summary: 'No matching PR type indicator found in title.', + text: `I saw a title of ${goQuote(title)}, which doesn't seem to have any of the acceptable prefixes. + +You need to have one of these as the prefix of your PR title: + +- Breaking change: ⚠ (\`:warning:\`) +- Non-breaking feature: ✨ (\`:sparkles:\`) +- Patch fix: πŸ› (\`:bug:\`) +- Docs: πŸ“– (\`:book:\`) +- Release: πŸš€ (\`:rocket:\`) +- Infra/Tests/Other: 🌱 (\`:seedling:\`) + +More details can be found at [sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md](https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md).`}; + } + return {name: 'PR Type', conclusion: 'success', title: 'PR Type in Title', + summary: `Found ${EMOJI[type]} PR (${type})`, text: `Final title:\n\n\t${finalTitle}\n`}; +} + +// verify/issue.go checkIssueInTitle; output title from verify/main.go. +function prIssueVerdict(rawTitle) { + const [, title] = prTypeFromTitle(String(rawTitle ?? '')); + if (/#[0-9]+\b/.test(title)) { + return {name: 'PR Issue', conclusion: 'failure', title: 'Issue/PR tag in PR title', + summary: 'Your PR has an Issue or PR number in the title.', + text: `The title should just be descriptive. +Issue numbers belong in the PR body as either \`Fixes #XYZ\` (if it closes the issue or PR), or something like \`Related to #XYZ\` (if it's just related).`}; + } + return {name: 'PR Issue', conclusion: 'success', title: 'Issue/PR tag in PR title', + summary: 'Your PR title does not contain any Issue or PR tags', text: ''}; +} + function verifyContents(title) { - const trimmed = title.replace(/^\W?WIP\W/iu, '').trim().replace(/^\[[\w.-]*\]/, '').trim(); - const prefixes = ['✨', 'πŸ›', 'πŸ“–', '🌱', '⚠', 'πŸš€', 'πŸƒ', - ':sparkles:', ':bug:', ':book:', ':seedling:', ':warning:', ':rocket:', ':running:']; - if (!prefixes.some(prefix => trimmed.startsWith(prefix)) || /#[0-9]+\b/.test(title)) { + if (prTypeVerdict(title).conclusion !== 'success' || prIssueVerdict(title).conclusion !== 'success') { throw new Error(`PR content policy rejected title; immutable source ${SOURCE}`); } } @@ -155,4 +216,4 @@ async function verify(kind, subject, members, github) { } } -module.exports = {SOURCE, row, digest, snapshot, verifyContents, verify}; +module.exports = {SOURCE, row, digest, snapshot, prTypeVerdict, prIssueVerdict, verifyContents, verify}; diff --git a/.github/scripts/integration-governance.test.cjs b/.github/scripts/integration-governance.test.cjs index 5884d0ba43..98506d9d60 100644 --- a/.github/scripts/integration-governance.test.cjs +++ b/.github/scripts/integration-governance.test.cjs @@ -26,6 +26,23 @@ test('pinned pure verifier actually validates the immutable upstream rules', () } }); +test('PR Type and PR Issue verdicts keep the upstream check text (#18764)', () => { + assert.deepEqual(policy.prTypeVerdict('WIP: [scope] ✨\uFE0F add thing'), {name: 'PR Type', + conclusion: 'success', title: 'PR Type in Title', summary: 'Found ✨ PR (feature)', + text: 'Final title:\n\n\tadd thing\n'}); + assert.equal(policy.prTypeVerdict(':running: chore').summary, 'Found 🌱 PR (infra)'); + const missing = policy.prTypeVerdict('πŸ“ proposal'); + assert.equal(missing.conclusion, 'failure'); + assert.equal(missing.summary, 'No matching PR type indicator found in title.'); + assert.ok(missing.text.startsWith("I saw a title of `πŸ“ proposal`, which doesn't seem to have")); + assert.deepEqual(policy.prIssueVerdict('πŸ› fix flake'), {name: 'PR Issue', conclusion: 'success', + title: 'Issue/PR tag in PR title', summary: 'Your PR title does not contain any Issue or PR tags', text: ''}); + const tagged = policy.prIssueVerdict('πŸ› fix #123'); + assert.equal(tagged.conclusion, 'failure'); + assert.equal(tagged.summary, 'Your PR has an Issue or PR number in the title.'); + assert.equal(policy.prIssueVerdict('πŸ› fix #12a').conclusion, 'success'); +}); + test('member snapshots never accept another head/repo/member or an unrelated merge', async () => { assert.equal((await policy.snapshot(client(member()), subject())).length, 1); const mutations = [ diff --git a/.github/scripts/pr-verifier.cjs b/.github/scripts/pr-verifier.cjs new file mode 100644 index 0000000000..c18c642054 --- /dev/null +++ b/.github/scripts/pr-verifier.cjs @@ -0,0 +1,89 @@ +'use strict'; + +// Records the required `PR Type` and `PR Issue` check runs for one PR head +// (#18764). Each check is ONE create call that already carries its final +// status, conclusion and output: no in_progress run is ever created, so no +// later update can fail and leave one hanging. The verdicts come from the +// in-repo port of the upstream verifier (integration-governance.cjs). +// +// A content verdict (a title without a type prefix, an issue number in the +// title) is reported by the check itself; the wrapper job stays green for it. +// Failing to RECORD a verdict is different: the job fails and says so, and a +// completed `failure` run carrying the reason is posted when GitHub allows it. + +const {prTypeVerdict, prIssueVerdict} = require('./integration-governance.cjs'); + +const CHECK_NAMES = ['PR Type', 'PR Issue']; + +const describe = error => + [error?.status, error?.message || String(error)].filter(Boolean).join(' '); + +async function create(github, owner, repo, headSha, verdict) { + const output = {title: verdict.title, summary: verdict.summary}; + if (verdict.text) output.text = verdict.text; + const {data} = await github.rest.checks.create({ + owner, repo, name: verdict.name, head_sha: headSha, + status: 'completed', conclusion: verdict.conclusion, output, + }); + // The response is GitHub's own record of the run: anything other than the + // completed verdict on this head means the required context is not settled. + if (data?.name !== verdict.name || data?.head_sha !== headSha || + data?.status !== 'completed' || data?.conclusion !== verdict.conclusion) { + throw new Error(`GitHub recorded ${JSON.stringify({name: data?.name, head_sha: data?.head_sha, + status: data?.status, conclusion: data?.conclusion})} instead of a completed ${verdict.conclusion}`); + } + return data; +} + +async function record({github, core, owner, repo, pr}) { + const headSha = pr?.head?.sha; + if (!/^[0-9a-f]{40}$/.test(headSha || '') || typeof pr?.title !== 'string') { + core.setFailed('PR Verifier could not record its result: the event carries no PR head SHA or title (#18764).'); + return {recorded: [], unrecorded: CHECK_NAMES.slice()}; + } + const verdicts = [prTypeVerdict(pr.title), prIssueVerdict(pr.title)]; + const recorded = [], unrecorded = []; + for (const verdict of verdicts) { + let error; + // One retry covers a transient API error; each attempt is a single, + // already-completed create, so a retry can never leave a run in progress. + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const run = await create(github, owner, repo, headSha, verdict); + core.info(`${verdict.name}: ${verdict.conclusion} β€” ${verdict.summary} (check run ${run.id} on ${headSha})`); + recorded.push(verdict.name); + error = undefined; + break; + } catch (caught) { + error = caught; + core.info(`${verdict.name}: attempt ${attempt} to record the result failed: ${describe(caught)}`); + } + } + if (!error) { + if (verdict.conclusion !== 'success') { + // Advisory for the wrapper job: the required context carries it. + core.warning(`${verdict.name}: ${verdict.summary} The required "${verdict.name}" check reports this; the wrapper job stays green.`); + } + continue; + } + unrecorded.push(verdict.name); + const reason = `PR Verifier could not record its "${verdict.name}" result on ${headSha}: ${describe(error)}`; + core.error(reason); + try { + await github.rest.checks.create({ + owner, repo, name: verdict.name, head_sha: headSha, + status: 'completed', conclusion: 'failure', + output: {title: verdict.title, summary: `${reason}. This is a verifier failure, not a verdict on the PR: re-run this PR Verifier run, or add or remove any label, to record a real result (#18764).`}, + }); + } catch (fallback) { + core.error(`${verdict.name}: the fallback failure run could not be posted either: ${describe(fallback)}`); + } + } + if (unrecorded.length) { + core.setFailed(`PR Verifier could not record its result for ${unrecorded.join(' and ')} on ${headSha}; ` + + 'the required check(s) may be missing or show no verdict for this head (#18764).'); + } + return {recorded, unrecorded}; +} + +module.exports = {CHECK_NAMES, record}; diff --git a/.github/scripts/pr-verifier.test.cjs b/.github/scripts/pr-verifier.test.cjs new file mode 100644 index 0000000000..8e40fcb853 --- /dev/null +++ b/.github/scripts/pr-verifier.test.cjs @@ -0,0 +1,143 @@ +'use strict'; +// #18764: PR Verifier records PR Type / PR Issue with one completed create +// each, and fails the job only when a result cannot be recorded. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const {CHECK_NAMES, record} = require('./pr-verifier.cjs'); + +const headSha = 'a'.repeat(40); +const pr = title => ({number: 7, title, head: {sha: headSha}}); + +function fakeCore() { + const core = {failed: [], warnings: [], errors: [], infos: []}; + core.setFailed = message => core.failed.push(message); + core.warning = message => core.warnings.push(message); + core.error = message => core.errors.push(message); + core.info = message => core.infos.push(message); + return core; +} + +// failures: name -> number of create calls for that name that throw. +function fakeGithub({failures = {}, record: mangle} = {}) { + const calls = []; + const remaining = {...failures}; + let id = 100; + const checks = { + create: async params => { + calls.push({method: 'create', ...params}); + if (remaining[params.name] > 0) { + remaining[params.name]--; + throw Object.assign(new Error('Not Found'), {status: 404}); + } + const data = {id: ++id, name: params.name, head_sha: params.head_sha, + status: params.status, conclusion: params.conclusion}; + return {data: mangle ? mangle(data) : data}; + }, + }; + for (const forbidden of ['update', 'get', 'listForRef', 'listForSuite']) { + checks[forbidden] = async () => { throw new Error(`checks.${forbidden} must not be called`); }; + } + return {calls, github: {rest: {checks}, request: async () => { throw new Error('raw requests must not be made'); }}}; +} + +const run = (github, core, title) => + record({github, core, owner: 'hivecommons', repo: 'hive', pr: pr(title)}); + +test('check names are the required contexts', () => { + assert.deepEqual(CHECK_NAMES, ['PR Type', 'PR Issue']); +}); + +test('a good title records two completed successes in one create each', async () => { + const {calls, github} = fakeGithub(); + const core = fakeCore(); + const result = await run(github, core, 'πŸ› fix: stuck checks'); + assert.deepEqual(result, {recorded: ['PR Type', 'PR Issue'], unrecorded: []}); + assert.equal(calls.length, 2); + assert.deepEqual(calls.map(c => c.name), ['PR Type', 'PR Issue']); + for (const call of calls) { + assert.equal(call.method, 'create'); + assert.equal(call.head_sha, headSha); + assert.equal(call.status, 'completed'); + assert.equal(call.conclusion, 'success'); + } + assert.equal(calls[0].output.title, 'PR Type in Title'); + assert.equal(calls[0].output.summary, 'Found πŸ› PR (bugfix)'); + assert.equal(calls[1].output.title, 'Issue/PR tag in PR title'); + assert.equal(calls[1].output.summary, 'Your PR title does not contain any Issue or PR tags'); + assert.equal(calls[1].output.text, undefined); + assert.deepEqual(core.failed, []); + assert.deepEqual(core.warnings, []); +}); + +test('a bad title is a completed failure on the check, advisory for the job', async () => { + const {calls, github} = fakeGithub(); + const core = fakeCore(); + await run(github, core, 'πŸ“ proposal for #12'); + assert.deepEqual(calls.map(c => [c.name, c.status, c.conclusion]), + [['PR Type', 'completed', 'failure'], ['PR Issue', 'completed', 'failure']]); + assert.equal(calls[0].output.summary, 'No matching PR type indicator found in title.'); + assert.equal(calls[1].output.summary, 'Your PR has an Issue or PR number in the title.'); + assert.deepEqual(core.failed, []); + assert.equal(core.warnings.length, 2); +}); + +test('one transient create error is retried with the real verdict', async () => { + const {calls, github} = fakeGithub({failures: {'PR Type': 1}}); + const core = fakeCore(); + const result = await run(github, core, '✨ feature'); + assert.deepEqual(result.unrecorded, []); + assert.deepEqual(calls.map(c => [c.name, c.conclusion]), + [['PR Type', 'success'], ['PR Type', 'success'], ['PR Issue', 'success']]); + assert.deepEqual(core.failed, []); +}); + +test('a result that cannot be recorded fails the job and posts a completed failure', async () => { + const {calls, github} = fakeGithub({failures: {'PR Issue': 2}}); + const core = fakeCore(); + const result = await run(github, core, '✨ feature'); + assert.deepEqual(result, {recorded: ['PR Type'], unrecorded: ['PR Issue']}); + const fallback = calls.at(-1); + assert.equal(fallback.name, 'PR Issue'); + assert.equal(fallback.status, 'completed'); + assert.equal(fallback.conclusion, 'failure'); + assert.match(fallback.output.summary, /could not record its "PR Issue" result/); + assert.match(fallback.output.summary, /404 Not Found/); + assert.equal(core.failed.length, 1); + assert.match(core.failed[0], /could not record its result for PR Issue/); +}); + +test('the job still fails when even the fallback cannot be posted', async () => { + const {github} = fakeGithub({failures: {'PR Type': 3, 'PR Issue': 3}}); + const core = fakeCore(); + const result = await run(github, core, '✨ feature'); + assert.deepEqual(result.unrecorded, ['PR Type', 'PR Issue']); + assert.equal(core.failed.length, 1); + assert.match(core.failed[0], /PR Type and PR Issue/); + assert.ok(core.errors.some(e => /fallback failure run could not be posted/.test(e))); +}); + +test('a create GitHub answers with anything but the completed verdict is not a recorded result', async () => { + const {github} = fakeGithub({record: data => ({...data, status: 'in_progress', conclusion: null})}); + const core = fakeCore(); + const result = await run(github, core, '✨ feature'); + assert.deepEqual(result.unrecorded, ['PR Type', 'PR Issue']); + assert.equal(core.failed.length, 1); +}); + +test('an event without a head SHA or title fails instead of passing silently', async () => { + const {calls, github} = fakeGithub(); + const core = fakeCore(); + await record({github, core, owner: 'o', repo: 'r', pr: {title: '✨ x', head: {}}}); + await record({github, core, owner: 'o', repo: 'r', pr: undefined}); + assert.equal(calls.length, 0); + assert.equal(core.failed.length, 2); +}); + +test('the verifier never updates or looks up a check run', () => { + const source = fs.readFileSync(path.join(__dirname, 'pr-verifier.cjs'), 'utf8'); + for (const forbidden of [/checks\.update/, /listForRef/, /PATCH/, /in_progress/, /status: 'queued'/]) { + assert.doesNotMatch(source.replace(/^\s*\/\/.*$/gm, ''), forbidden); + } +}); diff --git a/.github/scripts/test-native-queue-workflows.py b/.github/scripts/test-native-queue-workflows.py index 7ed06431dd..5fc577424c 100644 --- a/.github/scripts/test-native-queue-workflows.py +++ b/.github/scripts/test-native-queue-workflows.py @@ -124,7 +124,35 @@ def test_dependabot_runs_the_same_trusted_pr_verifier(self): self.assertEqual(job["name"], "verify PR contents") verify_step = next(step for step in job["steps"] if step.get("name") == "Verifier action") - self.assertIn("docker run --rm", verify_step["run"]) + # #18764: one completed create per check from the in-repo verifier, + # no image and no create-then-update. + self.assertTrue(verify_step["uses"].startswith("actions/github-script@")) + self.assertIn(".github/scripts/pr-verifier.cjs", verify_step["with"]["script"]) + self.assertNotIn("continue-on-error", verify_step) + + def test_pr_verifier_writes_completed_checks_and_fails_when_it_cannot(self): + # #18764: the image created PR Type / PR Issue in_progress and then + # PATCHed them; a 404 on the PATCH left the required runs in_progress + # for ever while the continue-on-error wrapper stayed green. + verifier = workflow("pr-verifier.yml") + job = verifier["jobs"]["verify"] + self.assertNotIn("packages", verifier["permissions"]) + self.assertEqual(verifier["permissions"]["checks"], "write") + for step in job["steps"]: + self.assertNotIn("continue-on-error", step, step.get("name")) + self.assertNotIn("docker", str(step.get("run", "")) + str(step.get("uses", ""))) + self.assertNotIn("env", job) + checkout = next(s for s in job["steps"] if str(s.get("uses", "")).startswith("actions/checkout@")) + # pull_request_target: github.sha is the default-branch tip, never PR code. + self.assertEqual(checkout["with"]["ref"], "${{ github.sha }}") + self.assertFalse(checkout["with"]["persist-credentials"]) + script = (ROOT / ".github/scripts/pr-verifier.cjs").read_text() + code = re.sub(r"(?m)^\s*//.*$", "", script) + self.assertIn("CHECK_NAMES = ['PR Type', 'PR Issue']", code) + self.assertIn("status: 'completed'", code) + for forbidden in ("checks.update", "listForRef", "PATCH", "in_progress"): + self.assertNotIn(forbidden, code) + self.assertIn("core.setFailed(", code) def test_autofix_push_deterministically_redispatches_the_verifier(self): # #16567: the autofix push landed after the last prow label and no @@ -146,7 +174,7 @@ def test_autofix_push_deterministically_redispatches_the_verifier(self): verify_step = next(s for s in steps if s.get("id") == "verifier") self.assertIn("steps.dispatch_event.outputs.stale != 'true'", verify_step["if"]) self.assertIn("steps.dispatch_event.outputs.event_path", verify_step["env"]["VERIFIER_EVENT_PATH"]) - self.assertIn('-v "$VERIFIER_EVENT_PATH:/github/event.json:ro"', verify_step["run"]) + self.assertIn("process.env.VERIFIER_EVENT_PATH", verify_step["with"]["script"]) guard = workflow("changelog-fragment-guard.yml") self.assertEqual(guard["permissions"]["actions"], "write") diff --git a/.github/workflows/pr-auto-update-test.yml b/.github/workflows/pr-auto-update-test.yml index 118e3ec806..58bd258e5c 100644 --- a/.github/workflows/pr-auto-update-test.yml +++ b/.github/workflows/pr-auto-update-test.yml @@ -8,6 +8,8 @@ on: - '.github/workflows/pr-auto-update*.yml' - '.github/scripts/native-queue-*.cjs' - '.github/scripts/diagnose-dco-provider*.cjs' + - '.github/scripts/pr-verifier*.cjs' + - '.github/scripts/integration-governance.cjs' - 'src/pkg/dashboard/static-src/js/config-helpers.js' - 'src/pkg/dashboard/static-src/js/config-features.js' - 'src/pkg/dashboard/static-src/js/tracker-native-queue.js' @@ -28,6 +30,8 @@ on: - '.github/workflows/pr-auto-update*.yml' - '.github/scripts/native-queue-*.cjs' - '.github/scripts/diagnose-dco-provider*.cjs' + - '.github/scripts/pr-verifier*.cjs' + - '.github/scripts/integration-governance.cjs' - 'src/pkg/dashboard/static-src/js/config-helpers.js' - 'src/pkg/dashboard/static-src/js/config-features.js' - 'src/pkg/dashboard/static-src/js/tracker-native-queue.js' @@ -61,6 +65,8 @@ jobs: node-version: '22' - name: Exercise native queue policy with mocked APIs run: node --test .github/scripts/native-queue-policy.test.cjs .github/scripts/native-queue-settings.test.cjs .github/scripts/native-queue-tracker.test.cjs .github/scripts/diagnose-dco-provider.test.cjs + - name: PR Verifier records completed PR Type / PR Issue results (#18764) + run: node --test .github/scripts/pr-verifier.test.cjs - name: Ensure YAML parser for workflow contracts run: | python3 -c 'import yaml' 2>/dev/null && exit 0 diff --git a/.github/workflows/pr-verifier.yml b/.github/workflows/pr-verifier.yml index e3fe5dcc90..34b69b732b 100644 --- a/.github/workflows/pr-verifier.yml +++ b/.github/workflows/pr-verifier.yml @@ -40,8 +40,9 @@ on: # - events that re-check the same head (labeled, unlabeled, edited, # reopened, ready_for_review) also carry github.run_id: each runs in its # own group and can neither cancel nor be cancelled. The wrapper is green -# whatever the label set (every verdict step is continue-on-error), and -# the label types stay for #17036, not for the verdict. +# whatever the label set or title verdict (the PR Type / PR Issue checks +# carry the verdict; the job fails only when it cannot record one, +# #18764), and the label types stay for #17036, not for the verdict. # - runs on a superseded commit finish (10 min cap) rather than being # cancelled; their verdict lands on that old SHA, which never merges. # .github/scripts/test-required-check-concurrency.py enforces this. @@ -51,23 +52,21 @@ concurrency: permissions: checks: write + contents: read pull-requests: read - packages: read jobs: verify: name: verify PR contents - # This pinned v0.4.3 mirror is linux/arm64 only. Reuse the native ARM64 - # override, never the general amd64 pool; forks stay on GitHub-hosted VMs. - runs-on: ${{ github.event.pull_request.head.repo.full_name == github.repository && fromJSON(vars.HIVE_ARM64_RUNNER || '["ubuntu-24.04-arm"]') || 'ubuntu-24.04-arm' }} + # Two API calls and no image: any runner will do. Forks never reach the + # self-hosted pool; a dispatch is only ever for a same-repo branch. + runs-on: ${{ (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 - env: - PR_VERIFIER_IMAGE: ghcr.io/hivecommons/pr-verifier@sha256:8c2b63817c4bfa7de510ab699b154637f27a42f87826c98083903daf2a539943 steps: - # A dispatched run has no pull_request payload. Rebuild one from the API - # for the verifier image, and bail out (green, nothing verified, said so) - # when the PR head is no longer this ref's tip: whoever moved it fired - # the normal events, and a verdict on a non-head commit is noise. + # A dispatched run has no pull_request payload. Rebuild one from the API, + # and bail out (green, nothing verified, said so) when the PR head is no + # longer this ref's tip: whoever moved it fired the normal events, and a + # verdict on a non-head commit is noise. - name: Reconstruct the pull_request payload for a dispatched run if: github.event_name == 'workflow_dispatch' id: dispatch_event @@ -97,65 +96,32 @@ jobs: '{action: "synchronize", number: $pr.number, pull_request: $pr, repository: $repo}' \ > "${RUNNER_TEMP}/pr-verifier-event.json" echo "event_path=${RUNNER_TEMP}/pr-verifier-event.json" >> "$GITHUB_OUTPUT" - # Login and pull are advisory for the same reason the verification below - # is: this check exists to report on PR CONTENTS, and a registry-side - # failure (org rename, package visibility change, GHCR outage β€” #6085 - # documents 11 consecutive runs red on `manifest unknown` after the - # kubestellarβ†’hivecommons move) says nothing about the PR. Failing here - # turns every open PR red at once before any content is examined, while - # the one step allowed to have an opinion is continue-on-error. When the - # image is unreachable the job logs it and stays green β€” an unreachable - # verifier is strictly less informative than a verifier that ran and - # objected, so it must not be treated more harshly. - - name: Login to GHCR - continue-on-error: true - uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 + # The verifier scripts only. On pull_request_target GitHub runs the + # DEFAULT branch's copy of this workflow and github.sha is that branch's + # tip, so this never checks out PR code; a dispatched run checks out its + # own same-repo branch, which already supplied this workflow file. + - name: Check out the verifier scripts + if: steps.dispatch_event.outputs.stale != 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Pull verifier image - id: pull - continue-on-error: true - run: | - if [ "$RUNNER_ARCH" != "ARM64" ]; then - echo "::error::The pinned PR Verifier requires native ARM64; HIVE_ARM64_RUNNER must not select $RUNNER_ARCH." - exit 1 - fi - docker pull "$PR_VERIFIER_IMAGE" - image_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$PR_VERIFIER_IMAGE")" - if [ "$image_platform" != "linux/arm64" ]; then - echo "::error::PR Verifier platform $image_platform does not match the required linux/arm64." - exit 1 - fi - - name: Report unreachable verifier image - if: steps.pull.outcome != 'success' - run: | - echo "::warning::PR Verifier image $PR_VERIFIER_IMAGE was unavailable or incompatible β€” verification was SKIPPED, not passed. Inspect the pull/platform error; this is a runner/registry/image problem, not a PR problem (#6085)." + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + # One completed create per check, never create-then-update (#18764). + # Not continue-on-error: a title verdict leaves this step green (the + # required check carries it), and the step fails only when a result + # could not be recorded. - name: Verifier action - if: steps.pull.outcome == 'success' && steps.dispatch_event.outputs.stale != 'true' + if: steps.dispatch_event.outputs.stale != 'true' id: verifier - continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - # A dispatched run presents itself to the image as the - # pull_request_target run it stands in for (see dispatch_event). - VERIFIER_EVENT_NAME: ${{ github.event_name == 'workflow_dispatch' && 'pull_request_target' || github.event_name }} VERIFIER_EVENT_PATH: ${{ steps.dispatch_event.outputs.event_path || github.event_path }} - run: | - docker run --rm \ - -e INPUT_GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ - -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ - -e GITHUB_ACTIONS="true" \ - -e GITHUB_EVENT_NAME="$VERIFIER_EVENT_NAME" \ - -e GITHUB_EVENT_PATH="/github/event.json" \ - -e GITHUB_REPOSITORY="$GITHUB_REPOSITORY" \ - -e GITHUB_SHA="$GITHUB_SHA" \ - -e GITHUB_REF="$GITHUB_REF" \ - -e GITHUB_WORKSPACE="/github/workspace" \ - -v "$VERIFIER_EVENT_PATH:/github/event.json:ro" \ - -v "$GITHUB_WORKSPACE:/github/workspace:ro" \ - "$PR_VERIFIER_IMAGE" - - name: Report verifier failure - if: steps.verifier.outcome == 'failure' - run: | - echo "::warning::PR Verifier returned a failure. This wrapper is advisory: inspect the individual PR checks and verifier log; a green wrapper does not prove those checks passed." + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('node:fs'); + const path = require('node:path'); + const {record} = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/pr-verifier.cjs')); + const event = JSON.parse(fs.readFileSync(process.env.VERIFIER_EVENT_PATH, 'utf8')); + await record({github, core, owner: context.repo.owner, repo: context.repo.repo, pr: event.pull_request}); diff --git a/changelog.d/fixed-18764-pr-verifier-single-post.md b/changelog.d/fixed-18764-pr-verifier-single-post.md new file mode 100644 index 0000000000..6cb33901d7 --- /dev/null +++ b/changelog.d/fixed-18764-pr-verifier-single-post.md @@ -0,0 +1 @@ +- **The required `PR Type` and `PR Issue` checks always conclude, and the PR Verifier goes red when it cannot record them** (#18764). The pinned verifier image created each check `in_progress` and then updated it with the verdict; when GitHub answered that update with 404 (#18667), the required checks stayed `in_progress` for ever, Tide reported "Jobs PR Issue, PR Type have not succeeded", and the `verify PR contents` job still went green because the image step was `continue-on-error`. The workflow now computes both verdicts in-repo (`.github/scripts/pr-verifier.cjs`, reusing the existing port of the upstream v0.4.3 title rules, with the same check names, output titles and summaries) and records each check with one create call that is already completed β€” nothing is ever left in progress. A title the rules reject is still a failed `PR Type`/`PR Issue` check with a green wrapper job; a result that cannot be recorded now fails `verify PR contents` with the reason, after posting a completed `failure` check that says so. No image is pulled any more, so the job runs on the ordinary runner pool instead of a native ARM64 runner. To recover a PR whose checks are already stuck, add or remove any label on it, or for a same-repo PR dispatch `pr-verifier.yml` on its branch with its `pr_number`, so a fresh completed pair lands on its head (re-running an old run re-runs the old workflow). diff --git a/src/deploy/test_pr_verifier_advisory.sh b/src/deploy/test_pr_verifier_advisory.sh index d0bc48f5d0..03afb50097 100755 --- a/src/deploy/test_pr_verifier_advisory.sh +++ b/src/deploy/test_pr_verifier_advisory.sh @@ -19,18 +19,20 @@ # observable after merge. A static assertion is the only way to demonstrate it # in CI, and the only thing that stops the combination drifting back. # -# THE INVARIANT IS RELATIVE, not a pinned pair of values. Either coherent -# posture is allowed: +# #18764 removed the image, and with it the registry: the verdicts are now +# computed in-repo (.github/scripts/pr-verifier.cjs) and each required check +# is recorded with one already-completed create. The image had created each +# run in_progress and then updated it; when GitHub 404'd the update, the +# required runs hung in_progress for ever while the continue-on-error wrapper +# stayed green. So the posture this suite pins is now: # -# - verification advisory => registry access advisory too (today's answer) -# - verification fatal => registry access may be fatal as well -# -# What is forbidden is the split that was there: registry access treated MORE -# harshly than the verification it exists to enable. An unreachable verifier is -# strictly less informative than a verifier that ran and objected, so it must -# not fail harder. #6085 says as much: if a missing verifier should block, the -# continue-on-error on the verification is the thing to reconsider -- the split -# is the one combination that cannot be right either way. +# - nothing touches a registry or a container image (the #6085 failure +# mode cannot come back with it); +# - the verdict about the PR's TITLE stays advisory for the wrapper job -- +# the required PR Type / PR Issue checks carry it (pr-verifier.test.cjs); +# - failing to RECORD a verdict is not advisory: the verifier step is not +# continue-on-error, so the job goes red with the reason instead of green +# beside a check that never concluded. # # Run: bash src/deploy/test_pr_verifier_advisory.sh # Exit codes: 0 the posture is coherent, 1 it is not. @@ -45,7 +47,7 @@ WORKFLOW="$(cd "$HERE/../../.github/workflows" && pwd)/pr-verifier.yml" ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } bad() { echo " FAIL: $1"; [ -n "${2:-}" ] && echo " $2"; FAIL=$((FAIL + 1)); } -echo "=== #6085: PR Verifier reports on the PR, not on registry reachability ===" +echo "=== #6085/#18764: PR Verifier reports on the PR and never hides a lost result ===" if [ ! -f "$WORKFLOW" ]; then bad "workflow not found: $WORKFLOW" @@ -60,130 +62,49 @@ if ! command -v python3 >/dev/null 2>&1 || ! python3 -c 'import yaml' >/dev/null echo; echo "=== $PASS passed, $FAIL failed ==="; exit 1 fi -# The whole verdict is computed in one python pass over the PARSED workflow -- -# not greps -- because `continue-on-error: true` and an `if:` expression are -# structure, and a substring check cannot tell which step owns them. +# One python pass over the PARSED workflow -- not greps -- because +# `continue-on-error` and an `if:` expression are structure, and a substring +# check cannot tell which step owns them. verdict="$(python3 - "$WORKFLOW" <<'PY' -import os, re, subprocess, sys, yaml +import sys, yaml doc = yaml.safe_load(open(sys.argv[1])) jobs = doc.get("jobs") or {} problems = [] notes = [] +if "packages" in (doc.get("permissions") or {}): + problems.append("workflow still asks for packages permission -- nothing here pulls an image any more (#18764)") + for job_id, job in jobs.items(): steps = job.get("steps") or [] - def find(pred): - return [s for s in steps if pred(s)] - - # The verification step: the one that runs the verifier image. - verifier = find(lambda s: s.get("id") == "verifier") + verifier = [s for s in steps if s.get("id") == "verifier"] if not verifier: problems.append(f"{job_id}: no step with id 'verifier' -- this suite can no longer find the verification it is about") continue verifier = verifier[0] - verifier_advisory = bool(verifier.get("continue-on-error")) - - # Keep the known single-platform digest on a native runner, without - # letting an untrusted fork opt into the repository's self-hosted override. - selector = "${{ github.event.pull_request.head.repo.full_name == github.repository && fromJSON(vars.HIVE_ARM64_RUNNER || '[\"ubuntu-24.04-arm\"]') || 'ubuntu-24.04-arm' }}" - if re.sub(r"\s+", "", str(job.get("runs-on", ""))) != re.sub(r"\s+", "", selector): - problems.append(f"{job_id}: verifier must select native ARM64 and keep forks off the self-hosted override") - image = str((job.get("env") or {}).get("PR_VERIFIER_IMAGE", "")) - if not re.fullmatch(r"ghcr\.io/hivecommons/pr-verifier@sha256:[0-9a-f]{64}", image): - problems.append(f"{job_id}: verifier must retain an immutable canonical image digest") - - # Registry-touching steps: the GHCR login, and the pull. - registry = find(lambda s: - "login" in str(s.get("name", "")).lower() - or "docker pull" in str(s.get("run", "")) - or "login-action" in str(s.get("uses", ""))) - if not registry: - problems.append(f"{job_id}: no registry login/pull step found -- the suite would assert nothing") - continue - notes.append(f"{job_id}: verification advisory={verifier_advisory}, registry steps={len(registry)}") - - # THE INVARIANT: registry access must not be treated more harshly than the - # verification it exists to enable. - if verifier_advisory: - for s in registry: - if not s.get("continue-on-error"): - problems.append( - f"{job_id}: step {s.get('name')!r} touches the registry and is FATAL, " - f"while the verification it enables is advisory -- a registry hiccup fails every PR") - - # The verifier must not run against an image that was not pulled. - pull = find(lambda s: "docker pull" in str(s.get("run", ""))) - if pull: - pull_id = pull[0].get("id") - if not pull_id: - problems.append(f"{job_id}: the pull step has no id, so nothing can gate on whether it succeeded") - elif f"steps.{pull_id}.outcome" not in str(verifier.get("if", "")): - problems.append( - f"{job_id}: the verification does not gate on the pull's outcome -- " - f"it would run against an image that was never pulled") - - # A skip has to be legible. Green-because-nothing-ran and - # green-because-it-passed must not look identical. - announced = [s for s in steps - if pull_id and f"steps.{pull_id}.outcome" in str(s.get("if", "")) - and "::warning" in str(s.get("run", ""))] - if not announced: - problems.append( - f"{job_id}: a skipped verification is silent -- no step warns when the image " - f"could not be pulled, so 'nothing ran' reads as 'it passed'") - - # Exercise the actual pull/preflight script, with no registry or - # engine access. A bad override must stop before even pulling. - mock_docker = r''' -docker() { - if [ "$1" = "pull" ]; then - echo MOCK_PULL - return "$MOCK_PULL_EXIT" - elif [ "$1" = "image" ] && [ "$2" = "inspect" ]; then - printf '%s\n' "$MOCK_IMAGE_PLATFORM" - else - echo "unexpected docker command" >&2 - return 90 - fi -} -''' - cases = [ - ("ARM64", "linux/arm64", "0", True, True), - ("X64", "linux/arm64", "0", False, False), - ("ARM", "linux/arm64", "0", False, False), - ("ARM64", "linux/amd64", "0", False, True), - ("ARM64", "windows/arm64", "0", False, True), - ("ARM64", "", "0", False, True), - ("ARM64", "linux/arm64", "1", False, True), - ] - for arch, platform, pull_exit, want_ok, want_pull in cases: - result = subprocess.run( - ["bash", "-e", "-c", mock_docker + "\n" + str(pull[0].get("run", ""))], - env={ - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "RUNNER_ARCH": arch, - "PR_VERIFIER_IMAGE": image, - "MOCK_IMAGE_PLATFORM": platform, - "MOCK_PULL_EXIT": pull_exit, - }, - capture_output=True, text=True, timeout=10, - ) - if (result.returncode == 0) != want_ok or ("MOCK_PULL" in result.stdout) != want_pull: - problems.append( - f"{job_id}: platform preflight accepted/rejected the wrong case " - f"runner={arch}, image={platform!r}, pull_exit={pull_exit}: {result.stdout} {result.stderr}") - notes.append(f"{job_id}: native runner/image preflight exercised in {len(cases)} mocked cases") - - failed_verifier_announced = [ - s for s in steps - if "steps.verifier.outcome" in str(s.get("if", "")) - and "failure" in str(s.get("if", "")) - and "::warning" in str(s.get("run", "")) - ] - if not failed_verifier_announced: - problems.append(f"{job_id}: a failed verifier must not silently look like successful verification") + + # A result that could not be recorded must fail the job (#18764). + if verifier.get("continue-on-error"): + problems.append(f"{job_id}: the verifier step is continue-on-error, so a result it could not record reads as green") + if not str(verifier.get("uses", "")).startswith("actions/github-script@"): + problems.append(f"{job_id}: the verifier step must be the in-repo github-script verifier, not an image") + if ".github/scripts/pr-verifier.cjs" not in str((verifier.get("with") or {}).get("script", "")): + problems.append(f"{job_id}: the verifier step does not run .github/scripts/pr-verifier.cjs") + if "steps.dispatch_event.outputs.stale != 'true'" not in str(verifier.get("if", "")): + problems.append(f"{job_id}: the verifier must not record a verdict on a dispatched run whose head moved on") + + # No registry, no image: the #6085 failure mode stays gone. + for s in steps: + blob = " ".join(str(s.get(k, "")) for k in ("name", "run", "uses")) + if "docker" in blob or "login-action" in blob or "ghcr.io" in blob: + problems.append(f"{job_id}: step {s.get('name')!r} touches a registry or container image") + if s.get("continue-on-error"): + problems.append(f"{job_id}: step {s.get('name')!r} is continue-on-error; nothing in this job may hide a failure") + if "PR_VERIFIER_IMAGE" in str(job.get("env", "")): + problems.append(f"{job_id}: the job still pins a verifier image") + notes.append(f"{job_id}: verifier={verifier.get('uses')}, steps={len(steps)}") for n in notes: print("NOTE " + n) @@ -205,9 +126,9 @@ while IFS= read -r line; do done <<<"$verdict" if ! printf '%s' "$verdict" | grep -q '^PROBLEM'; then - ok "registry access is not treated more harshly than the verification it enables" - ok "the verification is gated on the pull having succeeded" - ok "a skipped verification announces itself rather than passing silently" + ok "no registry or image stands between the PR and its verdict" + ok "a result the verifier could not record fails the job" + ok "a dispatched run whose head moved on records nothing" fi echo diff --git a/src/docs/hivecommons-migration.md b/src/docs/hivecommons-migration.md index 81ea0b6eff..717d862f8e 100644 --- a/src/docs/hivecommons-migration.md +++ b/src/docs/hivecommons-migration.md @@ -79,6 +79,11 @@ closing them. and source version, then a reviewed immutable-digest update in Hive's `.github/workflows/pr-verifier.yml` and the infra reusable consumer. Do not substitute an unverified tag or make forks use self-hosted runners. + Since [#18764](https://github.com/hivecommons/hive/issues/18764) Hive's own + `.github/workflows/pr-verifier.yml` no longer pulls this image: it records + `PR Type` and `PR Issue` from the in-repo port of the same v0.4.3 rules + (`.github/scripts/pr-verifier.cjs`), so the ARM64 constraint above no + longer applies to Hive's verifier. 6. **Hosted hub web domain** β€” In progress under [#5951](https://github.com/hivecommons/hive/issues/5951); the canonical hub is `https://hive.hivecommons.dev`, while `https://hive.kubestellar.io`