From e2f84dd8619782f70ca5e14f9eb1b5548eb4c8ad Mon Sep 17 00:00:00 2001 From: clubanderson Date: Tue, 3 Nov 2026 15:45:56 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20port=20DCO=20provider?= =?UTF-8?q?-unavailable=20classification=20to=20v6=20policy=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backport of the policy-script half of #16442 (head 0ac19dfa3) to v6: native-queue-policy.cjs gains ProviderUnavailable and the narrow dcoEnumerationUnavailable classifier (applied only after the App-1861 / exact-head / completed checks; still fails admission), its regression tests, the read-only diagnose-dco-provider reproducer and its tests, and the operator runbook. Why v6 first: .github/workflows/native-queue-policy.yml exists only on v5 and is the default-branch definition GitHub uses for every pull_request_target run, while the job checks out the PR base branch's scripts. Once #16442's yml lands on v5 it imports ProviderUnavailable from the checked-out script; on a v6 base that must exist, otherwise 'error instanceof undefined' throws inside the catch and breaks the bounded wait for pending evidence on every v6-base PR. Landing this first keeps v6-base admission intact and lets v6 report the ceiling case as UNKNOWN honestly. No workflow, permission, protection or trust change. Refs #16404. Signed-off-by: clubanderson --- .github/scripts/diagnose-dco-provider.cjs | 115 ++++++++++++++++++ .../scripts/diagnose-dco-provider.test.cjs | 86 +++++++++++++ .github/scripts/native-queue-policy.cjs | 23 ++++ .github/scripts/native-queue-policy.test.cjs | 43 +++++++ docs/operations/dco-provider-unavailable.md | 101 +++++++++++++++ 5 files changed, 368 insertions(+) create mode 100644 .github/scripts/diagnose-dco-provider.cjs create mode 100644 .github/scripts/diagnose-dco-provider.test.cjs create mode 100644 docs/operations/dco-provider-unavailable.md diff --git a/.github/scripts/diagnose-dco-provider.cjs b/.github/scripts/diagnose-dco-provider.cjs new file mode 100644 index 0000000000..c7b96b4d4e --- /dev/null +++ b/.github/scripts/diagnose-dco-provider.cjs @@ -0,0 +1,115 @@ +'use strict'; + +// Read-only enumeration reproducer, NOT a DCO evaluator or status publisher. +// Selection mirrors dcoapp/app index.js at 828cc0b29a141821ddecd674bca0d8e634234c18. +const { execFileSync } = require('node:child_process'); +const QUERY = `query PullRequestCommits( + $owner: String!, $repo: String!, $pullNumber: Int!, $cursor: String, $parentLimit: Int! +) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $pullNumber) { + headRefOid + commits(first: 100, after: $cursor) { + totalCount + nodes { commit { + oid url message + author { name email date user { login __typename } } + committer { name email date user { login __typename } } + signature { isValid state signature payload } + parents(first: $parentLimit) { nodes { oid url } } + } } + pageInfo { hasNextPage endCursor } + } + } + } +}`; +const SHA = /^[a-f0-9]{40}$/; + +function ghJSON(args, input) { + return JSON.parse(execFileSync('gh', args, { + input: input && JSON.stringify(input), encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024, + })); +} + +async function diagnose({ owner, repo, number, parentLimit = 100 }, api = ghJSON) { + if (!/^[\w.-]+$/.test(owner) || !/^[\w.-]+$/.test(repo) || + !Number.isSafeInteger(number) || number <= 0 || ![2, 100].includes(parentLimit)) { + throw new Error('Usage: node .github/scripts/diagnose-dco-provider.cjs OWNER REPO PR [--parents=2|100]'); + } + const endpoint = `repos/${owner}/${repo}/pulls/${number}`; + const pr = await api(['api', endpoint]); + const head = pr.head?.sha; + if (!SHA.test(head) || !Number.isSafeInteger(pr.commits) || pr.commits <= 0) { + throw new Error('Missing PR head or expected commit count'); + } + const restPages = await api(['api', `${endpoint}/commits?per_page=100`, '--paginate', '--slurp']); + const restCommits = restPages.flat(); + const commits = new Map(); + const cursors = new Set(); + let cursor = null; + let expected; + for (let page = 0; ; page++) { + // gh uses POST for GraphQL queries; this query only reads repository data. + const response = await api(['api', 'graphql', '--input', '-'], { + query: QUERY, variables: { owner, repo, pullNumber: number, cursor, parentLimit }, + }); + if (response.errors?.length) { + throw new Error(`GraphQL errors: ${JSON.stringify(response.errors)}`); + } + const current = response.data?.repository?.pullRequest; + if (current?.headRefOid !== head) throw new Error('PR head changed or unavailable during GraphQL enumeration'); + const connection = current.commits; + if (!Number.isSafeInteger(connection?.totalCount) || connection.totalCount !== pr.commits || + (expected !== undefined && connection.totalCount !== expected)) { + throw new Error('Incomplete or changed GraphQL totalCount'); + } + expected = connection.totalCount; + if (!Array.isArray(connection.nodes)) throw new Error('Missing GraphQL commit nodes'); + for (const node of connection.nodes) { + const commit = node?.commit; + if (!SHA.test(commit?.oid) || commits.has(commit.oid)) { + throw new Error('Malformed or duplicate GraphQL commit'); + } + commits.set(commit.oid, commit); + } + const info = connection.pageInfo; + if (typeof info?.hasNextPage !== 'boolean') throw new Error('Missing GraphQL pageInfo'); + if (!info.hasNextPage) break; + if (!info.endCursor || cursors.has(info.endCursor) || page + 1 >= Math.ceil(expected / 100) + 1) { + throw new Error('GraphQL pagination did not advance or exceeded expected pages'); + } + cursor = info.endCursor; + cursors.add(cursor); + } + if (commits.size !== expected || !commits.has(head) || + restCommits.some(commit => !commits.has(commit.sha))) { + throw new Error(`Incomplete history: GraphQL returned ${commits.size} of ${expected} commits, or REST/head mismatch`); + } + const final = await api(['api', endpoint]); + if (final.head?.sha !== head || final.commits !== expected || final.base?.sha !== pr.base?.sha) { + throw new Error('PR head/base/count changed during diagnosis'); + } + return { + head, restCount: restCommits.length, graphQLCount: commits.size, parentLimit, + // Full records remain available to callers inspecting mapping/sign-off + // failures beyond commit 250; the CLI only prints enumeration metadata. + commits: [...commits.values()], + }; +} + +if (require.main === module) { + const [owner, repo, pr, option, ...extra] = process.argv.slice(2); + const parentLimit = option === undefined ? 100 : Number(option.replace(/^--parents=/, '')); + Promise.resolve().then(() => { + if (extra.length || (option && !/^--parents=(2|100)$/.test(option))) throw new Error('Invalid arguments'); + return diagnose({ owner, repo, number: Number(pr), parentLimit }); + }).then(({ commits, ...result }) => { + console.log(JSON.stringify({ ...result, note: 'Enumeration only; no DCO verdict or admission proof.' }, null, 2)); + }).catch(error => { + console.error('DCO enumeration UNKNOWN:', error.stderr?.toString() || error.message); + process.exitCode = 1; + }); +} + +module.exports = { diagnose, QUERY }; diff --git a/.github/scripts/diagnose-dco-provider.test.cjs b/.github/scripts/diagnose-dco-provider.test.cjs new file mode 100644 index 0000000000..71a9a713d2 --- /dev/null +++ b/.github/scripts/diagnose-dco-provider.test.cjs @@ -0,0 +1,86 @@ +'use strict'; +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { diagnose } = require('./diagnose-dco-provider.cjs'); +const sha = n => n.toString(16).padStart(40, '0'); + +function fixture() { + const state = { + pr: { head: { sha: sha(251) }, base: { sha: sha(999) }, commits: 251 }, + // Commit 251 must survive enumeration, including its invalid sign-off. + commits: Array.from({ length: 251 }, (_, i) => ({ oid: sha(i + 1), message: i === 250 ? 'unsigned' : 'signed' })), + calls: [], reads: 0, + }; + const api = async (args, input) => { + state.calls.push({ args, input }); + if (args[1] === 'graphql') { + const start = Number(input.variables.cursor || 0); + const end = Math.min(start + 100, state.commits.length); + const result = { data: { repository: { pullRequest: { + headRefOid: state.pr.head.sha, + commits: { + totalCount: state.pr.commits, nodes: state.commits.slice(start, end).map(commit => ({ commit })), + pageInfo: { hasNextPage: end < state.commits.length, endCursor: String(end) }, + }, + } } } }; + state.mutate?.(result, start); + return result; + } + if (args[1].includes('/commits?')) { + return [state.commits.slice(0, 100), state.commits.slice(100, 200), state.commits.slice(200, 250)] + .map(page => page.map(commit => ({ sha: commit.oid }))); + } + state.reads++; + const pr = structuredClone(state.pr); + if (state.reads === 2) state.changeFinal?.(pr); + return pr; + }; + return { state, api }; +} +const options = { owner: 'acme', repo: 'widget', number: 1 }; + +test('full history beyond the REST ceiling is retained with both parent selections', async () => { + for (const parentLimit of [100, 2]) { + const { state, api } = fixture(); + const result = await diagnose({ ...options, parentLimit }, api); + assert.equal(result.restCount, 250); + assert.equal(result.graphQLCount, 251); + assert.equal(result.commits[250].message, 'unsigned'); + const queries = state.calls.filter(c => c.input); + assert.deepEqual(queries.map(c => c.input.variables.cursor), [null, '100', '200']); + assert.ok(queries.every(c => c.input.variables.parentLimit === parentLimit)); + assert.ok(queries.every(c => c.input.query.startsWith('query ') && !c.input.query.includes('mutation'))); + assert.ok(state.calls.every(c => c.args[0] === 'api' && + (c.args[1] === 'graphql' || c.args[1].startsWith('repos/acme/widget/pulls/1')))); + } +}); + +test('partial data, omitted tail, duplicates, changed head/count and nonadvancing pages stay UNKNOWN', async () => { + const mutations = [ + r => { r.errors = [{ message: 'provider unavailable' }]; }, + r => { r.data.repository.pullRequest.headRefOid = sha(500); }, + r => { r.data.repository.pullRequest.commits.totalCount = 250; }, + r => { r.data.repository.pullRequest.commits.pageInfo.hasNextPage = false; }, + (r, start) => { if (start) r.data.repository.pullRequest.commits.nodes[0].commit = { oid: sha(1) }; }, + r => { r.data.repository.pullRequest.commits.pageInfo.endCursor = '100'; }, + r => { r.data.repository.pullRequest.commits.pageInfo = null; }, + ]; + for (const mutate of mutations) { + const { state, api } = fixture(); + state.mutate = mutate; + await assert.rejects(diagnose(options, api)); + } + for (const changeFinal of [ + pr => { pr.head.sha = sha(500); }, pr => { pr.base.sha = sha(500); }, pr => { pr.commits++; }, + ]) { + const { state, api } = fixture(); + state.changeFinal = changeFinal; + await assert.rejects(diagnose(options, api), /changed during diagnosis/); + } +}); + +test('transport failures and malformed invocation do not produce an enumeration result', async () => { + await assert.rejects(diagnose(options, async () => { throw new Error('403 forbidden'); }), /403 forbidden/); + await assert.rejects(diagnose({ ...options, parentLimit: 1 }), /Usage/); + await assert.rejects(diagnose({ ...options, owner: '../other' }), /Usage/); +}); diff --git a/.github/scripts/native-queue-policy.cjs b/.github/scripts/native-queue-policy.cjs index 0818e259c4..d26d416b13 100644 --- a/.github/scripts/native-queue-policy.cjs +++ b/.github/scripts/native-queue-policy.cjs @@ -24,6 +24,27 @@ const MAX_QUEUE_PAGES = 10; class PendingEvidence extends Error {} +// A provider's inability to evaluate is not a negative sign-off verdict. It +// still blocks admission, and is deliberately not PendingEvidence: polling or +// retrying the admission job cannot repair the external provider. +class ProviderUnavailable extends Error { + constructor(run, head) { + super(`DCO provider unavailable on ${head}: verdict UNKNOWN (App ${DCO_APP_ID}, ` + + `check run ${run.id}, conclusion ${run.conclusion}). Admission remains blocked. ` + + 'An operator must investigate the DCO provider enumeration path; see ' + + 'docs/operations/dco-provider-unavailable.md. No PR source change is requested.'); + } +} + +function dcoEnumerationUnavailable(run) { + // This text only classifies a diagnostic, never supplies passing evidence. + // Call only after validating provider, head and completed status. + return run.name === 'DCO' && run.conclusion === 'failure' && + typeof run.output?.summary === 'string' && + run.output.summary.startsWith('The DCO check could not be evaluated because the complete pull request commit list could not be retrieved.') && + run.output.summary.includes('That fallback did not complete, so the DCO verdict is unknown.'); +} + function membersForGroup(entries, group) { if (!SHA.test(group.head_sha) || !SHA.test(group.base_sha) || !group.base_ref?.startsWith('refs/heads/') || @@ -72,6 +93,7 @@ function evidenceForHead(runs, head, checks) { throw new Error(`Untrusted provider or wrong head for ${name}`); } if (run.status !== 'completed') throw new PendingEvidence(`${name} is pending`); + if (dcoEnumerationUnavailable(run)) throw new ProviderUnavailable(run, head); // Existing PR path scoping legitimately skips technical jobs only. Policy // proofs, especially external DCO, must have actually succeeded. const scopedSkip = Object.hasOwn(BUILD_CHECKS, name) && run.conclusion === 'skipped'; @@ -217,4 +239,5 @@ async function evaluate({ github, context }) { module.exports = { ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS, PendingEvidence, membersForGroup, assertEligible, evidenceForHead, evaluate, + ProviderUnavailable, }; diff --git a/.github/scripts/native-queue-policy.test.cjs b/.github/scripts/native-queue-policy.test.cjs index b3661bcd74..98e44e5867 100644 --- a/.github/scripts/native-queue-policy.test.cjs +++ b/.github/scripts/native-queue-policy.test.cjs @@ -4,9 +4,14 @@ const assert = require('node:assert/strict'); const { ACTIONS_APP_ID, DCO_APP_ID, POLICY_CHECKS, BUILD_CHECKS, PendingEvidence, membersForGroup, assertEligible, evidenceForHead, evaluate, + ProviderUnavailable, } = require('./native-queue-policy.cjs'); const sha = n => n.toString(16).padStart(40, '0'); +// App 1861 check run 125732155592 on #16378, head 1989d20aa8d0d92cce761786ac05825aef1d9d3a. +const unavailableSummary = 'The DCO check could not be evaluated because the complete pull request commit list could not be retrieved.\n\n' + + 'GitHub returned 250 commits from the REST API, so the app attempted to retrieve the full list through GraphQL before evaluating the pull request. ' + + 'That fallback did not complete, so the DCO verdict is unknown.\n\nHTTP status: unknown\nGitHub request ID: unavailable'; const group = { base_sha: sha(1), head_sha: sha(4), base_ref: 'refs/heads/v5', head_ref: 'refs/heads/gh-readonly-queue/v5/pr-2-example', @@ -156,6 +161,44 @@ test('no Actions-generated DCO, stale success, or skipped policy proof', () => { assert.equal(evidenceForHead(technical, sha(11), BUILD_CHECKS).length, 6); }); +test('provider enumeration UNKNOWN blocks both admission paths without retry or substituted proof', async () => { + for (const eventName of ['pull_request_target', 'merge_group']) { + const f = fixture(eventName); + const runs = f.state.runs.get(sha(11)); + const dco = runs.find(r => r.name === 'DCO'); + Object.assign(dco, { conclusion: 'failure', output: { summary: unavailableSummary } }); + // Neither an older authoritative success nor a lowercase status-like + // check can replace the latest failed authoritative run. + runs.push({ ...dco, id: 1, conclusion: 'success' }, + { ...dco, id: 999, name: 'dco', conclusion: 'success', app: { id: 999 } }); + await assert.rejects(evaluate(f), error => { + assert.ok(error instanceof ProviderUnavailable); + assert.ok(!(error instanceof PendingEvidence)); + assert.match(error.message, /verdict UNKNOWN.*App 1861/); + assert.match(error.message, /Admission remains blocked/); + assert.match(error.message, /No PR source change is requested/); + return true; + }); + dco.conclusion = 'success'; + assert.match(await evaluate(f), /policy providers succeeded|validated all/); + } +}); + +test('UNKNOWN reporting requires trusted exact-head completed DCO and the specific enumeration diagnostic', () => { + const dco = checks(sha(11), { DCO: null })[0]; + const unknown = { ...dco, conclusion: 'failure', output: { summary: unavailableSummary } }; + for (const change of [ + { app: { id: ACTIONS_APP_ID } }, { head_sha: sha(99) }, { status: 'in_progress' }, + { output: undefined }, { output: { summary: 'The sign-off is missing.' } }, + { output: { summary: 'DCO verdict is unknown.' } }, { conclusion: 'action_required' }, + ]) { + assert.throws(() => evidenceForHead([{ ...unknown, ...change }], sha(11), { DCO: null }), + error => !(error instanceof ProviderUnavailable)); + } + assert.throws(() => evidenceForHead([{ ...dco, conclusion: 'failure' }], sha(11), { DCO: null }), + error => !(error instanceof ProviderUnavailable) && error.message === 'DCO: failure'); +}); + test('draft, closed, held and moved-head/base PRs cannot pass', () => { for (const change of [ { draft: true }, { state: 'closed' }, { head: { sha: sha(99) } }, { base: { ref: 'v6' } }, diff --git a/docs/operations/dco-provider-unavailable.md b/docs/operations/dco-provider-unavailable.md new file mode 100644 index 0000000000..4e154835a2 --- /dev/null +++ b/docs/operations/dco-provider-unavailable.md @@ -0,0 +1,101 @@ +# DCO provider unavailable on large forward merges + +`native-queue-policy` requires the latest completed **DCO** check from App +**1861** to succeed on the exact current PR head. When that provider reports +that it could not retrieve the complete PR commit list, admission is blocked +with **verdict UNKNOWN**. The Actions job remains failed and its summary +identifies provider unavailability. This is an operator/provider investigation; +it does not request a source change from the PR author. + +An ordinary negative DCO verdict still requires sign-off remediation. The +UNKNOWN classification applies only to the provider's explicit incomplete-list +diagnostic, after the App, head and completed status have been checked. A +changed or unrecognized provider message continues to fail closed as the +original check conclusion. No diagnostic text supplies passing evidence. + +## Observed evidence and provider path + +Investigation for [#16404](https://github.com/hivecommons/hive/issues/16404): + +| Evidence | Observation | +| --- | --- | +| [Check run 125732155592](https://github.com/hivecommons/hive/runs/125732155592) | App 1861, head `1989d20aa8d0d92cce761786ac05825aef1d9d3a`, completed `failure`; summary explicitly says the DCO verdict is unknown because GraphQL fallback did not complete. | +| [Check run 126045650867](https://github.com/hivecommons/hive/runs/126045650867) | Same diagnostic on head `66143614dbed95c6239f225719f4ec7e374f947b`; #16378 had 305 commits when inspected on 2026-11-03. | +| [Provider source](https://github.com/dcoapp/app/blob/828cc0b29a141821ddecd674bca0d8e634234c18/index.js) | `listPullRequestCommits` escalates exactly 250 REST commits to `fetchCompletePullRequestCommits`. It pages GraphQL by 100, checks stable totals, uniqueness, cursor progress and completeness, then maps the records before running the DCO evaluator. | +| [Provider issue #327](https://github.com/dcoapp/app/issues/327) | Reports the same incomplete-list failure. | +| [Provider PR #339](https://github.com/dcoapp/app/pull/339) | Proposed repair reduces nested `parents(first: 100)` to `parents(first: 2)` and exposes the underlying GraphQL error. Open and unmerged when inspected on 2026-11-03; deployment is not established. | + +250 is the REST endpoint's enumeration ceiling, not a Git ancestry limit. +The provider already attempts a full-history fallback. Its error summary hides +the actual exception and can report HTTP status `unknown` and request ID +`unavailable`, including for locally detected completeness/mapping errors. +Those fields alone do not prove an API permission, timeout or query-limit +failure. The nested parent selection is the upstream repair candidate; only +provider logs or a live query comparison can establish the cause for this +installation. A repository token's successful query would not prove that the +App installation token can perform the same query. + +## Read-only full-history reproducer + +From a checkout, with Node.js, `gh` and an existing `GH_TOKEN` that can read the +repository: + +```sh +node .github/scripts/diagnose-dco-provider.cjs hivecommons hive 16378 +node .github/scripts/diagnose-dco-provider.cjs hivecommons hive 16378 --parents=2 +``` + +The default reproduces the provider's commit fields and 100-parent selection; +the second run compares the proposed smaller parent selection. Each reads the +REST list, then **all** GraphQL pages, verifies unique commit IDs against the +reported total and REST list, and checks head/base/count stability. The CLI +prints only enumeration metadata; it does not evaluate sign-offs, publish +checks/statuses, modify the PR, or certify DCO. It exits nonzero on incomplete +history, partial GraphQL errors, duplicate IDs or a moved PR. Capture both +outputs and errors with the head SHA for the provider operator. Do not compare +runs across different heads as if they were one experiment. + +GraphQL queries use a read-only `query` over HTTP POST. Contributor environments +that disallow `gh api` POST cannot run this reproducer; use an operator's normal +read-capable environment. Do not circumvent that gate. The investigation above +verified public check evidence and provider source, not a successful live +full-history GraphQL run from such a restricted environment. + +The companion test fixture has 251 unique commits with an unsigned final +commit beyond the REST ceiling. It verifies that the final record survives +enumeration, and that truncation, partial errors and changed history stay +UNKNOWN. Mocked tests do not establish that GitHub accepts the provider query +or that the hosted provider has deployed a fix. + +## Operator recovery + +1. Record the current PR head and the latest App-1861 DCO run, including its + output, timestamps and check-run ID: + + ```sh + GH_TOKEN="$GH_TOKEN" gh api repos/hivecommons/hive/pulls/16378 --jq '{head: .head.sha, base: .base.sha, commits}' + # Replace HEAD_SHA with the exact head just read. + GH_TOKEN="$GH_TOKEN" gh api 'repos/hivecommons/hive/commits/HEAD_SHA/check-runs?per_page=100' --paginate --jq '.check_runs[] | select(.name == "DCO") | {id, head_sha, app: .app.id, status, conclusion, output}' + ``` + +2. Have the DCO App operator inspect logs for that delivery/time/head. The + provider logs the original exception as `failed to evaluate DCO check`. + Capture the underlying GraphQL errors, request ID, page/cursor and count, or + mapping exception. Confirm installation access and the deployed provider + revision. Track #327/#339 with that evidence; this repository cannot deploy + a change to the hosted App. For a self-hosted provider, its operator must + review and deploy a repair preserving complete enumeration and the existing + DCO evaluator. Follow the provider's [administration guide](https://github.com/dcoapp/app/blob/828cc0b29a141821ddecd674bca0d8e634234c18/docs/administration.md). +3. After the cause is repaired, an authorized operator can use GitHub's **DCO + check re-run** UI on the current PR head. The provider supports + `check_run.rerequested` and resolves the PR again, rejecting superseded heads. + This does not require granting Hive `checks:write`. Do not use **Set DCO to + pass**, copied statuses, blind retry loops or a policy override. +4. Verify a fresh App-1861 DCO `success` on the still-current head. Then rerun + **Native queue policy** through the Actions UI; it reads current evidence + again. If either head or evidence changed, investigate that state instead. + +Preserve the true merge ancestry of #16378. Splitting/rebasing the forward +merge, adding empty commits, accepting the lowercase prow `dco` status, or +weakening protection does not repair this provider failure. Until authoritative +current-head DCO success exists, the correct admission outcome remains blocked.