Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 223 additions & 0 deletions .github/integration-candidate.template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
name: Integration candidate
run-name: Integration ${{ fromJSON(inputs.manifest).id }} ${{ fromJSON(inputs.manifest).dispatchClaim.owner.nonce }}

on:
workflow_dispatch:
inputs:
manifest:
description: Exact versioned subject emitted by an enrolled trusted materializer.
required: true
type: string

permissions:
contents: read
pull-requests: read

concurrency:
group: integration-candidate-${{ github.run_id }}
cancel-in-progress: false

jobs:
subject:
name: gate
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
metadata_digest: ${{ steps.metadata.outputs.metadata_digest }}
steps:
- name: Validate immutable ordered subject before assigning technical jobs
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SUBJECT_MANIFEST: ${{ inputs.manifest }}
with:
script: |
const crypto = require('node:crypto');
const MAX_MANIFEST_BYTES = 32 * 1024;
const MAX_CANDIDATE_MEMBERS = 8;
const EXPECTED_PARENTS = 2;
const fail = message => { throw new Error(message); };
const sha = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value) && !/^0+$/.test(value);
const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value) && !/^0+$/.test(value);
const raw = process.env.SUBJECT_MANIFEST || '';
if (Buffer.byteLength(raw, 'utf8') > MAX_MANIFEST_BYTES) fail('Candidate manifest exceeds the bounded protocol');
const subject = JSON.parse(raw);
const canonical = value => {
if (value === null || typeof value !== 'object') return JSON.stringify(value);
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
};
if (subject.schema !== 1 || subject.protocol !== 'integration-prefix-v1' ||
subject.engine !== 'hive' || !digest(subject.id) ||
subject.repo !== `${context.repo.owner}/${context.repo.repo}` ||
!sha(subject.baseSHA) || !digest(subject.policy) ||
!sha(subject.workflowTree) || !Number.isSafeInteger(subject.generation) ||
subject.generation < 1 || typeof subject.base !== 'string' || !subject.base.length) {
fail('Invalid integration subject identity; no candidate jobs admitted');
}
const {id, generation, preparedBy, dispatchClaim, ...immutable} = subject;
if (crypto.createHash('sha256').update(canonical(immutable)).digest('hex') !== id) {
fail('Subject ID does not bind its exact immutable membership, commits, trees and policy');
}
const claimKey = crypto.createHash('sha256').update(`integration-ci-initial-v1/${subject.id}`).digest('hex');
if (!dispatchClaim || dispatchClaim.state !== 'claimed' ||
dispatchClaim.key !== claimKey || dispatchClaim.ref !== `heads/hive-integration-intents/${claimKey}` ||
!sha(dispatchClaim.commit) || !dispatchClaim.owner ||
typeof dispatchClaim.owner.nonce !== 'string' || !/^[a-f0-9]{32}$/.test(dispatchClaim.owner.nonce) ||
!/^[1-9][0-9]*$/.test(dispatchClaim.owner.runID) || !/^[1-9][0-9]*$/.test(dispatchClaim.owner.attempt)) {
fail('Candidate has no exact shared single-attempt dispatch owner');
}
if (context.eventName !== 'workflow_dispatch' ||
context.ref !== `refs/tags/hive-integration/${subject.id}` ||
!Array.isArray(subject.members) || !Array.isArray(subject.prefixes) ||
subject.members.length < 1 || subject.members.length > MAX_CANDIDATE_MEMBERS ||
subject.members.length !== subject.prefixes.length) {
fail('Candidate dispatch/ref or bounded ordered membership mismatch');
}
const numbers = new Set();
const heads = new Set();
const variables = {owner: context.repo.owner, name: context.repo.repo,
claimRef: `refs/${dispatchClaim.ref}`};
const declarations = ['$owner:String!', '$name:String!', '$policy:String!', '$claimRef:String!'];
const objects = [];
for (let i = 0; i < subject.members.length; i++) {
const member = subject.members[i], prefix = subject.prefixes[i];
if (!member || !Number.isSafeInteger(member.pr) || member.pr < 1 ||
numbers.has(member.pr) || heads.has(member.head) || !sha(member.head) ||
!prefix || !sha(prefix.commit) || !sha(prefix.tree)) {
fail('Invalid or duplicate ordered candidate member');
}
numbers.add(member.pr);
heads.add(member.head);
declarations.push(`$prefix${i}:String!`);
variables[`prefix${i}`] = prefix.commit;
objects.push(`prefix${i}:object(expression:$prefix${i}){... on Commit{oid tree{oid} parents(first:${EXPECTED_PARENTS + 1}){nodes{oid}}}}`);
}
if (subject.prefixes.at(-1).commit !== context.sha) {
fail('Dispatch does not execute the actual tested candidate commit');
}
variables.policy = `${context.sha}:.github`;
const query = `query(${declarations.join(',')}){repository(owner:$owner,name:$name){nameWithOwner claim:ref(qualifiedName:$claimRef){target{oid ... on Commit{message}}} policy:object(expression:$policy){... on Tree{oid}} ${objects.join(' ')}}}`;
const {repository} = await github.graphql(query, variables);
if (repository?.nameWithOwner !== subject.repo || repository.policy?.oid !== subject.workflowTree) {
fail('Repository or producer workflow closure changed');
}
if (repository.claim?.target?.oid !== dispatchClaim.commit) {
fail('Shared dispatch authority changed or is unavailable; no TTL takeover');
}
const records = (repository.claim.target.message || '').split('\n').filter(line => line.startsWith('{'));
if (records.length !== 1) fail('Shared dispatch authority record is malformed');
const owner = JSON.parse(records[0]);
if (owner.schema !== 1 || owner.kind !== 'integration-ci-single-attempt-v1' ||
owner.key !== claimKey || owner.subjectID !== subject.id ||
owner.owner?.nonce !== dispatchClaim.owner.nonce ||
owner.owner?.runID !== dispatchClaim.owner.runID ||
owner.owner?.attempt !== dispatchClaim.owner.attempt) {
fail('Shared dispatch owner does not match the candidate execution');
}
let parent = subject.baseSHA;
for (let i = 0; i < subject.members.length; i++) {
const actual = repository[`prefix${i}`], expected = subject.prefixes[i];
const parents = actual?.parents?.nodes;
if (actual?.oid !== expected.commit || actual.tree?.oid !== expected.tree ||
!Array.isArray(parents) || parents.length !== EXPECTED_PARENTS ||
parents[0].oid !== parent || parents[1].oid !== subject.members[i].head) {
fail(`Candidate prefix ${i + 1} is not its exact ordered two-parent subject`);
}
parent = actual.oid;
}
core.info(`Validated immutable subject ${subject.id}; this is not promotion authorization.`);
core.info(`Manifest SHA256 ${crypto.createHash('sha256').update(raw).digest('hex')}`);

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- name: Bind exact current member governance metadata
id: metadata
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3
env:
SUBJECT_MANIFEST: ${{ inputs.manifest }}
with:
script: |
const {snapshot, digest} = require('./.github/scripts/integration-governance.cjs');
const subject = JSON.parse(process.env.SUBJECT_MANIFEST);
core.setOutput('metadata_digest', digest(await snapshot(github, subject)));

receipt:
name: integration producer receipt
needs: [subject]
if: ${{ always() && needs.subject.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
actions: read
checks: read
id-token: write
steps:
- name: Bind exact receipt bytes to this isolated finalizer job
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SUBJECT_MANIFEST: ${{ inputs.manifest }}
TECHNICAL_OUTCOMES: ${{ toJSON(needs) }}
REQUIRED_CONTEXT_JOBS: '{}' # Filled from the actual generated DAG.
with:
script: |
const crypto = require('node:crypto');
const fs = require('node:fs');
const subject = JSON.parse(process.env.SUBJECT_MANIFEST);
const needs = JSON.parse(process.env.TECHNICAL_OUTCOMES);
const metadataDigest = needs.subject.outputs.metadata_digest;
if (!/^[a-f0-9]{64}$/.test(metadataDigest || '')) throw new Error('Missing exact member governance binding');
for (const [id, job] of Object.entries(needs)) {
if (id.startsWith('governance-') && job.outputs.metadata_digest !== metadataDigest) {
throw new Error('Member metadata drifted during governance verification');
}
}
const runID = Number(process.env.GITHUB_RUN_ID), attempt = Number(process.env.GITHUB_RUN_ATTEMPT);
const args = {...context.repo, run_id: runID};
const {data: run} = await github.rest.actions.getWorkflowRun(args);
const {data: jobs} = await github.rest.actions.listJobsForWorkflowRunAttempt({...args, attempt_number: attempt, per_page: 100});
const {data: checks} = await github.rest.checks.listForRef({...context.repo, ref: context.sha, filter: 'all', per_page: 100});
if (run.run_attempt !== attempt || run.head_sha !== context.sha ||
jobs.total_count !== jobs.jobs.length || checks.total_count !== checks.check_runs.length) {
throw new Error('Latest exact-attempt job/check cohort is unknown or truncated');
}
const contexts = {};
for (const [name, id] of Object.entries(JSON.parse(process.env.REQUIRED_CONTEXT_JOBS))) {
const matches = jobs.jobs.filter(job => job.name === name);
if (matches.length !== 1) throw new Error(`Missing/ambiguous actual required job ${name}`);
const job = matches[0], checkID = Number(job.check_run_url.split('/').at(-1));
const check = checks.check_runs.find(item => item.id === checkID);
if (!check || job.run_id !== runID || job.head_sha !== context.sha ||
check.name !== name || check.head_sha !== context.sha || check.check_suite.id !== run.check_suite_id ||
check.app.id !== 15368 || job.status !== 'completed' || check.status !== 'completed' ||
check.conclusion !== job.conclusion || job.conclusion !== needs[id].result) {
throw new Error(`Required context ${name} is not its actual exact-subject Actions producer`);
}
contexts[name] = {jobID: job.id, checkID, appID: check.app.id, suiteID: check.check_suite.id, conclusion: job.conclusion};
}
const payload = Buffer.from(JSON.stringify({
schema: 2,
protocol: 'integration-producer-receipt-v2',
subject,
runID: process.env.GITHUB_RUN_ID,
runAttempt: process.env.GITHUB_RUN_ATTEMPT,
metadataDigest,
contexts,
}), 'utf8');
const digest = crypto.createHash('sha256').update(payload).digest('hex');
const token = await core.getIDToken(`hive-integration-producer-v1/${digest}`);
core.setSecret(token);
fs.mkdirSync('.integration-producer', {recursive: true});
fs.writeFileSync('.integration-producer/receipt.json', JSON.stringify({
schema: 1, payload: payload.toString('base64'), token,
}), {flag: 'wx', mode: 0o600});
core.info(`Recorded producer identity for ${subject.id}; passing CI and live policy require independent verification.`);
- name: Publish immutable attempt-specific producer receipt
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: integration-producer-receipt-${{ github.run_attempt }}
path: .integration-producer/receipt.json
include-hidden-files: true
if-no-files-found: error
retention-days: 30
104 changes: 103 additions & 1 deletion .github/scripts/generate-integration-ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,16 @@
"# Generated by .github/scripts/generate-integration-ci.py; do not edit.\n"
"# Candidate-only technical CI. Ordinary and native CI remain in the source.\n"
)
CONTEXTS = {
"build-and-test", "test", "coverage", "golangci-lint",
"gosec (insecure patterns)", "govulncheck (known vulnerabilities)",
}
GOVERNANCE = {
"dco": "DCO",
"changelog": "changelog-fragment-guard",
"contents": "verify PR contents",
"attribution": "Sign-off survives the squash",
}


class WorkflowLoader(yaml.SafeLoader):
Expand Down Expand Up @@ -137,16 +147,99 @@ def render(source, document):
document, Dumper=WorkflowDumper, sort_keys=False, width=10000
)

def compose(documents):
template = ROOT / ".github" / "integration-candidate.template.yml"
entry = yaml.load(template.read_text(), Loader=WorkflowLoader)
jobs = entry["jobs"]
receipt = jobs.pop("receipt")
bindings = {"gate": "subject"}
for source, document in documents.items():
lane = {"v2-ci.yml": "ci", "v2-tests.yml": "tests",
"go-security-analysis.yml": "security"}[source]
generated = yaml.load(render(source, document), Loader=WorkflowLoader)
mapping = {key: f"{lane}-{key}" for key in generated["jobs"]}

def rewrite(value):
if isinstance(value, str):
return re.sub(r"\bneeds\.([A-Za-z0-9_-]+)",
lambda match: "needs." + mapping.get(match[1], match[1]), value)
if isinstance(value, list):
return [rewrite(item) for item in value]
if isinstance(value, dict):
return {key: rewrite(item) for key, item in value.items()}
return value

for key, original in generated["jobs"].items():
job = rewrite(original)
dependencies = original.get("needs", [])
if isinstance(dependencies, str):
dependencies = [dependencies]
job["needs"] = ["subject"] + [mapping[value] for value in dependencies]
condition = str(job.get("if", "${{ success() }}")).strip()
if condition.startswith("${{") and condition.endswith("}}"):
condition = condition[3:-2].strip()
job["if"] = "${{ always() && needs.subject.result == 'success' && (" + condition + ") }}"
job["name"] = original.get("name", key)
job["env"] = {**generated.get("env", {}), **job.get("env", {})}
if job["name"] in CONTEXTS:
if job["name"] in bindings:
raise ValueError(f"duplicate required context {job['name']}")
bindings[job["name"]] = mapping[key]
if source == "v2-tests.yml" and key == "test":
for step in job["steps"]:
if step.get("name") == "Check shard results":
step["run"] = step["run"].replace(
'if [ "${#cancelled[@]}" -gt 0 ]; then',
'if [ "${#cancelled[@]}" -gt 0 ]; then\n echo "::error::Candidate shards must all complete"; exit 1')
jobs[mapping[key]] = job
if set(bindings) != CONTEXTS | {"gate"}:
raise ValueError("actual technical producer context coverage changed")
for kind, name in GOVERNANCE.items():
key = f"governance-{kind}"
bindings[name] = key
jobs[key] = {
"name": name, "needs": ["subject"], "runs-on": "ubuntu-latest",
"timeout-minutes": 15,
"outputs": {"metadata_digest": "${{ steps.verify.outputs.metadata_digest }}"},
"steps": [
{"uses": "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1",
"with": {"fetch-depth": 0, "persist-credentials": False}},
{"name": "Verify exact current members with trusted readonly policy", "id": "verify",
"uses": "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3",
"env": {"SUBJECT_MANIFEST": "${{ inputs.manifest }}",
"EXPECTED_METADATA_DIGEST": "${{ needs.subject.outputs.metadata_digest }}",
"GITHUB_TOKEN": "${{ github.token }}"},
"with": {"script": (
"const policy = require('./.github/scripts/integration-governance.cjs');\n"
"const subject = JSON.parse(process.env.SUBJECT_MANIFEST);\n"
"const members = await policy.snapshot(github, subject);\n"
"const actual = policy.digest(members);\n"
"if (actual !== process.env.EXPECTED_METADATA_DIGEST) throw new Error('Member metadata drifted');\n"
"core.setOutput('metadata_digest', actual);\n"
f"await policy.verify('{kind}', subject, members, github);\n"
)}},
],
}
receipt["needs"] = list(jobs)
for step in receipt["steps"]:
if "REQUIRED_CONTEXT_JOBS" in step.get("env", {}):
import json
step["env"]["REQUIRED_CONTEXT_JOBS"] = json.dumps(bindings, sort_keys=True)
jobs["receipt"] = receipt
return HEADER + yaml.dump(entry, Dumper=WorkflowDumper, sort_keys=False, width=10000)


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
different = False
documents = {}
for source in SOURCES:
original = ROOT / ".github" / "workflows" / source
output = original.with_name(f"integration-technical-{source}")
expected = render(source, yaml.load(original.read_text(), Loader=WorkflowLoader))
documents[source] = yaml.load(original.read_text(), Loader=WorkflowLoader)
expected = render(source, documents[source])
if args.check:
actual = output.read_text() if output.exists() else ""
if actual != expected:
Expand All @@ -158,6 +251,15 @@ def main():
else:
output.write_text(expected)
print(output.relative_to(ROOT))
output = ROOT / ".github" / "workflows" / "integration-candidate.yml"
expected = compose(documents)
if args.check:
if not output.exists() or output.read_text() != expected:
different = True
print("integration-candidate.yml differs from its trusted template/composed producers")
else:
output.write_text(expected)
print(output.relative_to(ROOT))
if different:
raise SystemExit("Run python3 .github/scripts/generate-integration-ci.py")

Expand Down
Loading
Loading