diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index d5f6e7fc..73a728dc 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -27,7 +27,9 @@ jobs: # formal legs, the merge of an up-to-date pull request for all of them) finds # the receipt and does not repeat the work; a formal leg that failed on the # same inputs is refuted in seconds instead of failing again an hour later - # (re-run the job to observe it anyway). Each leg uploads only its outcome; + # (re-run the job to observe it anyway). Before it runs, a leg deletes every + # file it does not declare, so an undeclared input fails the leg instead of + # proving a receipt it should not. Each leg uploads only its outcome; # factory.yml turns outcomes into receipts from the default branch. template-gate: name: acceptance / ${{ matrix.leg }} / ${{ matrix.runner }} @@ -64,6 +66,12 @@ jobs: if: steps.route.outputs.route != 'observe' && steps.route.outputs.route != 'proved' shell: bash run: exit 1 + - name: Keep only this leg's declared inputs + if: steps.route.outputs.route == 'observe' + shell: bash + env: + OBLIGATION: ${{ matrix.leg }}-${{ matrix.runner }} + run: node scripts/factory.mjs materialize "$OBLIGATION" - name: Pull the locked SDK, retrying the registry if: steps.route.outputs.route == 'observe' shell: bash diff --git a/factory/README.md b/factory/README.md index 2186c409..acb987a3 100644 --- a/factory/README.md +++ b/factory/README.md @@ -15,18 +15,36 @@ that orders remaining work. Here the decisions are a small Node host | Obligation | Kind | Reads | Carried out by | | --- | --- | --- | --- | | `rest-` (2) | fetch | everything | bootstrap `template-gate` | -| `artifacts`, `claims`, `varint`, `reproducibility`, `integrity` | hermetic | the formal set | bootstrap `template-gate` | +| `artifacts` | hermetic | the LexLean project: `src/` and its project files | bootstrap `template-gate` | +| `varint` | hermetic | the import closure of `Uorc.Varint`, `Uorc.VarintContract`; the project files, xtask, scripts | bootstrap `template-gate` | +| `integrity` | hermetic | the import closure of every `Uorc.Integrity` module, `Uorc.Wire.ArchiveIntegrity`, `Uorc.Wire.OwnedRead` (99 of 340 modules); `validation/integrity/`, `standards/`; the project files, xtask, scripts | bootstrap `template-gate` | +| `claims`, `reproducibility` | hermetic | the formal set | bootstrap `template-gate` | | `offline` | fetch | the formal set | offline-validation | -| `export` | fetch, host | the import closure of `Uorc.Client`, the LexLean project files, the export scripts | `node scripts/client-export.mjs` on a workstation | +| `export` | fetch, host | the import closure of `Uorc.Client` (client/src over src), the LexLean project files, the export scripts | `node scripts/client-export.mjs` on a workstation | | `gate` | gate | the identities of the seven bootstrap legs | proved when all seven are | | `native-` (2), `wasm` | fetch | the client set (+ `crates/uorc-wasm/`) | release `native`, `wasm` | | `bench` | fetch | `scripts/bench.mjs`, release.yml, `native-x86_64` | release `bench` | | `publish` | fetch | everything, `@mode`, `@run` | release `publish`, `pages` | -The formal set is the tree without the product client and the release plumbing -(the same scope the formal legs had before). The client set is what `cargo build --p uorc` reads: the workspace manifests and lock, the toolchain, `.cargo/`, -`crates/uorc/` and `generated/rust/`. +The formal set is the tree without the product client and the release plumbing. +The legs that build xtask (every formal leg but `artifacts`) read the product +crates' Cargo manifests only (`manifest:` inputs): the closure copy gives their +targets empty files, so cargo resolves the workspace without reading their code. +The client set is what `cargo build -p uorc` reads: the workspace manifests and +lock, the toolchain, `.cargo/`, `crates/uorc/` and `generated/rust/`. + +A registry or claims edit therefore re-runs `artifacts`, `claims` and +`reproducibility` (they read all of `src/` or the model), not `integrity` or +`varint`; a change to `model/`, `features/` or `CONFORMANCE.md` does not re-run +`artifacts`, `integrity` or `varint`. + +## Closure copy + +Before a bootstrap leg runs, `factory.mjs materialize` deletes every tracked +file the leg does not declare (PrismPM runs each obligation in a copy of its +closure). An undeclared input fails the leg instead of proving a receipt it +should not. Adding the copy found one: `just model-write` reads `README.md`, +which the old κ scope left out. ## Identity @@ -38,7 +56,8 @@ name kind hermetic|fetch|gate environment run -input one line per declared input (`!prefix` excludes, `@key` is a context, `closure:Module` an import closure) +input one line per declared input (`!prefix` excludes, `@key` is a context, + `closure:ROOTS@DIRS` an import closure, `manifest:PATH` a Cargo manifest only) needs one line per need closure ``` diff --git a/factory/graph.json b/factory/graph.json index 9e898698..6eaff621 100644 --- a/factory/graph.json +++ b/factory/graph.json @@ -1,19 +1,22 @@ { "schema": "uorc/factory-graph/1", "sets": { - "formal": ["", "!client/", "!crates/uorc/", "!crates/uorc-wasm/", "!generated/", "!web/", "!vendor/hologram-brand/", "!scripts/site.", "!Cargo.toml", "!Cargo.lock", "!README.md", "!scripts/bench.mjs", "!scripts/client-export.mjs", "!factory/", "!scripts/factory.", "!.github/workflows/release.yml", "!.github/workflows/pages.yml", "!.github/workflows/factory.yml"], + "formal": ["", "!client/", "!crates/uorc/", "!crates/uorc-wasm/", "!generated/", "!web/", "!vendor/hologram-brand/", "!scripts/site.", "!scripts/bench.mjs", "!scripts/client-export.mjs", "!factory/", "!scripts/factory.", "!.github/workflows/release.yml", "!.github/workflows/pages.yml", "!.github/workflows/factory.yml", "manifest:crates/conformance/Cargo.toml", "manifest:crates/uorc/Cargo.toml", "manifest:crates/uorc-wasm/Cargo.toml", "manifest:generated/rust/uorc_compress/Cargo.toml", "manifest:generated/rust/uorc_decompress/Cargo.toml"], + "lexlean": ["lexlean.toml", "lexlean.lock", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "prismpm.lock", "Justfile", ".github/workflows/bootstrap.yml", "validation/lexlean-verify-hints.json"], + "xtask": ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", ".cargo/", "xtask/", "crates/model/", "manifest:crates/conformance/Cargo.toml", "manifest:crates/uorc/Cargo.toml", "manifest:crates/uorc-wasm/Cargo.toml", "manifest:generated/rust/uorc_compress/Cargo.toml", "manifest:generated/rust/uorc_decompress/Cargo.toml"], + "scripts": ["scripts/", "!scripts/factory.", "!scripts/site.", "!scripts/bench.mjs", "!scripts/client-export.mjs"], "client": ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", ".cargo/", "crates/uorc/", "generated/rust/", ".github/workflows/release.yml"] }, "obligations": [ {"name": "rest-ubuntu-24.04", "kind": "fetch", "environment": "ubuntu-24.04", "run": "just lexlean-authority template-check fmt-check model lint test features bdd deny", "inputs": [""], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "rest-ubuntu-24.04-arm", "kind": "fetch", "environment": "ubuntu-24.04-arm", "run": "just lexlean-authority template-check fmt-check model lint test features bdd deny", "inputs": [""], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, - {"name": "artifacts-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just lexlean-artifacts", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "artifacts-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just lexlean-artifacts", "inputs": ["&lexlean", "src/"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "claims-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just claim-dispositions formal-tests", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, - {"name": "varint-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just varint-proofs", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "varint-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just varint-proofs", "inputs": ["&lexlean", "&xtask", "&scripts", "closure:Uorc.Varint,Uorc.VarintContract"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "reproducibility-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just reproducibility", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, - {"name": "integrity-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just integrity", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "integrity-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just integrity", "inputs": ["&lexlean", "&xtask", "&scripts", "closure:Uorc.Integrity.*,Uorc.Wire.ArchiveIntegrity,Uorc.Wire.OwnedRead", "validation/integrity/", "standards/", "standards.lock"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "offline", "kind": "fetch", "environment": "ubuntu-24.04", "run": "node scripts/offline-validation.mjs acquire, isolated first and second-distinct-root, compare-offline", "inputs": ["&formal"], "needs": [], "workflow": "offline-validation.yml", "job": "offline-compare"}, - {"name": "export", "kind": "fetch", "environment": "host", "run": "node scripts/client-export.mjs", "inputs": ["closure:Uorc.Client", "lexlean.toml", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "scripts/client-export.mjs", "scripts/semantic-source.mjs"], "needs": [], "record": "generated/rust/factory.identity"}, + {"name": "export", "kind": "fetch", "environment": "host", "run": "node scripts/client-export.mjs", "inputs": ["closure:Uorc.Client@client/src,src", "lexlean.toml", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "scripts/client-export.mjs", "scripts/semantic-source.mjs"], "needs": [], "record": "generated/rust/factory.identity"}, {"name": "gate", "kind": "gate", "environment": "-", "run": "-", "inputs": [], "needs": ["rest-ubuntu-24.04", "rest-ubuntu-24.04-arm", "artifacts-ubuntu-24.04", "claims-ubuntu-24.04", "varint-ubuntu-24.04", "reproducibility-ubuntu-24.04", "integrity-ubuntu-24.04"], "workflow": "release.yml", "job": "factory"}, {"name": "native-x86_64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target x86_64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-x86_64-unknown-linux-gnu"}, {"name": "native-aarch64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04-arm", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target aarch64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-aarch64-unknown-linux-gnu"}, diff --git a/generated/rust/factory.identity b/generated/rust/factory.identity index 74113b3c..3edbfdca 100644 --- a/generated/rust/factory.identity +++ b/generated/rust/factory.identity @@ -1 +1 @@ -export 2c94bf8be48d5269b913c6ea52fea4ba1d2516072e4100947225037c50d0d0e1 +export c128b01d319be60d3ea704e483e337bcec8ade796fe2e9539ea808b42bcf1c3b diff --git a/scripts/factory.mjs b/scripts/factory.mjs index af4a9e9f..af93c2cc 100644 --- a/scripts/factory.mjs +++ b/scripts/factory.mjs @@ -21,6 +21,7 @@ // node scripts/factory.mjs plan [--rev REV] [--context KEY=VALUE]... // node scripts/factory.mjs identity NAME [--rev REV] [--context KEY=VALUE]... // node scripts/factory.mjs route NAME... [--context KEY=VALUE]... [--expect-run TEXT] +// node scripts/factory.mjs materialize NAME (CI: delete what NAME does not declare) // node scripts/factory.mjs outcome NAME IDENTITY success|failure DIR // node scripts/factory.mjs publish DIR (factory.yml only: needs GH_TOKEN) // node scripts/factory.mjs export-check | export-record @@ -69,14 +70,23 @@ export function loadGraph(dir) { return graph; } -// `closure:Module` is the module's import closure, each module read from -// client/src when it has one there and from src otherwise (as the export stages it). -function closurePaths(module, entries) { - const paths = new Set(entries.map(e => e.path)); const found = []; const seen = new Set(); const stack = [module]; +// `closure:ROOTS@DIRS` is the import closure of ROOTS (comma-separated module +// names; `Uorc.Integrity.*` names every module under that directory), each +// module read from the first of DIRS (default `src`) that has it, as the +// export stages client/src over src. +function closurePaths(spec, entries) { + const [rootList, dirList = 'src'] = spec.split('@'); + const dirs = dirList.split(','); + const paths = new Set(entries.map(e => e.path)); const found = []; const seen = new Set(); + const stack = rootList.split(',').flatMap(r => { + if (!r.endsWith('.*')) return [r]; + const prefix = r.slice(0, -2).split('.').join('/') + '/'; + return [...paths].flatMap(p => dirs.map(d => `${d}/`).filter(d => p.startsWith(d + prefix) && p.endsWith('.lex.tex')).map(d => p.slice(d.length, -8).split('/').join('.'))); + }); while (stack.length) { const m = stack.pop(); if (seen.has(m)) continue; seen.add(m); const rel = m.split('.').join('/') + '.lex.tex'; - const p = paths.has(`client/src/${rel}`) ? `client/src/${rel}` : paths.has(`src/${rel}`) ? `src/${rel}` : fail(`closure ${module}: module ${m} is not tracked`); + const p = dirs.map(d => `${d}/${rel}`).find(x => paths.has(x)) ?? fail(`closure ${spec}: module ${m} is not tracked`); found.push(p); const object = entries.find(e => e.path === p).object.split(':')[1]; for (const i of git(['cat-file', 'blob', object]).toString().matchAll(/\\importmodule\{([^}]+)\}/g)) stack.push(i[1]); @@ -84,12 +94,18 @@ function closurePaths(module, entries) { return new Set(found); } +// `manifest:PATH` is a Cargo manifest read for workspace resolution only: the +// manifest is an input, and a closure copy gives each target it declares an +// empty file when the target's own sources are not inputs. +const manifestPath = i => i.slice('manifest:'.length); + /// An entry is covered when the longest inclusion it starts with is longer /// than the longest exclusion (`!prefix`) it starts with. export function covered(inputs, p, exact) { let include = -1, exclude = -1; for (const i of inputs) { if (i.startsWith('@') || i.startsWith('closure:')) continue; + if (i.startsWith('manifest:')) { if (p === manifestPath(i)) include = Math.max(include, p.length); continue; } if (i.startsWith('!')) { if (p.startsWith(i.slice(1))) exclude = Math.max(exclude, i.length - 1); } else if (p.startsWith(i)) include = Math.max(include, i.length); } @@ -97,6 +113,10 @@ export function covered(inputs, p, exact) { return include > exclude; } +function exactPaths(o, entries) { + return new Set(o.inputs.filter(i => i.startsWith('closure:')).flatMap(i => [...closurePaths(i.slice(8), entries)])); +} + /// Identities of every obligation at REV: PrismPM's `prism-factory/obligation/1`. export function identities(graph, entries, contexts) { const ids = {}; @@ -104,7 +124,7 @@ export function identities(graph, entries, contexts) { // An obligation that reads a context nobody supplied, or needs one that // does, has no identity here (a pull request run never plans `publish`). if (o.inputs.some(i => i.startsWith('@') && contexts[i.slice(1)] === undefined) || o.needs.some(n => ids[n] === null)) { ids[o.name] = null; continue; } - const exact = new Set(o.inputs.filter(i => i.startsWith('closure:')).flatMap(i => [...closurePaths(i.slice(8), entries)])); + const exact = exactPaths(o, entries); let closure = ''; for (const e of entries) if (covered(o.inputs, e.path, exact)) closure += `file ${e.object} ${e.path}\n`; for (const i of o.inputs) if (i.startsWith('@')) { @@ -268,6 +288,35 @@ function main(argv) { fs.writeFileSync(path.join(out, `${name}.outcome`), `outcome ${name} ${identity} ${result === 'success' ? 'pass' : 'fail'} ${git(['rev-parse', 'HEAD']).toString().trim()}\n`); return; } + case 'materialize': { + // A closure copy in place (PrismPM runs each obligation in a copy of its + // closure): delete every tracked file the obligation does not declare, + // so an undeclared input fails the run instead of proving a false receipt. + const o = byName(opts.names[0]); + if (o.inputs.includes('') && !o.inputs.some(i => i.startsWith('!'))) { console.log(`${o.name} reads the whole tree`); return; } + const entries = tree('HEAD', dir), exact = exactPaths(o, entries); + let removed = 0, kept = 0; + for (const e of entries) { + if (covered(o.inputs, e.path, exact)) { kept++; continue; } + if (e.path.startsWith('factory/') || e.path.startsWith('scripts/factory.')) continue; // the host itself, for the outcome step + fs.rmSync(path.join(dir, e.path), {force: true}); removed++; + } + const stubs = []; + for (const i of o.inputs.filter(x => x.startsWith('manifest:'))) { + const manifest = manifestPath(i), crate = path.dirname(manifest); + const text = fs.readFileSync(path.join(dir, manifest), 'utf8'); + const targets = [...text.matchAll(/^path\s*=\s*"([^"]+)"/gm)].map(m => m[1]); + if (!targets.length) targets.push('src/lib.rs'); + for (const t of targets) { + const file = path.join(dir, crate, t); + if (fs.existsSync(file)) continue; + fs.mkdirSync(path.dirname(file), {recursive: true}); fs.writeFileSync(file, ''); + stubs.push(path.join(crate, t)); + } + } + console.log(`${o.name}: kept ${kept} declared files, removed ${removed}, stubbed ${stubs.length} target(s) of manifest-only crates`); + return; + } case 'publish': return publish(graph, dir, opts.names[0]); case 'export-check': { const ids = identities(graph, tree('HEAD', dir), plannedContexts); diff --git a/scripts/factory.test.mjs b/scripts/factory.test.mjs index c816835f..855a3e68 100644 --- a/scripts/factory.test.mjs +++ b/scripts/factory.test.mjs @@ -60,3 +60,10 @@ test('weights stay in 1..16: a failure raises priority, a pass lowers it, and sc const r = route(graph, ids, parseStore('')); assert.deepEqual(schedule(graph, r, w).map(s => s.name), ['leg', 'build', 'bench', 'net']); }); + +test('a manifest input covers only that manifest, even under an exclusion', () => { + const inputs = ['', '!crates/uorc/', 'manifest:crates/uorc/Cargo.toml']; + assert.equal(covered(inputs, 'crates/uorc/Cargo.toml', new Set()), true); + assert.equal(covered(inputs, 'crates/uorc/src/lib.rs', new Set()), false); + assert.equal(covered(inputs, 'crates/model/src/lib.rs', new Set()), true); +}); diff --git a/template.lock b/template.lock index b3c3200a..1ed43d14 100644 --- a/template.lock +++ b/template.lock @@ -1 +1 @@ -{"contract_digest":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5","policy_files":[{"path":".devcontainer/devcontainer.json","sha256":"sha256:faad010f6e2561877feeac664d30c9b20da5c264de8f0007d485cc3c5a3505f2"},{"path":".github/workflows/bootstrap.yml","sha256":"sha256:5501ae07dd47f3be16bdd9d2afbf3832f811ebfb1693bf971cc1fde2e3ed9dba"},{"path":"AGENTS.md","sha256":"sha256:9a1fe3f73ab2963812c35976909c331292e1ca39ce47c0711ec1aafe8b869d87"},{"path":"VERIFICATION.md","sha256":"sha256:0912821fea354276fff5faa2b6b6ded0cdf4ef86112ad035b41a0a7a0513bd21"},{"path":"prismpm.lock","sha256":"sha256:97f3ca37a1077a7819a6ac6206c9132672a05820d547cc834ade18503e5643bf"},{"path":"template-contract.json","sha256":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5"}],"policy_tree_sha256":"sha256:f1146a46c3401d3b014463d7e31e785466312450c2e2edcb56f71b9c121cdcbc","schema":"uor/template-lock/1","sdk_image":"ghcr.io/uor-foundation/prismpm-sdk-candidate@sha256:0519fe6e2f1d731fc0d475c70ad21d00396d9089051f0c703d477e1ffe39a11c","template_repository":"https://github.com/UOR-Foundation/template","template_revision":"0f1245367d317d439e6752be277047eef76a9e90"} \ No newline at end of file +{"contract_digest":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5","policy_files":[{"path":".devcontainer/devcontainer.json","sha256":"sha256:faad010f6e2561877feeac664d30c9b20da5c264de8f0007d485cc3c5a3505f2"},{"path":".github/workflows/bootstrap.yml","sha256":"sha256:d0dbf29693b8417aa3c353bff99a789abefd954935c53e0f1ab3411487ae2f6a"},{"path":"AGENTS.md","sha256":"sha256:9a1fe3f73ab2963812c35976909c331292e1ca39ce47c0711ec1aafe8b869d87"},{"path":"VERIFICATION.md","sha256":"sha256:0912821fea354276fff5faa2b6b6ded0cdf4ef86112ad035b41a0a7a0513bd21"},{"path":"prismpm.lock","sha256":"sha256:97f3ca37a1077a7819a6ac6206c9132672a05820d547cc834ade18503e5643bf"},{"path":"template-contract.json","sha256":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5"}],"policy_tree_sha256":"sha256:ab8c95bcfed0dec567336db35afbc1672d449f4e2c617cb6347d08f53452d11c","schema":"uor/template-lock/1","sdk_image":"ghcr.io/uor-foundation/prismpm-sdk-candidate@sha256:0519fe6e2f1d731fc0d475c70ad21d00396d9089051f0c703d477e1ffe39a11c","template_repository":"https://github.com/UOR-Foundation/template","template_revision":"0f1245367d317d439e6752be277047eef76a9e90"} \ No newline at end of file