From 8be7e139703f851536ab4e0e390bcd82ededff64 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:52:19 +0000 Subject: [PATCH 1/3] Key every CI and release obligation on its own inputs (factory route) Adopts the PrismPM factory route (PrismPM #566, #579-#581, #586) for uorc: factory/graph.json declares each obligation's inputs; scripts/factory.mjs computes its prism-factory/obligation/1 identity, routes it from receipts on the factory-receipts branch (proved, refuted, blocked, observe) and keeps a 16-level weight per obligation. - bootstrap and offline legs route by their own identities, refute a hermetic failure on unchanged inputs (a manual re-run observes it again), retry the SDK pull, and upload only their outcome. - factory.yml (workflow_run, default branch) turns outcomes into receipts, so no pull-request job holds a write token. - release.yml: refuses a tag that does not name the workspace version, requires the bootstrap gate on the tag's tree, reuses native/wasm/bench artifacts from the run that proved them, and gains a rehearsal mode, so a tag of a rehearsed commit only packages and publishes. - client-export.mjs records the export identity (the Uorc.Client import closure and the LexLean project files) and skips an export whose inputs are unchanged. - scripts/ci-receipt.sh is replaced. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3LdDSxZc5dyuEbM7w7 --- .github/workflows/bootstrap.yml | 58 ++-- .github/workflows/factory.yml | 43 +++ .github/workflows/offline-validation.yml | 41 ++- .github/workflows/release.yml | 314 +++++++++++++----- factory/README.md | 81 +++++ factory/graph.json | 24 ++ generated/rust/factory.txt | 1 + scripts/ci-receipt.sh | 57 ---- scripts/client-export.mjs | 14 + scripts/factory.mjs | 381 ++++++++++++++++++++++ scripts/factory.test.mjs | 62 ++++ template.lock | 2 +- tests/sdk-preflight/test_source_format.py | 4 +- 13 files changed, 907 insertions(+), 175 deletions(-) create mode 100644 .github/workflows/factory.yml create mode 100644 factory/README.md create mode 100644 factory/graph.json create mode 100644 generated/rust/factory.txt delete mode 100755 scripts/ci-receipt.sh create mode 100644 scripts/factory.mjs create mode 100644 scripts/factory.test.mjs diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index cd5c810e..d5f6e7fc 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -9,7 +9,6 @@ on: permissions: contents: read - actions: read # A newer push to a pull request supersedes its older run. concurrency: @@ -22,11 +21,14 @@ jobs: # normative x86-64 host; the arm host runs the host-dependent `rest` leg. # Wall time is the slowest leg, not the sum. # - # Every leg but `rest` is a formal obligation: it runs once per content - # identity κ of its inputs (scripts/ci-receipt.sh) and uploads a receipt when - # it passes. A later run with the same κ (a product-only change, a rerun, the - # merge of an up-to-date pull request) finds the receipt and does not repeat - # the work. `rest` checks the product and runs on every revision. + # Every leg is a factory obligation (factory/graph.json, scripts/factory.mjs): + # its identity is the SHA-256 of the inputs it declares, and it runs once per + # identity. A later run with the same identity (a product-only change for the + # formal legs, the merge of an up-to-date pull request for all of them) finds + # the receipt and does not repeat the work; a formal leg that failed on the + # same inputs is refuted in seconds instead of failing again an hour later + # (re-run the job to observe it anyway). Each leg uploads only its outcome; + # factory.yml turns outcomes into receipts from the default branch. template-gate: name: acceptance / ${{ matrix.leg }} / ${{ matrix.runner }} strategy: @@ -51,16 +53,31 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Look up a receipt for this leg's inputs - id: receipt - if: matrix.leg != 'rest' + - name: Route this leg by its own inputs + id: route shell: bash env: - GH_TOKEN: ${{ github.token }} OBLIGATION: ${{ matrix.leg }}-${{ matrix.runner }} - run: scripts/ci-receipt.sh lookup "$OBLIGATION" >> "$GITHUB_OUTPUT" + LEG_RECIPES: ${{ matrix.recipes }} + run: node scripts/factory.mjs route "$OBLIGATION" --expect-run "just $LEG_RECIPES" >> "$GITHUB_OUTPUT" + - name: Stop on refuted or blocked inputs + if: steps.route.outputs.route != 'observe' && steps.route.outputs.route != 'proved' + shell: bash + run: exit 1 + - name: Pull the locked SDK, retrying the registry + if: steps.route.outputs.route == 'observe' + shell: bash + run: | + set -euo pipefail + image=$(sed -n 's/.*"sdk_image":"\([^"]*\)".*/\1/p' prismpm.lock) + for attempt in 1 2 3 4; do + if docker pull --quiet "$image"; then exit 0; fi + sleep $((attempt * 15)) + done + exit 1 - name: Run this leg of the template gate through the locked SDK - if: steps.receipt.outputs.proved != 'true' + id: run + if: steps.route.outputs.route == 'observe' shell: bash env: LEG_RECIPES: ${{ matrix.recipes }} @@ -103,20 +120,21 @@ jobs: devcontainer --version exec just $LEG_RECIPES ' - - name: Record the receipt for this leg's inputs - if: steps.receipt.outputs.proved == 'false' + - name: Record this leg's outcome + if: always() && steps.route.outputs.route == 'observe' && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') shell: bash env: OBLIGATION: ${{ matrix.leg }}-${{ matrix.runner }} - KAPPA: ${{ steps.receipt.outputs.kappa }} - run: scripts/ci-receipt.sh record "$OBLIGATION" "$KAPPA" + IDENTITY: ${{ steps.route.outputs.identity }} + OUTCOME: ${{ steps.run.outcome }} + run: node scripts/factory.mjs outcome "$OBLIGATION" "$IDENTITY" "$OUTCOME" .tmp/factory - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a - if: steps.receipt.outputs.proved == 'false' + if: always() && steps.route.outputs.route == 'observe' && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') with: - name: receipt-${{ matrix.leg }}-${{ matrix.runner }}-${{ steps.receipt.outputs.kappa }} - path: .tmp/receipt/receipt.json + name: factory-outcome-${{ matrix.leg }}-${{ matrix.runner }} + path: .tmp/factory/ if-no-files-found: error - retention-days: 90 + retention-days: 3 # The single required signal per host: it passes only when every leg passed. acceptance: diff --git a/.github/workflows/factory.yml b/.github/workflows/factory.yml new file mode 100644 index 00000000..79baf622 --- /dev/null +++ b/.github/workflows/factory.yml @@ -0,0 +1,43 @@ +# Records receipts. After a bootstrap, offline or release run from this +# repository completes, this turns the outcomes its jobs uploaded into receipts +# on the factory-receipts branch (scripts/factory.mjs publish). It runs from the +# default branch with its own token, so no pull-request code ever gets write +# access, and an outcome from a fork is never recorded. Runs that finish +# together each rebuild on the branch's newest head and retry the push. +name: factory + +on: + workflow_run: + workflows: [bootstrap, SDK offline reproducibility, release] + types: [completed] + +permissions: + contents: read + +jobs: + receipts: + name: record receipts + if: github.event.workflow_run.head_repository.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: factory-outcome-* + path: .tmp/outcomes + merge-multiple: true + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + - name: Publish the run's receipts and reweigh the schedule + env: + GH_TOKEN: ${{ github.token }} + FACTORY_RUN_ID: ${{ github.event.workflow_run.id }} + FACTORY_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: node scripts/factory.mjs publish .tmp/outcomes diff --git a/.github/workflows/offline-validation.yml b/.github/workflows/offline-validation.yml index 173c9bb3..ff004e9b 100644 --- a/.github/workflows/offline-validation.yml +++ b/.github/workflows/offline-validation.yml @@ -8,7 +8,6 @@ on: permissions: contents: read - actions: read # A newer push to a pull request supersedes its older run. concurrency: @@ -16,31 +15,29 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # The whole workflow is one formal obligation: it runs once per content - # identity κ of its inputs (scripts/ci-receipt.sh). When an earlier run with - # the same κ passed, every job below is skipped. + # The whole workflow is one factory obligation, `offline` (factory/graph.json): + # it runs once per identity of its declared inputs. When an earlier run with + # the same identity passed, every job below is skipped. route: name: offline formal construction / receipt runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - proved: ${{ steps.receipt.outputs.proved }} - kappa: ${{ steps.receipt.outputs.kappa }} + route: ${{ steps.route.outputs.route }} + identity: ${{ steps.route.outputs.identity }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 persist-credentials: false - - name: Look up a receipt for this workflow's inputs - id: receipt + - name: Route this workflow by its own inputs + id: route shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: scripts/ci-receipt.sh lookup offline >> "$GITHUB_OUTPUT" + run: node scripts/factory.mjs route offline >> "$GITHUB_OUTPUT" regenerate: name: review-only generated projections needs: route - if: needs.route.outputs.proved != 'true' + if: needs.route.outputs.route == 'observe' runs-on: ubuntu-24.04 timeout-minutes: 20 steps: @@ -60,7 +57,7 @@ jobs: set -euo pipefail image=$(node -p 'JSON.parse(require("fs").readFileSync("prismpm.lock")).sdk_image') [[ "$image" =~ ^[a-z0-9.-]+(:[0-9]{1,5})?/[a-z0-9./_-]+@sha256:[0-9a-f]{64}$ ]] - docker pull "$image" + for attempt in 1 2 3 4; do docker pull "$image" && break; test "$attempt" -lt 4; sleep $((attempt * 15)); done docker run --rm --user "$(id -u):$(id -g)" \ --volume "$PWD:$PWD" --workdir "$PWD" \ --env PRISMPM_EPHEMERAL_HOME=1 \ @@ -75,7 +72,7 @@ jobs: offline-setup: name: offline formal construction / dependency setup needs: route - if: needs.route.outputs.proved != 'true' + if: needs.route.outputs.route == 'observe' runs-on: ubuntu-24.04 timeout-minutes: 30 steps: @@ -95,7 +92,7 @@ jobs: set -euo pipefail image=$(node -p 'JSON.parse(require("fs").readFileSync("prismpm.lock")).sdk_image') [[ "$image" =~ ^[a-z0-9.-]+(:[0-9]{1,5})?/[a-z0-9./_-]+@sha256:[0-9a-f]{64}$ ]] - docker pull "$image" + for attempt in 1 2 3 4; do docker pull "$image" && break; test "$attempt" -lt 4; sleep $((attempt * 15)); done docker run --rm --user "$(id -u):$(id -g)" \ --group-add "$(stat -c '%g' /var/run/docker.sock)" \ --volume "$PWD:$PWD" --workdir "$PWD" \ @@ -146,7 +143,7 @@ jobs: mkdir -p .tmp && tar -C .tmp -xf .prism/offline-setup/offline-acquisition.tar && rm -rf .prism/offline-setup image=$(node -p 'JSON.parse(require("fs").readFileSync("prismpm.lock")).sdk_image') [[ "$image" =~ ^[a-z0-9.-]+(:[0-9]{1,5})?/[a-z0-9./_-]+@sha256:[0-9a-f]{64}$ ]] - docker pull "$image" + for attempt in 1 2 3 4; do docker pull "$image" && break; test "$attempt" -lt 4; sleep $((attempt * 15)); done # Only orchestration sees the daemon. The verifier containers have # no socket, credentials, parent checkout mount, or network interface. docker run --rm --user "$(id -u):$(id -g)" \ @@ -188,14 +185,14 @@ jobs: name: offline-comparison path: .prism/uorc/offline-comparison.json if-no-files-found: error - - name: Record the receipt for this workflow's inputs + - name: Record this workflow's outcome shell: bash env: - KAPPA: ${{ needs.route.outputs.kappa }} - run: scripts/ci-receipt.sh record offline "$KAPPA" + IDENTITY: ${{ needs.route.outputs.identity }} + run: node scripts/factory.mjs outcome offline "$IDENTITY" success .tmp/factory - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: - name: receipt-offline-${{ needs.route.outputs.kappa }} - path: .tmp/receipt/receipt.json + name: factory-outcome-offline + path: .tmp/factory/ if-no-files-found: error - retention-days: 90 + retention-days: 3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d60b2c8..4a3deca9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,14 @@ # Builds the uorc client from the committed LexLean-exported Rust (generated/rust) # and publishes it: native binaries, the library source bundle, the WebAssembly # module, a Canterbury and Silesia benchmark run, and the GitHub Pages site that runs the client. +# +# Every stage is a factory obligation (factory/graph.json, scripts/factory.mjs) +# keyed on its own inputs. A stage proved by a receipt is not run again: its +# artifact is taken from the run that proved it. So a rehearsal (dispatch with +# mode rehearsal, on main or any ref) builds, benchmarks and packages a commit +# without publishing, and the tag of that commit then only packages and +# publishes. A failed publish or Pages step re-runs only what is left. +# Publication is blocked unless the bootstrap gate passed on exactly this tree. name: release on: @@ -8,58 +16,171 @@ on: tags: ["v*"] workflow_dispatch: inputs: - tag: - description: Existing tag to (re)publish, e.g. v0.3.0 + ref: + description: The tag to publish (e.g. v0.4.0), or for a rehearsal any branch, tag or commit required: true type: string + mode: + description: publish creates or updates the release; rehearsal builds and packages only + required: true + default: publish + type: choice + options: [publish, rehearsal] permissions: contents: read concurrency: - group: release-${{ github.event.inputs.tag || github.ref_name }} + group: release-${{ github.event.inputs.ref || github.ref_name }} cancel-in-progress: false env: - TAG: ${{ github.event.inputs.tag || github.ref_name }} + REF: ${{ github.event.inputs.ref || github.ref_name }} + MODE: ${{ github.event.inputs.mode || 'publish' }} CARGO_TERM_COLOR: never jobs: + # Decides every stage's route from the receipts, and refuses a publish whose + # tag does not name the workspace version (v0.2.0 shipped binaries that said + # 0.1.0) or whose tree has not passed the bootstrap gate. + factory: + name: factory route + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + tag: ${{ steps.policy.outputs.tag }} + native_matrix: ${{ steps.matrix.outputs.native }} + native_x86_identity: ${{ steps.route.outputs.native_x86_64_unknown_linux_gnu_identity }} + native_x86_run: ${{ steps.route.outputs.native_x86_64_unknown_linux_gnu_run }} + native_arm_identity: ${{ steps.route.outputs.native_aarch64_unknown_linux_gnu_identity }} + native_arm_run: ${{ steps.route.outputs.native_aarch64_unknown_linux_gnu_run }} + wasm: ${{ steps.route.outputs.wasm_route }} + wasm_identity: ${{ steps.route.outputs.wasm_identity }} + wasm_run: ${{ steps.route.outputs.wasm_run }} + bench: ${{ steps.route.outputs.bench_route }} + bench_identity: ${{ steps.route.outputs.bench_identity }} + bench_run: ${{ steps.route.outputs.bench_run }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ env.REF }} + fetch-depth: 0 + persist-credentials: false + - name: Check the release policy + id: policy + run: | + set -euo pipefail + version=$(sed -n '/^\[workspace.package\]/,/^\[/s/^version = "\(.*\)"/\1/p' Cargo.toml) + if [ "$MODE" = publish ]; then + git rev-parse --verify --quiet "refs/tags/$REF" >/dev/null || { echo "::error::$REF is not a tag; publish needs one (rehearse a branch with mode rehearsal)"; exit 1; } + [ "$REF" = "v$version" ] || { echo "::error::tag $REF does not name the workspace version $version; bump [workspace.package] version in Cargo.toml first"; exit 1; } + fi + echo "tag=v$version" >> "$GITHUB_OUTPUT" + - name: Route every stage by its own inputs + id: route + run: | + set -euo pipefail + node --test scripts/factory.test.mjs >/dev/null + node scripts/factory.mjs route gate native-x86_64-unknown-linux-gnu native-aarch64-unknown-linux-gnu wasm bench publish \ + --context "mode=$MODE" --context "run=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" | tee -a "$GITHUB_OUTPUT" + - name: List the native targets to build + id: matrix + env: + X86: ${{ steps.route.outputs.native_x86_64_unknown_linux_gnu_route }} + ARM: ${{ steps.route.outputs.native_aarch64_unknown_linux_gnu_route }} + X86_IDENTITY: ${{ steps.route.outputs.native_x86_64_unknown_linux_gnu_identity }} + ARM_IDENTITY: ${{ steps.route.outputs.native_aarch64_unknown_linux_gnu_identity }} + run: | + rows=() + [ "$X86" = observe ] && rows+=("{\"runner\":\"ubuntu-24.04\",\"target\":\"x86_64-unknown-linux-gnu\",\"identity\":\"$X86_IDENTITY\"}") + [ "$ARM" = observe ] && rows+=("{\"runner\":\"ubuntu-24.04-arm\",\"target\":\"aarch64-unknown-linux-gnu\",\"identity\":\"$ARM_IDENTITY\"}") + echo "native={\"include\":[$(IFS=,; echo "${rows[*]}")]}" >> "$GITHUB_OUTPUT" + - name: Require the bootstrap gate on this tree + if: steps.route.outputs.gate_route != 'proved' || steps.route.outputs.publish_route != 'observe' + run: | + echo "::error::the bootstrap legs have not all passed on this exact tree; let bootstrap finish on this commit (receipts land a minute after it completes), then re-run" + exit 1 + native: name: native / ${{ matrix.target }} + needs: factory + if: needs.factory.outputs.native_matrix != '{"include":[]}' strategy: fail-fast: false - matrix: - include: - - { runner: ubuntu-24.04, target: x86_64-unknown-linux-gnu } - - { runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu } + matrix: ${{ fromJSON(needs.factory.outputs.native_matrix) }} runs-on: ${{ matrix.runner }} timeout-minutes: 45 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ env.TAG }} + ref: ${{ env.REF }} fetch-depth: 0 persist-credentials: false - name: Build and test the client + id: run run: | set -euo pipefail rustup show active-toolchain cargo test --locked --release -p uorc cargo build --locked --release -p uorc --target ${{ matrix.target }} - - name: Package + mkdir -p build && cp "target/${{ matrix.target }}/release/uorc" build/ + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: factory-native-${{ matrix.target }} + path: build/uorc + if-no-files-found: error + retention-days: 90 + - name: Record the outcome + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') + env: + IDENTITY: ${{ matrix.identity }} + OUTCOME: ${{ steps.run.outcome }} + run: node scripts/factory.mjs outcome "native-${{ matrix.target }}" "$IDENTITY" "$OUTCOME" .tmp/factory + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') + with: + name: factory-outcome-native-${{ matrix.target }} + path: .tmp/factory/ + if-no-files-found: error + retention-days: 3 + + wasm: + name: wasm + needs: factory + if: needs.factory.outputs.wasm == 'observe' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ env.REF }} + fetch-depth: 0 + persist-credentials: false + - name: Build the WebAssembly client + id: run run: | set -euo pipefail - name="uorc-${TAG}-${{ matrix.target }}" - mkdir -p "dist/$name" - cp "target/${{ matrix.target }}/release/uorc" LICENSE-APACHE LICENSE-MIT "dist/$name/" - tar -C dist -czf "dist/$name.tar.gz" "$name" - (cd dist && sha256sum "$name.tar.gz" > "$name.tar.gz.sha256") + cargo build --locked --release -p uorc-wasm --target wasm32-unknown-unknown + mkdir -p build && cp target/wasm32-unknown-unknown/release/uorc_wasm.wasm build/uorc.wasm + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: factory-wasm + path: build/uorc.wasm + if-no-files-found: error + retention-days: 90 + - name: Record the outcome + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') + env: + IDENTITY: ${{ needs.factory.outputs.wasm_identity }} + OUTCOME: ${{ steps.run.outcome }} + run: node scripts/factory.mjs outcome wasm "$IDENTITY" "$OUTCOME" .tmp/factory - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') with: - name: native-${{ matrix.target }} - path: dist/*.tar.gz* + name: factory-outcome-wasm + path: .tmp/factory/ if-no-files-found: error + retention-days: 3 # Measurement only (UC-PERF-01): the release binary on the Canterbury and # Silesia corpora against standard and context-mixing compressors, beside @@ -67,127 +188,174 @@ jobs: # pull requests, and its table goes into the release notes and the Pages site. bench: name: benchmark - needs: native + needs: [factory, native] + if: always() && needs.factory.outputs.bench == 'observe' && (needs.native.result == 'success' || needs.native.result == 'skipped') runs-on: ubuntu-24.04 timeout-minutes: 120 + permissions: + contents: read + actions: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ env.TAG }} + ref: ${{ env.REF }} fetch-depth: 0 persist-credentials: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: - name: native-x86_64-unknown-linux-gnu - path: dist + name: factory-native-x86_64-unknown-linux-gnu + path: build + run-id: ${{ needs.factory.outputs.native_x86_run }} + github-token: ${{ needs.factory.outputs.native_x86_run != github.run_id && github.token || '' }} - name: Fetch the corpora and the comparison tools run: | set -euo pipefail - sudo apt-get update -qq - sudo apt-get install -y -qq zstd brotli zpaq 7zip >/dev/null - tar -C dist -xzf "dist/uorc-${TAG}-x86_64-unknown-linux-gnu.tar.gz" + chmod +x build/uorc + for attempt in 1 2 3; do + sudo apt-get update -qq && sudo apt-get install -y -qq zstd brotli zpaq 7zip >/dev/null && break + test "$attempt" -lt 3; sleep $((attempt * 20)) + done mkdir -p corpus/canterbury corpus/silesia - curl -fsSL --retry 3 https://corpus.canterbury.ac.nz/resources/cantrbry.tar.gz | tar -xz -C corpus/canterbury - curl -fsSL --retry 3 -o silesia.zip https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip + curl -fsSL --retry 5 --retry-all-errors https://corpus.canterbury.ac.nz/resources/cantrbry.tar.gz | tar -xz -C corpus/canterbury + curl -fsSL --retry 5 --retry-all-errors -o silesia.zip https://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip unzip -q -d corpus/silesia silesia.zip find corpus/silesia -mindepth 2 -type f -exec mv -t corpus/silesia {} + - name: Benchmark the release binary - run: node scripts/bench.mjs --uorc "dist/uorc-${TAG}-x86_64-unknown-linux-gnu/uorc" --corpus corpus/canterbury,corpus/silesia --steps 0 --out bench + id: run + run: node scripts/bench.mjs --uorc build/uorc --corpus corpus/canterbury,corpus/silesia --steps 0 --out bench - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: - name: bench + name: factory-bench path: | bench/bench.json bench/bench.md bench/bench.html if-no-files-found: error + retention-days: 90 + - name: Record the outcome + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') + env: + IDENTITY: ${{ needs.factory.outputs.bench_identity }} + OUTCOME: ${{ steps.run.outcome }} + run: node scripts/factory.mjs outcome bench "$IDENTITY" "$OUTCOME" .tmp/factory + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() && (steps.run.outcome == 'success' || steps.run.outcome == 'failure') + with: + name: factory-outcome-bench + path: .tmp/factory/ + if-no-files-found: error + retention-days: 3 - wasm: - name: wasm and site - needs: bench + # Packages what the stages proved (from this run or the runs that proved + # them) under the tag, and publishes it; a rehearsal keeps the package as an + # artifact instead. + publish: + name: ${{ github.event.inputs.mode == 'rehearsal' && 'package (rehearsal)' || 'publish release' }} + needs: [factory, native, wasm, bench] + if: always() && needs.factory.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 15 + permissions: + contents: write + actions: read + env: + TAG: ${{ needs.factory.outputs.tag }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ env.TAG }} + ref: ${{ env.REF }} fetch-depth: 0 persist-credentials: false - - name: Build the WebAssembly client - run: cargo build --locked --release -p uorc-wasm --target wasm32-unknown-unknown - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: - name: bench + name: factory-native-x86_64-unknown-linux-gnu + path: build/x86_64-unknown-linux-gnu + run-id: ${{ needs.factory.outputs.native_x86_run }} + github-token: ${{ needs.factory.outputs.native_x86_run != github.run_id && github.token || '' }} + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: factory-native-aarch64-unknown-linux-gnu + path: build/aarch64-unknown-linux-gnu + run-id: ${{ needs.factory.outputs.native_arm_run }} + github-token: ${{ needs.factory.outputs.native_arm_run != github.run_id && github.token || '' }} + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: factory-wasm + path: build/wasm + run-id: ${{ needs.factory.outputs.wasm_run }} + github-token: ${{ needs.factory.outputs.wasm_run != github.run_id && github.token || '' }} + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: factory-bench path: bench - - name: Assemble the site and the wasm bundle + run-id: ${{ needs.factory.outputs.bench_run }} + github-token: ${{ needs.factory.outputs.bench_run != github.run_id && github.token || '' }} + - name: Package the binaries, the source bundle, the wasm bundle and the site run: | set -euo pipefail - mkdir -p site dist + mkdir -p dist site + for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do + name="uorc-${TAG}-${target}" + mkdir -p "dist/$name" + install -m 755 "build/$target/uorc" "dist/$name/uorc" + cp LICENSE-APACHE LICENSE-MIT "dist/$name/" + tar -C dist -czf "dist/$name.tar.gz" "$name" + (cd dist && sha256sum "$name.tar.gz" > "$name.tar.gz.sha256") + rm -rf "dist/$name" + done cp web/* site/ - cp target/wasm32-unknown-unknown/release/uorc_wasm.wasm site/uorc.wasm + cp build/wasm/uorc.wasm site/uorc.wasm cp bench/bench.html bench/bench.json bench/bench.md site/ printf '%s\n' "$TAG" > site/version.txt name="uorc-${TAG}-wasm" mkdir -p "dist/$name" cp site/uorc.wasm site/uorc.js LICENSE-APACHE LICENSE-MIT "dist/$name/" tar -C dist -czf "dist/$name.tar.gz" "$name" + rm -rf "dist/$name" git archive --format=tar.gz --prefix="uorc-${TAG}-src/" -o "dist/uorc-${TAG}-src.tar.gz" HEAD \ Cargo.toml Cargo.lock rust-toolchain.toml crates/uorc crates/uorc-wasm generated/rust LICENSE-APACHE LICENSE-MIT cp bench/bench.json "dist/uorc-${TAG}-bench.json" - (cd dist && sha256sum *.tar.gz *.json > SHA256SUMS-wasm-src) + (cd dist && sha256sum uorc-*-wasm.tar.gz uorc-*-src.tar.gz *.json > SHA256SUMS-wasm-src) + ls -la dist - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: env.MODE == 'rehearsal' with: - name: wasm-src + name: release-rehearsal path: dist/* if-no-files-found: error + retention-days: 7 + - name: Create or update the GitHub release + if: env.MODE == 'publish' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + { printf 'uorc %s\n\n' "$TAG"; printf 'Client: https://%s.github.io/%s/\n\n' "${GITHUB_REPOSITORY_OWNER,,}" "${GITHUB_REPOSITORY#*/}"; cat bench/bench.md; } > notes.md + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" dist/* --clobber + else + gh release create "$TAG" --verify-tag --title "uorc $TAG" --notes-file notes.md dist/* + fi # The Pages artifact is packed here rather than by actions/upload-pages-artifact, # which calls an unpinned upload-artifact that the organization's action policy rejects. - name: Pack the Pages site + if: env.MODE == 'publish' run: tar --dereference --hard-dereference -C site -cf "$RUNNER_TEMP/artifact.tar" . - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: env.MODE == 'publish' with: name: github-pages path: ${{ runner.temp }}/artifact.tar retention-days: 1 if-no-files-found: error - publish: - name: publish release - needs: [native, bench, wasm] - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: write - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - ref: ${{ env.TAG }} - fetch-depth: 0 - persist-credentials: false - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - path: assets - merge-multiple: true - - name: Create or update the GitHub release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - ls -la assets - { printf 'uorc %s\n\n' "$TAG"; printf 'Client: https://%s.github.io/%s/\n\n' "${GITHUB_REPOSITORY_OWNER,,}" "${GITHUB_REPOSITORY#*/}"; cat assets/bench.md; } > notes.md - files=$(find assets -maxdepth 1 -type f \( -name '*.tar.gz' -o -name '*.sha256' -o -name 'SHA256SUMS*' -o -name '*-bench.json' \)) - if gh release view "$TAG" >/dev/null 2>&1; then - gh release upload "$TAG" $files --clobber - else - gh release create "$TAG" --verify-tag --title "uorc $TAG" --notes-file notes.md $files - fi - # The github-pages environment accepts only the default branch, so a tag-push # run hands its site to pages.yml, dispatched there (v0.2.0's in-run deploy # was refused before it started). pages: name: deploy pages - needs: [wasm, publish] + needs: publish + if: github.event.inputs.mode != 'rehearsal' runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: diff --git a/factory/README.md b/factory/README.md new file mode 100644 index 00000000..337fa504 --- /dev/null +++ b/factory/README.md @@ -0,0 +1,81 @@ +# uorc factory + +The release factory keys every CI and release obligation on its own declared +inputs, so a change re-runs only the obligations that read it. It follows the +PrismPM factory route (PrismPM #566, #579-#581, #586): the same obligation +identity text (`prism-factory/obligation/1`), receipts on a `factory-receipts` +branch, refutation of hermetic failures, retried fetches, and a 16-level weight +that orders remaining work. Here the decisions are a small Node host +(`scripts/factory.mjs`, tests in `scripts/factory.test.mjs`) over the graph in +`graph.json`, not a kernel-checked model: uorc's graph is fifteen rows. + +## The graph + +| Obligation | Kind | Reads | Carried out by | +| --- | --- | --- | --- | +| `rest-` (2) | fetch | everything | bootstrap `template-gate` | +| `artifacts`, `claims`, `varint`, `reproducibility`, `integrity` | hermetic | the formal set | bootstrap `template-gate` | +| `offline` | fetch | the formal set | offline-validation | +| `export` | fetch, host | the import closure of `Uorc.Client`, the LexLean project files, the export scripts | `node scripts/client-export.mjs` on a workstation | +| `gate` | gate | the identities of the seven bootstrap legs | proved when all seven are | +| `native-` (2), `wasm` | fetch | the client set (+ `crates/uorc-wasm/`) | release `native`, `wasm` | +| `bench` | fetch | `scripts/bench.mjs`, release.yml, `native-x86_64` | release `bench` | +| `publish` | fetch | everything, `@mode`, `@run` | release `publish`, `pages` | + +The formal set is the tree without the product client and the release plumbing +(the same scope the formal legs had before). The client set is what `cargo build +-p uorc` reads: the workspace manifests and lock, the toolchain, `.cargo/`, +`crates/uorc/` and `generated/rust/`. + +## Identity + +An obligation's identity is the SHA-256 (hex) of + +```text +prism-factory/obligation/1 +name +kind hermetic|fetch|gate +environment +run +input one line per declared input (`!prefix` excludes, `@key` is a context, `closure:Module` an import closure) +needs one line per need +closure +``` + +where the closure is `file : ` for each tracked entry the +inputs cover, in tree order, then `file context: @` per context. An +entry is covered when the longest inclusion it starts with is longer than the +longest exclusion it starts with. + +## Routes + +- **proved**: a passing receipt for this identity exists. The job is skipped; + release stages take their artifact from the run that proved them. +- **refuted**: a hermetic obligation failed on exactly these inputs. It fails in + seconds instead of an hour later. Re-running the job (attempt 2+) observes it again. +- **blocked**: something it needs is not proved and will not run. +- **observe**: run it. + +Each job uploads only a `factory-outcome-*` artifact. `factory.yml` runs from +the default branch after a same-repository run completes and turns outcomes into +receipts, so no pull-request job holds a write token. Receipts for stages with +artifacts expire a day before the artifact does. + +## Releasing + +1. Bump `[workspace.package] version` in `Cargo.toml` (publish refuses a tag that + does not name it). +2. Optional: dispatch **release** with `mode: rehearsal` and `ref: main`. It + builds, benchmarks and packages without publishing. +3. Tag `vX.Y.Z`. The tag run requires the bootstrap gate on that tree, reuses + every stage the rehearsal (or an earlier attempt) proved, packages, publishes, + and dispatches the Pages deployment on the default branch. + +## Locally + +```sh +node scripts/factory.mjs plan # every obligation's route for HEAD, and the schedule +node scripts/factory.mjs export-check # is generated/rust exported from the current client closure? +node scripts/client-export.mjs # exits at once when it is; --force exports anyway +node --test scripts/factory.test.mjs +``` diff --git a/factory/graph.json b/factory/graph.json new file mode 100644 index 00000000..92d8bbc3 --- /dev/null +++ b/factory/graph.json @@ -0,0 +1,24 @@ +{ + "schema": "uorc/factory-graph/1", + "sets": { + "formal": ["", "!client/", "!crates/uorc/", "!crates/uorc-wasm/", "!generated/", "!web/", "!Cargo.toml", "!Cargo.lock", "!README.md", "!scripts/bench.mjs", "!scripts/client-export.mjs", "!factory/", "!scripts/factory.", "!.github/workflows/release.yml", "!.github/workflows/pages.yml", "!.github/workflows/factory.yml"], + "client": ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", ".cargo/", "crates/uorc/", "generated/rust/", ".github/workflows/release.yml"] + }, + "obligations": [ + {"name": "rest-ubuntu-24.04", "kind": "fetch", "environment": "ubuntu-24.04", "run": "just lexlean-authority template-check fmt-check model lint test features bdd deny", "inputs": [""], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "rest-ubuntu-24.04-arm", "kind": "fetch", "environment": "ubuntu-24.04-arm", "run": "just lexlean-authority template-check fmt-check model lint test features bdd deny", "inputs": [""], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "artifacts-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just lexlean-artifacts", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "claims-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just claim-dispositions formal-tests", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "varint-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just varint-proofs", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "reproducibility-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just reproducibility", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "integrity-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just integrity", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, + {"name": "offline", "kind": "fetch", "environment": "ubuntu-24.04", "run": "node scripts/offline-validation.mjs acquire, isolated first and second-distinct-root, compare-offline", "inputs": ["&formal"], "needs": [], "workflow": "offline-validation.yml", "job": "offline-compare"}, + {"name": "export", "kind": "fetch", "environment": "host", "run": "node scripts/client-export.mjs", "inputs": ["closure:Uorc.Client", "lexlean.toml", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "scripts/client-export.mjs", "scripts/semantic-source.mjs"], "needs": [], "record": "generated/rust/factory.txt"}, + {"name": "gate", "kind": "gate", "environment": "-", "run": "-", "inputs": [], "needs": ["rest-ubuntu-24.04", "rest-ubuntu-24.04-arm", "artifacts-ubuntu-24.04", "claims-ubuntu-24.04", "varint-ubuntu-24.04", "reproducibility-ubuntu-24.04", "integrity-ubuntu-24.04"], "workflow": "release.yml", "job": "factory"}, + {"name": "native-x86_64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target x86_64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-x86_64-unknown-linux-gnu"}, + {"name": "native-aarch64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04-arm", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target aarch64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-aarch64-unknown-linux-gnu"}, + {"name": "wasm", "kind": "fetch", "environment": "ubuntu-24.04", "run": "cargo build --locked --release -p uorc-wasm --target wasm32-unknown-unknown", "inputs": ["&client", "crates/uorc-wasm/"], "needs": [], "workflow": "release.yml", "job": "wasm", "evidence": "factory-wasm"}, + {"name": "bench", "kind": "fetch", "environment": "ubuntu-24.04", "run": "node scripts/bench.mjs --corpus corpus/canterbury,corpus/silesia --steps 0", "inputs": ["scripts/bench.mjs", ".github/workflows/release.yml"], "needs": ["native-x86_64-unknown-linux-gnu"], "workflow": "release.yml", "job": "bench", "evidence": "factory-bench"}, + {"name": "publish", "kind": "fetch", "environment": "ubuntu-24.04", "run": "package, gh release, pages", "inputs": ["", "@mode", "@run"], "needs": ["gate", "native-x86_64-unknown-linux-gnu", "native-aarch64-unknown-linux-gnu", "wasm", "bench"], "workflow": "release.yml", "job": "publish"} + ] +} diff --git a/generated/rust/factory.txt b/generated/rust/factory.txt new file mode 100644 index 00000000..bcf53c38 --- /dev/null +++ b/generated/rust/factory.txt @@ -0,0 +1 @@ +export c518791b8a839e42cb5eb428c3153b0342c8423fac7ecc001d4b513bdd07b0d9 diff --git a/scripts/ci-receipt.sh b/scripts/ci-receipt.sh deleted file mode 100755 index 00064f89..00000000 --- a/scripts/ci-receipt.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Content-addressed CI receipts: each slow obligation runs once per content -# identity of its inputs, and a later run with the same identity reuses that -# result instead of proving it again. -# -# kappa print κ, the SHA-256 of the tracked tree minus the -# paths no formal leg reads (the product client and -# release plumbing below) -# lookup print kappa=…, proved=true|false and run=… for -# $GITHUB_OUTPUT. proved=true when an unexpired -# receipt artifact for this obligation and κ exists -# from a same-repository run -# record <κ> write .tmp/receipt/receipt.json for upload -# -# A receipt artifact is uploaded only after its obligation passed, so its -# existence is the proof. Any lookup failure means "not proved": the work runs. -set -euo pipefail - -# The product client and release plumbing. Product changes are checked by the -# `rest` leg on every run; none of these paths is read by the formal legs. -EXCLUDE='^(client/|crates/uorc/|crates/uorc-wasm/|generated/|web/|Cargo\.toml$|Cargo\.lock$|README\.md$|scripts/bench\.mjs$|scripts/client-export\.mjs$|scripts/ci-receipt\.sh$|\.github/workflows/release\.yml$|\.github/workflows/pages\.yml$)' - -kappa() { - git ls-tree -r --full-tree HEAD | awk -F'\t' -v re="$EXCLUDE" '$2 !~ re' | sha256sum | cut -c1-64 -} - -case "${1:-}" in - kappa) - kappa - ;; - lookup) - obligation=${2:?obligation} - k=$(kappa) - echo "kappa=$k" - run=$(gh api "repos/$GITHUB_REPOSITORY/actions/artifacts?name=receipt-$obligation-$k&per_page=100" \ - --jq '[.artifacts[] | select(.expired | not) | select(.workflow_run.head_repository_id == .workflow_run.repository_id)] | first | .workflow_run.id // empty' \ - 2>/dev/null || true) - if [[ $run =~ ^[0-9]+$ ]]; then - echo "proved=true" - echo "run=$run" - echo "::notice title=$obligation proved by receipt::κ $k passed in run $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$run; its inputs are unchanged, so it is not run again." >&2 - else - echo "proved=false" - fi - ;; - record) - obligation=${2:?obligation} - k=${3:?kappa} - mkdir -p .tmp/receipt - printf '{"schema":"uorc/ci-receipt/1","obligation":"%s","kappa":"%s","sha":"%s","run":"%s","attempt":"%s"}\n' \ - "$obligation" "$k" "$(git rev-parse HEAD)" "${GITHUB_RUN_ID:-}" "${GITHUB_RUN_ATTEMPT:-}" > .tmp/receipt/receipt.json - ;; - *) - echo "usage: $0 kappa | lookup | record " >&2 - exit 64 - ;; -esac diff --git a/scripts/client-export.mjs b/scripts/client-export.mjs index 38542e5f..8c1bae82 100644 --- a/scripts/client-export.mjs +++ b/scripts/client-export.mjs @@ -25,11 +25,18 @@ // field read, and the in-place byte update primitive (LexLean#443) gives the // codec O(1) table writes. The pin moves to a LexLean release once both ship. // Set LEXLEAN to an already built binary of the same revision to skip the build. +// +// The export is the factory obligation `export` (factory/graph.json): its +// identity covers only what it reads (the import closure of Uorc.Client, the +// LexLean project files and these scripts), recorded in generated/rust/factory.txt. +// When the recorded identity matches, the export is already proved and this +// exits without running; pass --force to export anyway. import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import {spawnSync} from 'node:child_process'; import {dataForm, notation} from './semantic-source.mjs'; +import {loadGraph, identities, workingTree} from './factory.mjs'; const root = process.cwd(); const out = path.join(root, 'generated/rust'); @@ -131,6 +138,12 @@ function wrapBoxedReturns(dir, modules) { } } +const exportRecord = path.join(out, 'factory.txt'); +const exportIdentity = identities(loadGraph(root), workingTree(root), {}).export; +if (!process.argv.includes('--force') && fs.existsSync(exportRecord) && fs.readFileSync(exportRecord, 'utf8').trim() === `export ${exportIdentity}`) { + console.log(`export proved: generated/rust was exported from these inputs (${exportIdentity}); --force exports anyway`); + process.exit(0); +} const lexlean = lexleanBinary(); const stage = fs.mkdtempSync(path.join(os.tmpdir(), 'uorc-client-export-')); try { @@ -154,5 +167,6 @@ try { for (const [name] of roots) fs.cpSync(path.join(bundle, 'packages/rust-std', name), path.join(out, name), {recursive: true}); fs.writeFileSync(path.join(out, 'request.json'), JSON.stringify(request, null, 1) + '\n'); fs.copyFileSync(path.join(bundle, 'result.json'), path.join(out, 'result.json')); + fs.writeFileSync(exportRecord, `export ${exportIdentity}\n`); console.log(`exported ${roots.length} packages from ${modules.length} modules (result ${result.result_id ?? result.artifacts[0]})`); } finally { fs.rmSync(stage, {recursive: true, force: true}); } diff --git a/scripts/factory.mjs b/scripts/factory.mjs new file mode 100644 index 00000000..af4a9e9f --- /dev/null +++ b/scripts/factory.mjs @@ -0,0 +1,381 @@ +#!/usr/bin/env node +// The uorc release factory: the PrismPM factory route (PrismPM #566, #579-#581, +// #586) applied to this repository's CI and release. +// +// Each obligation in factory/graph.json declares its own inputs. Its identity is +// the SHA-256 of those inputs' tracked content, its kind, environment and run, +// and the identities of what it needs, in PrismPM's `prism-factory/obligation/1` +// text, so a change re-runs only the obligations that read it. Outcomes become +// receipts on the `factory-receipts` branch, written only by factory.yml from +// the default branch after a same-repository run completes. Routing: +// proved a passing receipt for this identity exists; the work is skipped and +// its evidence (if any) comes from the run that proved it +// refuted a hermetic obligation failed on exactly these inputs; it is not +// run again (a manual re-run, attempt > 1, observes it again) +// blocked something it needs is neither proved nor runnable +// observe run it +// A failed fetch obligation is observed again, because the network is not an +// input. Priority is a 16-level weight per obligation: a failure raises it by +// one, a pass lowers it by one, and `plan` orders the remaining work by it. +// +// node scripts/factory.mjs plan [--rev REV] [--context KEY=VALUE]... +// node scripts/factory.mjs identity NAME [--rev REV] [--context KEY=VALUE]... +// node scripts/factory.mjs route NAME... [--context KEY=VALUE]... [--expect-run TEXT] +// node scripts/factory.mjs outcome NAME IDENTITY success|failure DIR +// node scripts/factory.mjs publish DIR (factory.yml only: needs GH_TOKEN) +// node scripts/factory.mjs export-check | export-record +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import {spawnSync} from 'node:child_process'; +import {fileURLToPath} from 'node:url'; + +export const BRANCH = process.env.FACTORY_RECEIPTS_BRANCH || 'factory-receipts'; +const LEVELS = 16, DEFAULT_LEVEL = 4; +const sha = data => crypto.createHash('sha256').update(data).digest('hex'); +const fail = message => { throw new Error(message); }; + +export function git(args, {cwd = process.cwd(), input, allowFail = false, env} = {}) { + const r = spawnSync('git', args, {cwd, input, env: env ? {...process.env, ...env} : process.env, maxBuffer: 1 << 30}); + if (r.status !== 0 && !allowFail) fail(`git ${args.join(' ')}: ${r.stderr}`); + return r.status === 0 ? r.stdout : null; +} +const root = () => git(['rev-parse', '--show-toplevel']).toString().trim(); + +/// Every tracked entry at REV, in Git tree order: its path and `mode:object`. +export function tree(rev, cwd) { + const entries = []; + for (const row of git(['ls-tree', '-r', '-z', '--full-tree', rev], {cwd}).toString().split('\0')) { + if (!row) continue; + const tab = row.indexOf('\t'); const [mode, , object] = row.slice(0, tab).split(' '); + const p = row.slice(tab + 1); if (p.includes('\n')) fail(`unsupported path ${JSON.stringify(p)}`); + entries.push({path: p, object: `${mode}:${object}`}); + } + return entries; +} + +export function loadGraph(dir) { + const graph = JSON.parse(fs.readFileSync(path.join(dir, 'factory/graph.json'), 'utf8')); + if (graph.schema !== 'uorc/factory-graph/1') fail('factory/graph.json: unknown schema'); + const seen = new Set(); + for (const o of graph.obligations) { + if (seen.has(o.name)) fail(`obligation ${o.name} is declared twice`); + for (const need of o.needs) if (!seen.has(need)) fail(`${o.name} needs ${need}, which is not declared before it`); + if (!['hermetic', 'fetch', 'gate'].includes(o.kind)) fail(`${o.name}: unknown kind ${o.kind}`); + seen.add(o.name); + o.inputs = o.inputs.flatMap(i => i.startsWith('&') ? (graph.sets[i.slice(1)] ?? fail(`${o.name}: unknown set ${i}`)) : [i]); + } + return graph; +} + +// `closure:Module` is the module's import closure, each module read from +// client/src when it has one there and from src otherwise (as the export stages it). +function closurePaths(module, entries) { + const paths = new Set(entries.map(e => e.path)); const found = []; const seen = new Set(); const stack = [module]; + while (stack.length) { + const m = stack.pop(); if (seen.has(m)) continue; seen.add(m); + const rel = m.split('.').join('/') + '.lex.tex'; + const p = paths.has(`client/src/${rel}`) ? `client/src/${rel}` : paths.has(`src/${rel}`) ? `src/${rel}` : fail(`closure ${module}: module ${m} is not tracked`); + found.push(p); + const object = entries.find(e => e.path === p).object.split(':')[1]; + for (const i of git(['cat-file', 'blob', object]).toString().matchAll(/\\importmodule\{([^}]+)\}/g)) stack.push(i[1]); + } + return new Set(found); +} + +/// An entry is covered when the longest inclusion it starts with is longer +/// than the longest exclusion (`!prefix`) it starts with. +export function covered(inputs, p, exact) { + let include = -1, exclude = -1; + for (const i of inputs) { + if (i.startsWith('@') || i.startsWith('closure:')) continue; + if (i.startsWith('!')) { if (p.startsWith(i.slice(1))) exclude = Math.max(exclude, i.length - 1); } + else if (p.startsWith(i)) include = Math.max(include, i.length); + } + if (exact.has(p)) include = Math.max(include, p.length); + return include > exclude; +} + +/// Identities of every obligation at REV: PrismPM's `prism-factory/obligation/1`. +export function identities(graph, entries, contexts) { + const ids = {}; + for (const o of graph.obligations) { + // An obligation that reads a context nobody supplied, or needs one that + // does, has no identity here (a pull request run never plans `publish`). + if (o.inputs.some(i => i.startsWith('@') && contexts[i.slice(1)] === undefined) || o.needs.some(n => ids[n] === null)) { ids[o.name] = null; continue; } + const exact = new Set(o.inputs.filter(i => i.startsWith('closure:')).flatMap(i => [...closurePaths(i.slice(8), entries)])); + let closure = ''; + for (const e of entries) if (covered(o.inputs, e.path, exact)) closure += `file ${e.object} ${e.path}\n`; + for (const i of o.inputs) if (i.startsWith('@')) { + closure += `file context:${contexts[i.slice(1)]} ${i}\n`; + } + let text = `prism-factory/obligation/1\nname ${o.name}\nkind ${o.kind}\nenvironment ${o.environment}\nrun ${o.run}\n`; + for (const i of o.inputs) text += `input ${i}\n`; + for (const n of o.needs) text += `needs ${n} ${ids[n]}\n`; + ids[o.name] = sha(text + `closure ${sha(closure)}\n`); + } + return ids; +} + +// Receipts: the index of the receipts branch, one line per receipt: +// receipt IDENTITY NAME pass|fail RUN ATTEMPT COMMIT EXPIRES +// and its weights: `weight NAME LEVEL`. +export function parseStore(text) { + const receipts = [], weights = {}; + for (const line of (text ?? '').split('\n')) { + const f = line.split(' '); + if (f[0] === 'receipt' && f.length === 8) receipts.push({identity: f[1], name: f[2], result: f[3], run: f[4], attempt: f[5], commit: f[6], expires: f[7]}); + if (f[0] === 'weight' && f.length === 3) weights[f[1]] = Math.min(LEVELS, Math.max(1, Number(f[2]) || DEFAULT_LEVEL)); + } + return {receipts, weights}; +} + +export function fetchStore(cwd) { + if (process.env.FACTORY_STORE) return fs.readFileSync(process.env.FACTORY_STORE, 'utf8'); + if (git(['fetch', '--quiet', '--depth=1', 'origin', `+refs/heads/${BRANCH}:refs/factory/receipts`], {cwd, allowFail: true}) === null) { + console.error(`factory: no ${BRANCH} branch could be read; every obligation is observed`); + return ''; + } + return (git(['cat-file', 'blob', 'refs/factory/receipts:index'], {cwd, allowFail: true}) ?? '').toString(); +} + +/// Routes in graph order. `inRun` treats an `observe` need as passing later in +/// the same run (a release run builds native before bench); without it a need +/// must already be proved. +export function route(graph, ids, store, {now = new Date(), attempt = 1, inRun = true} = {}) { + const routes = {}; + for (const o of graph.obligations) { + if (ids[o.name] === null) { routes[o.name] = {route: 'unplanned', identity: null}; continue; } + const mine = store.receipts.filter(r => r.identity === ids[o.name] && r.name === o.name && (r.expires === 'never' || new Date(r.expires) > now)); + const pass = mine.filter(r => r.result === 'pass').at(-1), failed = mine.filter(r => r.result === 'fail').at(-1); + const needs = o.needs.map(n => routes[n].route); + const ready = needs.every(r => r === 'proved' || (inRun && r === 'observe')); + let r; + if (o.kind === 'gate') r = needs.every(x => x === 'proved') ? {route: 'proved'} : {route: 'blocked'}; + else if (!ready) r = {route: 'blocked'}; + else if (pass) r = {route: 'proved', run: pass.run}; + else if (o.kind === 'hermetic' && failed && attempt < 2) r = {route: 'refuted', run: failed.run}; + else r = {route: 'observe'}; + routes[o.name] = {...r, identity: ids[o.name]}; + } + return routes; +} + +export function schedule(graph, routes, weights) { + return graph.obligations.filter(o => routes[o.name].route === 'observe') + .map((o, index) => ({name: o.name, level: weights[o.name] ?? DEFAULT_LEVEL, index})) + .sort((a, b) => b.level - a.level || a.index - b.index); +} + +export function reweigh(weights, outcomes) { + const next = {...weights}; + for (const {name, result} of outcomes) { + const level = next[name] ?? DEFAULT_LEVEL; + next[name] = result === 'pass' ? Math.max(1, level - 1) : Math.min(LEVELS, level + 1); + } + return next; +} + +function exportRoute(graph, ids, dir) { + const o = graph.obligations.find(x => x.name === 'export'); + const file = path.join(dir, o.record); + const recorded = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim() : null; + return {route: recorded === `export ${ids.export}` ? 'proved' : 'observe', identity: ids.export, recorded, file}; +} + +// The graph names each obligation's job; a renamed job or a changed leg recipe +// must change the graph too, or the route refuses to decide. +function checkWorkflows(graph, dir) { + for (const o of graph.obligations) { + if (!o.workflow) continue; + const text = fs.readFileSync(path.join(dir, '.github/workflows', o.workflow), 'utf8'); + if (!text.includes(`\n ${o.job}:\n`)) fail(`${o.name}: job ${o.job} is not in ${o.workflow}`); + } +} + +function options(argv) { + const out = {names: [], contexts: {}, rev: 'HEAD'}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--context') { const [k, ...v] = argv[++i].split('='); if (!k || !v.length) fail('--context needs KEY=VALUE'); out.contexts[k] = v.join('='); } + else if (a === '--rev') out.rev = argv[++i]; + else if (a === '--expect-run') out.expectRun = argv[++i]; + else if (a.startsWith('--')) fail(`unknown option ${a}`); + else out.names.push(a); + } + return out; +} + +const outputKey = name => name.replace(/[^A-Za-z0-9]/g, '_'); +const server = () => `${process.env.GITHUB_SERVER_URL ?? 'https://github.com'}/${process.env.GITHUB_REPOSITORY ?? 'UOR-Foundation/compression'}`; + +function main(argv) { + const [command, ...rest] = argv; + const dir = root(); + const opts = options(rest); + const graph = loadGraph(dir); + const byName = name => graph.obligations.find(o => o.name === name) ?? fail(`unknown obligation ${name}`); + const plannedContexts = {mode: 'rehearsal', run: 'local', ...opts.contexts}; + switch (command) { + case 'identity': { + const ids = identities(graph, tree(opts.rev, dir), plannedContexts); + for (const n of opts.names) console.log(ids[byName(n).name]); + return; + } + case 'plan': { + const ids = identities(graph, tree(opts.rev, dir), plannedContexts); + const store = parseStore(fetchStore(dir)); + const routes = route(graph, ids, store, {inRun: false}); + if (opts.rev === 'HEAD') routes.export = exportRoute(graph, ids, dir); + for (const o of graph.obligations) { + const r = routes[o.name]; + console.log(`obligation ${o.name} ${r.route} ${r.identity}${r.run ? ` run ${server()}/actions/runs/${r.run}` : ''}`); + } + for (const s of schedule(graph, routes, store.weights)) console.log(`schedule ${s.name} ${s.level}`); + console.log(`releasable ${routes.gate.route === 'proved' ? 'yes' : 'no'}`); + return; + } + case 'route': { + // GitHub Actions: write each named obligation's route to $GITHUB_OUTPUT. + checkWorkflows(graph, dir); + if (!opts.names.length) fail('route needs at least one obligation'); + if (opts.expectRun !== undefined && (opts.names.length !== 1 || byName(opts.names[0]).run !== opts.expectRun)) + fail(`the workflow runs "${opts.expectRun}" but factory/graph.json declares "${byName(opts.names[0]).run}" for ${opts.names[0]}`); + const ids = identities(graph, tree('HEAD', dir), opts.contexts); + const routes = route(graph, ids, parseStore(fetchStore(dir)), {attempt: Number(process.env.GITHUB_RUN_ATTEMPT ?? 1)}); + const lines = []; + for (const n of opts.names) { + const r = routes[byName(n).name]; + if (r.identity === null) fail(`${n} reads a context; pass it with --context`); + const key = opts.names.length === 1 ? '' : `${outputKey(n)}_`; + lines.push(`${key}route=${r.route}`, `${key}identity=${r.identity}`, `${key}run=${r.run ?? process.env.GITHUB_RUN_ID ?? ''}`); + const where = r.run ? ` in ${server()}/actions/runs/${r.run}` : ''; + if (r.route === 'proved') console.error(`::notice title=${n} proved by receipt::identity ${r.identity} passed${where}; its inputs are unchanged, so it is not run again.`); + if (r.route === 'refuted') console.error(`::error title=${n} refuted::identity ${r.identity} failed${where} on exactly these inputs. Change an input, or re-run the job to observe it again.`); + if (r.route === 'blocked') console.error(`::error title=${n} blocked::${byName(n).needs.filter(x => routes[x].route !== 'proved' && routes[x].route !== 'observe').join(', ') || byName(n).needs.join(', ')} not proved for this tree.`); + } + process.stdout.write(lines.join('\n') + '\n'); + return; + } + case 'outcome': { + // A job's own record of what it established, for factory.yml to publish. + const [name, identity, result, out] = opts.names; + byName(name); + if (!/^[0-9a-f]{64}$/.test(identity ?? '')) fail('outcome needs the identity the route printed'); + if (!['success', 'failure'].includes(result)) fail('outcome result is success or failure'); + fs.mkdirSync(out, {recursive: true}); + fs.writeFileSync(path.join(out, `${name}.outcome`), `outcome ${name} ${identity} ${result === 'success' ? 'pass' : 'fail'} ${git(['rev-parse', 'HEAD']).toString().trim()}\n`); + return; + } + case 'publish': return publish(graph, dir, opts.names[0]); + case 'export-check': { + const ids = identities(graph, tree('HEAD', dir), plannedContexts); + const r = exportRoute(graph, ids, dir); + if (r.route === 'proved') console.log(`export proved: generated/rust was exported from these inputs (${r.identity})`); + else console.log(`export observe: the client closure changed since generated/rust was exported (${r.recorded ?? 'no record'} -> export ${r.identity}); run node scripts/client-export.mjs`); + process.exitCode = r.route === 'proved' ? 0 : 3; + return; + } + case 'export-record': { + // Identity of the working tree's export inputs (client-export.mjs runs on + // the working tree, so it records what it actually read). + const ids = identities(graph, workingTree(dir), plannedContexts); + const o = byName('export'); + fs.writeFileSync(path.join(dir, o.record), `export ${ids.export}\n`); + console.log(`recorded export ${ids.export}`); + return; + } + default: + process.stderr.write(fs.readFileSync(fileURLToPath(import.meta.url), 'utf8').split('\n').filter(l => l.startsWith('// node')).join('\n') + '\n'); + process.exitCode = 64; + } +} + +/// The working tree as a tree listing: tracked and untracked (not ignored) files +/// with their blob ids as `git add` would write them. +export function workingTree(dir) { + const files = git(['ls-files', '-z', '--cached', '--others', '--exclude-standard'], {cwd: dir}).toString().split('\0').filter(Boolean) + .filter(p => fs.existsSync(path.join(dir, p))); + const unique = [...new Set(files)].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b))); + const objects = git(['hash-object', '-w', '--stdin-paths'], {cwd: dir, input: unique.join('\n') + '\n'}).toString().trim().split('\n'); + return unique.map((p, i) => { + const st = fs.lstatSync(path.join(dir, p)); + const mode = st.isSymbolicLink() ? '120000' : (st.mode & 0o111 ? '100755' : '100644'); + return {path: p, object: `${mode}:${objects[i]}`}; + }); +} + +// factory.yml: turn a completed run's outcome files into receipts on the +// receipts branch, reweigh the schedule, and push; another run may push first, +// so rebuild on the newest head and retry. +export async function publish(graph, dir, outcomes) { + const run = process.env.FACTORY_RUN_ID ?? fail('FACTORY_RUN_ID is required'); + const attempt = process.env.FACTORY_RUN_ATTEMPT ?? fail('FACTORY_RUN_ATTEMPT is required'); + const token = process.env.GH_TOKEN ?? fail('GH_TOKEN is required'); + const repo = process.env.GITHUB_REPOSITORY ?? fail('GITHUB_REPOSITORY is required'); + const files = fs.existsSync(outcomes) ? fs.readdirSync(outcomes).filter(f => f.endsWith('.outcome')).sort() : []; + const fresh = files.flatMap(f => { + const [tag, name, identity, result, commit] = fs.readFileSync(path.join(outcomes, f), 'utf8').trim().split(' '); + if (tag !== 'outcome' || !/^[a-z0-9._-]+$/.test(name ?? '') || !/^[0-9a-f]{64}$/.test(identity ?? '') || !['pass', 'fail'].includes(result) || !/^[0-9a-f]{40}$/.test(commit ?? '')) fail(`${f}: malformed outcome`); + // A pull request may declare an obligation the default branch does not know yet. + if (!graph.obligations.some(o => o.name === name)) { console.error(`::warning::${name} is not in the default branch's factory/graph.json; no receipt`); return []; } + return [{name, identity, result, commit}]; + }); + if (!fresh.length) { console.log('factory: this run established nothing new'); return; } + // A receipt whose obligation carries evidence expires a day before its evidence does. + const listing = await (globalThis.factoryFetch ?? fetch)(`https://api.github.com/repos/${repo}/actions/runs/${run}/artifacts?per_page=100`, {headers: {authorization: `Bearer ${token}`, accept: 'application/vnd.github+json'}}); + const artifacts = listing.ok ? (await listing.json()).artifacts : fail(`listing the artifacts of run ${run}: ${listing.status}`); + const expiry = name => { + const o = graph.obligations.find(x => x.name === name); + if (!o.evidence) return 'never'; + const a = artifacts.find(x => x.name === o.evidence && !x.expired); + return a ? new Date(Date.parse(a.expires_at) - 86400000).toISOString() : null; + }; + const header = `AUTHORIZATION: basic ${Buffer.from(`x-access-token:${token}`).toString('base64')}`; + const auth = ['-c', `http.https://github.com/.extraheader=${header}`]; + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'factory-receipts-')); + for (let tries = 1; tries <= 5; tries++) { + const index = path.join(work, `index-${tries}`); const env = {GIT_INDEX_FILE: index}; + let parent = []; + if (git([...auth, 'fetch', '--quiet', '--depth=1', 'origin', `+refs/heads/${BRANCH}:refs/factory/receipts`], {cwd: dir, allowFail: true}) !== null) { + git(['read-tree', 'refs/factory/receipts'], {cwd: dir, env}); + parent = ['-p', git(['rev-parse', 'refs/factory/receipts'], {cwd: dir}).toString().trim()]; + } + const old = parentIndex(dir, env); + const store = parseStore(old); + const lines = []; + for (const o of fresh) { + const expires = o.result === 'pass' ? expiry(o.name) : 'never'; + if (expires === null) { console.error(`::warning::${o.name} passed but its evidence ${graph.obligations.find(x => x.name === o.name).evidence} is not in run ${run}; no receipt`); continue; } + const line = `receipt ${o.identity} ${o.name} ${o.result} ${run} ${attempt} ${o.commit} ${expires}`; + if (store.receipts.some(r => r.identity === o.identity && r.name === o.name && r.run === run && r.attempt === attempt)) continue; + lines.push(line); + const blob = git(['hash-object', '-w', '--stdin'], {cwd: dir, input: line + '\n'}).toString().trim(); + git(['update-index', '--add', '--cacheinfo', `100644,${blob},receipts/${o.name}/${o.identity}-${run}-${attempt}.receipt`], {cwd: dir, env}); + } + const weights = reweigh(store.weights, fresh); + const now = new Date(); + const kept = store.receipts.filter(r => r.expires === 'never' || new Date(r.expires) > now) + .map(r => `receipt ${r.identity} ${r.name} ${r.result} ${r.run} ${r.attempt} ${r.commit} ${r.expires}`); + const text = [...kept, ...lines, ...Object.keys(weights).sort().map(n => `weight ${n} ${weights[n]}`)].join('\n') + '\n'; + const blob = git(['hash-object', '-w', '--stdin'], {cwd: dir, input: text}).toString().trim(); + git(['update-index', '--add', '--cacheinfo', `100644,${blob},index`], {cwd: dir, env}); + const treeId = git(['write-tree'], {cwd: dir, env}).toString().trim(); + const commit = git(['-c', 'user.name=uorc-factory', '-c', 'user.email=uorc-factory@users.noreply.github.com', 'commit-tree', treeId, ...parent, '-m', `Receipts of run ${run} attempt ${attempt}`], {cwd: dir}).toString().trim(); + if (git([...auth, 'push', '--quiet', 'origin', `${commit}:refs/heads/${BRANCH}`], {cwd: dir, allowFail: true}) !== null) { + console.log(`factory: recorded ${lines.length} receipt(s) of run ${run}:\n${lines.join('\n')}`); + return; + } + await new Promise(r => setTimeout(r, tries * 5000)); + } + fail(`could not push receipts to ${BRANCH}`); +} + +function parentIndex(dir, env) { + return (git(['cat-file', 'blob', ':index'], {cwd: dir, env, allowFail: true}) ?? '').toString(); +} + +if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { await main(process.argv.slice(2)); } catch (error) { console.error(`factory: ${error.message}`); process.exitCode = 1; } +} diff --git a/scripts/factory.test.mjs b/scripts/factory.test.mjs new file mode 100644 index 00000000..c816835f --- /dev/null +++ b/scripts/factory.test.mjs @@ -0,0 +1,62 @@ +// node --test scripts/factory.test.mjs +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {covered, route, reweigh, schedule, parseStore} from './factory.mjs'; + +const graph = {obligations: [ + {name: 'leg', kind: 'hermetic', needs: [], inputs: []}, + {name: 'net', kind: 'fetch', needs: [], inputs: []}, + {name: 'build', kind: 'fetch', needs: [], inputs: []}, + {name: 'bench', kind: 'fetch', needs: ['build'], inputs: []}, + {name: 'gate', kind: 'gate', needs: ['leg', 'net'], inputs: []}, +]}; +const ids = {leg: 'a', net: 'b', build: 'c', bench: 'd', gate: 'e'}; +const receipt = (identity, name, result, run, expires = 'never') => `receipt ${identity} ${name} ${result} ${run} 1 ${'0'.repeat(40)} ${expires}`; + +test('coverage: the longest inclusion must beat the longest exclusion', () => { + const inputs = ['', '!client/', 'client/src/Keep.lex.tex', '@mode']; + assert.equal(covered(inputs, 'src/A.lex.tex', new Set()), true); + assert.equal(covered(inputs, 'client/gen/x.py', new Set()), false); + assert.equal(covered(inputs, 'client/src/Keep.lex.tex', new Set()), true); + assert.equal(covered(['crates/uorc/'], 'Cargo.toml', new Set()), false); + assert.equal(covered(['!x/'], 'x/y', new Set(['x/y'])), true); +}); + +test('a pass proves, a hermetic failure refutes, a fetch failure is observed again', () => { + const store = parseStore([receipt('a', 'leg', 'fail', 1), receipt('b', 'net', 'fail', 2), receipt('c', 'build', 'pass', 3)].join('\n')); + const r = route(graph, ids, store); + assert.equal(r.leg.route, 'refuted'); + assert.equal(r.leg.run, '1'); + assert.equal(r.net.route, 'observe'); + assert.equal(r.build.route, 'proved'); + assert.equal(r.build.run, '3'); + assert.equal(r.bench.route, 'observe'); + assert.equal(r.gate.route, 'blocked'); +}); + +test('a manual re-run observes a refuted obligation; a later pass outweighs a failure', () => { + const failed = parseStore(receipt('a', 'leg', 'fail', 1)); + assert.equal(route(graph, ids, failed, {attempt: 2}).leg.route, 'observe'); + const both = parseStore([receipt('a', 'leg', 'fail', 1), receipt('a', 'leg', 'pass', 4), receipt('b', 'net', 'pass', 5)].join('\n')); + const r = route(graph, ids, both); + assert.equal(r.leg.route, 'proved'); + assert.equal(r.gate.route, 'proved'); +}); + +test('a receipt for other inputs or past its evidence proves nothing', () => { + const store = parseStore([receipt('zz', 'build', 'pass', 1), receipt('c', 'build', 'pass', 2, '2000-01-01T00:00:00Z')].join('\n')); + assert.equal(route(graph, ids, store).build.route, 'observe'); +}); + +test('outside a run, a need that is not yet proved blocks', () => { + assert.equal(route(graph, ids, parseStore(''), {inRun: false}).bench.route, 'blocked'); + assert.equal(route(graph, ids, parseStore(''), {inRun: true}).bench.route, 'observe'); +}); + +test('weights stay in 1..16: a failure raises priority, a pass lowers it, and schedule orders by it', () => { + let w = {}; + for (let i = 0; i < 20; i++) w = reweigh(w, [{name: 'leg', result: 'fail'}, {name: 'net', result: 'pass'}]); + assert.deepEqual(w, {leg: 16, net: 1}); + const r = route(graph, ids, parseStore('')); + assert.deepEqual(schedule(graph, r, w).map(s => s.name), ['leg', 'build', 'bench', 'net']); +}); diff --git a/template.lock b/template.lock index 9b69b950..b3c3200a 100644 --- a/template.lock +++ b/template.lock @@ -1 +1 @@ -{"contract_digest":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5","policy_files":[{"path":".devcontainer/devcontainer.json","sha256":"sha256:faad010f6e2561877feeac664d30c9b20da5c264de8f0007d485cc3c5a3505f2"},{"path":".github/workflows/bootstrap.yml","sha256":"sha256:ae8b1bba4cb1cba61c258c5a78f73877f5152e85b12b451dc81ba3b5af1b9689"},{"path":"AGENTS.md","sha256":"sha256:9a1fe3f73ab2963812c35976909c331292e1ca39ce47c0711ec1aafe8b869d87"},{"path":"VERIFICATION.md","sha256":"sha256:0912821fea354276fff5faa2b6b6ded0cdf4ef86112ad035b41a0a7a0513bd21"},{"path":"prismpm.lock","sha256":"sha256:97f3ca37a1077a7819a6ac6206c9132672a05820d547cc834ade18503e5643bf"},{"path":"template-contract.json","sha256":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5"}],"policy_tree_sha256":"sha256:0034c428f0f9a8fa81bf58e845b07a2a31f7de7775170f49a89fb317e399ac5c","schema":"uor/template-lock/1","sdk_image":"ghcr.io/uor-foundation/prismpm-sdk-candidate@sha256:0519fe6e2f1d731fc0d475c70ad21d00396d9089051f0c703d477e1ffe39a11c","template_repository":"https://github.com/UOR-Foundation/template","template_revision":"0f1245367d317d439e6752be277047eef76a9e90"} \ No newline at end of file +{"contract_digest":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5","policy_files":[{"path":".devcontainer/devcontainer.json","sha256":"sha256:faad010f6e2561877feeac664d30c9b20da5c264de8f0007d485cc3c5a3505f2"},{"path":".github/workflows/bootstrap.yml","sha256":"sha256:5501ae07dd47f3be16bdd9d2afbf3832f811ebfb1693bf971cc1fde2e3ed9dba"},{"path":"AGENTS.md","sha256":"sha256:9a1fe3f73ab2963812c35976909c331292e1ca39ce47c0711ec1aafe8b869d87"},{"path":"VERIFICATION.md","sha256":"sha256:0912821fea354276fff5faa2b6b6ded0cdf4ef86112ad035b41a0a7a0513bd21"},{"path":"prismpm.lock","sha256":"sha256:97f3ca37a1077a7819a6ac6206c9132672a05820d547cc834ade18503e5643bf"},{"path":"template-contract.json","sha256":"sha256:67a3292cfe6e6400bc18718b6a62e8741dec8d6d6b062cb0e4077fc8d9b616f5"}],"policy_tree_sha256":"sha256:f1146a46c3401d3b014463d7e31e785466312450c2e2edcb56f71b9c121cdcbc","schema":"uor/template-lock/1","sdk_image":"ghcr.io/uor-foundation/prismpm-sdk-candidate@sha256:0519fe6e2f1d731fc0d475c70ad21d00396d9089051f0c703d477e1ffe39a11c","template_repository":"https://github.com/UOR-Foundation/template","template_revision":"0f1245367d317d439e6752be277047eef76a9e90"} \ No newline at end of file diff --git a/tests/sdk-preflight/test_source_format.py b/tests/sdk-preflight/test_source_format.py index b97d8ec9..852d516f 100644 --- a/tests/sdk-preflight/test_source_format.py +++ b/tests/sdk-preflight/test_source_format.py @@ -233,9 +233,9 @@ def audit_workflow(self, workflow): self.assertIn('node scripts/offline-validation.mjs compare-offline .prism/offline/first .prism/offline/second-distinct-root', workflow) self.assertIn('name: review-only generated projections', workflow) self.assertEqual([line.strip() for line in workflow.splitlines() if 'if:' in line], [ - "if: needs.route.outputs.proved != 'true'", "if: needs.route.outputs.proved != 'true'", + "if: needs.route.outputs.route == 'observe'", "if: needs.route.outputs.route == 'observe'", 'if: always()', 'if: always()']) - self.assertIn('run: scripts/ci-receipt.sh lookup offline >> "$GITHUB_OUTPUT"', workflow) + self.assertIn('run: node scripts/factory.mjs route offline >> "$GITHUB_OUTPUT"', workflow) def test_acceptance_legs_run_every_vv_recipe_exactly_once(self): From d9d2da5491a6cfe7cebc3ad0f08cdd52d121c312 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:58:51 +0000 Subject: [PATCH 2/3] factory: mark the README non-authoritative; record the export identity outside prose files Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3LdDSxZc5dyuEbM7w7 --- factory/README.md | 1 + factory/graph.json | 2 +- generated/rust/{factory.txt => factory.identity} | 0 scripts/client-export.mjs | 4 ++-- 4 files changed, 4 insertions(+), 3 deletions(-) rename generated/rust/{factory.txt => factory.identity} (100%) diff --git a/factory/README.md b/factory/README.md index 337fa504..2186c409 100644 --- a/factory/README.md +++ b/factory/README.md @@ -1,3 +1,4 @@ + # uorc factory The release factory keys every CI and release obligation on its own declared diff --git a/factory/graph.json b/factory/graph.json index 92d8bbc3..265a4f00 100644 --- a/factory/graph.json +++ b/factory/graph.json @@ -13,7 +13,7 @@ {"name": "reproducibility-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just reproducibility", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "integrity-ubuntu-24.04", "kind": "hermetic", "environment": "ubuntu-24.04", "run": "just integrity", "inputs": ["&formal"], "needs": [], "workflow": "bootstrap.yml", "job": "template-gate"}, {"name": "offline", "kind": "fetch", "environment": "ubuntu-24.04", "run": "node scripts/offline-validation.mjs acquire, isolated first and second-distinct-root, compare-offline", "inputs": ["&formal"], "needs": [], "workflow": "offline-validation.yml", "job": "offline-compare"}, - {"name": "export", "kind": "fetch", "environment": "host", "run": "node scripts/client-export.mjs", "inputs": ["closure:Uorc.Client", "lexlean.toml", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "scripts/client-export.mjs", "scripts/semantic-source.mjs"], "needs": [], "record": "generated/rust/factory.txt"}, + {"name": "export", "kind": "fetch", "environment": "host", "run": "node scripts/client-export.mjs", "inputs": ["closure:Uorc.Client", "lexlean.toml", "lakefile.toml", "lake-manifest.json", "lean-toolchain", "scripts/client-export.mjs", "scripts/semantic-source.mjs"], "needs": [], "record": "generated/rust/factory.identity"}, {"name": "gate", "kind": "gate", "environment": "-", "run": "-", "inputs": [], "needs": ["rest-ubuntu-24.04", "rest-ubuntu-24.04-arm", "artifacts-ubuntu-24.04", "claims-ubuntu-24.04", "varint-ubuntu-24.04", "reproducibility-ubuntu-24.04", "integrity-ubuntu-24.04"], "workflow": "release.yml", "job": "factory"}, {"name": "native-x86_64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target x86_64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-x86_64-unknown-linux-gnu"}, {"name": "native-aarch64-unknown-linux-gnu", "kind": "fetch", "environment": "ubuntu-24.04-arm", "run": "cargo test --locked --release -p uorc && cargo build --locked --release -p uorc --target aarch64-unknown-linux-gnu", "inputs": ["&client"], "needs": [], "workflow": "release.yml", "job": "native", "evidence": "factory-native-aarch64-unknown-linux-gnu"}, diff --git a/generated/rust/factory.txt b/generated/rust/factory.identity similarity index 100% rename from generated/rust/factory.txt rename to generated/rust/factory.identity diff --git a/scripts/client-export.mjs b/scripts/client-export.mjs index 8c1bae82..eb906c22 100644 --- a/scripts/client-export.mjs +++ b/scripts/client-export.mjs @@ -28,7 +28,7 @@ // // The export is the factory obligation `export` (factory/graph.json): its // identity covers only what it reads (the import closure of Uorc.Client, the -// LexLean project files and these scripts), recorded in generated/rust/factory.txt. +// LexLean project files and these scripts), recorded in generated/rust/factory.identity. // When the recorded identity matches, the export is already proved and this // exits without running; pass --force to export anyway. import fs from 'node:fs'; @@ -138,7 +138,7 @@ function wrapBoxedReturns(dir, modules) { } } -const exportRecord = path.join(out, 'factory.txt'); +const exportRecord = path.join(out, 'factory.identity'); const exportIdentity = identities(loadGraph(root), workingTree(root), {}).export; if (!process.argv.includes('--force') && fs.existsSync(exportRecord) && fs.readFileSync(exportRecord, 'utf8').trim() === `export ${exportIdentity}`) { console.log(`export proved: generated/rust was exported from these inputs (${exportIdentity}); --force exports anyway`); From c60a95a9d2cd9ecf2978e5833e068a83d90471d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 20:58:52 +0000 Subject: [PATCH 3/3] generated: record the export identity for the current client closure Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3LdDSxZc5dyuEbM7w7 --- generated/rust/factory.identity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated/rust/factory.identity b/generated/rust/factory.identity index bcf53c38..74113b3c 100644 --- a/generated/rust/factory.identity +++ b/generated/rust/factory.identity @@ -1 +1 @@ -export c518791b8a839e42cb5eb428c3153b0342c8423fac7ecc001d4b513bdd07b0d9 +export 2c94bf8be48d5269b913c6ea52fea4ba1d2516072e4100947225037c50d0d0e1