diff --git a/.github/workflows/regenerate.yml b/.github/workflows/regenerate.yml new file mode 100755 index 0000000000..e639c89868 --- /dev/null +++ b/.github/workflows/regenerate.yml @@ -0,0 +1,299 @@ +# The write recipes of SPEC.md §9.2 on a runner (§9.4): every file the gate +# compares with a committed oracle, regenerated from a clean checkout of the +# dispatched ref and returned as one binary patch. Not part of the gate. It +# reads the repository and writes nothing to it; the patch is reviewed and +# committed like any other written output (§28.3). +# +# git push origin :refs/heads/regenerate/ # or regenerate/atlas/ +# gh workflow run regenerate.yml --ref [-f atlas=true] # once on the default branch +# gh run download --name regenerated- +# git apply --index regenerated.patch +# +# A push to `regenerate/**` starts it from any commit, because GitHub +# dispatches only a workflow its default branch carries. +name: regenerate + +on: + push: + branches: ['regenerate/**'] + workflow_dispatch: + inputs: + atlas: + description: Rewrite UOR Atlas's verification records too (its whole proof, in the records job) + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: regenerate-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Each writer runs from its own clean checkout, so the three run at once; + # the paths they write are disjoint, which `combine` checks. + writer: + name: write (${{ matrix.writer }}) + if: github.repository == 'UOR-Foundation/LexLean' + strategy: + fail-fast: false + matrix: + writer: [production-export, records, oracles] + runs-on: ubuntu-24.04 + timeout-minutes: 360 + env: + # The profile of the `examples` leg (§22.11): a schedule, never a byte + # of what is written, which is that of width 1. + LEXLEAN_VERIFY_WIDTH: "2" + LEXLEAN_VERIFY_MEMORY_KIB: "12582912" + LEXLEAN_VERIFY_HINTS: ${{ github.workspace }}/.github/verify-hints.json + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install the pinned Rust toolchain + run: rustup toolchain install + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + with: + tool: just + - name: Install elan and the pinned Lean toolchain + run: | + set -euo pipefail + curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + "$HOME/.elan/bin/elan" toolchain install "$(cat lean-toolchain)" + - name: Reserve bounded overflow memory for the proofs + run: | + set -euo pipefail + sudo fallocate -l 4G "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo chmod 600 "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo mkswap "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo swapon "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo sysctl vm.swappiness=10 + - name: Run the write recipes + shell: bash + env: + ATLAS: ${{ inputs.atlas || startsWith(github.ref_name, 'regenerate/atlas/') }} + run: | + set -euo pipefail + case "${{ matrix.writer }}" in + production-export) recipes="production-export-write" ;; + records) + recipes="verify-write" + if [ "$ATLAS" != "true" ]; then + export LEXLEAN_EXAMPLES_PART=without:uor-atlas + fi + ;; + oracles) recipes="golden-write fixtures-write calculus-write model-write" ;; + esac + just --timestamp --timestamp-format '%s%.3f' $recipes + - name: Capture what the recipes wrote + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/part" + git add --all + git diff --cached --binary > "$RUNNER_TEMP/part/${{ matrix.writer }}.patch" + git diff --cached --name-only | tee "$RUNNER_TEMP/part/${{ matrix.writer }}.files" + echo "${{ matrix.writer }}: $(wc -l < "$RUNNER_TEMP/part/${{ matrix.writer }}.files") paths" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: regenerate-part-${{ matrix.writer }} + path: ${{ runner.temp }}/part + if-no-files-found: error + retention-days: 7 + + # The parts applied in turn to a clean checkout of the same commit: two parts + # that write one path fail here, rather than one silently replacing the other. + combine: + name: combine + needs: [writer] + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: regenerate-part-* + path: ${{ runner.temp }}/parts + merge-multiple: true + - name: Apply every part + shell: bash + run: | + set -euo pipefail + for part in production-export records oracles; do + # A writer that changed nothing returns an empty patch. + if [ -s "$RUNNER_TEMP/parts/$part.patch" ]; then + git apply --index "$RUNNER_TEMP/parts/$part.patch" + fi + done + mkdir -p "$RUNNER_TEMP/combined" + git diff --cached --binary > "$RUNNER_TEMP/combined/combined.patch" + echo "combined: $(git diff --cached --name-only | wc -l) paths" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: regenerate-combined + path: ${{ runner.temp }}/combined + if-no-files-found: error + retention-days: 7 + + # What follows from the regenerated tree: the counts the bootstrap count + # cases assert (`cargo xtask counts`, from runs of their parts under + # `LEXLEAN_COUNTS=fast`), the model documents, and the stack bound of each + # recursive package the record lacks (PD-08), measured by the cases that run + # it. A measuring run is a whole run of its case, so it also checks the + # counts just written. + finish: + name: counts and stack bounds + needs: [combine] + runs-on: ubuntu-24.04 + timeout-minutes: 360 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install the pinned Rust toolchain + run: rustup toolchain install + - name: Install elan and the pinned Lean toolchain + run: | + set -euo pipefail + curl https://elan.lean-lang.org/elan-init.sh -sSf | sh -s -- -y --default-toolchain none + echo "$HOME/.elan/bin" >> "$GITHUB_PATH" + "$HOME/.elan/bin/elan" toolchain install "$(cat lean-toolchain)" + - name: Reserve bounded overflow memory for the proofs + run: | + set -euo pipefail + sudo fallocate -l 4G "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo chmod 600 "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo mkswap "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo swapon "$RUNNER_TEMP/lexlean-regenerate.swap" + sudo sysctl vm.swappiness=10 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: regenerate-combined + path: ${{ runner.temp }}/combined + - name: Apply the regenerated tree + shell: bash + run: | + set -euo pipefail + if [ -s "$RUNNER_TEMP/combined/combined.patch" ]; then + git apply --index "$RUNNER_TEMP/combined/combined.patch" + fi + - name: Write the moved counts + shell: bash + run: | + set -uo pipefail + mkdir -p "$RUNNER_TEMP/regenerated" + # The parts of `.github/bdd-parts.json` that hold a case with a + # `crate::counts!` check; each runs only up to its counts. + parts=$(python3 - <<'PY' + import json, pathlib, re + ids = set() + for path in pathlib.Path("crates/conformance/src").rglob("*.rs"): + ids |= set(re.findall(r'counts!\(\s*"([A-Z]+-[0-9]+)"', path.read_text())) + plan = json.loads(pathlib.Path(".github/bdd-parts.json").read_text()) + parts = plan["parts"] if "parts" in plan else plan + print(" ".join(sorted(name for name, members in parts.items() + if isinstance(members, list) and ids & set(members)))) + PY + ) + if [ -z "$parts" ]; then + echo "no case of this tree has a \`crate::counts!\` check: no counts written" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "parts with counts: $parts" + for part in $parts; do + LEXLEAN_COUNTS=fast LEXLEAN_BDD_PART="$part" cargo xtask bdd > "$RUNNER_TEMP/fast-$part.log" 2>&1 + echo "$part: exit $?" + done + cargo xtask counts --write 2>&1 | grep -v -E '^\s*(Finished|Running|Compiling)' | tee "$RUNNER_TEMP/regenerated/counts.txt" + test "${PIPESTATUS[0]}" -eq 0 + - name: Write the model documents + run: cargo xtask validate-model --write + - name: Measure the stack bound of each recursive package the record lacks + shell: bash + env: + LEXLEAN_VERIFY_WIDTH: "2" + LEXLEAN_VERIFY_MEMORY_KIB: "12582912" + run: | + set -euo pipefail + # Rows for packages no longer in the exports are dropped; for each + # recursive package without a row, the case whose source names its + # root runs whole and measures it (`LEXLEAN_MEASURE_STACK=all`). + cases=$(python3 - <<'PY' + import json, pathlib, re + record_path = pathlib.Path("tests/stack-bounds.json") + record = json.loads(record_path.read_text()) + recursive = set() + for corpus in ("bootstrap", "stdlib"): + for provenance in pathlib.Path("tests/production-export", corpus, "packages").rglob("provenance.json"): + if "recursion" in json.loads(provenance.read_text()).get("effects", []): + recursive.add(f"{corpus}/{provenance.parent.name}") + rows = {key for key in record["rows"] if key != "bootstrap/evaluator"} + for stale in sorted(rows - recursive): + print(f"dropping the row of {stale}", file=__import__("sys").stderr) + del record["rows"][stale] + if rows - recursive: + record_path.write_text(json.dumps(record, indent=2) + "\n") + sources = list(pathlib.Path("crates/conformance/src/cases").rglob("*.rs")) + cases = set() + for package in sorted(recursive - rows): + name = package.split("/", 1)[1] + root = name.split("_")[0] + "".join(part.title() for part in name.split("_")[1:]) + owners = set() + for source in sources: + text = source.read_text() + # The file's first line names its own case. + own = re.match(r'//! `([A-Z]+-[0-9]+)`', text) + if f'"{root}"' in text and own: + owners.add(own.group(1)) + if not owners: + raise SystemExit(f"{package}: no case names the root `{root}`; measure it by hand") + print(f"{package}: measured by {', '.join(sorted(owners))}", file=__import__("sys").stderr) + cases |= owners + print(" ".join("conformance_" + case.lower().replace("-", "_") for case in sorted(cases))) + PY + ) + if [ -z "$cases" ]; then + echo "every recursive package has its row" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "measuring: $cases" | tee -a "$GITHUB_STEP_SUMMARY" + LEXLEAN_MEASURE_STACK=all cargo test -p repo-conformance --all-features --test conformance -- $cases --nocapture 2>&1 \ + | grep -E --line-buffered 'debug [0-9]+ bytes|test result|panicked|^test ' + - name: Return the regenerated tree + if: ${{ !cancelled() }} + shell: bash + run: | + set -euo pipefail + out="$RUNNER_TEMP/regenerated" + mkdir -p "$out" + git add --all + git diff --cached --binary > "$out/regenerated.patch" + git diff --cached --name-only > "$out/regenerated-files.txt" + echo "$GITHUB_SHA" > "$out/base-commit.txt" + (cd "$out" && sha256sum regenerated.patch regenerated-files.txt base-commit.txt > SHA256SUMS) + { + echo "### Regenerated $(wc -l < "$out/regenerated-files.txt") paths on $GITHUB_SHA" + echo + if [ -s "$out/counts.txt" ]; then + echo '#### Counts (the writer lists here anything it could not place)' + echo '```text' + cat "$out/counts.txt" + echo '```' + fi + echo '```bash' + echo "gh run download $GITHUB_RUN_ID --repo $GITHUB_REPOSITORY --name regenerated-$GITHUB_SHA" + echo "git apply --index regenerated.patch" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ !cancelled() }} + with: + name: regenerated-${{ github.sha }} + path: ${{ runner.temp }}/regenerated + if-no-files-found: error + retention-days: 7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 17a425bd30..39f2c77eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ workspace carried at those versions without a tag. ## Unreleased +- `regenerate.yml` (SPEC.md §9.4): a push to `regenerate/`, or a dispatch, runs the write recipes on runners, the production + export, the verification records and the other oracles as three parallel jobs, and returns one + reviewed patch, so that a change which moves committed output is no longer regenerated on a + workstation. + A fourth job then writes the moved counts of the bootstrap count cases, the model documents, + and the stack bound of each recursive package that lacks one, so the patch is the whole landing. - `rustix` 1.1.6 (RUSTSEC-2026-0486: the extended-attribute functions marked unwritten bytes as initialized for a zero-length buffer), so `cargo deny` passes; `errno` and `tempfile` resolve `windows-sys` 0.59, the version `fs4` 0.13 and `rustix` 1.1.6 both accept. diff --git a/SPEC.md b/SPEC.md index 6e1c370fbe..0a19772ddc 100644 --- a/SPEC.md +++ b/SPEC.md @@ -707,6 +707,8 @@ The Intel host factory is separate from the normative BDD and Atlas partitions a `reproducibility.yml` MUST perform two builds in distinct absolute directories and compare every platform-independent artifact byte-for-byte after path normalization. +`regenerate.yml` is not part of the gate: it runs the write recipes of §9.2 on runners, so that a change which moves committed output is regenerated from a clean checkout rather than on a workstation. A push to a branch under `regenerate/`, or a `workflow_dispatch` on a ref, starts three jobs at once, each from its own clean checkout of that commit: `production-export-write`; `verify-write` over every example but UOR Atlas, or over every example when the branch is under `regenerate/atlas/` or the `atlas` input is set; and `golden-write fixtures-write calculus-write model-write`. They verify with the profile of the `examples` leg, which changes no byte of what is written (§22.11). Each returns what its recipes wrote as a binary patch, and the job `combine` applies the patches in turn to a clean checkout of the same commit, failing when two write one path. The job `finish` then applies that tree and writes what follows from it: the counts the bootstrap count cases assert, by `cargo xtask counts --write` over runs of the parts of `.github/bdd-parts.json` that hold a `crate::counts!` check under `LEXLEAN_COUNTS=fast`; the model documents, by `validate-model --write`; and the row of `tests/stack-bounds.json` of each recursive package of the bootstrap and stdlib exports that has none (`PD-08`), measured by a whole run under `LEXLEAN_MEASURE_STACK=all` of the case whose source names the package's root, which also checks the counts just written, while a row of a package no longer exported is dropped. It returns the result as one patch with the list of its paths, the counts the writer reported, and the commit it applies to. The workflow reads the repository and writes nothing to it (`contents: read`, no persisted credentials); its patch is reviewed and committed like any other written output (§28.3), and the gate then verifies the tree that carries it. + --- ## 10. Project files