diff --git a/.github/bdd-parts.json b/.github/bdd-parts.json index 34462fb14f..0109d5372d 100644 --- a/.github/bdd-parts.json +++ b/.github/bdd-parts.json @@ -246,5 +246,8 @@ "preservation-std": [ "SP-29" ] + }, + "targets": { + "honesty": ["bdd"] } } diff --git a/.github/ci-plan.sh b/.github/ci-plan.sh index 2adbb7db1b..ffbc44dea1 100755 --- a/.github/ci-plan.sh +++ b/.github/ci-plan.sh @@ -2,8 +2,8 @@ # What a run of ci.yml is (SPEC.md §9.4), decided from the event and the # repository alone, before any job that costs a runner starts. # -# pull request, draft the `rest` leg only: a development signal -# pull request, ready, base the `rest` leg only, the same signal: only a pull +# pull request, draft honesty then the `rest` leg only: a development signal +# pull request, ready, base honesty then the `rest` leg only, the same signal: only a pull # not main or release/* request into a line can land on it # pull request, ready, base the whole gate # main or release/* @@ -40,7 +40,9 @@ out=${GITHUB_OUTPUT:-/dev/stdout} summary=${GITHUB_STEP_SUMMARY:-/dev/null} emit() { - printf 'full=%s\nrows=%s\n' "$1" "$2" >> "$out" + printf 'full=%s\nrows=%s\nhonesty=%s\nfanout=%s\n' "$1" "$2" \ + "$(jq -c '[.[] | select(.part == "honesty")]' <<< "$2")" \ + "$(jq -c '[.[] | select(.part != "honesty")]' <<< "$2")" >> "$out" } whole_gate() { @@ -50,11 +52,11 @@ whole_gate() { # The id of a pull request run of this workflow on the second parent of HEAD # that ran every job of the gate and passed every one, when HEAD is a merge of -# that parent with the same tree. A draft run has only the `rest` leg, so it is +# that parent with the same tree. A draft run has honesty and the `rest` leg, so it is # never one: the run must hold the join of Atlas, every host of the matrix, and # every leg of the rows file, and no job that did not conclude `success`. verified_parent_run() { - local parent head_tree hosts runs id names part + local parent head_tree hosts runs id names part job parent=$(git rev-parse --verify --quiet 'HEAD^2') || return 1 head_tree=$(git rev-parse 'HEAD^{tree}') [ "$head_tree" = "$(git rev-parse "$parent^{tree}")" ] || return 1 @@ -65,10 +67,12 @@ verified_parent_run() { names=$(gh api "repos/$repo/actions/runs/$id/jobs?per_page=100" --paginate \ --jq '.jobs[] | "\(.name)\t\(.conclusion)"') || continue if printf '%s\n' "$names" | awk -F'\t' '$2 != "success" { bad = 1 } END { exit bad }' \ - && printf '%s\n' "$names" | grep -qxF $'atlas (join)\tsuccess' \ + && printf '%s\n' "$names" | grep -xF $'atlas (join)\tsuccess' >/dev/null \ && [ "$(printf '%s\n' "$names" | grep -c '^build and non-Lean tests (')" -eq "$hosts" ]; then for part in $(jq -r '.[].part' "$rows_file"); do - printf '%s\n' "$names" | grep -qxF "just vv ($part)"$'\tsuccess' || continue 2 + job="just vv ($part)" + [ "$part" != honesty ] || job=honesty + printf '%s\n' "$names" | grep -xF "$job"$'\tsuccess' >/dev/null || continue 2 done printf '%s\n' "$id" return 0 @@ -80,8 +84,8 @@ verified_parent_run() { case "$event" in pull_request) if [ "$draft" = true ]; then - emit false "$(jq -c '[.[] | select(.part == "rest")]' "$rows_file")" - printf '%s\n' 'Draft pull request: the `rest` leg only. This run is not the gate; marking the pull request ready runs the whole gate.' >> "$summary" + emit false "$(jq -c '[.[] | select(.part == "honesty" or .part == "rest")]' "$rows_file")" + printf '%s\n' 'Draft pull request: honesty then the `rest` leg only. This run is not the gate; marking the pull request ready runs the whole gate.' >> "$summary" else case "$base" in # An unknown base is not known to be stacked: the whole gate. @@ -89,8 +93,8 @@ case "$event" in whole_gate 'Pull request ready for review: the whole gate.' ;; *) - emit false "$(jq -c '[.[] | select(.part == "rest")]' "$rows_file")" - printf 'Pull request into `%s`, which is neither `main` nor a release line: the `rest` leg only. A stacked pull request cannot land on a line; the pull request into `main` or a release line runs the whole gate.\n' "$base" >> "$summary" + emit false "$(jq -c '[.[] | select(.part == "honesty" or .part == "rest")]' "$rows_file")" + printf 'Pull request into `%s`, which is neither `main` nor a release line: honesty then the `rest` leg only. A stacked pull request cannot land on a line; the pull request into `main` or a release line runs the whole gate.\n' "$base" >> "$summary" ;; esac fi diff --git a/.github/release-gate.sh b/.github/release-gate.sh index 50330ebe56..85ca2376dc 100755 --- a/.github/release-gate.sh +++ b/.github/release-gate.sh @@ -12,7 +12,7 @@ # tree is the tree of its second parent, a run on that parent. A run counts # when every job of it concluded `success` and it holds the join of Atlas, # every host of the matrix, and every leg of `.github/vv-rows.json`; a -# draft run, which has only the `rest` leg, never does. A tag on a commit no +# draft run, which has only honesty and the `rest` leg, never does. A tag on a commit no # such run verified is refused, naming what was looked for. # # Inputs, all from the environment: REPO (owner/name), SHA (the tagged @@ -42,14 +42,16 @@ jobs_of() { # Whether a jobs listing is the whole gate, passed. whole_gate_passed() { - local jobs=$1 part + local jobs=$1 part job [ -n "$jobs" ] || return 1 printf '%s\n' "$jobs" | awk -F'\t' '$2 != "success" { bad = 1 } END { exit bad }' || return 1 # Quiet grep can close the pipe early and make printf fail under pipefail. printf '%s\n' "$jobs" | grep -xP 'atlas \(join\)\tsuccess\t.*' >/dev/null || return 1 [ "$(printf '%s\n' "$jobs" | grep -c '^build and non-Lean tests (')" -eq "$hosts" ] || return 1 for part in $parts; do - printf '%s\n' "$jobs" | grep -P "^just vv \(\Q$part\E\)\tsuccess\t" >/dev/null || return 1 + job="just vv ($part)" + [ "$part" != honesty ] || job=honesty + printf '%s\n' "$jobs" | grep -P "^\Q$job\E\tsuccess\t" >/dev/null || return 1 done } diff --git a/.github/vv-rows.json b/.github/vv-rows.json index b618e90f49..648d05088c 100644 --- a/.github/vv-rows.json +++ b/.github/vv-rows.json @@ -1,4 +1,13 @@ [ + { + "part": "honesty", + "recipes": "fmt-check model spec-links bdd", + "examples_part": "", + "bdd_part": "honesty", + "width": "", + "memory_kib": "", + "hints": "" + }, { "part": "examples", "recipes": "examples", @@ -10,7 +19,7 @@ }, { "part": "rest", - "recipes": "fmt-check model spec-links lint test features bdd inventory golden repro deny", + "recipes": "lint test features bdd inventory golden repro deny", "examples_part": "", "bdd_part": "rest", "width": "", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e705569bb4..dd50e592c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,9 +6,10 @@ # checkout", and a restored target directory is not one. CI runs its twelve # recipes as the legs of `.github/vv-rows.json` at once, each its own clean # checkout: `examples`, which here verifies every example but UOR Atlas, and -# `rest`, the other eleven in the order of `vv`, with the `bdd` rows below. +# `honesty`, format/model/spec-links and the BDD meta target first, then +# `rest`, the remaining recipes in order, with the `bdd` rows below. # Together they run every recipe of `vv` exactly once, so the gate is the same -# and the wall time is the longest leg rather than the sum. +# and its wall time includes honesty followed by the downstream critical path. # # `bdd`, the conformance suite, was most of the `rest` leg's time, and it runs # as the parts of `.github/bdd-parts.json` (§9.4): each `bdd-` row runs @@ -34,8 +35,8 @@ # from broken code. # # Which runs are the gate (§9.4): the `plan` job decides from the event alone. -# A pull request that is a draft runs the `rest` leg and nothing else, because -# a head pushed again before it is final should cost one job and not the whole +# A pull request that is a draft runs honesty then the `rest` leg, because +# a head pushed again before it is final should cost two jobs and not the whole # gate; it is a development signal and the pull request is not mergeable on it. # So does a ready pull request whose base is neither `main` nor `release/*`: a # stacked pull request cannot land on a line, and the pull request that carries @@ -109,6 +110,8 @@ jobs: outputs: full: ${{ steps.plan.outputs.full }} rows: ${{ steps.plan.outputs.rows }} + honesty: ${{ steps.plan.outputs.honesty }} + fanout: ${{ steps.plan.outputs.fanout }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -124,17 +127,55 @@ jobs: REPO: ${{ github.repository }} run: .github/ci-plan.sh + # The existing model/spec-link recipes and complete BDD meta target own + # this row. Claim drift fails before expensive proof and supported-host jobs. + honesty: + name: ${{ matrix.part }} + needs: [plan] + if: ${{ needs.plan.outputs.honesty != '[]' }} + strategy: + matrix: + include: ${{ fromJSON(needs.plan.outputs.honesty) }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + LEXLEAN_BDD_PART: ${{ matrix.bdd_part }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install the pinned Rust toolchain + run: rustup toolchain install + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 + with: + tool: just + - name: Run the honesty part of the acceptance gate + shell: bash + run: | + set -euo pipefail + mkdir -p ci-evidence + just --timestamp --timestamp-format '%s%.3f' ${{ matrix.recipes }} 2>&1 | tee ci-evidence/vv.txt + printf '[%s] %s\n' "$(date +%s.%3N)" 'vv part: every recipe passed' >> ci-evidence/vv.txt + - name: Summarize the run's timing and memory + if: ${{ !cancelled() }} + run: cargo xtask ci-summary --evidence ci-evidence + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: ${{ !cancelled() }} + with: + name: acceptance-gate-evidence-${{ matrix.part }} + path: ci-evidence + if-no-files-found: error + retention-days: 14 + vv: name: just vv (${{ matrix.part }}) - needs: [plan] - if: ${{ needs.plan.outputs.rows != '[]' }} + needs: [plan, honesty] + if: ${{ needs.plan.outputs.fanout != '[]' }} # A failing leg does not cancel the others: each leg's evidence is complete. # The legs are the rows of `.github/vv-rows.json`, or only its `rest` row # while the pull request is a draft. strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.plan.outputs.rows) }} + include: ${{ fromJSON(needs.plan.outputs.fanout) }} runs-on: ubuntu-24.04 # Complete proof coverage is intentionally the long pole. GitHub-hosted # jobs permit six hours; a shorter local-style timeout would turn complete @@ -257,7 +298,7 @@ jobs: atlas-prefix: name: atlas (prefix) if: ${{ needs.plan.outputs.full == 'true' }} - needs: [plan] + needs: [plan, honesty] runs-on: ubuntu-24.04 timeout-minutes: 360 env: @@ -806,7 +847,7 @@ jobs: # own `xtask` and `repo-conformance`, which no host in this job runs. host-matrix: name: build and non-Lean tests (${{ matrix.target }}) - needs: [plan] + needs: [plan, honesty] if: ${{ needs.plan.outputs.full == 'true' }} strategy: fail-fast: false diff --git a/.github/workflows/honesty.yml b/.github/workflows/honesty.yml deleted file mode 100644 index 40894d36d3..0000000000 --- a/.github/workflows/honesty.yml +++ /dev/null @@ -1,44 +0,0 @@ -# R1, R2, R3: the model gate, the specification-link gate, and the honesty -# meta-gate (SPEC.md §9.4), run separately from the code gate so that a red -# mark here reads as what it is: not broken code, but a claim that has -# drifted from its evidence. -# -# This job needs no Lean toolchain: `validate-model` and `validate-spec-links` -# read the model, the suites, and the sources; the meta-gate test binary -# (`crates/conformance/tests/bdd.rs`) checks register/scenario/test bijection, -# the Gherkin subset, and its own falsifiability. The Lean-backed conformance -# cases belong to `just bdd` inside `just vv` in ci.yml, where the pinned -# toolchain is installed (`just bdd` runs the whole `repo-conformance` suite, the -# meta-gate target among it; this job runs that target alone). -name: honesty - -on: - push: { branches: [main, "release/**"] } - pull_request: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: honesty-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -env: - CARGO_TERM_COLOR: always - -jobs: - honesty: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install the pinned Rust toolchain - run: rustup toolchain install - - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 - with: - tool: just - # The model is the single source; the §31 table and the register are - # bijective; every registered ID has one scenario and one named test. - - run: just model spec-links - - run: cargo test --locked -p repo-conformance --test bdd diff --git a/CONFORMANCE.md b/CONFORMANCE.md index d8bb95e2f5..61e2c8d1ea 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -500,7 +500,7 @@ the error registry does not sanction (R5, `ERRORS.md`). | `RP-11` | `build` | Every public README capability claim is tied to a registered model ID and honesty level. | | `RP-12` | `build` | A release is refused unless the complete release criterion and all required artifacts are satisfied. | | `RP-13` | `build` | The ordinary workspace test phase excludes repo-conformance, the bdd phase alone runs its complete suite, and a test inventory proves that the two phases together equal the whole workspace test set with no test lost and none run twice, and that the parts of the committed bdd plan together equal the bdd phase with every test in exactly one part. | -| `RP-14` | `build` | The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run the rest leg and no other job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. | +| `RP-14` | `build` | The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run honesty then the rest leg and no other gate job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. | | `RP-15` | `build` | Every match over a closed semantic enum (the IR, the production realization calculus, and its Rust AST) that names at least five of the enum's variants names all of them, with no wildcard or binding catch-all arm, so that a variant added to the enum is a compile error in each Lean, LaTeX, validation, charge, and calculus consumer and never a silent fallback; the audit rejects a catch-all planted in each consumer group and fails when a consumer it must read has no such match left. | | `RP-16` | `build` | The development container installs, at image build, exactly the Rust toolchain that rust-toolchain.toml pins and exactly the Lean toolchain that lean-toolchain pins and selects no other default, so that nothing is fetched when a session starts, and a Dockerfile that names another version of either, installs a second toolchain, or selects an unpinned default is refused. | | `RP-18` | `build` | The release workflow packages the lexlean crate twice to the same bytes, runs cargo publish --dry-run, and refuses a crate whose SHA-256 differs from the one release/checksums.txt lists for it or from the bytes cargo would upload, or whose archive lacks the metadata and size crates.io requires, and uploads to crates.io only when a CARGO_REGISTRY_TOKEN secret or trusted publishing authorises it; a commit of a release line release/vX.Y is published as the pre-release X.Y.0-dev.N, N being its number of first-parent commits, and a plant in each of these checks is refused. | diff --git a/Justfile b/Justfile index 73dba5bb4c..134cde1b38 100644 --- a/Justfile +++ b/Justfile @@ -43,7 +43,7 @@ features: # target with one test per registered ID. `test` leaves all of it to this # recipe, and `inventory` proves that between them nothing was lost or run # twice. The meta-gate target alone, which needs no Lean, is what the -# honesty workflow runs (`cargo test -p repo-conformance --test bdd`). +# initial same-run honesty part owns (by the complete Cargo test target). # `cargo xtask bdd` runs `cargo test -p repo-conformance --all-features`; in # CI, `LEXLEAN_BDD_PART` names the part of `.github/bdd-parts.json` a job runs # (§9.4), and unset, as here, the recipe runs the whole suite. diff --git a/SPEC.md b/SPEC.md index 8a32e4950c..e0bfaa464e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -283,7 +283,6 @@ The completed repository MUST have this layout. Additional files are allowed onl ├── .github/ │ └── workflows/ │ ├── ci.yml -│ ├── honesty.yml │ └── reproducibility.yml ├── AGENTS.md ├── CONFORMANCE.md @@ -663,7 +662,7 @@ deny cargo deny --all-features check vv all recipes above, in order ``` -`test` runs every test target of every workspace crate except `repo-conformance`, and `bdd` is the sole owner of the complete `repo-conformance` suite: its library tests and doctests, its `conformance` target (one test per registered ID, §27.8), and its `bdd` meta-gate target. `cargo xtask bdd` runs `cargo test -p repo-conformance --all-features`, the whole suite; when `LEXLEAN_BDD_PART` names a part of `.github/bdd-parts.json`, it runs the same command with that part's harness filters, which is how CI runs it (§9.4). The two phases partition the workspace test set: `inventory` lists the tests of `cargo test --workspace --all-features`, of `test`, and of `bdd` (RP-13), and requires the last two to be disjoint, their union to equal the first, and every test target of `repo-conformance` to be in `bdd`'s list, so that no test is lost and none runs twice in `vv`. It also lists the tests of every part of the `bdd` plan with that part's harness filters, and requires the parts' lists to be disjoint, their union to equal `bdd`'s list, and every test a part names to be in that part's list, so that every test of `bdd` runs in exactly one part. The honesty workflow (§9.4) runs the meta-gate target alone, as a separate signal that is not part of `vv`. `just model-write` regenerates `CONFORMANCE.md` and `ERRORS.md`. Golden files are rewritten only by the explicit `just golden-write` recipe. No acceptance recipe rewrites source or expected output. +`test` runs every test target of every workspace crate except `repo-conformance`, and `bdd` is the sole owner of the complete `repo-conformance` suite: its library tests and doctests, its `conformance` target (one test per registered ID, §27.8), and its `bdd` meta-gate target. `cargo xtask bdd` runs `cargo test -p repo-conformance --all-features`, the whole suite; when `LEXLEAN_BDD_PART` names a part of `.github/bdd-parts.json`, it runs the part with its Cargo target selectors and harness filters, which is how CI runs it (§9.4). The two phases partition the workspace test set: `inventory` lists the tests of `cargo test --workspace --all-features`, of `test`, and of `bdd` (RP-13), and requires the last two to be disjoint, their union to equal the first, and every test target of `repo-conformance` to be in `bdd`'s list, so that no test is lost and none runs twice in `vv`. It also lists the tests of every part of the `bdd` plan with that part's Cargo target selectors and harness filters, and requires the parts' lists to be disjoint, their union to equal `bdd`'s list, and every test a part names to be in that part's list, so that every test of `bdd` runs in exactly one part. The initial honesty part (§9.4) owns the complete meta-gate target within the same CI run, as a distinct signal before expensive jobs. `just model-write` regenerates `CONFORMANCE.md` and `ERRORS.md`. Golden files are rewritten only by the explicit `just golden-write` recipe. No acceptance recipe rewrites source or expected output. ### 9.3 Development container @@ -682,21 +681,21 @@ Its image MUST install the pinned Rust and Lean toolchains at build, so that not `ci.yml` MUST run every recipe of `just vv` from a clean checkout on Linux x86-64. It MUST install the exact Rust toolchain, Lean 4.32.1 through `elan`, `just`, and `cargo-deny`. -**What a run is.** `ci.yml` starts with the job `plan`, which decides from the event and the repository alone, through `.github/ci-plan.sh`, what the run is before any job that costs a runner starts. A pull request that is a draft is the `rest` leg and nothing else: a head pushed again before it is final costs one job and not the gate, the run is a development signal, and a draft is not mergeable on it; marking the pull request ready, and every push to it after, run the whole gate. So is a ready pull request whose base branch is neither `main` nor a `release/*` line (a stacked pull request, whose base is another feature branch): only a pull request into `main` or a release line can land on a line, that pull request runs the whole gate on the tree it lands, and the stacked run says in its step summary that the base is why it is the `rest` leg alone. A ready pull request into `main` or a release line, a `workflow_dispatch`, and a push to `main` or to a release line are the whole gate, every job below. One push to `main` or to a release line is not: when it is a merge commit whose tree is the tree of its second parent (a pull request head that already contained `main`) and a run of this workflow for that parent as a pull request of this repository passed every job, with none skipped, the join of Atlas, every host of §8.3, and every leg of `.github/vv-rows.json` among them, the push runs no job but the plan, which records the run: the tree it lands is byte for byte the tree that run verified, on the same runners and with the same pins. Any other push to `main` or to a release line, including one the plan cannot decide (a failed query, no run, a run of a draft, a merge that changed the tree, a push that is no merge), is the whole gate. `release.yml` runs `just vv` whole and reads no earlier run. `RP-14` runs the script against planted events, among them a ready pull request into a feature branch, into a release line, and into `main`, and a plant in it fails the case, and reads the three workflows' push triggers, which name `main` and `release/**`. +**What a run is.** `ci.yml` starts with the job `plan`, which decides from the event and the repository alone, through `.github/ci-plan.sh`, what the run is before any job that costs a runner starts. A pull request that is a draft runs the initial honesty part then the `rest` leg: a head pushed again before it is final costs these two jobs and not the full gate, the run is a development signal, and a draft is not mergeable on it; marking the pull request ready, and every push to it after, run the whole gate. So is a ready pull request whose base branch is neither `main` nor a `release/*` line (a stacked pull request, whose base is another feature branch): only a pull request into `main` or a release line can land on a line, that pull request runs the whole gate on the tree it lands, and the stacked run says in its step summary that the base is why it runs honesty then the `rest` leg. A ready pull request into `main` or a release line, a `workflow_dispatch`, and a push to `main` or to a release line are the whole gate, every job below. One push to `main` or to a release line is not: when it is a merge commit whose tree is the tree of its second parent (a pull request head that already contained `main`) and a run of this workflow for that parent as a pull request of this repository passed every job, with none skipped, the join of Atlas, every host of §8.3, and every leg of `.github/vv-rows.json` among them, the push runs no job but the plan, which records the run: the tree it lands is byte for byte the tree that run verified, on the same runners and with the same pins. Any other push to `main` or to a release line, including one the plan cannot decide (a failed query, no run, a run of a draft, a merge that changed the tree, a push that is no merge), is the whole gate. `release.yml` consumes the complete same-tree gate evidence described below. `RP-14` runs the script against planted events, among them a ready pull request into a feature branch, into a release line, and into `main`, and a plant in it fails the case, and reads the CI and reproducibility workflows' push triggers, which name `main` and `release/**`. **Release lines.** A release line is the branch `release/vX.Y` of the release X.Y: it is cut from `main` when work on X.Y begins, every pull request for the milestone X.Y targets it, each `vX.Y.Z` tag is made from it, and it is merged forward, by merge commit, into the next release line after each landing so that the next line carries every fix of the one before it. `main` is the newest released line: a release line is merged into `main` when its release is tagged, and nothing else lands on `main`. A release line is held to the same gate as `main`: a push to it is the whole gate under the rule above, and a pull request against it is a pull request of this repository. **The release's evidence.** `release.yml` runs no recipe of the gate. The CI evidence a release attaches (§30.3) is a run of `ci.yml` that was the whole gate and passed every job, on the tagged commit or, when the tagged commit is a merge whose tree is the tree of its second parent, on that parent; `.github/release-gate.sh` finds it under the rule `plan` applies to a push, and refuses, naming the commits it looked at, when no such run exists. The evidence names the run, every job of it with its conclusion, and the transcript of every leg the run still retains. The gate run whole on one hosted runner exceeds the six-hour job limit (the `bdd` recipe alone takes over four hours there, and Atlas follows it), so a release gate that re-ran it would never conclude, and the run of the line has already verified the tree. -In a run that is the whole gate, the job runs the recipes as the legs of one matrix, the rows of `.github/vv-rows.json`, at once, each from its own clean checkout: `examples`, which verifies every example but UOR Atlas; `rest`, the other eleven recipes in the order of `vv`; and one `bdd-` row for each named part of the `bdd` plan below, which runs `bdd` alone. Each leg runs `just` with its recipes, which stops at the first that fails, as `vv` does. The part of `examples` a job runs is named by its matrix row through `LEXLEAN_EXAMPLES_PART` and by nothing else: `without:uor-atlas` on the leg, `shard:uor-atlas:` and `join:uor-atlas` on the Atlas jobs below; unset or empty, which is what `just vv` runs, the recipe verifies every example whole. Together the legs and the Atlas jobs run every recipe of `vv`, every example, and every test of `bdd` exactly once, and the run passes only when every job passes, so it proves what one run of `just vv` proves; no recipe depends on another having run, and every recipe owns the files it builds. Its wall time is the longest path through the jobs rather than the sum. `just vv` remains the acceptance recipe of §9.2 for every other run, and `release.yml` runs it whole. +In a run that is the whole gate, the jobs run the recipes as the rows of `.github/vv-rows.json`, each from its own clean checkout: `examples`, which verifies every example but UOR Atlas; `honesty`, which runs `fmt-check model spec-links bdd` before any expensive root; `rest`, the other recipes in the order of `vv`; and one `bdd-` row for each named part of the `bdd` plan below, which runs `bdd` alone. Each leg runs `just` with its recipes, which stops at the first that fails, as `vv` does. The part of `examples` a job runs is named by its matrix row through `LEXLEAN_EXAMPLES_PART` and by nothing else: `without:uor-atlas` on the leg, `shard:uor-atlas:` and `join:uor-atlas` on the Atlas jobs below; unset or empty, which is what `just vv` runs, the recipe verifies every example whole. After honesty succeeds, the remaining rows run as one matrix and the supported-host matrix and Atlas prefix may start; a failed or skipped honesty part starts none of these expensive roots. Together the legs and the Atlas jobs run every recipe of `vv`, every example, and every test of `bdd` exactly once, and the run passes only when every job passes, so it proves what one run of `just vv` proves; no recipe depends on another having run, and every recipe owns the files it builds. Its wall time is the longest path through the jobs rather than the sum. `just vv` remains the acceptance recipe of §9.2 for every other run, and release acceptance requires the complete same-tree run. UOR Atlas is verified as the parts of one run. `.github/verify-shards.json` partitions its verification into shards, each naming the shards it needs, the modules it elaborates, and the modules it replays and audits, one of them named `join`; `cargo xtask ci-shards` checks, as the verifier would when a part ran, that every module is elaborated by exactly one shard and replayed by exactly one, that every import of a module a shard elaborates or replays is elaborated by that shard or by one it needs, directly or through another, and that `join` needs every other shard. Each shard is a job of its own clean checkout that runs the verifier's prelude whole (certificate generation, the preflights, staging, the generated-source audit, the probe), then elaborates, replays, and audits only the modules of its plan, reading the `.olean` files of the modules it imports from the output the jobs of the shards it needs uploaded in the same run, and uploads its own `.olean` files and process records as its output. The join runs the modules of its plan, reads every shard's records, requires each to name the executable the join itself hashed for the tool that produced it (`lean` for an elaboration or an audit member, `leanchecker` for a replay), judges each as the stage that produced it judges its own, and continues through policy, the attestation over every record in the canonical order of §22.11, publication, and the comparison of every normalized record with `expected/verify` (§29.5): what it publishes is what the whole verification publishes, record for record and attestation identity included (`VR-23`). A module without a record from any part, a record of a process no module owns, a record whose `executable_sha256` is not the join's digest of its tool (`LLI9001`, naming the record and both digests), or a shard whose plan leaves a module without the `.olean` it imports fails the run. Every process of the verification runs exactly once per run. The executable the Atlas jobs run is the one the `prefix` job built from its clean checkout, carried to the other jobs of the run as an artifact beside the shards' output; it, the output, and the records cross no run, and a job reads the artifacts of its own run only. The Atlas jobs run after the jobs of the shards they need, at the profile of the `examples` leg, and only in a run that is the whole gate. -The `bdd` recipe runs as the parts of one run as well. `.github/bdd-parts.json` names parts of the `repo-conformance` suite by conformance ID, and a remainder, `rest`, that names none; the cases that share a fixture built once per test process, such as a verified project, an exported package set, or a compiled environment, are named by one part, so that a fixture is built in one part and not once per part. `cargo xtask bdd` runs `cargo test -p repo-conformance --all-features` with the harness filters of the part `LEXLEAN_BDD_PART` names: `--exact` and the test of every ID the part names, or for the remainder `--exact` and a `--skip` of every test another part names, so that the remainder runs the library tests, the doctests, the meta-gate target, and every conformance test no other part names, an ID added to the register included. The part a job runs is named by its matrix row and by nothing else: a `bdd-` row names its part, the `rest` leg names the remainder, and the `examples` leg runs no `bdd` and names none; unset or empty, which is what `just vv` and `release.yml` run, the recipe runs the whole suite. The harness selects the tests, and no case decides whether it runs. `inventory` proves from the harness's own lists that the parts are disjoint and together are `bdd` (§9.2), and every part, the remainder included, is run by exactly one row. Each row builds from its own clean checkout; nothing passes between rows, and nothing crosses runs. +The `bdd` recipe runs as the parts of one run as well. `.github/bdd-parts.json` names parts by conformance ID, complete integration-test targets in `targets`, and a remainder, `rest`. Cases that share a fixture built once per test process belong to one part. The `honesty` part owns the complete `bdd` meta-gate target, including any tests added to that target; no copied test-name list selects it. `cargo xtask bdd` runs `cargo test -p repo-conformance --all-features` with the Cargo target selectors and harness filters of `LEXLEAN_BDD_PART`. A target part selects its targets through `--test`; an ID part uses `--exact` and the test of each ID. The remainder discovers all integration-test targets from Cargo metadata, selects every target no target part owns, and keeps the library and binary tests and doctests; Cargo requires doctests in a separate invocation when target selectors are present. Its `--exact` and `--skip` filters omit every conformance test an ID part owns. A new test in a target belongs to that target's part; a new integration target and a new conformance ID belong to the remainder until assigned. No case decides whether it runs. `inventory` lists the tests using the same selectors and filters, proves disjointness and equality with the whole `bdd` suite, and requires a target part to own every test of its target (§9.2). Exactly one row runs every part, the remainder included. The initial honesty row names its part; each later `bdd-` row and the `rest` row name theirs. Unset or empty, as in local `just vv`, the recipe runs the whole suite unchanged. Each row builds from its own clean checkout; no build or evidence crosses runs. -Every gate job runs on the standard GitHub-hosted runner, `ubuntu-24.04`; no larger runner is part of the gate. The verification resource profile (§22.11) of a job is named by its matrix row, or on an Atlas job by the job itself, and by nothing else, no repository or workflow variable and no input: the `examples` leg and every Atlas job verify at width 2, with the scheduling hints committed in `.github/verify-hints.json` (derived from a run's telemetry by `cargo xtask ci-hints`) and a memory budget of 12 GiB (12582912 KiB) for the summed expected peaks of the proof processes running at once; the other legs name no profile and run at the default. A profile is a schedule and never a verdict: every elaboration, replay, and audit member still runs on the pinned toolchain, the directory each project publishes is byte for byte the one of width 1 without hints or a budget (§22.11), and the leg and the join still require every example's normalized verification records to equal `expected/verify` (§29.5), so a profile that changed a byte would fail the gate rather than pass a different one. A stale hint reorders processes and never skips one. Width 1 is still verified whole on UOR Atlas by `release.yml`, which runs `just vv` with no profile. The budget is below the runner's 16 GB, with the 4 GiB swap file of the job as overflow, so two processes run together only when their expected peaks fit, and a process whose expected peak alone passes the budget runs alone. The gate has no path filter, no cache of a build directory, object, verification tree, attestation, or proof result, and no condition that skips a recipe of `just vv` but the plan's, which decides only what a run is, as above; the artifacts the jobs of one run exchange are read by that run alone, and every job starts from a clean checkout with the exact pinned toolchains. Each job writes its performance telemetry (§22.12) to `ci-evidence/` and uploads that directory as the artifact named after it, `acceptance-gate-evidence-` or `acceptance-gate-evidence-atlas-`: the gate log (`just --timestamp` stamps each recipe's first command on a leg, which is how a recipe is timed without changing it; an Atlas job's log is the recipe's own output), the host's memory and swap sampled every five seconds, the verifier's telemetry, and a summary of the wall time of each recipe and verification stage, the slowest modules and audit members, and the peak resident set and swap of the proof processes. The summary step also prints that summary to the job log and appends it to the step summary, and runs when the job failed, because an uploaded artifact cannot be read from the log; each table lists only its slowest rows and the part appended to the step summary is cut at 512 KiB, half of what GitHub accepts. None of it is an input of any recipe, and a timing is never a claim about a semantic property. +Every gate job runs on the standard GitHub-hosted runner, `ubuntu-24.04`; no larger runner is part of the gate. The verification resource profile (§22.11) of a job is named by its matrix row, or on an Atlas job by the job itself, and by nothing else, no repository or workflow variable and no input: the `examples` leg and every Atlas job verify at width 2, with the scheduling hints committed in `.github/verify-hints.json` (derived from a run's telemetry by `cargo xtask ci-hints`) and a memory budget of 12 GiB (12582912 KiB) for the summed expected peaks of the proof processes running at once; the other legs name no profile and run at the default. A profile is a schedule and never a verdict: every elaboration, replay, and audit member still runs on the pinned toolchain, the directory each project publishes is byte for byte the one of width 1 without hints or a budget (§22.11), and the leg and the join still require every example's normalized verification records to equal `expected/verify` (§29.5), so a profile that changed a byte would fail the gate rather than pass a different one. A stale hint reorders processes and never skips one. Local `just vv` verifies UOR Atlas whole at width 1 with no profile. The budget is below the runner's 16 GB, with the 4 GiB swap file of the job as overflow, so two processes run together only when their expected peaks fit, and a process whose expected peak alone passes the budget runs alone. The gate has no path filter, no cache of a build directory, object, verification tree, attestation, or proof result, and no condition that skips a recipe of `just vv` but the plan's, which decides only what a run is, as above; the artifacts the jobs of one run exchange are read by that run alone, and every job starts from a clean checkout with the exact pinned toolchains. Each proof or fan-out job writes its performance telemetry (§22.12) to `ci-evidence/` and uploads that directory as the artifact named after it, `acceptance-gate-evidence-` or `acceptance-gate-evidence-atlas-`: the gate log (`just --timestamp` stamps each recipe's first command on a leg, which is how a recipe is timed without changing it; an Atlas job's log is the recipe's own output), the host's memory and swap sampled every five seconds, the verifier's telemetry, and a summary of the wall time of each recipe and verification stage, the slowest modules and audit members, and the peak resident set and swap of the proof processes. The summary step also prints that summary to the job log and appends it to the step summary, and runs when the job failed, because an uploaded artifact cannot be read from the log; each table lists only its slowest rows and the part appended to the step summary is cut at 512 KiB, half of what GitHub accepts. The initial honesty job retains its recipe timing and summary in the same evidence format, without Lean processes or proof-resource samples. None of it is an input of any recipe, and a timing is never a claim about a semantic property. -`honesty.yml` MUST run the model/honesty gate separately so claim drift is distinguishable from code failure. +`ci.yml` MUST expose the initial `honesty` job (retaining the previous check context) separately so claim drift is distinguishable from code failure. It runs the existing format, model and specification-link recipes and the complete BDD meta-gate target, without Lean. The remaining rows, supported hosts and Atlas prefix MUST require its success; these recipes and meta tests run once in the normative gate. Both the plan's previous-run evidence and release evidence MUST require the honesty row, including its successful conclusion; missing, failed or skipped honesty never establishes full acceptance. The separate `honesty.yml` workflow is subsumed by this same-run job. `reproducibility.yml` MUST perform two builds in distinct absolute directories and compare every platform-independent artifact byte-for-byte after path normalization. @@ -10999,7 +10998,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `RP-11` | `repository` | Every public README capability claim is tied to a registered model ID and honesty level. | §27 | | `RP-12` | `repository` | A release is refused unless the complete release criterion and all required artifacts are satisfied. | §30 | | `RP-13` | `repository` | The ordinary workspace test phase excludes repo-conformance, the bdd phase alone runs its complete suite, and a test inventory proves that the two phases together equal the whole workspace test set with no test lost and none run twice, and that the parts of the committed bdd plan together equal the bdd phase with every test in exactly one part. | §9.2 | -| `RP-14` | `repository` | The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run the rest leg and no other job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. | §9.4 | +| `RP-14` | `repository` | The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run honesty then the rest leg and no other gate job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. | §9.4 | | `RP-15` | `repository` | Every match over a closed semantic enum (the IR, the production realization calculus, and its Rust AST) that names at least five of the enum's variants names all of them, with no wildcard or binding catch-all arm, so that a variant added to the enum is a compile error in each Lean, LaTeX, validation, charge, and calculus consumer and never a silent fallback; the audit rejects a catch-all planted in each consumer group and fails when a consumer it must read has no such match left. | §27.2 | | `RP-16` | `repository` | The development container installs, at image build, exactly the Rust toolchain that rust-toolchain.toml pins and exactly the Lean toolchain that lean-toolchain pins and selects no other default, so that nothing is fetched when a session starts, and a Dockerfile that names another version of either, installs a second toolchain, or selects an unpinned default is refused. | §9.3 | | `RP-18` | `repository` | The release workflow packages the lexlean crate twice to the same bytes, runs cargo publish --dry-run, and refuses a crate whose SHA-256 differs from the one release/checksums.txt lists for it or from the bytes cargo would upload, or whose archive lacks the metadata and size crates.io requires, and uploads to crates.io only when a CARGO_REGISTRY_TOKEN secret or trusted publishing authorises it; a commit of a release line release/vX.Y is published as the pre-release X.Y.0-dev.N, N being its number of first-parent commits, and a plant in each of these checks is refused. | §30.3 | diff --git a/VERIFICATION.md b/VERIFICATION.md index eab5eef1fb..9a8e40ff1c 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -21,6 +21,8 @@ How this repository's claims are checked, which recipe enforces which rule, and | `repro` | `cargo xtask check-reproducibility` | AR-13, §28.4: two clean `build`s in distinct absolute directories publish byte-identical trees with no absolute path inside | | `deny` | `cargo deny --all-features check` | advisories, bans, licenses, sources | +In CI, the initial `honesty` row owns `fmt-check model spec-links` and the complete `bdd` meta-gate target. The remaining rows, supported hosts and Atlas prefix require its success. The BDD remainder owns the library, doctests and unassigned integration targets; Cargo metadata discovers target names and the inventory lists the same target selectors and filters as execution. The separate honesty workflow is subsumed by this same-run job; local `just vv` retains its complete suite. + Outside `vv`: - `just fixtures` (`cargo xtask check-fixtures`) runs every §28.2 fixture under `tests/fixtures/` and `tests/negative/` through the CLI entry point and compares exit code, canonical command result, diagnostics, artifact list, and platform-independent hashes with `expected/`. `just fixtures-write` is the only rewrite path. @@ -12471,6 +12473,87 @@ This removes a redundant parse; it does not establish or resolve the cause of PR #403's earlier TC-03 CI stack overflow. No full acceptance run is claimed by this local check. + +## Existing honesty checks precede expensive CI fan-out (#411) + +The original #403 honesty run 41605700268, job 125814846351, started on +2026-11-03 at 08:46:05 UTC and failed at 08:48:58 with `VR-11: 8 steps`. +Its five BDD meta tests took 1.09 seconds; `model spec-links` had passed. +In CI run 41605700263, `rest` repeated the failure at 09:01:20. The Intel +host job 125815138918 started at 10:05:57, repeated it at 10:19:42, and +continued under `--no-fail-fast` until cancellation at 10:26:14. Atlas prefix +125815138772 independently ran 08:46:37–09:11:17 successfully. These are +observed job-log timestamps, not a sum of runner savings. The later run +41619096018 is separate evidence and does not establish this failure. + +The initial row now runs the existing format, model and specification-link +recipes and the existing complete BDD meta target, with the check name +`honesty`. Those recipes move out of `rest`; the meta target moves out of +the BDD remainder. All remaining matrix rows, the supported hosts and Atlas +prefix depend on this job, and the standalone honesty workflow is subsumed. +The five meta tests and their assertions are unchanged. In particular, the +six-step assertion remains in the existing BDD target: running model checks +alone would have missed the observed eight-step defect. + +The parts plan owns complete integration-test targets rather than a copy of +their test names. The remainder discovers unassigned targets through Cargo +metadata, retains library and binary tests, and invokes doctests separately +because Cargo forbids combining `--doc` with other target selectors. Both +commands run even if one fails; any failure fails the part. A newly added +test follows its target, and an unassigned target follows the remainder. +Local `just vv` still runs the complete suite. Supported-host workspace +coverage is unchanged, including the meta tests on each host. + +Focused local evidence on Linux x86-64 with the pinned Rust toolchain: + +| Check or plant | Observed | +| --- | --- | +| Actual harness inventory, `cargo xtask check-test-inventory` | 810 workspace tests = 296 ordinary + 514 BDD, disjoint with none lost; 23 parts partition all 514 BDD tests, honesty owns 5 and rest 324 | +| Existing BDD target through `LEXLEAN_BDD_PART=honesty cargo xtask bdd` | 5 passed | +| Restored complete initial row, `LEXLEAN_BDD_PART=honesty just fmt-check model spec-links bdd` | Passed: formatting, model audits (468 IDs / 86 codes), specification links (468 rows), and all 5 meta tests | +| VR-11's sixth step split into three, making eight steps; the initial `fmt-check model spec-links bdd` row | Refused by the unchanged `the_suites_are_inside_the_gherkin_subset`: `VR-11: 8 steps; a scenario states one behavior`; 4 passed, 1 failed; plant restored | +| RP-13 complete-target ownership plants | A repeated target test and meta tests silently reassigned to rest are each refused | +| RP-14 and RP-18 required-row plants | Missing, failed and skipped honesty each refuse whole-gate evidence in parent reuse and release validation | +| RP-14 long parent listing, 2,500 extra successful jobs | Before correcting the quiet join match, failed: plan returned `full=true` instead of reusing the passed parent; after consuming the complete listing, passed; a late failed job still refuses reuse | +| RP-18 large listing | Existing large-success acceptance and late-failure refusal retained and passed | + +The 41 conformance library tests, exact RP-13/RP-14/RP-18 cases, and +workspace/all-target/all-feature Clippy with warnings denied also passed. + +The parent-reuse correction is separate from #408's release-gate correction: +its join match also stopped reading before the producer finished under +`pipefail`. No production failure is attributed to that path. Both scripts +still require the Atlas join, every supported host, every required row and +every successful conclusion, and preserve same-tree matching. + +No compiler identity input changed: the embedded digest reads `language/`, +`schemas/`, and the axiom-parser and canonical-JSON goldens. RP-10 recomputed +that digest successfully; no language identity or verification oracle was +regenerated. This is focused evidence, not the complete final-head CI gate; +that gate remains required after the dependency lands. No post-change CI +runtime or aggregate savings is claimed. + + +### Hosted honesty matrix naming correction (#411) + +Draft run 41639924023 at head `1a5f87ac9` passed its initial checks, but +GitHub named job 125930780532 +`honesty (honesty, fmt-check model spec-links bdd, honesty)`. A static job +name inside the matrix acquired GitHub's matrix suffix. The source check +had accepted `name: honesty`; that check did not establish the hosted +context name. Both evidence consumers correctly refused the suffixed name +because they require exactly `honesty`. + +The job now uses `name: ${{ matrix.part }}`, the same matrix-based naming +mechanism as the existing fan-out rows, with the initial row's part equal to +`honesty`. RP-14 requires that expression and rejects the previous static +name. With the strengthened assertion and the old declaration, exact RP-14 +failed (exit 101); with the expression it passes. RP-14 and RP-18 additionally +plant the observed suffixed name and refuse it in parent-reuse and release +evidence. Neither evidence predicate was broadened. The corrected hosted +name must still be checked after publication; local source and mocked-job +checks alone do not establish GitHub's emitted context. + ## The canonical LaTeX of a tooling module is language-1.2 source (`BS-20`, #270) `bootstrap/src/ToolingAst.lex.tex` holds the closed schema of the tooling module diff --git a/crates/conformance/src/bdd_parts.rs b/crates/conformance/src/bdd_parts.rs index 35ed89d9ed..1d39d05e0a 100644 --- a/crates/conformance/src/bdd_parts.rs +++ b/crates/conformance/src/bdd_parts.rs @@ -6,8 +6,8 @@ //! share a fixture (a verified project, an export, a compiled environment, all //! built once per test process) run in one part and the fixture is not built //! once per part. One part, the remainder, names no ID: it runs every test of -//! the suite no other part names, its library tests, doctests, and meta-gate -//! target included, so an ID added to the register runs there until the plan +//! the suite no other part names, its library tests and doctests included; target parts own complete +//! integration targets such as the honesty meta-gate, so an ID added to the register runs there until the plan //! moves it. //! //! `cargo xtask bdd` runs `cargo test` with [`CARGO_ARGUMENTS`] and, when @@ -46,6 +46,8 @@ pub struct Plan { pub remainder: String, /// The named parts: each the conformance IDs whose tests it runs. pub parts: BTreeMap>, + /// Parts that own complete integration-test targets, including future tests. + pub targets: BTreeMap>, } fn is_name(name: &str) -> bool { @@ -77,7 +79,7 @@ impl Plan { .ok_or_else(|| format!("{PLAN_FILE}: an object is required"))?; if let Some(unknown) = object .keys() - .find(|key| !["remainder", "parts"].contains(&key.as_str())) + .find(|key| !["remainder", "parts", "targets"].contains(&key.as_str())) { return Err(format!( "{PLAN_FILE}: `{unknown}` is not a field of the plan" @@ -106,7 +108,16 @@ impl Plan { .collect::>()?; parts.insert(name.clone(), ids); } - let plan = Self { remainder, parts }; + let targets = match object.get("targets") { + Some(value) => serde_json::from_value::>>(value.clone()) + .map_err(|error| format!("{PLAN_FILE}: targets: {error}"))?, + None => BTreeMap::new(), + }; + let plan = Self { + remainder, + parts, + targets, + }; plan.check()?; Ok(plan) } @@ -139,7 +150,7 @@ impl Plan { self.remainder )); } - if self.parts.contains_key(&self.remainder) { + if self.parts.contains_key(&self.remainder) || self.targets.contains_key(&self.remainder) { return Err(format!( "{PLAN_FILE}: `{}` is both the remainder and a named part", self.remainder @@ -168,13 +179,37 @@ impl Plan { } } } + let mut target_owner = BTreeMap::new(); + for (part, targets) in &self.targets { + if !is_name(part) || self.parts.contains_key(part) || targets.is_empty() { + return Err(format!( + "{PLAN_FILE}: invalid or repeated target part `{part}`" + )); + } + for target in targets { + if !is_name(target) { + return Err(format!("{PLAN_FILE}: invalid test target `{target}`")); + } + if let Some(first) = target_owner.insert(target, part) { + return Err(format!( + "{PLAN_FILE}: target `{target}` belongs to `{first}` and `{part}`" + )); + } + } + } Ok(()) } /// Every part's name, the remainder last. #[must_use] pub fn names(&self) -> Vec { - let mut names: Vec = self.parts.keys().cloned().collect(); + let mut names: Vec = self + .parts + .keys() + .chain(self.targets.keys()) + .cloned() + .collect(); + names.sort(); names.push(self.remainder.clone()); names } @@ -189,6 +224,46 @@ impl Plan { .collect() } + /// Cargo target selectors for a part. Integration targets are discovered + /// from Cargo metadata rather than a list of tests that can become stale. + /// The remainder keeps every unassigned target and the library/doctests. + /// + /// # Errors + /// An unknown part or a target absent from Cargo's metadata. + pub fn cargo_targets( + &self, + part: &str, + integration: &BTreeSet, + ) -> Result>, String> { + for target in self.targets.values().flatten() { + if !integration.contains(target) { + return Err(format!( + "{PLAN_FILE}: test target `{target}` does not exist" + )); + } + } + if let Some(targets) = self.targets.get(part) { + return Ok(vec![targets + .iter() + .flat_map(|target| ["--test".to_owned(), target.clone()]) + .collect()]); + } + if part == self.remainder && !self.targets.is_empty() { + let assigned: BTreeSet<&String> = self.targets.values().flatten().collect(); + let mut selected = vec!["--lib".to_owned(), "--bins".to_owned()]; + for target in integration + .iter() + .filter(|target| !assigned.contains(target)) + { + selected.extend(["--test".to_owned(), target.clone()]); + } + // Cargo forbids --doc with another target selector. + return Ok(vec![selected, vec!["--doc".to_owned()]]); + } + self.harness_filters(part)?; + Ok(vec![Vec::new()]) + } + /// The harness arguments (after `--`) that select one part: `--exact` /// and the part's test names, or for the remainder `--exact` and a /// `--skip` of every test a named part runs. @@ -197,6 +272,9 @@ impl Plan { /// /// A part the plan does not have. pub fn harness_filters(&self, part: &str) -> Result, String> { + if self.targets.contains_key(part) { + return Ok(Vec::new()); + } let mut filters = vec!["--exact".to_owned()]; if part == self.remainder { for name in self.named_tests() { @@ -255,6 +333,48 @@ mod tests { assert!(unknown.contains("heavy, other, rest"), "{unknown}"); } + #[test] + fn complete_targets_move_without_copying_their_test_names() { + let plan = Plan::parse( + r#"{ + "remainder": "rest", "parts": {"first": ["RP-01"]}, + "targets": {"honesty": ["bdd"]} + }"#, + ) + .expect("a target part"); + let targets = ["bdd", "conformance", "future"] + .into_iter() + .map(str::to_owned) + .collect(); + assert_eq!( + plan.cargo_targets("honesty", &targets).expect("honesty"), + [vec!["--test", "bdd"]] + ); + assert!(plan + .harness_filters("honesty") + .expect("all tests") + .is_empty()); + assert_eq!( + plan.cargo_targets("rest", &targets).expect("rest"), + [ + vec![ + "--lib", + "--bins", + "--test", + "conformance", + "--test", + "future" + ], + vec!["--doc"] + ] + ); + assert!(plan.cargo_targets("honesty", &Default::default()).is_err()); + assert!(Plan::parse( + r#"{"remainder":"rest","parts":{},"targets":{"one":["bdd"],"two":["bdd"]}}"# + ) + .is_err()); + } + #[test] fn every_defect_of_a_plan_is_named() { for (text, expected) in [ diff --git a/crates/conformance/src/cases/repository.rs b/crates/conformance/src/cases/repository.rs index 042d563674..e8c834af3c 100644 --- a/crates/conformance/src/cases/repository.rs +++ b/crates/conformance/src/cases/repository.rs @@ -169,11 +169,13 @@ fn plan_decisions() { // the Atlas join, and every host. The shards do not matter to the plan: // the join needs them all. let mut green: Vec = vec!["plan\tsuccess".to_owned()]; - green.extend( - parts - .iter() - .map(|part| format!("just vv ({part})\tsuccess")), - ); + green.extend(parts.iter().map(|part| { + if part == "honesty" { + "honesty\tsuccess".to_owned() + } else { + format!("just vv ({part})\tsuccess") + } + })); green.push("atlas (prefix)\tsuccess".to_owned()); green.push("atlas (join)\tsuccess".to_owned()); green.extend(hosts.iter().map(|host| format!("{host}\tsuccess"))); @@ -320,6 +322,19 @@ fn plan_decisions() { }; let planned: Vec = serde_json::from_str(&value("rows")).expect("rows are JSON"); + let honesty: Vec = + serde_json::from_str(&value("honesty")).expect("honesty is JSON"); + let fanout: Vec = + serde_json::from_str(&value("fanout")).expect("fanout is JSON"); + assert!(honesty.iter().all(|row| row["part"] == "honesty")); + assert!(fanout.iter().all(|row| row["part"] != "honesty")); + let mut combined = honesty; + combined.extend(fanout); + assert_eq!( + combined, planned, + "honesty and fanout partition the planned rows" + ); + ( value("full"), planned @@ -329,9 +344,12 @@ fn plan_decisions() { ) }; let whole = ("true".to_owned(), parts.clone()); - let draft = ("false".to_owned(), vec!["rest".to_owned()]); + let draft = ( + "false".to_owned(), + vec!["honesty".to_owned(), "rest".to_owned()], + ); let nothing = ("false".to_owned(), Vec::::new()); - // A draft pull request is the `rest` leg, and so is a ready one stacked on + // A draft pull request runs honesty then `rest`, as does a ready one stacked on // a feature branch, which cannot land on a line; a ready one into `main` // or a release line, a request, and a push are the whole gate when no run // is known. A base the event does not name is not known to be stacked. @@ -392,6 +410,21 @@ fn plan_decisions() { decide("push", "", "main", &merge_equal, "4242", &green, false), nothing ); + // A complete jobs listing can exceed a pipe buffer. Matching the join + // and required rows must consume it without a producer SIGPIPE. + let mut large = green.clone(); + large.extend((0..2500).map(|index| format!("additional successful job {index}\tsuccess"))); + assert_eq!( + decide("push", "", "main", &merge_equal, "4242", &large, false), + nothing, + "a large successful parent gate still avoids repeating the gate" + ); + large.push("a late failed parent job\tfailure".to_owned()); + assert_eq!( + decide("push", "", "main", &merge_equal, "4242", &large, false), + whole, + "a late failure never establishes successful parent evidence" + ); // Every way the evidence can fall short runs the whole gate: no run, a // run that skipped or failed a job, a run without the Atlas join, one // host or one leg missing, the gate unreadable, and a merge that changed @@ -418,6 +451,33 @@ fn plan_decisions() { ("no Atlas join", without("atlas (join)")), ("no host", without(&hosts[0])), ("no leg", without("just vv (rest)")), + ("no honesty", without("honesty")), + ( + "suffixed honesty", + green + .iter() + .map(|job| { + job.replace( + "honesty\tsuccess", + "honesty (honesty, fmt-check model spec-links bdd, honesty)\tsuccess", + ) + }) + .collect(), + ), + ( + "failed honesty", + green + .iter() + .map(|job| job.replace("honesty\tsuccess", "honesty\tfailure")) + .collect(), + ), + ( + "skipped honesty", + green + .iter() + .map(|job| job.replace("honesty\tsuccess", "honesty\tskipped")) + .collect(), + ), ( "a draft run", vec![ @@ -1342,6 +1402,38 @@ pub(crate) fn run(id: &str) { }; crate::inventory::compare_parts(&owned, &parts(&first, &remainder), &plants) .expect("parts that partition bdd are accepted"); + let target_plan = crate::bdd_parts::Plan::parse( + r#"{"remainder":"rest","parts":{},"targets":{"honesty":["bdd"]}}"#, + ) + .expect("complete-target ownership"); + let target_parts = |honesty: crate::inventory::Parsed, + rest: crate::inventory::Parsed| { + vec![("honesty".to_owned(), honesty), ("rest".to_owned(), rest)] + }; + crate::inventory::compare_parts( + &owned, + &target_parts(list(&[bdd_target]), list(&[conformance])), + &target_plan, + ) + .expect("the complete target moves to honesty"); + assert!( + crate::inventory::compare_parts( + &owned, + &target_parts(list(&[bdd_target]), owned.clone()), + &target_plan + ) + .is_err(), + "the remainder cannot repeat honesty tests" + ); + assert!( + crate::inventory::compare_parts( + &owned, + &target_parts(Default::default(), owned.clone()), + &target_plan + ) + .is_err(), + "moving tests back to rest cannot silently empty honesty" + ); let twice = crate::inventory::compare_parts(&owned, &parts(&first, &owned), &plants) .expect_err("a test two parts run"); assert!(twice.contains("parts `first` and `rest`"), "{twice}"); @@ -1361,7 +1453,9 @@ pub(crate) fn run(id: &str) { && task.contains("std::env::var(PART_VARIABLE)") && task.contains("plan.harness_filters(part)") && task.contains("plan.harness_filters(&name)") - && task.contains("inventory::compare_parts("), + && task.contains("inventory::compare_parts(") + && task.contains("plan.cargo_targets(part,") + && task.contains("plan.cargo_targets(&name,"), "the bdd task and the inventory select a part through the plan's filters" ); @@ -1381,10 +1475,12 @@ pub(crate) fn run(id: &str) { "SPEC.md names `{recipe_command}`" ); } - // The honesty workflow keeps its own signal, and the host matrix + // The initial honesty job keeps its own signal, and the host matrix // (not part of vv) keeps the whole workspace on every host. - assert!(root_file(".github/workflows/honesty.yml") - .contains("cargo test --locked -p repo-conformance --test bdd")); + assert!(root_file(".github/workflows/ci.yml").contains(" honesty:\n")); + assert!(!support::repo_root() + .join(".github/workflows/honesty.yml") + .exists()); assert!(root_file(".github/workflows/ci.yml") .contains("cargo test --locked --workspace --all-features")); } @@ -1437,7 +1533,11 @@ pub(crate) fn run(id: &str) { ); // The plan decides what the run is and runs no recipe: it has // ten minutes, not the six hours of the proof jobs. - let limit = if key == "plan" { 10 } else { 360 }; + let limit = match key.as_str() { + "plan" => 10, + "honesty" => 30, + _ => 360, + }; assert!( body.contains(&format!("\n timeout-minutes: {limit}\n")), "§9.4: the job `{key}` has its limit of {limit} minutes" @@ -1527,6 +1627,16 @@ pub(crate) fn run(id: &str) { "§9.4: a leg names exactly its part, recipes, and profile: {row}" ); } + let initial: Vec<_> = rows + .iter() + .filter(|row| field(row, "part").as_deref() == Some("honesty")) + .collect(); + assert_eq!(initial.len(), 1, "one initial honesty row"); + assert_eq!( + field(initial[0], "recipes").as_deref(), + Some("fmt-check model spec-links bdd") + ); + assert_eq!(field(initial[0], "bdd_part").as_deref(), Some("honesty")); assert_eq!( rows.iter() .filter(|row| field(row, "part").as_deref() == Some("rest")) @@ -1676,7 +1786,7 @@ pub(crate) fn run(id: &str) { "just --timestamp --timestamp-format '%s%.3f' ${{ matrix.recipes }} 2>&1 | tee ci-evidence/vv.txt" ) .count(), - 1, + 2, "every leg runs its recipes through the one gate command" ); assert!( @@ -1891,7 +2001,7 @@ pub(crate) fn run(id: &str) { // §9.4: a release line is held to the same gate as `main`, so the // push trigger of every workflow names both; a workflow that named // `main` alone would leave a release line's pushes unverified. - for workflow in ["ci.yml", "honesty.yml", "reproducibility.yml"] { + for workflow in ["ci.yml", "reproducibility.yml"] { let text = root_file(&format!(".github/workflows/{workflow}")); assert!( text.contains(" push: { branches: [main, \"release/**\"] }\n"), @@ -1920,12 +2030,14 @@ pub(crate) fn run(id: &str) { let mut step_conditions = 0; let mut full_conditions = 0; let mut rows_conditions = 0; + let mut honesty_conditions = 0; for (index, line) in ci.lines().enumerate() { if line.trim_start().starts_with("if:") { match line { " if: ${{ !cancelled() }}" => step_conditions += 1, " if: ${{ needs.plan.outputs.full == 'true' }}" => full_conditions += 1, - " if: ${{ needs.plan.outputs.rows != '[]' }}" => rows_conditions += 1, + " if: ${{ needs.plan.outputs.fanout != '[]' }}" => rows_conditions += 1, + " if: ${{ needs.plan.outputs.honesty != '[]' }}" => honesty_conditions += 1, other => panic!( "ci.yml:{}: a condition other than a step's `!cancelled()` or a job's condition on the plan: {other}", index + 1 @@ -1935,7 +2047,7 @@ pub(crate) fn run(id: &str) { } assert_eq!( step_conditions, - 2 + 2 * atlas_jobs.len(), + 4 + 2 * atlas_jobs.len(), "the summary step and the evidence upload of every gate job carry it" ); assert_eq!( @@ -1944,15 +2056,36 @@ pub(crate) fn run(id: &str) { "every Atlas job and the host matrix, and no other job, wait for the plan to name the whole gate" ); assert_eq!(rows_conditions, 1, "the legs run when the plan names a leg"); + assert_eq!(honesty_conditions, 1, "honesty runs when the plan names it"); + honesty_dependencies(&ci).expect("expensive roots require successful same-run honesty"); + assert!( + honesty_dependencies(&ci.replacen("name: ${{ matrix.part }}", "name: honesty", 1)) + .is_err(), + "a static matrix name acquires a suffix instead of the exact honesty context" + ); + for root in ["vv", "atlas-prefix", "host-matrix"] { + let original = format!(" {root}:\n"); + let pos = ci.find(&original).expect("the root"); + let end = ci[pos + original.len()..] + .find("\n #") + .map_or(ci.len(), |next| pos + original.len() + next); + let mut planted = ci.clone(); + let body = ci[pos..end].replace("needs: [plan, honesty]", "needs: [plan]"); + planted.replace_range(pos..end, &body); + assert!( + honesty_dependencies(&planted).is_err(), + "{root} cannot bypass honesty" + ); + } assert!( - job("host-matrix").contains(" needs: [plan]\n") + job("host-matrix").contains(" needs: [plan, honesty]\n") && job("host-matrix") .contains(" if: ${{ needs.plan.outputs.full == 'true' }}\n"), "§8.3: the hosts are checked when the plan says the run is the whole gate" ); assert!( - vv.contains(" needs: [plan]\n") - && vv.contains(" include: ${{ fromJSON(needs.plan.outputs.rows) }}\n"), + vv.contains(" needs: [plan, honesty]\n") + && vv.contains(" include: ${{ fromJSON(needs.plan.outputs.fanout) }}\n"), "§9.4: the legs of vv are the rows the plan names" ); // The plan: one job, no recipe, ten minutes, reading the event and @@ -2022,7 +2155,14 @@ pub(crate) fn run(id: &str) { "the host matrix runs the workspace's tests once, and reports every failing test of the host, not the first binary's" ); // The honesty and reproducibility workflows stay separate. - assert!(root_file(".github/workflows/honesty.yml").contains("just model spec-links")); + let honesty = job("honesty"); + assert!( + honesty.contains("LEXLEAN_BDD_PART: ${{ matrix.bdd_part }}") + && honesty.contains("fromJSON(needs.plan.outputs.honesty)") + && honesty.contains("just --timestamp") + && !honesty.contains("elan"), + "the initial part needs no Lean" + ); assert!(root_file(".github/workflows/reproducibility.yml").contains("run: just repro")); } // R8: a `match` that names several variants of a closed enum names @@ -2854,6 +2994,26 @@ fn rp_18() { release_gate_decisions(); } +/// Claim drift must finish successfully before a costly root can start. +fn honesty_dependencies(ci: &str) -> Result<(), String> { + let initial = workflow_job(ci, "honesty")?; + if !initial.contains("name: ${{ matrix.part }}\n") + || !initial.contains("needs: [plan]\n") + || !initial.contains("if: ${{ needs.plan.outputs.honesty != '[]' }}") + { + return Err( + "RP-14: honesty must expose its matrix-resolved context and follow the plan".to_owned(), + ); + } + for root in ["vv", "atlas-prefix", "host-matrix"] { + let body = workflow_job(ci, root)?; + if !body.contains("needs: [plan, honesty]\n") || body.contains("always()") { + return Err(format!("RP-14: {root} must wait for successful honesty")); + } + } + Ok(()) +} + /// What `.github/release-gate.sh` accepts as the release's evidence (§9.4, /// §30.3): the script runs in a scratch repository with a stand-in for `gh` /// that answers with planted runs and jobs, so that the run it names is shown @@ -2890,7 +3050,11 @@ fn release_gate_decisions() { let mut green: Vec = vec![stamp("plan", "success")]; green.extend(rows.iter().map(|row| { stamp( - &format!("just vv ({})", row["part"].as_str().expect("a part")), + &if row["part"] == "honesty" { + "honesty".to_owned() + } else { + format!("just vv ({})", row["part"].as_str().expect("a part")) + }, "success", ) })); @@ -2909,6 +3073,28 @@ fn release_gate_decisions() { ) }) .collect(); + let without_honesty: Vec = green + .iter() + .filter(|job| !job.starts_with("honesty")) + .cloned() + .collect(); + let failed_honesty: Vec = green + .iter() + .map(|job| job.replace("honesty\tsuccess", "honesty\tfailure")) + .collect(); + let skipped_honesty: Vec = green + .iter() + .map(|job| job.replace("honesty\tsuccess", "honesty\tskipped")) + .collect(); + let suffixed_honesty: Vec = green + .iter() + .map(|job| { + job.replace( + "honesty\tsuccess", + "honesty (honesty, fmt-check model spec-links bdd, honesty)\tsuccess", + ) + }) + .collect(); let no_join: Vec = green .iter() .filter(|job| !job.starts_with("atlas (join)")) @@ -3105,6 +3291,26 @@ fn release_gate_decisions() { // No run, a run with a failed job, a run without the join, a draft run. for (why, runs, jobs) in [ ("no run", vec![], &green), + ( + "suffixed honesty", + vec![(head.as_str(), run_line("24"))], + &suffixed_honesty, + ), + ( + "missing honesty", + vec![(head.as_str(), run_line("21"))], + &without_honesty, + ), + ( + "failed honesty", + vec![(head.as_str(), run_line("22"))], + &failed_honesty, + ), + ( + "skipped honesty", + vec![(head.as_str(), run_line("23"))], + &skipped_honesty, + ), ( "a failed job", vec![(head.as_str(), run_line("14"))], diff --git a/crates/conformance/src/inventory.rs b/crates/conformance/src/inventory.rs index ef6828508c..7b50d3ec5f 100644 --- a/crates/conformance/src/inventory.rs +++ b/crates/conformance/src/inventory.rs @@ -291,6 +291,28 @@ pub fn compare_parts( } } } + for (part, targets) in &plan.targets { + let listed = parts + .iter() + .find(|(name, _)| name == part) + .ok_or_else(|| format!("RP-13: target part `{part}` is missing"))?; + for target in targets { + let expected: BTreeSet<&Entry> = owned + .entries + .iter() + .filter(|entry| crate_of_target(&entry.target) == target.replace('-', "_")) + .collect(); + if expected.is_empty() + || !expected + .iter() + .all(|entry| listed.1.entries.contains(*entry)) + { + return Err(format!( + "RP-13: part `{part}` does not own complete target `{target}`" + )); + } + } + } Ok(parts.len()) } diff --git a/features/suites/repository.feature b/features/suites/repository.feature index ea908d8bc7..c3cb5f0698 100644 --- a/features/suites/repository.feature +++ b/features/suites/repository.feature @@ -98,13 +98,13 @@ Feature: repository And the test lists of the parts of the bdd plan, each taken with the part's harness filters, are disjoint and their union equals the bdd recipe's list, so that a test two parts run or a test no part runs fails the inventory @RP-14 @build - Scenario: The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run the rest leg and no other job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. + Scenario: The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run honesty then the rest leg and no other gate job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing. Given .github/workflows/ci.yml, .github/vv-rows.json, .github/ci-plan.sh, .github/verify-shards.json, and .github/bdd-parts.json When the runner, the environment, the job steps, and the plan are read, and the plan script is run against planted events - Then every gate job runs on ubuntu-24.04 with the 360-minute limit, and each verification profile variable is set once per job, from the leg's matrix row or as the examples profile on an Atlas job + Then every gate job runs on ubuntu-24.04, honesty has a 30-minute limit and precedes the expensive jobs, proof jobs have the 360-minute limit, and each verification profile variable is set once per job, from the leg's matrix row or as the examples profile on an Atlas job And the leg that runs examples names width 2, a 12 GiB budget, and the committed hints, which the verifier parses, verifies every example but uor-atlas, and every other leg names no profile and no example part, and the legs run the recipes of vv, each exactly once but bdd, which runs once per part of the bdd plan on the one row that names the part, in the order of vv within a leg, through the one gate command And the Atlas jobs run every shard of the plan exactly once, each after the jobs of the shards it needs, from the one executable the prefix job built, the join asserts that Atlas verified, and the plan elaborates and replays every Atlas module exactly once with every import elaborated by the shard or by one it needs - And the vv job has no path filter, no cache action, and no condition on a recipe, the Atlas jobs and the host matrix run only when the plan names the whole gate, and each job uploads its ci-evidence under a name of its own, and the plan script names the rest leg alone for a draft pull request and for a ready one whose base is neither main nor a release line, and the whole gate for a ready one into main or a release line, a request, and a push to main or to a release line, and names no job for a merge whose tree is its second parent's only when a run of that parent passed every job, the Atlas join, every host, and every leg, and for a skipped or failed job, a missing join, host, or leg, a failed call, a changed tree, and a push that merges nothing it names the whole gate + And the vv job has no path filter, no cache action, and no condition on a recipe, the Atlas jobs and the host matrix run only when the plan names the whole gate, and each job uploads its ci-evidence under a name of its own, and the plan script names honesty then the rest leg for a draft pull request and for a ready one whose base is neither main nor a release line, and the whole gate for a ready one into main or a release line, a request, and a push to main or to a release line, and names no job for a merge whose tree is its second parent's only when a run of that parent passed every job, the Atlas join, every host, and every leg, and for a skipped or failed job, a missing join, host, or leg, a failed call, a changed tree, and a push that merges nothing it names the whole gate @RP-15 @build Scenario: Every match over a closed semantic enum (the IR, the production realization calculus, and its Rust AST) that names at least five of the enum's variants names all of them, with no wildcard or binding catch-all arm, so that a variant added to the enum is a compile error in each Lean, LaTeX, validation, charge, and calculus consumer and never a silent fallback; the audit rejects a catch-all planted in each consumer group and fails when a consumer it must read has no such match left. diff --git a/model/ids.toml b/model/ids.toml index c242b773a5..1311d8b0b9 100644 --- a/model/ids.toml +++ b/model/ids.toml @@ -90,7 +90,7 @@ statement = "The ordinary workspace test phase excludes repo-conformance, the bd id = "RP-14" level = "build" suite = "repository" -statement = "The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run the rest leg and no other job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing." +statement = "The normative CI job runs every recipe of just vv in parallel legs that each start from a clean checkout on the standard ubuntu-24.04 runner, each recipe exactly once but bdd, which runs once per part of its committed plan on the row that names the part, the examples leg at the verification resource profile its matrix row names and without UOR Atlas, which the Atlas jobs of the same run verify as the shards of the committed plan and a join at that profile, every process once and nothing across runs, and the other legs at the default, with no cache, path filter, or skipped recipe, and each job uploads its CI evidence; a plan job decides what a run is from the event alone: a draft pull request, and a ready pull request whose base is neither main nor a release line, run honesty then the rest leg and no other gate job, a ready pull request into main or a release line, a request, and a push to main or to a release line run every job, and a push to main or to a release line that merges a head whose tree it equals runs none when the whole gate passed on that head, and runs the whole gate when any job of that run did not pass, was skipped, or is missing." [[id]] id = "RP-15" diff --git a/xtask/src/ci.rs b/xtask/src/ci.rs index 97c3c5c8df..862a176cec 100644 --- a/xtask/src/ci.rs +++ b/xtask/src/ci.rs @@ -83,7 +83,7 @@ fn list_tests( /// --all-features`, the whole `repo-conformance` suite, or, when /// `LEXLEAN_BDD_PART` names a part of `.github/bdd-parts.json`, the same /// command with that part's harness filters. Unset or empty, which is what -/// `just vv` and `release.yml` run, the recipe is the command it always was. +/// local `just vv` runs, the recipe is the command it always was. /// The filters select tests; no case decides whether it runs, and the /// inventory proves from the harness's own lists that the parts are the whole /// suite, each test in one part (RP-13). @@ -92,44 +92,41 @@ pub fn bdd(root: &Path) -> Result<(), Fail> { let part = std::env::var(PART_VARIABLE) .ok() .filter(|value| !value.is_empty()); - let mut harness: Vec = Vec::new(); - if let Some(part) = &part { + let mut harness = Vec::new(); + let targets = if let Some(part) = &part { let plan = Plan::read(root)?; - harness.push("--".to_owned()); harness.extend(plan.harness_filters(part)?); - let what = if *part == plan.remainder { - format!( - "every test but the {} the other parts name", - plan.named_tests().len() - ) - } else { - format!("{} conformance cases", plan.parts[part].len()) - }; - println!("bdd: the part `{part}` of {PLAN_FILE}: {what} (§9.4)"); - } + println!("bdd: the part `{part}` of {PLAN_FILE} (§9.4)"); + plan.cargo_targets(part, &owned_targets(root)?.1)? + } else { + vec![Vec::new()] + }; let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); - let status = Command::new(cargo) - .current_dir(root) - .arg("test") - .args(CARGO_ARGUMENTS) - .args(&harness) - .stdin(Stdio::null()) - .status()?; - if !status.success() { - return Err(format!( - "bdd: `cargo test {}`{} failed", - CARGO_ARGUMENTS.join(" "), - part.map(|part| format!(" (the part `{part}`)")) - .unwrap_or_default() - ) - .into()); + let mut failed = false; + for selected in targets { + let status = Command::new(&cargo) + .current_dir(root) + .arg("test") + .args(CARGO_ARGUMENTS) + .args(&selected) + .arg("--") + .args(&harness) + .stdin(Stdio::null()) + .status()?; + if !status.success() { + eprintln!("bdd: cargo test failed for part {part:?}, targets {selected:?}"); + failed = true; + } + } + if failed { + return Err(format!("bdd: cargo test failed for part {part:?}").into()); } Ok(()) } /// The names of the binaries that carry the tests of `repo-conformance`: /// its library and each of its test targets, from `cargo metadata`. -fn owned_crates(root: &Path) -> Result, Fail> { +fn owned_targets(root: &Path) -> Result<(BTreeSet, BTreeSet), Fail> { let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); let output = Command::new(cargo) .current_dir(root) @@ -142,6 +139,7 @@ fn owned_crates(root: &Path) -> Result, Fail> { } let metadata: Value = serde_json::from_slice(&output.stdout)?; let mut crates = BTreeSet::new(); + let mut integration = BTreeSet::new(); for package in metadata["packages"].as_array().into_iter().flatten() { if package["name"] != inventory::OWNED_PACKAGE { continue; @@ -158,10 +156,13 @@ fn owned_crates(root: &Path) -> Result, Fail> { .any(|kind| matches!(*kind, "lib" | "rlib" | "bin" | "test")); if let (true, Some(name)) = (tested, target["name"].as_str()) { crates.insert(name.replace('-', "_")); + if kinds.contains(&"test") { + integration.insert(name.to_owned()); + } } } } - Ok(crates) + Ok((crates, integration)) } /// RP-13: `test` and `bdd` partition `cargo test --workspace --all-features`, @@ -185,11 +186,22 @@ pub fn check_test_inventory(root: &Path) -> Result<(), Fail> { let workspace = list_tests(root, &reference, &[])?; let ordinary = list_tests(root, &inventory::recipe_arguments(&ordinary_line)?, &[])?; let owned = list_tests(root, &owned_arguments, &[])?; - let summary = inventory::compare(&workspace, &ordinary, &owned, &owned_crates(root)?)?; + let (crates, integration) = owned_targets(root)?; + let summary = inventory::compare(&workspace, &ordinary, &owned, &crates)?; let plan = Plan::read(root)?; let mut parts = Vec::new(); for name in plan.names() { - let listed = list_tests(root, &owned_arguments, &plan.harness_filters(&name)?)?; + let mut listed = inventory::Parsed::default(); + for selected in plan.cargo_targets(&name, &integration)? { + let arguments: Vec = owned_arguments.iter().chain(&selected).cloned().collect(); + let command = list_tests(root, &arguments, &plan.harness_filters(&name)?)?; + listed.targets.extend(command.targets); + for entry in command.entries { + if !listed.entries.insert(entry.clone()) { + return Err(format!("RP-13: part `{name}` runs `{}` twice", entry.name).into()); + } + } + } parts.push((name, listed)); } let count = inventory::compare_parts(&owned, &parts, &plan)?;