diff --git a/.github/bdd-parts.json b/.github/bdd-parts.json index fe7db0138d..8e637287d5 100644 --- a/.github/bdd-parts.json +++ b/.github/bdd-parts.json @@ -37,6 +37,9 @@ "bootstrap-terms": [ "BS-10" ], + "bootstrap-content-id": [ + "BS-32" + ], "bootstrap-tooling-latex": [ "BS-20" ], diff --git a/.github/intel-host-lanes.json b/.github/intel-host-lanes.json index 8ebf28ab75..562de7cfc8 100644 --- a/.github/intel-host-lanes.json +++ b/.github/intel-host-lanes.json @@ -2,7 +2,7 @@ "producer": ["honesty"], "lanes": { "lower": ["bootstrap-lower"], - "print": ["bootstrap-print", "bootstrap-bodies", "bootstrap-succeed", "bootstrap-loops", "bootstrap-assemble", "bootstrap-package", "bootstrap-check", "bootstrap-gnaf", "bootstrap-runtime", "bootstrap-terms", "bootstrap-tooling-latex"], + "print": ["bootstrap-print", "bootstrap-bodies", "bootstrap-succeed", "bootstrap-loops", "bootstrap-assemble", "bootstrap-package", "bootstrap-check", "bootstrap-gnaf", "bootstrap-runtime", "bootstrap-terms", "bootstrap-tooling-latex", "bootstrap-content-id"], "validate": ["bootstrap-validate", "compiler", "preservation", "preservation-lists", "preservation-runtime", "preservation-support", "preservation-prims", "preservation-std", "production"], "moves": ["bootstrap-moves", "verification", "rest"] } diff --git a/.github/vv-rows.json b/.github/vv-rows.json index a0695b8575..7716020d0b 100644 --- a/.github/vv-rows.json +++ b/.github/vv-rows.json @@ -62,6 +62,15 @@ "memory_kib": "", "hints": "" }, + { + "part": "bdd-bootstrap-content-id", + "recipes": "bdd", + "examples_part": "", + "bdd_part": "bootstrap-content-id", + "width": "", + "memory_kib": "", + "hints": "" + }, { "part": "bdd-bootstrap-tooling-latex", "recipes": "bdd", diff --git a/CHANGELOG.md b/CHANGELOG.md index 68f6a6eb5e..72a1febf78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ workspace carried at those versions without a tag. ## Unreleased +- The labeled hash recipes of content identity as language-1.2 source (SPEC.md + §17.23, §21.1 to §21.5, `BS-32`, #274; no new diagnostic code, no change to the + language-1.2 identity). `treeDigest`, `sourceId`, `semanticId`, `buildId`, + `attestationId`, and `pdfRecipeId` of `ContentId` return what the recipes of + `artifact::content_id` return; none admits `recursion`. The register is 479 IDs. - Canonical JSON reading in the standard library, the fourth member the self-hosted compiler needs, #276 (SPEC.md §17.22, `SL-13`). The module `LexLeanStd.JsonRead` holds the production roots `readJson` and diff --git a/CONFORMANCE.md b/CONFORMANCE.md index c9b4937578..b7971bcfd4 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -77,6 +77,7 @@ the error registry does not sanction (R5, `ERRORS.md`). | `BS-15` | `build` | The package renderer of SPEC.md §17.16 (the checks `package::lower` makes of a manifest's own fields before it reads the program: the spec, the crate name against the Rust keywords and the reserved crates, the version, the profile, the form and the order of the sources, and the effects against the registry, with `allocation` refused on `rust-core`; the Cargo manifest over the scaffold record; and the provenance in canonical JSON with the digests of the runtime, the Cargo manifest, and the library) is language-1.2 source in the `bootstrap/` project, as two production roots of `RustPackage` that the host composes around the roots of `BS-04` to `BS-14`: `checkManifest`, which for the fields of a manifest and the registered effects returns whether the profile is `rust-std`, or the refusal of `package::lower` to the byte, and `renderPackage`, which for the scaffold record, the manifest's name, version, profile, sources, effects and exports, the program's identity, the compiler-semantics identity, the two runtime texts, and the library returns the `Cargo.toml` and the `provenance.json` of `package::package` to the byte; each is eligible for `rust-std` with exactly its declared effects and has no part of its certificate beyond 3,145,728 bytes; the oracle of `checkManifest` is `package::lower` on the manifest with its program and exports replaced by a pair every later check accepts, which agrees with each committed negative manifest's refusal, and the oracle of `renderPackage` is `package::package` on the 299 committed manifests it accepts and, on 137 hand-built inputs (names, exports, sources and effects that reach the escaper, and scaffold records that differ from the committed one) and 299 seeded mutants, `cargo_toml_of` and the canonical JSON of §21.7 by the recipe `package::package` is held to; on the normative host the lowered programs under the calculus evaluator agree with the oracle on the 318 committed manifests, 182 rule manifests and 2,000 seeded mutants, and on the 241 renders whose library is at most 16,384 bytes; the rendered packages agree with the oracle under the pinned rustc on all 3,190 runs; 21 planted defects in the source are each detected; and the certified packages the production export publishes for the project are the committed ones; the checks of the program, the exports and the crate, and the bytes of the library, which are the rows `BS-04` to `BS-14`, are outside the claim. | | `BS-16` | `build` | The GNAF request validation of SPEC.md §17.15 (`gnaf::validate`, with `universe_size`) is language-1.2 source in the `bootstrap/` project: `validateRequest` of `GnafValidate` is a production root eligible for `rust-std` with exactly its declared effects, over the projection of a request that validation reads (the length of the domain, the machine, the carrier's kind with each plan's preparations and the thresholds, the completeness evidence, the objective, the claim's class, and the scope) as first-order data, and returns the first violated rule in the model's order as the oracle's `serde_json` bytes, or nothing; on every host, its rendered `rust-std` package under the pinned rustc agrees with the Rust oracle `gnaf::validate` on the acceptance and on the rejection, to the byte, on the committed requests of `compiler/gnaf/`, the requests the committed certificate fixtures name, hand-built requests that reach each rejection, and seeded mutants of the committed requests; on the normative host, the root's lowered program under the calculus evaluator agrees with the oracle on the same corpus, each planted defect in the source is detected, and the certified package the production export publishes for the project is the committed one; loading a request (`gnaf::load`), the universe identity, the answer, and certification are outside the claim. | | `BS-20` | `build` | The canonical LaTeX of a tooling module of SPEC.md §17.20 (`backend::tooling::render_latex`: the fixed document frame and, for each declaration in order, its title, its kind, and the lines the Lean printer of the same file spells for its types, signatures, and terms, every line escaped by `tex_escape`) is language-1.2 source in the `bootstrap/` project: `renderToolingLatex` of `ToolingLatex`, over the tooling module as first-order data with each list carried as a chain of its own `Nil` and `Cons` constructors and with the typings the checker infers as an argument, writing through the standard library's `Buffer` with an explicit capacity, is a production root eligible for `rust-std` with exactly its declared effects; its rendered `rust-std` package under the pinned rustc and its lowered program under the calculus evaluator agree with the Rust oracle on the text, to the byte, and on the refusal of a write past the capacity, on the committed tooling modules (whose committed `.tex` is equal to the text), on hand-built modules that reach every declaration, type, signature, and term form and every byte class of the escaper, and on seeded mutants, every planted defect is detected, and the certified package the production export publishes for the project is the committed one; the checker, the decoder of the module, and the source map of the emitted pieces stay in the host. | +| `BS-32` | `build` | The labeled hash recipes of content identity of SPEC.md §21.1 to §21.5, §11.5, §19.8, and §22.9 (`artifact::content_id`: the frame of a label and a payload, each preceded by its big-endian length in four and eight bytes, under a domain and a NUL; `tree_digest` over path and content rows; `source_id` over the project manifest, the lock, and the sources; `semantic_id_for` over the compiler-semantics digest, the language, the pinned Lean toolchain, the linked IR, and the lexicon closure; `build_id`; `attestation_id`; and `pdf_recipe_id`) is language-1.2 source in the `bootstrap/` project, as six production roots of `ContentId` over the standard library's SHA-256: `treeDigest`, `sourceId`, `semanticId`, `buildId`, `attestationId`, and `pdfRecipeId`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built inputs that reach every frame of every recipe, lengths beyond one byte, and messages across blocks, on the trees and source inputs of the committed projects, on the source and semantic IDs of every committed build manifest, whose recorded build ID the oracle reproduces, and on seeded inputs; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; choosing and normalizing the files of a tree or a source ID, and serializing the IR, the closure, and the attestation body, are outside the claim. | ## calculus diff --git a/README.md b/README.md index 0f5db6830c..0d1c2c1635 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ just vv # the complete normative acceptance gate (SPEC.md §9.2) just release # vv, then the §30 release criterion; refused until 1.0.0 ``` -All 478 registered conformance IDs have their named tests; `just vv` checks them with the pinned toolchain installed. +All 479 registered conformance IDs have their named tests; `just vv` checks them with the pinned toolchain installed. The differential against the independent generator `lean4-prod` (SPEC.md §17.19, `OR-01`..`OR-08`) acquires it at its pinned commit: from the checkout `LEXLEAN_ORACLE_DIR` names, or by fetching the commit with `git` into the Cargo target directory (`LEXLEAN_ORACLE_CACHE` overrides), and builds it with the pinned Lean and Rust toolchains. It is validation evidence only and no input to any identity. An oracle that cannot be acquired, is not the pinned files, or fails is unavailable: that fails the gate on Linux x86-64 and is printed as `NOT VALIDATED` elsewhere, and is never reported as agreement. @@ -197,7 +197,7 @@ Every row is validated by `just vv`; the IDs link the claim to its register row, | The language-1.2 authoring notation: a readable `.lex.tex` text for a semantic module, in bijection with its canonical JSON, whose every position and variant is read from the schema, with readable forms for types and terms and a generic form for every other value, formatted canonically by `fmt` failing closed, the only form in which a language-1.2 source of the product is written (the data form is refused under language 1.2, and `lexlean::sdk` is the explicit conversion), the same for the compiler-tooling modules in the generic notation, with no compiler-semantics ID that depends on which form an embedded module is written in, with no product source owned by a generator but five named records, read and written by programs only through the authoring API (`lexlean::sdk`), and, for a native core module, a readable program over the shared table of kernel terms whose numbering is the order of its text | `AU-01`..`AU-13` | `build` | | The runtime of generated Lean as native modules: the portable, collection, model, and reasoning runtimes of language 1.2 are LexLean source that every generated module imports, migrated losslessly and audited against their exact axioms, the released runtime text of languages 1.0 and 1.1 is frozen data equal to their released output, and no other Lean runtime text remains in the compiler but the decoder of core module 2 | `RT-01`..`RT-04` | `build` | | The standard library: language-1.2 source beside the compiler whose members are certified production roots, each validated against its oracle by a seeded differential of its evaluator and rustc realizations; bytes and text over `Bytes`, an output buffer of a given capacity written in order; decimal and hexadecimal; the byte order and stable sorts over it; canonical JSON reading and writing; maps and sets over byte keys; SHA-256 against FIPS 180-4, the NIST CAVP vectors, and the `sha2` crate; and the `std-models` and `std-reasoning` source packages, generic over types, whose laws are kernel-checked theorems and whose application is the `helpdesk` example | `SL-01`, `SL-02`, `SL-03`, `SL-04`, `SL-05`, `SL-09`, `SL-10`, `SL-11`, `SL-13`, `SL-14` | `build` | -| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the GNAF request validation, and the LaTeX of a tooling module, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-20` | `build` | +| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the GNAF request validation, the LaTeX of a tooling module, and the hash recipes of content identity, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-32` | `build` | | Source packages: a language-1.2 project imports the modules of a locked, content-addressed package of its workspace: `[[source_package]]` and `lexlean-workspace.toml`, the pins in `lexlean.lock` recomputed by every command, imports closed at link, no re-export of a package's production roots, and the package's modules replayed by the importer's `verify` with their origin in the records | `PK-01`..`PK-05` | `build` | | The literal `nat-add-zero` example, the Lean-verified feature examples, and the complete negative fixture suite | `EX-01`..`EX-09` | `build` | | Language-1.2 text components: tokenizers and their round-trip laws, with the laws carried as one native core module the generated modules import rather than authored Lean text | `LM-01`, `LM-02`, `LM-03`, `LM-13` | `build` | diff --git a/SPEC.md b/SPEC.md index 9cedf934c4..c81d3c30f5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -8733,6 +8733,25 @@ the public marker and the visibility line, and the trimming of a line) are each pinned Lean the production export (§24.7) certifies the root beside the others and its package is the committed one. +The labeled hash recipes of content identity (§21.1 to §21.5) are the second component of the +project manager as source (`BS-32`, issue 274), the module `ContentId`, with six production roots +for `rust-std` over the standard library's `Sha256`: + +| Root | Takes | Returns | Oracle | +| --- | --- | --- | --- | +| `LexLeanBoot.ContentId.treeDigest` | path and content rows | the tree digest (§11.5) | `content_id::tree_digest` | +| `LexLeanBoot.ContentId.sourceId` | the manifest, the lock, and the sources | the source ID (§21.3) | `content_id::source_id` | +| `LexLeanBoot.ContentId.semanticId` | the compiler-semantics digest, the language, the IR, and the closure | the semantic ID (§21.4) | `content_id::semantic_id_for` | +| `LexLeanBoot.ContentId.buildId` | the source and semantic IDs | the build ID (§21.5) | `content_id::build_id` | +| `LexLeanBoot.ContentId.attestationId` | the attestation body | the attestation ID (§22.9) | `content_id::attestation_id` | +| `LexLeanBoot.ContentId.pdfRecipeId` | the PDF recipe's digests and JSON | the PDF recipe ID (§19.8) | `content_id::pdf_recipe_id` | + +Each root writes its domain, a NUL, and its frames (§21.1), each frame the big-endian four-byte +length of its label, the label, the big-endian eight-byte length of its payload, and the payload, +and returns the 32 bytes of the message's SHA-256. The semantic ID frames the pinned Lean toolchain. +None of the six admits `recursion`. Choosing and normalizing the files a digest covers, and +serializing the IR, the closure, and the attestation body, stay in the host. + ## 18. Lean backend ### 18.1 Output contract @@ -11654,6 +11673,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `BS-15` | `bootstrap` | The package renderer of SPEC.md §17.16 (the checks `package::lower` makes of a manifest's own fields before it reads the program: the spec, the crate name against the Rust keywords and the reserved crates, the version, the profile, the form and the order of the sources, and the effects against the registry, with `allocation` refused on `rust-core`; the Cargo manifest over the scaffold record; and the provenance in canonical JSON with the digests of the runtime, the Cargo manifest, and the library) is language-1.2 source in the `bootstrap/` project, as two production roots of `RustPackage` that the host composes around the roots of `BS-04` to `BS-14`: `checkManifest`, which for the fields of a manifest and the registered effects returns whether the profile is `rust-std`, or the refusal of `package::lower` to the byte, and `renderPackage`, which for the scaffold record, the manifest's name, version, profile, sources, effects and exports, the program's identity, the compiler-semantics identity, the two runtime texts, and the library returns the `Cargo.toml` and the `provenance.json` of `package::package` to the byte; each is eligible for `rust-std` with exactly its declared effects and has no part of its certificate beyond 3,145,728 bytes; the oracle of `checkManifest` is `package::lower` on the manifest with its program and exports replaced by a pair every later check accepts, which agrees with each committed negative manifest's refusal, and the oracle of `renderPackage` is `package::package` on the 299 committed manifests it accepts and, on 137 hand-built inputs (names, exports, sources and effects that reach the escaper, and scaffold records that differ from the committed one) and 299 seeded mutants, `cargo_toml_of` and the canonical JSON of §21.7 by the recipe `package::package` is held to; on the normative host the lowered programs under the calculus evaluator agree with the oracle on the 318 committed manifests, 182 rule manifests and 2,000 seeded mutants, and on the 241 renders whose library is at most 16,384 bytes; the rendered packages agree with the oracle under the pinned rustc on all 3,190 runs; 21 planted defects in the source are each detected; and the certified packages the production export publishes for the project are the committed ones; the checks of the program, the exports and the crate, and the bytes of the library, which are the rows `BS-04` to `BS-14`, are outside the claim. | §17.16, §17.23 | | `BS-16` | `bootstrap` | The GNAF request validation of SPEC.md §17.15 (`gnaf::validate`, with `universe_size`) is language-1.2 source in the `bootstrap/` project: `validateRequest` of `GnafValidate` is a production root eligible for `rust-std` with exactly its declared effects, over the projection of a request that validation reads (the length of the domain, the machine, the carrier's kind with each plan's preparations and the thresholds, the completeness evidence, the objective, the claim's class, and the scope) as first-order data, and returns the first violated rule in the model's order as the oracle's `serde_json` bytes, or nothing; on every host, its rendered `rust-std` package under the pinned rustc agrees with the Rust oracle `gnaf::validate` on the acceptance and on the rejection, to the byte, on the committed requests of `compiler/gnaf/`, the requests the committed certificate fixtures name, hand-built requests that reach each rejection, and seeded mutants of the committed requests; on the normative host, the root's lowered program under the calculus evaluator agrees with the oracle on the same corpus, each planted defect in the source is detected, and the certified package the production export publishes for the project is the committed one; loading a request (`gnaf::load`), the universe identity, the answer, and certification are outside the claim. | §17.15, §17.23 | | `BS-20` | `bootstrap` | The canonical LaTeX of a tooling module of SPEC.md §17.20 (`backend::tooling::render_latex`: the fixed document frame and, for each declaration in order, its title, its kind, and the lines the Lean printer of the same file spells for its types, signatures, and terms, every line escaped by `tex_escape`) is language-1.2 source in the `bootstrap/` project: `renderToolingLatex` of `ToolingLatex`, over the tooling module as first-order data with each list carried as a chain of its own `Nil` and `Cons` constructors and with the typings the checker infers as an argument, writing through the standard library's `Buffer` with an explicit capacity, is a production root eligible for `rust-std` with exactly its declared effects; its rendered `rust-std` package under the pinned rustc and its lowered program under the calculus evaluator agree with the Rust oracle on the text, to the byte, and on the refusal of a write past the capacity, on the committed tooling modules (whose committed `.tex` is equal to the text), on hand-built modules that reach every declaration, type, signature, and term form and every byte class of the escaper, and on seeded mutants, every planted defect is detected, and the certified package the production export publishes for the project is the committed one; the checker, the decoder of the module, and the source map of the emitted pieces stay in the host. | §17.20, §17.23 | +| `BS-32` | `bootstrap` | The labeled hash recipes of content identity of SPEC.md §21.1 to §21.5, §11.5, §19.8, and §22.9 (`artifact::content_id`: the frame of a label and a payload, each preceded by its big-endian length in four and eight bytes, under a domain and a NUL; `tree_digest` over path and content rows; `source_id` over the project manifest, the lock, and the sources; `semantic_id_for` over the compiler-semantics digest, the language, the pinned Lean toolchain, the linked IR, and the lexicon closure; `build_id`; `attestation_id`; and `pdf_recipe_id`) is language-1.2 source in the `bootstrap/` project, as six production roots of `ContentId` over the standard library's SHA-256: `treeDigest`, `sourceId`, `semanticId`, `buildId`, `attestationId`, and `pdfRecipeId`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built inputs that reach every frame of every recipe, lengths beyond one byte, and messages across blocks, on the trees and source inputs of the committed projects, on the source and semantic IDs of every committed build manifest, whose recorded build ID the oracle reproduces, and on seeded inputs; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; choosing and normalizing the files of a tree or a source ID, and serializing the IR, the closure, and the attestation body, are outside the claim. | §17.23, §21.1, §21.5 | | `PK-01` | `packages` | A language-1.2 project that declares a source package by its identifier, version, kind `path`, and the member directory a `lexlean-workspace.toml` lists locks it by the tree digest of its manifest and source files, its manifest digest, its version, its module prefix, and its own packages, and the configuration and the lock read back to the bytes written; a project without a source package keeps its bytes, and a package of another kind, a source package in another language, a changed byte or version (LLC0105), an undeclared package (LLC0106), a module prefix claimed twice (LLC0107), a project or a package outside the workspace (LLC0108), a path that escapes it, and a symlink (LLS8001) are each refused with the named code. | §10.5, §11.1, §25.1 | | `PK-02` | `packages` | A module of a project imports only modules of its own project and modules of the packages the project declares, and a module of a package only modules of itself and of the packages its own manifest declares: a package module is a module of the importing project under its own module prefix, a module of a package the project does not declare fails with LLR3009, and an import cycle through a package fails with LLR3003. | §10.5, §15.1 | | `PK-03` | `packages` | An importing project builds and verifies under the pinned Lean offline, and its package modules are generated, elaborated, replayed with `leanchecker`, and audited for their exact axioms under the package's module prefix with the package as their origin in the records; the source ID of the project changes with the pins, and no file of the package is written by `fmt`, `build`, or `verify`. | §10.5, §21.3, §22 | @@ -11670,7 +11690,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `LM-09` | `language-models` | No prompt is code: no member of the closed language maps `string` or `bytes` to a function, a term, or source — every operation of the primitive registry that takes a `string` or `bytes` operand is classified by the audit with a data result, the registry's schema enumeration and the production registry's primitive rows equal one another and equal that classified set, and the text declarations of `lexlean/semantic-module/2` state no `string`-typed member, so a prompt reaches a model only as the tokens `encodeText` makes of it; a planted operation or text member of that shape is caught by the audit with its name. | §17.12 | | `LM-13` | `language-models` | The text laws a tokenizer elaborates through are LexLean source and not authored text: the module that states them is one language-1.2 native core module of the embedded `language/text-1.2` project, the project holds no file of Lean source, no independently authored Lean file anywhere in the release tree declares the `LexLeanText` namespace or lies under its module root, and the one-time migration input is absent; the registry lists only that module, the declarations and their exact axiom policies are read from its native data, which the compiler embeds and hashes into the language-1.2 semantics, and a module that declares a text component imports the generated module rather than splicing any Lean text. | §17.10, §17.12 | -**Total required capability IDs:** 478. +**Total required capability IDs:** 479. No row may be downgraded to `some-true` or `open`. Upstream Lean facts are ledger/authority rows, not substitutions for these build behaviors. diff --git a/VERIFICATION.md b/VERIFICATION.md index 7412633b08..a747157687 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -11046,6 +11046,49 @@ writes them); `cargo clippy -p repo-conformance -p xtask --all-targets --all-features -D warnings`, `validate-model` (461 ids), `validate-spec-links` (461 rows), `check-test-inventory`, and the five `bdd` meta-gate tests pass. +### the hash recipes of content identity are source (BS-32) + +`BS-32` holds the six production roots of `LexLeanBoot.ContentId` +(`bootstrap/src/ContentId.lex.tex`) to the recipes of `artifact::content_id`: +`treeDigest` to `tree_digest`, `sourceId` to `source_id`, `semanticId` to +`semantic_id_for`, `buildId` to `build_id`, `attestationId` to +`attestation_id`, and `pdfRecipeId` to `pdf_recipe_id`. Each root writes its +recipe's domain, a NUL, and its frames, each frame the big-endian four-byte +length of its label, the label, the big-endian eight-byte length of its payload, +and the payload, and returns the 32 bytes of the message's SHA-256 under the +standard library's `Sha256`. The semantic ID frames the pinned Lean toolchain. +Each root admits exactly `allocation` and `overflow`, and none admits +`recursion`. The oracle is the existing public code of `content_id.rs`; no Rust +outside the case changes. The case holds the roots to the oracle on hand-built +inputs that reach every frame of every recipe, an empty tree, an empty path, a +path of 300 bytes, and a message across blocks; on the trees and source inputs of +the committed projects, the manifest, the lock, and the sources under `src/`, up +to 4,096 bytes under the evaluator and 65,536 under rustc; on the source and +semantic IDs of every committed build manifest, whose recorded build ID the case +asserts the oracle reproduces; and on seeded inputs. Each planted defect names +the one vector that catches it, and the case asserts that the site of each plant +occurs once. The row's statement carries no counts. + +Measured with the compiler binary of this tree, on a copy of the project whose +only entrypoint is `ContentId`: `lexlean check`, `lexlean build`, and `lexlean +verify` pass, and the eligibility report gives each root exactly `allocation` +and `overflow`. The register agrees with the §31 table on 479 IDs, by the same +parse `validate-spec-links` makes. + +The case was first compiled and run on the normative host over +`release/v0.4` at `cf76090d4` (`cargo test -p repo-conformance --test +conformance -- conformance_bs_32`, 225 s). Its corpus is 17 named vectors, the +362 committed projects (180 trees under the evaluator, 345 under rustc), the 26 +committed build manifests, each of whose build ID the oracle reproduces, and 100 +and 500 seeded runs of each root. The evaluator agrees with the oracle on 287 +trees, 283 source IDs, 102 semantic IDs, 128 build IDs, 102 attestation IDs, and +101 PDF recipe IDs, the rendered packages under the pinned rustc on 3,733 runs, +and each of the 15 planted defects is detected by its own vector. The first runs +found no fault of the recipes: Lean asked for exactly the axioms the module now +declares. The case then stops at the stack-bound step, `bootstrap/tree_digest is +no package of the committed exports`, as each bootstrap member did before its +regeneration, which produces the six packages. + ## The `bdd` recipe as the parts of one run (Issue #131) With UOR Atlas verified as the parts of one run, the `rest` leg became the diff --git a/bootstrap/README.md b/bootstrap/README.md index a9f234eb1a..c5e366f9d4 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -18,3 +18,6 @@ payloads a printer reads); `src/RustPrint.lex.tex` also imports the standard library's `Buffer` (`stdlib/src/Buffer.lex.tex`), which it writes through. `src/GnafValidate.lex.tex` (SPEC.md §17.15, `BS-16`) imports `Buffer` alone and writes the rejection of a GNAF request through it. + +`src/ContentId.lex.tex` (SPEC.md §17.23, `BS-32`) imports the standard library's `Numeral` and +`Sha256`, and computes the labeled hash recipes of content identity of SPEC.md §21. diff --git a/bootstrap/lexlean.lock b/bootstrap/lexlean.lock index dec1db451b..0572751130 100644 --- a/bootstrap/lexlean.lock +++ b/bootstrap/lexlean.lock @@ -1,7 +1,7 @@ spec = "lexlean/lock/2" language = "1.2" compiler_semantics = "635a25acb886587f8777525e778092e9599d1f2df26da59403ee39e9e299151a" -project_config_sha256 = "b2bd695c7fad3f83d9a41fdabb6f9f5eee0538452d59609e5170eff53d78f14f" +project_config_sha256 = "1a56b5a7067851b3aa0abc888df7e631c058a4c48a59d54c0659317b4b159268" lean_toolchain = "leanprover/lean4:v4.32.1" [[workspace_file]] diff --git a/bootstrap/lexlean.toml b/bootstrap/lexlean.toml index f57986a7e8..b5268d5d48 100644 --- a/bootstrap/lexlean.toml +++ b/bootstrap/lexlean.toml @@ -3,7 +3,7 @@ name = "bootstrap" language = "1.2" module_prefix = "LexLeanBoot" source_roots = ["src"] -entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/GnafValidate.lex.tex", "src/PrimitiveScan.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/TokenAudit.lex.tex", "src/ToolingAst.lex.tex", "src/ToolingLatex.lex.tex"] +entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/ContentId.lex.tex", "src/GnafValidate.lex.tex", "src/PrimitiveScan.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/TokenAudit.lex.tex", "src/ToolingAst.lex.tex", "src/ToolingLatex.lex.tex"] build_root = ".lexlean" lockfile = "lexlean.lock" lean_workspace = "." diff --git a/bootstrap/src/ContentId.lex.tex b/bootstrap/src/ContentId.lex.tex new file mode 100644 index 0000000000..cc284aa1c8 --- /dev/null +++ b/bootstrap/src/ContentId.lex.tex @@ -0,0 +1,145 @@ +\begin{lexlean}{ContentId} +\useglossary{lexlean.std.nat@1.2.0} +\importmodule{LexLeanStd.Numeral} +\importmodule{LexLeanStd.Sha256} +\title{Natural number addition} +\begin{semanticmodule} +definition{ + name: lit, + parameters: [literal : String], + result: Bytes, + body: (prim utf8_encode literal : Bytes), + executable: true +} + +definition{ + name: cat, + parameters: [left : Bytes, right : Bytes], + result: Bytes, + body: (prim append left right : Bytes), + executable: true +} + +definition{ + name: byteText, + parameters: [value : Nat], + result: Bytes, + body: (LexLeanStd.Numeral::pick (LexLeanStd.Numeral::byteValues) value), + executable: true +} + +definition{ + name: bigEndian, + parameters: [value : Nat, width : Nat], + result: Bytes, + body: (second (prim iterate (fun (state : (Nat * Bytes)) => ((prim quotient (first state) 256 0 : Nat), (cat (byteText (prim remainder (first state) 256 0 : Nat)) (second state)))) width (value, 0x) : (Nat * Bytes))), + executable: true +} + +definition{ + name: wordBytes, + parameters: [word : UInt32], + result: Bytes, + body: (cat (cat (cat (byteText (prim to_nat (LexLeanStd.Sha256::shiftRight word (24 : UInt32)) : Nat)) (byteText (prim to_nat (prim bit_and (LexLeanStd.Sha256::shiftRight word (16 : UInt32)) (255 : UInt32) : UInt32) : Nat))) (byteText (prim to_nat (prim bit_and (LexLeanStd.Sha256::shiftRight word (8 : UInt32)) (255 : UInt32) : UInt32) : Nat))) (byteText (prim to_nat (prim bit_and word (255 : UInt32) : UInt32) : Nat))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: digestBytes, + parameters: [digest : LexLeanStd.Sha256::Digest], + result: Bytes, + body: (cat (cat (cat (cat (cat (cat (cat (wordBytes (project digest h0)) (wordBytes (project digest h1))) (wordBytes (project digest h2))) (wordBytes (project digest h3))) (wordBytes (project digest h4))) (wordBytes (project digest h5))) (wordBytes (project digest h6))) (wordBytes (project digest h7))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: frame, + parameters: [label : Bytes, payload : Bytes], + result: Bytes, + body: (cat (cat (cat (bigEndian (prim length label : Nat) 4) label) (bigEndian (prim length payload : Nat) 8)) payload), + executable: true +} + +definition{ + name: framedHash, + parameters: [domain : Bytes, frames : Bytes], + result: Bytes, + body: (digestBytes (LexLeanStd.Sha256::sha256 (cat (cat domain 0x00) frames))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: treeDigest, + parameters: [files : (List (Bytes * Bytes))], + result: Bytes, + body: (framedHash (lit "lexlean-tree-v1") (prim list_fold (fun (acc : Bytes) (row : (Bytes * Bytes)) => (cat acc (frame (first row) (second row)))) 0x files : Bytes)), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: sourceId, + parameters: [projectToml : Bytes, lockText : Bytes, sources : (List (Bytes * Bytes))], + result: Bytes, + body: (framedHash (lit "lexlean-source-v1") (cat (cat (frame (lit "project") projectToml) (frame (lit "lock") lockText)) (prim list_fold (fun (acc : Bytes) (row : (Bytes * Bytes)) => (cat acc (cat (frame (lit "path") (first row)) (frame (lit "source") (second row))))) 0x sources : Bytes))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: semanticId, + parameters: [ + compilerSemantics : Bytes, + language : Bytes, + linkedIr : Bytes, + lexiconClosure : Bytes + ], + result: Bytes, + body: (framedHash (lit "lexlean-semantic-v1") (cat (cat (cat (cat (frame (lit "compiler-semantics") compilerSemantics) (frame (lit "language") language)) (frame (lit "toolchain") (lit "leanprover/lean4:v4.32.1"))) (frame (lit "linked-ir") linkedIr)) (frame (lit "lexicon-closure") lexiconClosure))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: buildId, + parameters: [sourceDigest : Bytes, semanticDigest : Bytes], + result: Bytes, + body: (framedHash (lit "lexlean-build-v1") (cat (frame (lit "source-id") sourceDigest) (frame (lit "semantic-id") semanticDigest))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: attestationId, + parameters: [attestationBody : Bytes], + result: Bytes, + body: (framedHash (lit "lexlean-attestation-v1") (frame (lit "attestation-body") attestationBody)), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: pdfRecipeId, + parameters: [ + texDigest : Bytes, + programDigest : Bytes, + versionDigest : Bytes, + compileArgv : Bytes, + resources : Bytes + ], + result: Bytes, + body: (framedHash (lit "lexlean-pdf-recipe-v1") (cat (cat (cat (cat (frame (lit "tex") texDigest) (frame (lit "program") programDigest)) (frame (lit "version-output") versionDigest)) (frame (lit "argv") compileArgv)) (frame (lit "resources") resources))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} +\end{semanticmodule} +\end{lexlean} diff --git a/crates/conformance/src/cases/bootstrap.rs b/crates/conformance/src/cases/bootstrap.rs index 49c5bbf14c..16255ff42f 100644 --- a/crates/conformance/src/cases/bootstrap.rs +++ b/crates/conformance/src/cases/bootstrap.rs @@ -13,6 +13,7 @@ use lexlean::production::lower::{linked_modules, lower_root, roots, Lowered}; use serde_json::Value as Json; mod calculus_check; +mod content_id; pub mod perf; use crate::production_export as harness; @@ -2457,6 +2458,7 @@ fn case(id: &str) { "BS-16" => gnaf_validate::case(), // §17.23, §17.20: the canonical LaTeX of a tooling module. "BS-20" => tooling_latex::case(), + "BS-32" => content_id::case(), other => panic!("no bootstrap case is wired for {other}"), } } diff --git a/crates/conformance/src/cases/bootstrap/content_id.rs b/crates/conformance/src/cases/bootstrap/content_id.rs new file mode 100644 index 0000000000..9ac83c0dea --- /dev/null +++ b/crates/conformance/src/cases/bootstrap/content_id.rs @@ -0,0 +1,724 @@ +//! `BS-32` (SPEC.md §11.5, §17.23, §19.8, §21.1–§21.5, §22.9): the labeled +//! hash recipes of content identity as language-1.2 source, +//! `LexLeanBoot.ContentId`, held to the byte to +//! `lexlean::artifact::content_id::{tree_digest, source_id, semantic_id_for, +//! build_id, attestation_id, pdf_recipe_id}`. +//! +//! Each root writes its recipe's domain, a NUL, and its frames, each frame a +//! big-endian four-byte label length, the label, a big-endian eight-byte +//! payload length, and the payload, and hashes the message with the standard +//! library's SHA-256. The committed build manifests are runs of `buildId`, +//! and each manifest's build ID is the oracle's. +//! +//! Each planted defect names the one vector that catches it. + +use std::collections::{BTreeMap, BTreeSet}; + +use lexlean::artifact::content_id::{ + attestation_id, build_id, pdf_recipe_id, semantic_id_for, source_id, tree_digest, +}; +use lexlean::calculus::Value; +use lexlean::Sha256Digest; +use serde_json::Value as Json2; + +use super::rust_validate::{eligibility_of, rustc_differential}; +use super::{bytes, list, pair, Run, Seeded}; +use crate::production_export as harness; +use crate::support::{self, repo_root}; + +const MODULE: &str = "LexLeanBoot.ContentId"; +const ROOTS: [(&str, &[&str]); 6] = [ + ("treeDigest", &["allocation", "overflow"]), + ("sourceId", &["allocation", "overflow"]), + ("semanticId", &["allocation", "overflow"]), + ("buildId", &["allocation", "overflow"]), + ("attestationId", &["allocation", "overflow"]), + ("pdfRecipeId", &["allocation", "overflow"]), +]; + +/// The most a part of a certificate of a root may be, as the row states. +const CERTIFICATE_BUDGET: u64 = 3_145_728; +/// Seeded runs of each root under the evaluator, and under rustc. +const EVALUATOR_SEEDED: usize = 100; +const RUSTC_SEEDED: usize = 500; +/// The most bytes a committed tree may hold to be a run under the evaluator, +/// and under rustc. +const EVALUATOR_TREE_BYTES: usize = 4_096; +const RUSTC_TREE_BYTES: usize = 65_536; + +// ---------------------------------------------------------------- the runs + +type Rows = Vec<(Vec, Vec)>; + +fn rows_value(rows: &Rows) -> Value { + list( + rows.iter() + .map(|(path, content)| pair(bytes(path), bytes(content))) + .collect(), + ) +} + +fn digest_of(value: &[u8]) -> Sha256Digest { + Sha256Digest(value.try_into().expect("a digest is 32 bytes")) +} + +fn out(digest: Sha256Digest) -> Json2 { + serde_json::to_value(bytes(&digest.0)).expect("a value is JSON") +} + +fn utf8(value: &[u8]) -> &str { + std::str::from_utf8(value).expect("the recipe takes text") +} + +fn tree_run(rows: &Rows) -> Run { + let borrowed: Vec<(&str, &[u8])> = rows + .iter() + .map(|(path, content)| (utf8(path), content.as_slice())) + .collect(); + (vec![rows_value(rows)], out(tree_digest(&borrowed))) +} + +fn source_run(project: &[u8], lock: &[u8], rows: &Rows) -> Run { + let owned: Vec<(String, Vec)> = rows + .iter() + .map(|(path, content)| (utf8(path).to_owned(), content.clone())) + .collect(); + ( + vec![bytes(project), bytes(lock), rows_value(rows)], + out(source_id(project, lock, &owned)), + ) +} + +fn semantic_run(semantics: &[u8], language: &str, ir: &str, closure: &str) -> Run { + ( + vec![ + bytes(semantics), + bytes(language.as_bytes()), + bytes(ir.as_bytes()), + bytes(closure.as_bytes()), + ], + out(semantic_id_for(digest_of(semantics), language, ir, closure)), + ) +} + +fn build_run(source: &[u8], semantic: &[u8]) -> Run { + ( + vec![bytes(source), bytes(semantic)], + out(build_id(digest_of(source), digest_of(semantic))), + ) +} + +fn attestation_run(body: &str) -> Run { + (vec![bytes(body.as_bytes())], out(attestation_id(body))) +} + +fn pdf_run(tex: &[u8], program: &[u8], version: &[u8], argv: &str, resources: &str) -> Run { + ( + vec![ + bytes(tex), + bytes(program), + bytes(version), + bytes(argv.as_bytes()), + bytes(resources.as_bytes()), + ], + out(pdf_recipe_id( + digest_of(tex), + digest_of(program), + digest_of(version), + argv, + resources, + )), + ) +} + +fn tree_bytes(rows: &Rows) -> usize { + rows.iter() + .map(|(path, content)| path.len() + content.len()) + .sum() +} + +// ------------------------------------------------------------------ corpora + +fn projects_under(dir: &std::path::Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if path.is_dir() { + if !matches!( + name.as_str(), + ".git" | "target" | ".lexlean" | "node_modules" + ) { + projects_under(&path, out); + } + } else if name == "lexlean.toml" { + out.push(dir.to_path_buf()); + } + } +} + +fn files_under(root: &std::path::Path, dir: &std::path::Path, rows: &mut Rows) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + files_under(root, &path, rows); + } else if let Ok(relative) = path.strip_prefix(root) { + let relative = relative.to_string_lossy().replace('\\', "/"); + rows.push(( + relative.into_bytes(), + std::fs::read(&path).expect("a committed file"), + )); + } + } +} + +/// Each committed project: its manifest, its lock or nothing, and its +/// sources sorted by path bytes, as the tree digest takes them. +fn committed_projects() -> Vec<(Vec, Vec, Rows)> { + let mut dirs = Vec::new(); + projects_under(repo_root().as_std_path(), &mut dirs); + dirs.iter() + .map(|dir| { + let mut rows = Vec::new(); + files_under(dir, &dir.join("src"), &mut rows); + for (path, _) in &mut rows { + let mut prefixed = b"src/".to_vec(); + prefixed.extend_from_slice(path); + *path = prefixed; + } + rows.retain(|(path, _)| std::str::from_utf8(path).is_ok()); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + ( + std::fs::read(dir.join("lexlean.toml")).expect("a manifest"), + std::fs::read(dir.join("lexlean.lock")).unwrap_or_default(), + rows, + ) + }) + .collect() +} + +/// The source, semantic, and build IDs of each committed build manifest. +fn committed_builds() -> Vec<(Sha256Digest, Sha256Digest, Sha256Digest)> { + let mut dirs = Vec::new(); + projects_under(repo_root().as_std_path(), &mut dirs); + let mut out = Vec::new(); + for dir in dirs { + let Ok(bytes) = std::fs::read(dir.join("expected/build/manifest.json")) else { + continue; + }; + let json: Json2 = serde_json::from_slice(&bytes).expect("a build manifest"); + let id = |key: &str| { + Sha256Digest::from_hex(json[key].as_str().expect("an ID")).expect("a digest") + }; + out.push((id("source_id"), id("semantic_id"), id("build_id"))); + } + out +} + +// ------------------------------------------------------------------ vectors + +fn d(byte: u8) -> Vec { + (0..32u8).map(|index| byte.wrapping_add(index)).collect() +} + +fn row(path: &str, content: &[u8]) -> (Vec, Vec) { + (path.as_bytes().to_vec(), content.to_vec()) +} + +struct Vector { + label: &'static str, + root: usize, + run: Run, +} + +#[allow(clippy::too_many_lines)] +fn named() -> Vec { + let long_path = "p".repeat(300); + let mut out = Vec::new(); + let mut add = |label: &'static str, root: usize, run: Run| { + out.push(Vector { label, root, run }); + }; + add("an empty tree", 0, tree_run(&Vec::new())); + add("one empty file", 0, tree_run(&vec![row("a", b"")])); + add( + "one file", + 0, + tree_run(&vec![row("src/Main.lex.tex", b"hello\n")]), + ); + add( + "two files", + 0, + tree_run(&vec![row("a", b"first"), row("b/c", b"second")]), + ); + add( + "a file with an empty path", + 0, + tree_run(&vec![row("", b"x")]), + ); + add( + "a file across blocks", + 0, + tree_run(&vec![row("f", &[0xa5; 200])]), + ); + add( + "a path of 300 bytes", + 0, + tree_run(&vec![row(&long_path, b"")]), + ); + add( + "no sources", + 1, + source_run(b"[project]\n", b"spec = 1\n", &Vec::new()), + ); + add( + "two sources", + 1, + source_run( + b"[project]\nname = \"p\"\n", + b"", + &vec![row("src/A.lex.tex", b"a"), row("src/B.lex.tex", b"b")], + ), + ); + add( + "an empty project and lock", + 1, + source_run(b"", b"", &vec![row("x", b"")]), + ); + add( + "the language 1.2", + 2, + semantic_run(&d(0x11), "1.2", "{\"modules\":[]}", "[]"), + ); + add("an empty IR", 2, semantic_run(&d(0x22), "1.1", "", "")); + add("a build of zero digests", 3, build_run(&[0; 32], &[0; 32])); + add("two distinct digests", 3, build_run(&d(0x01), &d(0x80))); + add("an empty body", 4, attestation_run("")); + add("a body of 300 bytes", 4, attestation_run(&"b".repeat(300))); + add( + "a recipe", + 5, + pdf_run( + &d(0x30), + &d(0x40), + &d(0x50), + "[\"lualatex\",\"main.tex\"]", + "[]", + ), + ); + out +} + +// ------------------------------------------------------------------- seeded + +fn drawn_bytes(seeded: &mut Seeded, most: usize) -> Vec { + let count = seeded.below(most + 1); + (0..count) + .map(|_| u8::try_from(seeded.below(256)).expect("a byte")) + .collect() +} + +fn drawn_text(seeded: &mut Seeded, most: usize) -> String { + let count = seeded.below(most + 1); + (0..count) + .map(|_| *seeded.pick(&['a', 'z', '0', '"', '{', '}', '\n', '\u{e9}', '\u{1f600}'])) + .collect() +} + +fn drawn_digest(seeded: &mut Seeded) -> Vec { + (0..32) + .map(|_| u8::try_from(seeded.below(256)).expect("a byte")) + .collect() +} + +fn drawn_rows(seeded: &mut Seeded) -> Rows { + let mut rows: Rows = (0..seeded.below(5)) + .map(|_| { + ( + drawn_text(seeded, 12).into_bytes(), + drawn_bytes(seeded, 150), + ) + }) + .collect(); + rows.sort_by(|a, b| a.0.cmp(&b.0)); + rows.dedup_by(|a, b| a.0 == b.0); + rows +} + +fn seeded_runs(count: usize, seed: u64) -> [Vec; 6] { + let mut seeded = Seeded(seed); + let mut out: [Vec; 6] = Default::default(); + for _ in 0..count { + out[0].push(tree_run(&drawn_rows(&mut seeded))); + let project = drawn_bytes(&mut seeded, 80); + let lock = drawn_bytes(&mut seeded, 80); + out[1].push(source_run(&project, &lock, &drawn_rows(&mut seeded))); + let semantics = drawn_digest(&mut seeded); + let language = drawn_text(&mut seeded, 4); + let ir = drawn_text(&mut seeded, 120); + let closure = drawn_text(&mut seeded, 40); + out[2].push(semantic_run(&semantics, &language, &ir, &closure)); + let source = drawn_digest(&mut seeded); + out[3].push(build_run(&source, &drawn_digest(&mut seeded))); + out[4].push(attestation_run(&drawn_text(&mut seeded, 200))); + let tex = drawn_digest(&mut seeded); + let program = drawn_digest(&mut seeded); + let version = drawn_digest(&mut seeded); + let argv = drawn_text(&mut seeded, 40); + out[5].push(pdf_run( + &tex, + &program, + &version, + &argv, + &drawn_text(&mut seeded, 40), + )); + } + out +} + +// ------------------------------------------------------------------- plants + +struct Plant { + label: &'static str, + root: usize, + site: &'static str, + replacement: &'static str, + vector: &'static str, +} + +const PLANTS: &[Plant] = &[ + Plant { + label: "the domain has no NUL", + root: 0, + site: "(cat (cat domain 0x00) frames)", + replacement: "(cat domain frames)", + vector: "an empty tree", + }, + Plant { + label: "the label length is two bytes", + root: 0, + site: "(bigEndian (prim length label : Nat) 4)", + replacement: "(bigEndian (prim length label : Nat) 2)", + vector: "one file", + }, + Plant { + label: "the payload length is four bytes", + root: 4, + site: "(bigEndian (prim length payload : Nat) 8)", + replacement: "(bigEndian (prim length payload : Nat) 4)", + vector: "an empty body", + }, + Plant { + label: "the lengths are little-endian", + root: 0, + site: "(cat (byteText (prim remainder (first state) 256 0 : Nat)) (second state))", + replacement: "(cat (second state) (byteText (prim remainder (first state) 256 0 : Nat)))", + vector: "a path of 300 bytes", + }, + Plant { + label: "the high byte of a word is dropped", + root: 0, + site: "(byteText (prim to_nat (LexLeanStd.Sha256::shiftRight word (24 : UInt32)) : Nat))", + replacement: "(byteText 0)", + vector: "one empty file", + }, + Plant { + label: "the low byte of a word is its third", + root: 0, + site: "(prim bit_and word (255 : UInt32) : UInt32)", + replacement: "(prim bit_and (LexLeanStd.Sha256::shiftRight word (8 : UInt32)) (255 : UInt32) : UInt32)", + vector: "two files", + }, + Plant { + label: "the digest ends a word early", + root: 3, + site: "(wordBytes (project digest h7))", + replacement: "0x", + vector: "a build of zero digests", + }, + Plant { + label: "the tree is hashed in the source domain", + root: 0, + site: "(lit \"lexlean-tree-v1\")", + replacement: "(lit \"lexlean-source-v1\")", + vector: "a file with an empty path", + }, + Plant { + label: "a source is not framed with its path", + root: 1, + site: "(cat (frame (lit \"path\") (first row)) (frame (lit \"source\") (second row)))", + replacement: "(frame (lit \"source\") (second row))", + vector: "two sources", + }, + Plant { + label: "the lock is framed as the project", + root: 1, + site: "(frame (lit \"lock\") lockText)", + replacement: "(frame (lit \"project\") lockText)", + vector: "no sources", + }, + Plant { + label: "the toolchain is another", + root: 2, + site: "(lit \"leanprover/lean4:v4.32.1\")", + replacement: "(lit \"leanprover/lean4:v4.32.0\")", + vector: "the language 1.2", + }, + Plant { + label: "the language is not framed", + root: 2, + site: "(frame (lit \"language\") language)", + replacement: "0x", + vector: "an empty IR", + }, + Plant { + label: "the build digests are swapped", + root: 3, + site: "(cat (frame (lit \"source-id\") sourceDigest) (frame (lit \"semantic-id\") semanticDigest))", + replacement: "(cat (frame (lit \"source-id\") semanticDigest) (frame (lit \"semantic-id\") sourceDigest))", + vector: "two distinct digests", + }, + Plant { + label: "the attestation label is another", + root: 4, + site: "(lit \"attestation-body\")", + replacement: "(lit \"attestation\")", + vector: "a body of 300 bytes", + }, + Plant { + label: "the argv is framed as the resources", + root: 5, + site: "(frame (lit \"argv\") compileArgv)", + replacement: "(frame (lit \"argv\") resources)", + vector: "a recipe", + }, +]; + +fn plants(vectors: &[Vector]) { + let mut missed = Vec::new(); + for plant in PLANTS { + let vector = vectors + .iter() + .find(|vector| vector.label == plant.vector) + .unwrap_or_else(|| panic!("{}: no vector `{}`", plant.label, plant.vector)); + assert_eq!( + vector.root, plant.root, + "{}: the vector is a run of the root planted", + plant.label + ); + let planted = support::P::bootstrap(); + let source = planted.read_raw("src/ContentId.lex.tex"); + assert_eq!( + source.matches(plant.site).count(), + 1, + "{}: the plant site occurs once", + plant.label + ); + planted.write( + "src/ContentId.lex.tex", + &source.replacen(plant.site, plant.replacement, 1), + ); + let checked = support::checked_project(&planted); + let low = super::lower(&checked, &planted, MODULE, ROOTS[plant.root].0); + match super::evaluate(&low, std::slice::from_ref(&vector.run), 1).first() { + Some(first) => eprintln!( + "BS-32: plant detected: {}: {}", + plant.label, + first.chars().take(160).collect::() + ), + None => missed.push(plant.label), + } + } + assert!( + missed.is_empty(), + "planted defects their own vector does not detect: {missed:?}" + ); +} + +fn export(lowered: &BTreeMap<&str, super::Lowered>) { + let exported = harness::exported("bootstrap"); + let bundle = exported.bundle(); + let mut realized = 0usize; + for exported_root in &exported.output.result.roots { + let Some((member, low)) = exported_root + .root + .strip_prefix(&format!("{MODULE}.")) + .and_then(|member| lowered.get(member).map(|low| (member, low))) + else { + continue; + }; + let largest = exported_root + .outputs + .iter() + .flat_map(|output| &output.certification.certificates) + .map(|certificate| certificate.file.byte_length) + .max() + .expect("a certificate"); + eprintln!( + "BS-32: {}: largest certificate part {largest} bytes", + exported_root.root + ); + assert!( + largest <= CERTIFICATE_BUDGET, + "{}: a certificate part of {largest} bytes is beyond {CERTIFICATE_BUDGET}", + exported_root.root + ); + for output in &exported_root.outputs { + let package: Json2 = serde_json::from_slice( + &std::fs::read( + bundle + .join(&output.package.directory) + .join("package.json") + .as_std_path(), + ) + .expect("package.json"), + ) + .expect("package JSON"); + assert_eq!( + package["program"], + serde_json::to_value(&low.program).expect("a program is JSON"), + "{member}: the exported package realizes the program the differential ran" + ); + realized += 1; + } + } + assert_eq!(realized, ROOTS.len(), "one certified package per root"); + let packages = harness::tree(&bundle.join("packages")); + let committed_packages = harness::tree(&harness::fixture_dir("bootstrap").join("packages")); + assert_eq!( + packages.keys().collect::>(), + committed_packages.keys().collect::>(), + "the exported package files are the committed ones" + ); + for (relative, content) in &packages { + assert!( + committed_packages.get(relative) == Some(content), + "packages/{relative} differs from the committed package" + ); + } +} + +#[allow(clippy::too_many_lines)] +pub(super) fn case() { + let started = std::time::Instant::now(); + let project = support::P::bootstrap(); + let checked_units = project.check_ok(); + assert!( + checked_units.units.contains_key("ContentId"), + "ContentId is a linked module of the project" + ); + eligibility_of(&project, "ContentId", MODULE, &ROOTS); + + let vectors = named(); + let labels: BTreeSet<&str> = vectors.iter().map(|vector| vector.label).collect(); + assert_eq!(labels.len(), vectors.len(), "each vector has its own label"); + let mut runs: [Vec; 6] = Default::default(); + for vector in &vectors { + runs[vector.root].push(vector.run.clone()); + } + + // The committed projects' trees and source inputs, and the committed + // build manifests, each of whose build ID the oracle reproduces. + let projects = committed_projects(); + let builds = committed_builds(); + for (source, semantic, recorded) in &builds { + assert_eq!( + build_id(*source, *semantic), + *recorded, + "a committed build ID is the build recipe of its source and semantic IDs" + ); + } + let mut evaluator_corpus: [Vec; 6] = Default::default(); + let mut rustc_corpus: [Vec; 6] = Default::default(); + for (manifest, lock, rows) in &projects { + let size = tree_bytes(rows) + manifest.len() + lock.len(); + if size > RUSTC_TREE_BYTES { + continue; + } + let tree = tree_run(rows); + let source = source_run(manifest, lock, rows); + if size <= EVALUATOR_TREE_BYTES { + evaluator_corpus[0].push(tree.clone()); + evaluator_corpus[1].push(source.clone()); + } + rustc_corpus[0].push(tree); + rustc_corpus[1].push(source); + } + for (source, semantic, _) in &builds { + let run = build_run(&source.0, &semantic.0); + evaluator_corpus[3].push(run.clone()); + rustc_corpus[3].push(run); + } + let normative = support::normative_host(); + let (evaluator_seeded, rustc_seeded) = if normative { + ( + seeded_runs(EVALUATOR_SEEDED, 0x0B55_0032), + seeded_runs(RUSTC_SEEDED, 0x0B55_1032), + ) + } else { + (Default::default(), Default::default()) + }; + eprintln!( + "BS-32: {} vectors, {} committed projects ({} trees under the evaluator, {} under rustc), {} committed builds, {} and {} seeded runs of each root ({:?})", + vectors.len(), + projects.len(), + evaluator_corpus[0].len(), + rustc_corpus[0].len(), + builds.len(), + EVALUATOR_SEEDED, + RUSTC_SEEDED, + started.elapsed() + ); + + let checked = support::checked_project(&project); + let lowered: BTreeMap<&str, super::Lowered> = ROOTS + .iter() + .map(|(root, _)| (*root, super::lower(&checked, &project, MODULE, root))) + .collect(); + if normative { + for (index, (root, _)) in ROOTS.iter().enumerate() { + let mut all_runs: Vec = runs[index].clone(); + all_runs.extend(evaluator_corpus[index].iter().cloned()); + all_runs.extend(evaluator_seeded[index].iter().cloned()); + let disagreements = super::evaluate(&lowered[root], &all_runs, 5); + assert!( + disagreements.is_empty(), + "{root} and the Rust oracle disagree: {disagreements:?}" + ); + eprintln!( + "BS-32: {root}: the evaluator agrees on {} runs ({:?})", + all_runs.len(), + started.elapsed() + ); + } + plants(&vectors); + } else { + eprintln!("BS-32: platform-bound host: the evaluator, the seeded runs, and the planted defects run on the normative host"); + } + + let mut compiled: [Vec; 6] = Default::default(); + for index in 0..6 { + compiled[index].extend(runs[index].iter().cloned()); + compiled[index].extend(rustc_corpus[index].iter().cloned()); + compiled[index].extend(rustc_seeded[index].iter().cloned()); + } + let roots: Vec<(&str, &super::Lowered, &[Run])> = ROOTS + .iter() + .enumerate() + .map(|(index, (root, _))| (*root, &lowered[root], compiled[index].as_slice())) + .collect(); + let total = rustc_differential("BS-32", &roots, "oracle"); + eprintln!("BS-32: {total} rustc runs agree ({:?})", started.elapsed()); + + if !support::lean_backed("BS-32") { + return; + } + export(&lowered); + eprintln!("BS-32: passed in {:?}", started.elapsed()); +} diff --git a/crates/conformance/src/cases/repository.rs b/crates/conformance/src/cases/repository.rs index d542a57dea..10772fbae2 100644 --- a/crates/conformance/src/cases/repository.rs +++ b/crates/conformance/src/cases/repository.rs @@ -898,7 +898,7 @@ pub(crate) fn run(id: &str) { let model = repo_model::Model::load(&root.join("model").into_std_path_buf()) .expect("the model loads"); let table = spec_table(); - assert_eq!(table.len(), 478, "§31 has 478 rows"); + assert_eq!(table.len(), 479, "§31 has 479 rows"); assert_eq!(model.ids.id.len(), table.len(), "register row count"); for ((spec_id, spec_suite, spec_statement), row) in table.iter().zip(model.ids.id.iter()) diff --git a/crates/conformance/tests/conformance.rs b/crates/conformance/tests/conformance.rs index 992ea8a7b3..8f2e743aec 100644 --- a/crates/conformance/tests/conformance.rs +++ b/crates/conformance/tests/conformance.rs @@ -2320,6 +2320,11 @@ fn conformance_bs_20() { repo_conformance::cases::run("BS-20"); } +#[test] +fn conformance_bs_32() { + repo_conformance::cases::run("BS-32"); +} + #[test] fn conformance_pk_01() { repo_conformance::cases::run("PK-01"); diff --git a/features/suites/bootstrap.feature b/features/suites/bootstrap.feature index 31570df912..b4ff3e66e4 100644 --- a/features/suites/bootstrap.feature +++ b/features/suites/bootstrap.feature @@ -119,3 +119,10 @@ Feature: bootstrap When the project is checked and built, the lowered program of the root is run by the evaluator on those modules and the mutants, and its rendered rust-std package is run by the pinned rustc on the modules, each with the capacity that fits and capacities that do not Then the root is eligible for rust-std with exactly its declared effects and no run disagrees with the oracle text, to the byte, or with its refusal And every planted defect is detected, and the exported package is byte-identical to its committed package and realizes the lowered program + + @BS-32 @build + Scenario: The labeled hash recipes of content identity of SPEC.md §21.1 to §21.5, §11.5, §19.8, and §22.9 (`artifact::content_id`: the frame of a label and a payload, each preceded by its big-endian length in four and eight bytes, under a domain and a NUL; `tree_digest` over path and content rows; `source_id` over the project manifest, the lock, and the sources; `semantic_id_for` over the compiler-semantics digest, the language, the pinned Lean toolchain, the linked IR, and the lexicon closure; `build_id`; `attestation_id`; and `pdf_recipe_id`) is language-1.2 source in the `bootstrap/` project, as six production roots of `ContentId` over the standard library's SHA-256: `treeDigest`, `sourceId`, `semanticId`, `buildId`, `attestationId`, and `pdfRecipeId`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built inputs that reach every frame of every recipe, lengths beyond one byte, and messages across blocks, on the trees and source inputs of the committed projects, on the source and semantic IDs of every committed build manifest, whose recorded build ID the oracle reproduces, and on seeded inputs; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; choosing and normalizing the files of a tree or a source ID, and serializing the IR, the closure, and the attestation body, are outside the claim. + Given the committed `bootstrap` project, hand-built recipe inputs, the trees and source inputs of the committed projects, the IDs of every committed build manifest, and seeded inputs + When the project is checked and built, and the lowered program and the rendered package of each root are run on those inputs + Then each root is eligible for rust-std with exactly its declared effects, no certificate part is beyond 3,145,728 bytes, and the six roots equal the content-identity recipes of `artifact::content_id` on every run, and each committed build ID is the build recipe of its source and semantic IDs + And every planted defect is detected by its own vector, and the exported packages are byte-identical to the committed packages and realize the programs the differential ran diff --git a/model/ids.toml b/model/ids.toml index 69eb399d8a..c47286aa62 100644 --- a/model/ids.toml +++ b/model/ids.toml @@ -2786,6 +2786,12 @@ level = "build" suite = "bootstrap" statement = "The canonical LaTeX of a tooling module of SPEC.md §17.20 (`backend::tooling::render_latex`: the fixed document frame and, for each declaration in order, its title, its kind, and the lines the Lean printer of the same file spells for its types, signatures, and terms, every line escaped by `tex_escape`) is language-1.2 source in the `bootstrap/` project: `renderToolingLatex` of `ToolingLatex`, over the tooling module as first-order data with each list carried as a chain of its own `Nil` and `Cons` constructors and with the typings the checker infers as an argument, writing through the standard library's `Buffer` with an explicit capacity, is a production root eligible for `rust-std` with exactly its declared effects; its rendered `rust-std` package under the pinned rustc and its lowered program under the calculus evaluator agree with the Rust oracle on the text, to the byte, and on the refusal of a write past the capacity, on the committed tooling modules (whose committed `.tex` is equal to the text), on hand-built modules that reach every declaration, type, signature, and term form and every byte class of the escaper, and on seeded mutants, every planted defect is detected, and the certified package the production export publishes for the project is the committed one; the checker, the decoder of the module, and the source map of the emitted pieces stay in the host." +[[id]] +id = "BS-32" +level = "build" +suite = "bootstrap" +statement = "The labeled hash recipes of content identity of SPEC.md §21.1 to §21.5, §11.5, §19.8, and §22.9 (`artifact::content_id`: the frame of a label and a payload, each preceded by its big-endian length in four and eight bytes, under a domain and a NUL; `tree_digest` over path and content rows; `source_id` over the project manifest, the lock, and the sources; `semantic_id_for` over the compiler-semantics digest, the language, the pinned Lean toolchain, the linked IR, and the lexicon closure; `build_id`; `attestation_id`; and `pdf_recipe_id`) is language-1.2 source in the `bootstrap/` project, as six production roots of `ContentId` over the standard library's SHA-256: `treeDigest`, `sourceId`, `semanticId`, `buildId`, `attestationId`, and `pdfRecipeId`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built inputs that reach every frame of every recipe, lengths beyond one byte, and messages across blocks, on the trees and source inputs of the committed projects, on the source and semantic IDs of every committed build manifest, whose recorded build ID the oracle reproduces, and on seeded inputs; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; choosing and normalizing the files of a tree or a source ID, and serializing the IR, the closure, and the attestation body, are outside the claim." + [[id]] id = "PK-01" level = "build" diff --git a/tests/production-export/bootstrap/roots.json b/tests/production-export/bootstrap/roots.json index 4f911a0892..d4ab95e86a 100644 --- a/tests/production-export/bootstrap/roots.json +++ b/tests/production-export/bootstrap/roots.json @@ -16,6 +16,14 @@ ], "version": "1.0.0" }, + { + "name": "attestation_id", + "root": "LexLeanBoot.ContentId.attestationId", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "audit_tokens", "root": "LexLeanBoot.TokenAudit.auditTokens", @@ -24,6 +32,14 @@ ], "version": "1.0.0" }, + { + "name": "build_id", + "root": "LexLeanBoot.ContentId.buildId", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "check_arguments", "root": "LexLeanBoot.CalculusCheck.checkArguments", @@ -152,6 +168,14 @@ ], "version": "1.0.0" }, + { + "name": "pdf_recipe_id", + "root": "LexLeanBoot.ContentId.pdfRecipeId", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "print_crate", "root": "LexLeanBoot.RustPrint.printCrate", @@ -200,6 +224,14 @@ ], "version": "1.0.0" }, + { + "name": "semantic_id", + "root": "LexLeanBoot.ContentId.semanticId", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "site_trees", "root": "LexLeanBoot.CalculusCheck.siteTrees", @@ -208,6 +240,14 @@ ], "version": "1.0.0" }, + { + "name": "source_id", + "root": "LexLeanBoot.ContentId.sourceId", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "succeed_bodies", "root": "LexLeanBoot.RustSucceed.succeedBodies", @@ -216,6 +256,14 @@ ], "version": "1.0.0" }, + { + "name": "tree_digest", + "root": "LexLeanBoot.ContentId.treeDigest", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "use_trees", "root": "LexLeanBoot.CalculusCheck.useTrees", diff --git a/xtask/src/spec_links.rs b/xtask/src/spec_links.rs index d3b755a3d4..962afcafd8 100644 --- a/xtask/src/spec_links.rs +++ b/xtask/src/spec_links.rs @@ -61,8 +61,8 @@ fn parse_table(spec: &str) -> Result, Fail> { pub fn validate(root: &Path) -> Result<(), Fail> { let spec = std::fs::read_to_string(root.join("SPEC.md"))?; let rows = parse_table(&spec)?; - if rows.len() != 478 { - return Err(format!("RP-07: the §31 table has {} rows, not 478", rows.len()).into()); + if rows.len() != 479 { + return Err(format!("RP-07: the §31 table has {} rows, not 479", rows.len()).into()); } // The prose total under the table is part of the contract a reader // relies on; a row added without it would leave §31 contradicting itself.