diff --git a/.github/bdd-parts.json b/.github/bdd-parts.json index 645b591a09..5d2ea4dee9 100644 --- a/.github/bdd-parts.json +++ b/.github/bdd-parts.json @@ -40,6 +40,9 @@ "bootstrap-lock-text": [ "BS-31" ], + "bootstrap-project-text": [ + "BS-33" + ], "bootstrap-tooling-latex": [ "BS-20" ], diff --git a/.github/intel-host-lanes.json b/.github/intel-host-lanes.json index 806ea36428..6a8868055f 100644 --- a/.github/intel-host-lanes.json +++ b/.github/intel-host-lanes.json @@ -2,7 +2,7 @@ "producer": ["honesty"], "lanes": { "lower": ["bootstrap-lower"], - "print": ["bootstrap-print", "bootstrap-bodies", "bootstrap-succeed", "bootstrap-loops", "bootstrap-assemble", "bootstrap-package", "bootstrap-check", "bootstrap-gnaf", "bootstrap-runtime", "bootstrap-terms", "bootstrap-tooling-latex", "bootstrap-lock-text"], + "print": ["bootstrap-print", "bootstrap-bodies", "bootstrap-succeed", "bootstrap-loops", "bootstrap-assemble", "bootstrap-package", "bootstrap-check", "bootstrap-gnaf", "bootstrap-runtime", "bootstrap-terms", "bootstrap-tooling-latex", "bootstrap-lock-text", "bootstrap-project-text"], "validate": ["bootstrap-validate", "compiler", "preservation", "preservation-lists", "preservation-runtime", "preservation-support", "preservation-prims", "preservation-std", "production"], "moves": ["bootstrap-moves", "verification", "rest"] } diff --git a/.github/vv-rows.json b/.github/vv-rows.json index 3910c86f82..af4f098a84 100644 --- a/.github/vv-rows.json +++ b/.github/vv-rows.json @@ -71,6 +71,15 @@ "memory_kib": "", "hints": "" }, + { + "part": "bdd-bootstrap-project-text", + "recipes": "bdd", + "examples_part": "", + "bdd_part": "bootstrap-project-text", + "width": "", + "memory_kib": "", + "hints": "" + }, { "part": "bdd-bootstrap-tooling-latex", "recipes": "bdd", diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a5725f7f0..e6e27825a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,16 @@ workspace carried at those versions without a tag. ## Unreleased +- The canonical text of the project manifest and its digest as language-1.2 + source (SPEC.md §10.1, §17.23, `BS-33`, #274; no new diagnostic code, no change + to the language-1.2 identity). `canonicalProject` and `configSha256` of + `ProjectText` return what `ProjectConfig::canonical_toml` and `config_sha256` + return; neither admits `recursion`. The register is 480 IDs. - The canonical text of the lock file as language-1.2 source (SPEC.md §11.2, §17.23, `BS-31`, #274; no new diagnostic code, no change to the language-1.2 identity). `canonicalLock` and `tomlString` of `LockText` return what `Lock::canonical_bytes` and `config::toml_string` return; neither admits - `recursion`. The register is 479 IDs. + `recursion`. - Canonical JSON reading in the standard library, the fourth member the self-hosted compiler needs, #276 (SPEC.md §17.22, `SL-13`). The module `LexLeanStd.JsonRead` holds the production roots `readJson` and diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4b46f5155d..45caa174e5 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -78,6 +78,7 @@ the error registry does not sanction (R5, `ERRORS.md`). | `BS-16` | `build` | The GNAF request validation of SPEC.md §17.15 (`gnaf::validate`, with `universe_size`) is language-1.2 source in the `bootstrap/` project: `validateRequest` of `GnafValidate` is a production root eligible for `rust-std` with exactly its declared effects, over the projection of a request that validation reads (the length of the domain, the machine, the carrier's kind with each plan's preparations and the thresholds, the completeness evidence, the objective, the claim's class, and the scope) as first-order data, and returns the first violated rule in the model's order as the oracle's `serde_json` bytes, or nothing; on every host, its rendered `rust-std` package under the pinned rustc agrees with the Rust oracle `gnaf::validate` on the acceptance and on the rejection, to the byte, on the committed requests of `compiler/gnaf/`, the requests the committed certificate fixtures name, hand-built requests that reach each rejection, and seeded mutants of the committed requests; on the normative host, the root's lowered program under the calculus evaluator agrees with the oracle on the same corpus, each planted defect in the source is detected, and the certified package the production export publishes for the project is the committed one; loading a request (`gnaf::load`), the universe identity, the answer, and certification are outside the claim. | | `BS-20` | `build` | The canonical LaTeX of a tooling module of SPEC.md §17.20 (`backend::tooling::render_latex`: the fixed document frame and, for each declaration in order, its title, its kind, and the lines the Lean printer of the same file spells for its types, signatures, and terms, every line escaped by `tex_escape`) is language-1.2 source in the `bootstrap/` project: `renderToolingLatex` of `ToolingLatex`, over the tooling module as first-order data with each list carried as a chain of its own `Nil` and `Cons` constructors and with the typings the checker infers as an argument, writing through the standard library's `Buffer` with an explicit capacity, is a production root eligible for `rust-std` with exactly its declared effects; its rendered `rust-std` package under the pinned rustc and its lowered program under the calculus evaluator agree with the Rust oracle on the text, to the byte, and on the refusal of a write past the capacity, on the committed tooling modules (whose committed `.tex` is equal to the text), on hand-built modules that reach every declaration, type, signature, and term form and every byte class of the escaper, and on seeded mutants, every planted defect is detected, and the certified package the production export publishes for the project is the committed one; the checker, the decoder of the module, and the source map of the emitted pieces stay in the host. | | `BS-31` | `build` | The canonical text of the lock file of SPEC.md §11.2 (`lock::Lock::canonical_bytes`: the header keys in their fixed order with the lock spec of the language and the pinned Lean toolchain, the workspace pins sorted by path, the packages sorted by identifier and version, the source packages sorted by identifier, and the PDF record with its resources sorted by path, each digest in lower-case hex; and `config::toml_string`: the TOML basic string of a text, with the quote, the backslash, and every control escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `LockText`: `canonicalLock` and `tomlString`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built locks and texts, which reach every table, both lock specs, every order of rows the sort decides, and every class of escape, on every committed `lexlean.lock` that parses and every text in them, and on seeded locks and texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading a lock, resolving its packages, and hashing its files are outside the claim. | +| `BS-33` | `build` | The canonical text of the project manifest of SPEC.md §10.1 and its digest (`config::ProjectConfig::canonical_toml`: the ten header keys in their fixed order with the pinned Lean toolchain, the lexicon sources sorted by package or the empty array when there is none, the artifact sources and the source packages in their order, the eleven limits in their fixed order, and the PDF provider; and `config_sha256`: the lower-case hex SHA-256 of that text) is language-1.2 source in the `bootstrap/` project, as two production roots of `ProjectText` over the TOML writers of `BS-31`: `canonicalProject` and `configSha256`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built configurations, which reach every kind of lexicon source, the empty source list, equal packages, every optional table, escapes, and the largest limits, on every committed `lexlean.toml` that parses, and on seeded configurations; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading and validating a manifest is outside the claim. | ## calculus diff --git a/README.md b/README.md index 1e14e1dca3..1db5a31f48 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ just vv # the complete normative acceptance gate (SPEC.md §9.2) just release # vv, then the §30 release criterion; refused until 1.0.0 ``` -All 479 registered conformance IDs have their named tests; `just vv` checks them with the pinned toolchain installed. +All 480 registered conformance IDs have their named tests; `just vv` checks them with the pinned toolchain installed. The differential against the independent generator `lean4-prod` (SPEC.md §17.19, `OR-01`..`OR-08`) acquires it at its pinned commit: from the checkout `LEXLEAN_ORACLE_DIR` names, or by fetching the commit with `git` into the Cargo target directory (`LEXLEAN_ORACLE_CACHE` overrides), and builds it with the pinned Lean and Rust toolchains. It is validation evidence only and no input to any identity. An oracle that cannot be acquired, is not the pinned files, or fails is unavailable: that fails the gate on Linux x86-64 and is printed as `NOT VALIDATED` elsewhere, and is never reported as agreement. @@ -197,7 +197,7 @@ Every row is validated by `just vv`; the IDs link the claim to its register row, | The language-1.2 authoring notation: a readable `.lex.tex` text for a semantic module, in bijection with its canonical JSON, whose every position and variant is read from the schema, with readable forms for types and terms and a generic form for every other value, formatted canonically by `fmt` failing closed, the only form in which a language-1.2 source of the product is written (the data form is refused under language 1.2, and `lexlean::sdk` is the explicit conversion), the same for the compiler-tooling modules in the generic notation, with no compiler-semantics ID that depends on which form an embedded module is written in, with no product source owned by a generator but five named records, read and written by programs only through the authoring API (`lexlean::sdk`), and, for a native core module, a readable program over the shared table of kernel terms whose numbering is the order of its text | `AU-01`..`AU-13` | `build` | | The runtime of generated Lean as native modules: the portable, collection, model, and reasoning runtimes of language 1.2 are LexLean source that every generated module imports, migrated losslessly and audited against their exact axioms, the released runtime text of languages 1.0 and 1.1 is frozen data equal to their released output, and no other Lean runtime text remains in the compiler but the decoder of core module 2 | `RT-01`..`RT-04` | `build` | | The standard library: language-1.2 source beside the compiler whose members are certified production roots, each validated against its oracle by a seeded differential of its evaluator and rustc realizations; bytes and text over `Bytes`, an output buffer of a given capacity written in order; decimal and hexadecimal; the byte order and stable sorts over it; canonical JSON reading and writing; maps and sets over byte keys; SHA-256 against FIPS 180-4, the NIST CAVP vectors, and the `sha2` crate; and the `std-models` and `std-reasoning` source packages, generic over types, whose laws are kernel-checked theorems and whose application is the `helpdesk` example | `SL-01`, `SL-02`, `SL-03`, `SL-04`, `SL-05`, `SL-09`, `SL-10`, `SL-11`, `SL-13`, `SL-14` | `build` | -| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the GNAF request validation, the LaTeX of a tooling module, and the canonical text of the lock file, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-31` | `build` | +| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the GNAF request validation, the LaTeX of a tooling module, and the canonical text of the lock file and of the project manifest, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-33` | `build` | | Source packages: a language-1.2 project imports the modules of a locked, content-addressed package of its workspace: `[[source_package]]` and `lexlean-workspace.toml`, the pins in `lexlean.lock` recomputed by every command, imports closed at link, no re-export of a package's production roots, and the package's modules replayed by the importer's `verify` with their origin in the records | `PK-01`..`PK-05` | `build` | | The literal `nat-add-zero` example, the Lean-verified feature examples, and the complete negative fixture suite | `EX-01`..`EX-09` | `build` | | Language-1.2 text components: tokenizers and their round-trip laws, with the laws carried as one native core module the generated modules import rather than authored Lean text | `LM-01`, `LM-02`, `LM-03`, `LM-13` | `build` | diff --git a/SPEC.md b/SPEC.md index 480e7dc3d7..190f24d770 100644 --- a/SPEC.md +++ b/SPEC.md @@ -8748,6 +8748,20 @@ identifier and version. `tomlString` escapes the quote and the backslash with a control below 0x20 and 0x7F as `\u` and four upper-case hex digits. Neither admits `recursion`. Reading a lock, resolving its packages, and hashing its files stay in the host. +The canonical text of the project manifest (§10.1) and its digest are the next component of the +project manager as source (`BS-33`, issue 274), the module `ProjectText` over `BS-31`'s +`LockText`, with two production roots for `rust-std`: + +| Root | Takes | Returns | Oracle | +| --- | --- | --- | --- | +| `LexLeanBoot.ProjectText.canonicalProject` | a configuration | its canonical TOML | `config::ProjectConfig::canonical_toml` | +| `LexLeanBoot.ProjectText.configSha256` | a configuration | the hex SHA-256 of that TOML | `config::ProjectConfig::config_sha256` | + +`canonicalProject` writes the ten header keys, the lexicon sources sorted by package with the +stable merge sort of the standard library's `Order` (or `lexicon_source = []` when there is none), +the artifact sources and the source packages in their order, the `[limits]` table, and the PDF +provider. Neither root admits `recursion`. Reading and validating a manifest stay in the host. + ## 18. Lean backend ### 18.1 Output contract @@ -11670,6 +11684,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `BS-16` | `bootstrap` | The GNAF request validation of SPEC.md §17.15 (`gnaf::validate`, with `universe_size`) is language-1.2 source in the `bootstrap/` project: `validateRequest` of `GnafValidate` is a production root eligible for `rust-std` with exactly its declared effects, over the projection of a request that validation reads (the length of the domain, the machine, the carrier's kind with each plan's preparations and the thresholds, the completeness evidence, the objective, the claim's class, and the scope) as first-order data, and returns the first violated rule in the model's order as the oracle's `serde_json` bytes, or nothing; on every host, its rendered `rust-std` package under the pinned rustc agrees with the Rust oracle `gnaf::validate` on the acceptance and on the rejection, to the byte, on the committed requests of `compiler/gnaf/`, the requests the committed certificate fixtures name, hand-built requests that reach each rejection, and seeded mutants of the committed requests; on the normative host, the root's lowered program under the calculus evaluator agrees with the oracle on the same corpus, each planted defect in the source is detected, and the certified package the production export publishes for the project is the committed one; loading a request (`gnaf::load`), the universe identity, the answer, and certification are outside the claim. | §17.15, §17.23 | | `BS-20` | `bootstrap` | The canonical LaTeX of a tooling module of SPEC.md §17.20 (`backend::tooling::render_latex`: the fixed document frame and, for each declaration in order, its title, its kind, and the lines the Lean printer of the same file spells for its types, signatures, and terms, every line escaped by `tex_escape`) is language-1.2 source in the `bootstrap/` project: `renderToolingLatex` of `ToolingLatex`, over the tooling module as first-order data with each list carried as a chain of its own `Nil` and `Cons` constructors and with the typings the checker infers as an argument, writing through the standard library's `Buffer` with an explicit capacity, is a production root eligible for `rust-std` with exactly its declared effects; its rendered `rust-std` package under the pinned rustc and its lowered program under the calculus evaluator agree with the Rust oracle on the text, to the byte, and on the refusal of a write past the capacity, on the committed tooling modules (whose committed `.tex` is equal to the text), on hand-built modules that reach every declaration, type, signature, and term form and every byte class of the escaper, and on seeded mutants, every planted defect is detected, and the certified package the production export publishes for the project is the committed one; the checker, the decoder of the module, and the source map of the emitted pieces stay in the host. | §17.20, §17.23 | | `BS-31` | `bootstrap` | The canonical text of the lock file of SPEC.md §11.2 (`lock::Lock::canonical_bytes`: the header keys in their fixed order with the lock spec of the language and the pinned Lean toolchain, the workspace pins sorted by path, the packages sorted by identifier and version, the source packages sorted by identifier, and the PDF record with its resources sorted by path, each digest in lower-case hex; and `config::toml_string`: the TOML basic string of a text, with the quote, the backslash, and every control escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `LockText`: `canonicalLock` and `tomlString`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built locks and texts, which reach every table, both lock specs, every order of rows the sort decides, and every class of escape, on every committed `lexlean.lock` that parses and every text in them, and on seeded locks and texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading a lock, resolving its packages, and hashing its files are outside the claim. | §11.2, §17.23 | +| `BS-33` | `bootstrap` | The canonical text of the project manifest of SPEC.md §10.1 and its digest (`config::ProjectConfig::canonical_toml`: the ten header keys in their fixed order with the pinned Lean toolchain, the lexicon sources sorted by package or the empty array when there is none, the artifact sources and the source packages in their order, the eleven limits in their fixed order, and the PDF provider; and `config_sha256`: the lower-case hex SHA-256 of that text) is language-1.2 source in the `bootstrap/` project, as two production roots of `ProjectText` over the TOML writers of `BS-31`: `canonicalProject` and `configSha256`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built configurations, which reach every kind of lexicon source, the empty source list, equal packages, every optional table, escapes, and the largest limits, on every committed `lexlean.toml` that parses, and on seeded configurations; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading and validating a manifest is outside the claim. | §10.1, §17.23 | | `PK-01` | `packages` | A language-1.2 project that declares a source package by its identifier, version, kind `path`, and the member directory a `lexlean-workspace.toml` lists locks it by the tree digest of its manifest and source files, its manifest digest, its version, its module prefix, and its own packages, and the configuration and the lock read back to the bytes written; a project without a source package keeps its bytes, and a package of another kind, a source package in another language, a changed byte or version (LLC0105), an undeclared package (LLC0106), a module prefix claimed twice (LLC0107), a project or a package outside the workspace (LLC0108), a path that escapes it, and a symlink (LLS8001) are each refused with the named code. | §10.5, §11.1, §25.1 | | `PK-02` | `packages` | A module of a project imports only modules of its own project and modules of the packages the project declares, and a module of a package only modules of itself and of the packages its own manifest declares: a package module is a module of the importing project under its own module prefix, a module of a package the project does not declare fails with LLR3009, and an import cycle through a package fails with LLR3003. | §10.5, §15.1 | | `PK-03` | `packages` | An importing project builds and verifies under the pinned Lean offline, and its package modules are generated, elaborated, replayed with `leanchecker`, and audited for their exact axioms under the package's module prefix with the package as their origin in the records; the source ID of the project changes with the pins, and no file of the package is written by `fmt`, `build`, or `verify`. | §10.5, §21.3, §22 | @@ -11686,7 +11701,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `LM-09` | `language-models` | No prompt is code: no member of the closed language maps `string` or `bytes` to a function, a term, or source — every operation of the primitive registry that takes a `string` or `bytes` operand is classified by the audit with a data result, the registry's schema enumeration and the production registry's primitive rows equal one another and equal that classified set, and the text declarations of `lexlean/semantic-module/2` state no `string`-typed member, so a prompt reaches a model only as the tokens `encodeText` makes of it; a planted operation or text member of that shape is caught by the audit with its name. | §17.12 | | `LM-13` | `language-models` | The text laws a tokenizer elaborates through are LexLean source and not authored text: the module that states them is one language-1.2 native core module of the embedded `language/text-1.2` project, the project holds no file of Lean source, no independently authored Lean file anywhere in the release tree declares the `LexLeanText` namespace or lies under its module root, and the one-time migration input is absent; the registry lists only that module, the declarations and their exact axiom policies are read from its native data, which the compiler embeds and hashes into the language-1.2 semantics, and a module that declares a text component imports the generated module rather than splicing any Lean text. | §17.10, §17.12 | -**Total required capability IDs:** 479. +**Total required capability IDs:** 480. No row may be downgraded to `some-true` or `open`. Upstream Lean facts are ledger/authority rows, not substitutions for these build behaviors. diff --git a/VERIFICATION.md b/VERIFICATION.md index 7b9fed48f5..d70f6143b0 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -11091,6 +11091,47 @@ stops at the stack-bound step, `bootstrap/toml_string is no package of the committed exports`, as each bootstrap member did before its regeneration, which produces the two packages. +### the canonical project manifest and its digest are source (BS-33) + +`BS-33` holds the two production roots of `LexLeanBoot.ProjectText` +(`bootstrap/src/ProjectText.lex.tex`) to `config::ProjectConfig`: +`canonicalProject` to `canonical_toml`, the ten header keys in their fixed order +with the pinned Lean toolchain, the lexicon sources sorted by package with the +stable merge sort of the standard library's `Order` (or `lexicon_source = []` +when there is none), the artifact sources and the source packages in their +order, the eleven limits, and the PDF provider; and `configSha256` to the +lower-case hex of `config_sha256`, the SHA-256 of that text under the standard +library's `Sha256`. The module writes through the TOML writers of `BS-31`'s +`LockText`. Each root admits exactly `allocation` and `overflow`, and neither +admits `recursion`. The oracle is the existing public code of `config.rs`; no +Rust outside the case changes. The case holds the roots to the oracle on +hand-built configurations that reach every kind of lexicon source, the empty +source list, two sources of one package, every optional table, escapes in the +name, and the largest limit; on every committed `lexlean.toml` that parses; and +on seeded configurations. Each planted defect names the one vector that catches +it, and the case asserts that the site of each plant occurs once. The row's +statement carries no counts. + +Measured with the compiler binary of this tree, on a copy of the project whose +only entrypoint is `ProjectText`: `lexlean check`, `lexlean build`, and `lexlean +verify` pass, and the eligibility report gives each root exactly `allocation` +and `overflow`. The register agrees with the §31 table on 480 IDs, by the same +parse `validate-spec-links` makes. + +The case was first compiled and run on the normative host on `BS-31` over +`release/v0.4` at `cf76090d4` (`cargo test -p repo-conformance --test +conformance -- conformance_bs_33`, 224 s). Its corpus is 12 named vectors (10 +texts, 2 digests), the 351 of the 362 committed manifests that parse, each of +which is already its own canonical text, and 200 and 1,000 seeded runs of each +root. The evaluator agrees with the oracle on 561 texts and 553 digests, the +rendered packages under the pinned rustc on 2,714 runs, and each of the 12 +planted defects is detected by its own vector. The first runs found no fault of +the writer: Lean asked for exactly the axioms the module now declares. The case +then stops at the stack-bound step, `bootstrap/canonical_project is no package +of the committed exports`, as each bootstrap member did before its +regeneration, which produces the two packages. The branch is stacked on `BS-31` +and lands after it. + ## The `bdd` recipe as the parts of one run (Issue #131) With UOR Atlas verified as the parts of one run, the `rest` leg became the diff --git a/bootstrap/README.md b/bootstrap/README.md index 46bcfa61c2..e47db88d92 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -21,3 +21,7 @@ writes the rejection of a GNAF request through it. `src/LockText.lex.tex` (SPEC.md §17.23, `BS-31`) imports the standard library's `Numeral`, `Order`, and `Text`, and writes the canonical lock text of SPEC.md §11.2. + +`src/ProjectText.lex.tex` (SPEC.md §17.23, `BS-33`) imports `LockText` (`BS-31`), whose TOML +writers it uses, and the standard library's `Numeral`, `Order`, and `Sha256`, and writes the +canonical project manifest of SPEC.md §10.1 and its digest. diff --git a/bootstrap/lexlean.lock b/bootstrap/lexlean.lock index 3ecbcb30c2..29f4299d64 100644 --- a/bootstrap/lexlean.lock +++ b/bootstrap/lexlean.lock @@ -1,7 +1,7 @@ spec = "lexlean/lock/2" language = "1.2" compiler_semantics = "635a25acb886587f8777525e778092e9599d1f2df26da59403ee39e9e299151a" -project_config_sha256 = "556ae88888d0da424b6160ea14de260c39a729b7bb26ba45fcf023c7cb24b724" +project_config_sha256 = "88d8b7d83e3259d37419ae6e0d7727e6cdbd2529e98dd6b21a4e7029bcdbc038" lean_toolchain = "leanprover/lean4:v4.32.1" [[workspace_file]] diff --git a/bootstrap/lexlean.toml b/bootstrap/lexlean.toml index d3c093b10b..b1cedea918 100644 --- a/bootstrap/lexlean.toml +++ b/bootstrap/lexlean.toml @@ -3,7 +3,7 @@ name = "bootstrap" language = "1.2" module_prefix = "LexLeanBoot" source_roots = ["src"] -entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/GnafValidate.lex.tex", "src/LockText.lex.tex", "src/PrimitiveScan.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/TokenAudit.lex.tex", "src/ToolingAst.lex.tex", "src/ToolingLatex.lex.tex"] +entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/GnafValidate.lex.tex", "src/LockText.lex.tex", "src/PrimitiveScan.lex.tex", "src/ProjectText.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/TokenAudit.lex.tex", "src/ToolingAst.lex.tex", "src/ToolingLatex.lex.tex"] build_root = ".lexlean" lockfile = "lexlean.lock" lean_workspace = "." diff --git a/bootstrap/src/ProjectText.lex.tex b/bootstrap/src/ProjectText.lex.tex new file mode 100644 index 0000000000..a2bac22e06 --- /dev/null +++ b/bootstrap/src/ProjectText.lex.tex @@ -0,0 +1,252 @@ +\begin{lexlean}{ProjectText} +\useglossary{lexlean.std.nat@1.2.0} +\importmodule{LockText} +\importmodule{LexLeanStd.Numeral} +\importmodule{LexLeanStd.Order} +\importmodule{LexLeanStd.Sha256} +\title{Natural number addition} +\begin{semanticmodule} +inductive{ + name: LexiconSource, + type_parameters: [], + parameters: [], + constructors: [ + {name: builtin, fields: [Bytes]}, + {name: path, fields: [Bytes, Bytes]}, + {name: git, fields: [Bytes, Bytes, Bytes, Bytes]} + ] +} + +structure{ + name: ArtifactSource, + type_parameters: [], + parameters: [], + fields: [{name: digest, type: Bytes}, {name: location, type: Bytes}] +} + +structure{ + name: SourcePackage, + type_parameters: [], + parameters: [], + fields: [ + {name: packageName, type: Bytes}, + {name: version, type: Bytes}, + {name: location, type: Bytes} + ] +} + +structure{ + name: Limits, + type_parameters: [], + parameters: [], + fields: [ + {name: maxFileBytes, type: Nat}, + {name: maxTotalSourceBytes, type: Nat}, + {name: maxPrimitiveAtoms, type: Nat}, + {name: maxTokenLatticeEdges, type: Nat}, + {name: maxParseStates, type: Nat}, + {name: maxIrNodes, type: Nat}, + {name: maxScopeDepth, type: Nat}, + {name: maxImportDepth, type: Nat}, + {name: maxDiagnostics, type: Nat}, + {name: maxChildOutputBytes, type: Nat}, + {name: childTimeoutMs, type: Nat} + ] +} + +structure{ + name: PdfProvider, + type_parameters: [], + parameters: [], + fields: [ + {name: program, type: Bytes}, + {name: programSha256, type: Bytes}, + {name: versionArgv, type: (List Bytes)}, + {name: versionStdoutSha256, type: Bytes}, + {name: compileArgv, type: (List Bytes)}, + {name: output, type: Bytes}, + {name: resources, type: (List Bytes)} + ] +} + +structure{ + name: ProjectConfig, + type_parameters: [], + parameters: [], + fields: [ + {name: projectName, type: Bytes}, + {name: language, type: Bytes}, + {name: modulePrefix, type: Bytes}, + {name: sourceRoots, type: (List Bytes)}, + {name: entrypoints, type: (List Bytes)}, + {name: buildRoot, type: Bytes}, + {name: lockfile, type: Bytes}, + {name: leanWorkspace, type: Bytes}, + {name: lexiconSources, type: (List LexiconSource)}, + {name: artifactSources, type: (List ArtifactSource)}, + {name: sourcePackages, type: (List SourcePackage)}, + {name: limits, type: Limits}, + {name: pdf, type: (Option PdfProvider)} + ] +} + +definition{ + name: lit, + parameters: [literal : String], + result: Bytes, + body: (prim utf8_encode literal : Bytes), + executable: true +} + +definition{ + name: cat, + parameters: [left : Bytes, right : Bytes], + result: Bytes, + body: (prim append left right : Bytes), + executable: true +} + +definition{ + name: stringLine, + parameters: [key : String, value : Bytes], + result: Bytes, + body: (LockText::stringLine (lit key) value), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: arrayLine, + parameters: [key : String, values : (List Bytes)], + result: Bytes, + body: (LockText::line (lit key) (LockText::tomlArray values)), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: digestLine, + parameters: [key : String, digest : Bytes], + result: Bytes, + body: (LockText::digestLine (lit key) digest), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: numberLine, + parameters: [key : String, value : Nat], + result: Bytes, + body: (LockText::line (lit key) (LexLeanStd.Numeral::decimal value)), + executable: true +} + +definition{ + name: header, + parameters: [table : String], + result: Bytes, + body: (LockText::header (lit table)), + executable: true +} + +definition{ + name: packageOf, + parameters: [source : LexiconSource], + result: Bytes, + body: (match source + | LexiconSource.builtin named => named + | LexiconSource.path named location => named + | LexiconSource.git named url revision subdirectory => named), + executable: true +} + +definition{ + name: sourcesSorted, + parameters: [rows : (List LexiconSource)], + result: (List LexiconSource), + body: (prim list_fold (fun (acc : (List LexiconSource)) (entry : (Bytes * LexiconSource)) => (cons (second entry) acc)) (nil LexiconSource) (LexLeanStd.Order::reverseList [(Bytes * LexiconSource)] (LexLeanStd.Order::sortByKey [LexiconSource] (LexLeanStd.Order::reverseList [(Bytes * LexiconSource)] (prim list_fold (fun (acc : (List (Bytes * LexiconSource))) (row : LexiconSource) => (cons ((packageOf row), row) acc)) (nil (Bytes * LexiconSource)) rows : (List (Bytes * LexiconSource)))))) : (List LexiconSource)), + axioms: [propext], + executable: true +} + +definition{ + name: sourceText, + parameters: [source : LexiconSource], + result: Bytes, + body: (cat (header "[[lexicon_source]]") (match source + | LexiconSource.builtin named => (cat (stringLine "package" named) (stringLine "kind" (lit "builtin"))) + | LexiconSource.path named location => (cat (cat (stringLine "package" named) (stringLine "kind" (lit "path"))) (stringLine "path" location)) + | LexiconSource.git named url revision subdirectory => (cat (cat (cat (cat (stringLine "package" named) (stringLine "kind" (lit "git"))) (stringLine "url" url)) (stringLine "revision" revision)) (stringLine "subdirectory" subdirectory)))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: sourcesText, + parameters: [rows : (List LexiconSource)], + result: Bytes, + body: (match rows + | List.nil => (cat (lit "lexicon_source = []") 0x0a) + | List.cons leading others => (prim list_fold (fun (acc : Bytes) (row : LexiconSource) => (cat acc (sourceText row))) 0x (sourcesSorted rows) : Bytes)), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: artifactText, + parameters: [artifact : ArtifactSource], + result: Bytes, + body: (cat (cat (header "[[artifact_source]]") (digestLine "sha256" (project artifact digest))) (stringLine "path" (project artifact location))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: packageText, + parameters: [member : SourcePackage], + result: Bytes, + body: (cat (cat (cat (cat (header "[[source_package]]") (stringLine "package" (project member packageName))) (stringLine "version" (project member version))) (stringLine "kind" (lit "path"))) (stringLine "path" (project member location))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: limitsText, + parameters: [given : Limits], + result: Bytes, + body: (cat (cat (cat (cat (cat (cat (cat (cat (cat (cat (cat (header "[limits]") (numberLine "max_file_bytes" (project given maxFileBytes))) (numberLine "max_total_source_bytes" (project given maxTotalSourceBytes))) (numberLine "max_primitive_atoms" (project given maxPrimitiveAtoms))) (numberLine "max_token_lattice_edges" (project given maxTokenLatticeEdges))) (numberLine "max_parse_states" (project given maxParseStates))) (numberLine "max_ir_nodes" (project given maxIrNodes))) (numberLine "max_scope_depth" (project given maxScopeDepth))) (numberLine "max_import_depth" (project given maxImportDepth))) (numberLine "max_diagnostics" (project given maxDiagnostics))) (numberLine "max_child_output_bytes" (project given maxChildOutputBytes))) (numberLine "child_timeout_ms" (project given childTimeoutMs))), + executable: true +} + +definition{ + name: pdfText, + parameters: [provider : PdfProvider], + result: Bytes, + body: (cat (cat (cat (cat (cat (cat (cat (cat (header "[pdf]") (stringLine "mode" (lit "external"))) (stringLine "program" (project provider program))) (digestLine "program_sha256" (project provider programSha256))) (arrayLine "version_argv" (project provider versionArgv))) (digestLine "version_stdout_sha256" (project provider versionStdoutSha256))) (arrayLine "compile_argv" (project provider compileArgv))) (stringLine "output" (project provider output))) (arrayLine "resources" (project provider resources))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: canonicalProject, + parameters: [config : ProjectConfig], + result: Bytes, + body: (let head : Bytes := (cat (cat (cat (cat (cat (cat (cat (cat (cat (stringLine "spec" (lit "lexlean/project/1")) (stringLine "name" (project config projectName))) (stringLine "language" (project config language))) (stringLine "module_prefix" (project config modulePrefix))) (arrayLine "source_roots" (project config sourceRoots))) (arrayLine "entrypoints" (project config entrypoints))) (stringLine "build_root" (project config buildRoot))) (stringLine "lockfile" (project config lockfile))) (stringLine "lean_workspace" (project config leanWorkspace))) (stringLine "lean_toolchain" (LockText::leanToolchain))); (let artifacts : Bytes := (prim list_fold (fun (acc : Bytes) (row : ArtifactSource) => (cat acc (artifactText row))) 0x (project config artifactSources) : Bytes); (let packages : Bytes := (prim list_fold (fun (acc : Bytes) (row : SourcePackage) => (cat acc (packageText row))) 0x (project config sourcePackages) : Bytes); (let tail : Bytes := (match (project config pdf) + | Option.some provider => (pdfText provider) + | Option.none => 0x); (cat (cat (cat (cat (cat head (sourcesText (project config lexiconSources))) artifacts) packages) (limitsText (project config limits))) tail))))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} + +definition{ + name: configSha256, + parameters: [config : ProjectConfig], + result: Bytes, + body: (LexLeanStd.Sha256::sha256Hex (canonicalProject config)), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} +\end{semanticmodule} +\end{lexlean} diff --git a/crates/conformance/src/cases/bootstrap.rs b/crates/conformance/src/cases/bootstrap.rs index 51d4c3bced..58029de784 100644 --- a/crates/conformance/src/cases/bootstrap.rs +++ b/crates/conformance/src/cases/bootstrap.rs @@ -23,6 +23,7 @@ use crate::support::{self, P}; mod gnaf_validate; mod lock_text; +mod project_text; mod runtime_validate; mod rust_assemble; mod rust_bodies; @@ -2459,6 +2460,7 @@ fn case(id: &str) { // §17.23, §17.20: the canonical LaTeX of a tooling module. "BS-20" => tooling_latex::case(), "BS-31" => lock_text::case(), + "BS-33" => project_text::case(), other => panic!("no bootstrap case is wired for {other}"), } } diff --git a/crates/conformance/src/cases/bootstrap/project_text.rs b/crates/conformance/src/cases/bootstrap/project_text.rs new file mode 100644 index 0000000000..ff05ff566e --- /dev/null +++ b/crates/conformance/src/cases/bootstrap/project_text.rs @@ -0,0 +1,829 @@ +//! `BS-33` (SPEC.md §10.1, §17.23): the canonical text of the project +//! manifest and its digest as language-1.2 source, `LexLeanBoot.ProjectText`, +//! held to the byte to `lexlean::config::ProjectConfig::{canonical_toml, +//! config_sha256}`. +//! +//! `canonicalProject` writes a configuration as `lexlean lock` canonicalizes +//! it: the ten header keys in their fixed order with the pinned Lean +//! toolchain, the lexicon sources sorted by package (or the empty array when +//! there is none), the artifact sources and the source packages in their +//! order, the eleven limits, and the PDF provider; `configSha256` is the +//! lower-case hex SHA-256 of that text. It writes through the TOML writers of +//! `BS-31`'s `LockText`. Every committed `lexlean.toml` that parses is a run. +//! +//! Each planted defect names the one vector that catches it. + +use std::collections::{BTreeMap, BTreeSet}; + +use lexlean::calculus::Value; +use lexlean::config::{ + parse_project, ArtifactSource, LexiconSource, Limits, PdfProvider, ProjectConfig, SourcePackage, +}; +use lexlean::Sha256Digest; +use serde_json::Value as Json2; + +use super::rust_validate::{eligibility_of, rustc_differential}; +use super::{bytes, list, option, text, texts, Run, Seeded}; +use crate::production_export as harness; +use crate::support::{self, repo_root}; + +const MODULE: &str = "LexLeanBoot.ProjectText"; +const ROOTS: [(&str, &[&str]); 2] = [ + ("canonicalProject", &["allocation", "overflow"]), + ("configSha256", &["allocation", "overflow"]), +]; + +/// The most a part of a certificate of a root may be, as the row states. +const CERTIFICATE_BUDGET: u64 = 3_145_728; +/// Seeded runs of each root under the evaluator, and under rustc. +const EVALUATOR_SEEDED: usize = 200; +const RUSTC_SEEDED: usize = 1_000; + +// ------------------------------------------------------------- the encoders + +fn nat(value: u64) -> Value { + Value::Nat { + value: value.to_string(), + } +} + +fn adt(constructor: u64, fields: Vec) -> Value { + Value::Adt { + constructor, + fields, + } +} + +fn enc_source(source: &LexiconSource) -> Value { + match source { + LexiconSource::Builtin { package } => adt(0, vec![text(package)]), + LexiconSource::Path { package, path } => adt(1, vec![text(package), text(path)]), + LexiconSource::Git { + package, + url, + revision, + subdirectory, + } => adt( + 2, + vec![text(package), text(url), text(revision), text(subdirectory)], + ), + } +} + +fn enc_limits(limits: &Limits) -> Value { + adt( + 0, + vec![ + nat(limits.max_file_bytes), + nat(limits.max_total_source_bytes), + nat(limits.max_primitive_atoms), + nat(limits.max_token_lattice_edges), + nat(limits.max_parse_states), + nat(limits.max_ir_nodes), + nat(limits.max_scope_depth), + nat(limits.max_import_depth), + nat(limits.max_diagnostics), + nat(limits.max_child_output_bytes), + nat(limits.child_timeout_ms), + ], + ) +} + +fn enc_pdf(pdf: &PdfProvider) -> Value { + adt( + 0, + vec![ + text(&pdf.program), + bytes(&pdf.program_sha256.0), + texts(&pdf.version_argv), + bytes(&pdf.version_stdout_sha256.0), + texts(&pdf.compile_argv), + text(&pdf.output), + texts(&pdf.resources), + ], + ) +} + +fn enc(config: &ProjectConfig) -> Value { + adt( + 0, + vec![ + text(&config.name), + text(&config.language), + text(&config.module_prefix), + texts(&config.source_roots), + texts(&config.entrypoints), + text(&config.build_root), + text(&config.lockfile), + text(&config.lean_workspace), + list(config.lexicon_sources.iter().map(enc_source).collect()), + list( + config + .artifact_sources + .iter() + .map(|artifact| adt(0, vec![bytes(&artifact.sha256.0), text(&artifact.path)])) + .collect(), + ), + list( + config + .source_packages + .iter() + .map(|package| { + adt( + 0, + vec![ + text(&package.package), + text(&package.version), + text(&package.path), + ], + ) + }) + .collect(), + ), + enc_limits(&config.limits), + option(config.pdf.as_ref().map(enc_pdf)), + ], + ) +} + +fn text_run(config: &ProjectConfig) -> Run { + ( + vec![enc(config)], + serde_json::to_value(text(&config.canonical_toml())).expect("a value is JSON"), + ) +} + +fn digest_run(config: &ProjectConfig) -> Run { + ( + vec![enc(config)], + serde_json::to_value(text(&config.config_sha256().to_hex())).expect("a value is JSON"), + ) +} + +// ------------------------------------------------------------------ corpora + +fn manifests_under(dir: &std::path::Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by_key(std::fs::DirEntry::file_name); + for entry in entries { + let name = entry.file_name().to_string_lossy().into_owned(); + let path = entry.path(); + if path.is_dir() { + if !matches!( + name.as_str(), + ".git" | "target" | ".lexlean" | "node_modules" + ) { + manifests_under(&path, out); + } + } else if name == "lexlean.toml" { + out.push(path); + } + } +} + +/// Every committed manifest that parses, how many committed manifests there +/// are, and how many of those that parse are their own canonical text. +fn committed() -> (Vec, usize, usize) { + let mut paths = Vec::new(); + manifests_under(repo_root().as_std_path(), &mut paths); + let found = paths.len(); + let mut canonical = 0usize; + let configs = paths + .iter() + .filter_map(|path| { + let bytes = std::fs::read(path).expect("a committed manifest"); + let config = parse_project(&path.to_string_lossy(), &bytes).ok()?; + if config.canonical_toml().as_bytes() == bytes.as_slice() { + canonical += 1; + } + Some(config) + }) + .collect(); + (configs, found, canonical) +} + +// ------------------------------------------------------------------ vectors + +fn sha(byte: u8) -> Sha256Digest { + let mut out = [0u8; 32]; + for (index, slot) in out.iter_mut().enumerate() { + *slot = byte.wrapping_add(u8::try_from(index).expect("a byte")); + } + Sha256Digest(out) +} + +fn strings(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} + +fn limits() -> Limits { + Limits { + max_file_bytes: 4_194_304, + max_total_source_bytes: 67_108_864, + max_primitive_atoms: 1_000_000, + max_token_lattice_edges: 2_000_000, + max_parse_states: 3_000_000, + max_ir_nodes: 4_000_000, + max_scope_depth: 256, + max_import_depth: 64, + max_diagnostics: 100, + max_child_output_bytes: 16_777_216, + child_timeout_ms: 300_000, + } +} + +fn minimal() -> ProjectConfig { + ProjectConfig { + name: "demo".to_owned(), + language: "1.2".to_owned(), + module_prefix: "Demo".to_owned(), + source_roots: strings(&["src"]), + entrypoints: strings(&["src/Main.lex.tex"]), + build_root: ".lexlean/build".to_owned(), + lockfile: "lexlean.lock".to_owned(), + lean_workspace: ".lexlean/lean".to_owned(), + lexicon_sources: Vec::new(), + artifact_sources: Vec::new(), + source_packages: Vec::new(), + limits: limits(), + pdf: None, + } +} + +fn builtin(package: &str) -> LexiconSource { + LexiconSource::Builtin { + package: package.to_owned(), + } +} + +fn pdf() -> PdfProvider { + PdfProvider { + program: "lualatex".to_owned(), + program_sha256: sha(0x30), + version_argv: strings(&["--version"]), + version_stdout_sha256: sha(0x40), + compile_argv: strings(&["-interaction=nonstopmode", "main.tex"]), + output: "main.pdf".to_owned(), + resources: strings(&["fonts/a.otf", "logo.png"]), + } +} + +struct Vector { + label: &'static str, + root: usize, + run: Run, +} + +#[allow(clippy::too_many_lines)] +fn named() -> Vec { + let mut out = Vec::new(); + let mut add = |label: &'static str, root: usize, run: Run| { + out.push(Vector { label, root, run }); + }; + add("a minimal project", 0, text_run(&minimal())); + add( + "one builtin source", + 0, + text_run(&ProjectConfig { + lexicon_sources: vec![builtin("lexlean.std.nat")], + ..minimal() + }), + ); + add( + "sources out of order", + 0, + text_run(&ProjectConfig { + lexicon_sources: vec![builtin("z.pkg"), builtin("a.pkg"), builtin("m.pkg")], + ..minimal() + }), + ); + add( + "every kind of source", + 0, + text_run(&ProjectConfig { + lexicon_sources: vec![ + LexiconSource::Git { + package: "c.git".to_owned(), + url: "https://example.org/r.git".to_owned(), + revision: "0123456789abcdef0123456789abcdef01234567".to_owned(), + subdirectory: "lexicon".to_owned(), + }, + LexiconSource::Path { + package: "b.path".to_owned(), + path: "../lexicon".to_owned(), + }, + builtin("a.builtin"), + ], + ..minimal() + }), + ); + add( + "equal packages keep their order", + 0, + text_run(&ProjectConfig { + lexicon_sources: vec![ + LexiconSource::Path { + package: "same".to_owned(), + path: "first".to_owned(), + }, + LexiconSource::Path { + package: "same".to_owned(), + path: "second".to_owned(), + }, + ], + ..minimal() + }), + ); + add( + "an artifact source", + 0, + text_run(&ProjectConfig { + artifact_sources: vec![ArtifactSource { + sha256: sha(0xab), + path: "models/m.bin".to_owned(), + }], + ..minimal() + }), + ); + add( + "a source package", + 0, + text_run(&ProjectConfig { + source_packages: vec![SourcePackage { + package: "lexlean.stdlib".to_owned(), + version: "1.2.0".to_owned(), + path: "../stdlib".to_owned(), + }], + ..minimal() + }), + ); + add( + "a pdf provider", + 0, + text_run(&ProjectConfig { + pdf: Some(pdf()), + ..minimal() + }), + ); + add( + "escapes in the name", + 0, + text_run(&ProjectConfig { + name: "a \"quoted\"\\name\twith\u{7f}".to_owned(), + ..minimal() + }), + ); + add( + "limits at their largest", + 0, + text_run(&ProjectConfig { + limits: Limits { + max_file_bytes: u64::MAX, + child_timeout_ms: 0, + ..limits() + }, + ..minimal() + }), + ); + add("the digest of a minimal project", 1, digest_run(&minimal())); + add( + "the digest of a pdf project", + 1, + digest_run(&ProjectConfig { + pdf: Some(pdf()), + ..minimal() + }), + ); + out +} + +// ------------------------------------------------------------------- seeded + +const PIECES: &[&str] = &[ + "a", "z", "0", ".", "/", "-", " ", "\"", "\\", "\n", "\u{1f}", "\u{7f}", "\u{e9}", +]; +const PACKAGES: &[&str] = &["a", "a.b", "b", "lexlean.std.nat", ""]; + +fn drawn_text(seeded: &mut Seeded) -> String { + let count = seeded.below(7); + (0..count).map(|_| *seeded.pick(PIECES)).collect() +} + +fn drawn_texts(seeded: &mut Seeded) -> Vec { + let count = seeded.below(4); + (0..count).map(|_| drawn_text(seeded)).collect() +} + +fn drawn_sha(seeded: &mut Seeded) -> Sha256Digest { + let mut out = [0u8; 32]; + for slot in &mut out { + *slot = u8::try_from(seeded.below(256)).expect("a byte"); + } + Sha256Digest(out) +} + +fn drawn_number(seeded: &mut Seeded) -> u64 { + match seeded.below(4) { + 0 => 0, + 1 => u64::try_from(seeded.below(1_000)).expect("a number"), + 2 => seeded.next(), + _ => u64::MAX, + } +} + +fn drawn_config(seeded: &mut Seeded) -> ProjectConfig { + let lexicon_sources = (0..seeded.below(5)) + .map(|_| { + let package = (*seeded.pick(PACKAGES)).to_owned(); + match seeded.below(3) { + 0 => LexiconSource::Builtin { package }, + 1 => LexiconSource::Path { + package, + path: drawn_text(seeded), + }, + _ => LexiconSource::Git { + package, + url: drawn_text(seeded), + revision: drawn_text(seeded), + subdirectory: drawn_text(seeded), + }, + } + }) + .collect(); + let artifact_sources = (0..seeded.below(3)) + .map(|_| ArtifactSource { + sha256: drawn_sha(seeded), + path: drawn_text(seeded), + }) + .collect(); + let source_packages = (0..seeded.below(3)) + .map(|_| SourcePackage { + package: drawn_text(seeded), + version: drawn_text(seeded), + path: drawn_text(seeded), + }) + .collect(); + let limits = Limits { + max_file_bytes: drawn_number(seeded), + max_total_source_bytes: drawn_number(seeded), + max_primitive_atoms: drawn_number(seeded), + max_token_lattice_edges: drawn_number(seeded), + max_parse_states: drawn_number(seeded), + max_ir_nodes: drawn_number(seeded), + max_scope_depth: drawn_number(seeded), + max_import_depth: drawn_number(seeded), + max_diagnostics: drawn_number(seeded), + max_child_output_bytes: drawn_number(seeded), + child_timeout_ms: drawn_number(seeded), + }; + let pdf = if seeded.chance(30) { + Some(PdfProvider { + program: drawn_text(seeded), + program_sha256: drawn_sha(seeded), + version_argv: drawn_texts(seeded), + version_stdout_sha256: drawn_sha(seeded), + compile_argv: drawn_texts(seeded), + output: drawn_text(seeded), + resources: drawn_texts(seeded), + }) + } else { + None + }; + ProjectConfig { + name: drawn_text(seeded), + language: (*seeded.pick(&["1.0", "1.1", "1.2"])).to_owned(), + module_prefix: drawn_text(seeded), + source_roots: drawn_texts(seeded), + entrypoints: drawn_texts(seeded), + build_root: drawn_text(seeded), + lockfile: drawn_text(seeded), + lean_workspace: drawn_text(seeded), + lexicon_sources, + artifact_sources, + source_packages, + limits, + pdf, + } +} + +fn seeded_runs(count: usize, seed: u64) -> [Vec; 2] { + let mut seeded = Seeded(seed); + let mut out: [Vec; 2] = [Vec::new(), Vec::new()]; + for _ in 0..count { + let config = drawn_config(&mut seeded); + out[0].push(text_run(&config)); + out[1].push(digest_run(&config)); + } + out +} + +// ------------------------------------------------------------------- plants + +struct Plant { + label: &'static str, + root: usize, + site: &'static str, + replacement: &'static str, + vector: &'static str, +} + +const PLANTS: &[Plant] = &[ + Plant { + label: "the empty source list is omitted", + root: 0, + site: "(cat (lit \"lexicon_source = []\") 0x0a)", + replacement: "0x", + vector: "a minimal project", + }, + Plant { + label: "the toolchain line is missing", + root: 0, + site: "(stringLine \"lean_toolchain\" (LockText::leanToolchain))", + replacement: "0x", + vector: "one builtin source", + }, + Plant { + label: "the sources are not sorted", + root: 0, + site: "0x (sourcesSorted rows) : Bytes)", + replacement: "0x rows : Bytes)", + vector: "sources out of order", + }, + Plant { + label: "equal packages are reordered", + root: 0, + site: "(LexLeanStd.Order::sortByKey [LexiconSource] (LexLeanStd.Order::reverseList [(Bytes * LexiconSource)] (prim list_fold", + replacement: "(LexLeanStd.Order::sortByKey [LexiconSource] (LexLeanStd.Order::sortByKey [LexiconSource] (prim list_fold", + vector: "equal packages keep their order", + }, + Plant { + label: "a git source has no subdirectory", + root: 0, + site: "(stringLine \"subdirectory\" subdirectory)", + replacement: "0x", + vector: "every kind of source", + }, + Plant { + label: "the artifact path is its digest", + root: 0, + site: "(stringLine \"path\" (project artifact location))", + replacement: "(stringLine \"path\" (project artifact digest))", + vector: "an artifact source", + }, + Plant { + label: "a source package has no kind", + root: 0, + site: "(stringLine \"version\" (project member version))) (stringLine \"kind\" (lit \"path\")))", + replacement: "(stringLine \"version\" (project member version))) 0x)", + vector: "a source package", + }, + Plant { + label: "the scope depth is the import depth", + root: 0, + site: "(numberLine \"max_scope_depth\" (project given maxScopeDepth))", + replacement: "(numberLine \"max_scope_depth\" (project given maxImportDepth))", + vector: "a minimal project", + }, + Plant { + label: "the pdf mode is internal", + root: 0, + site: "(lit \"external\")", + replacement: "(lit \"internal\")", + vector: "a pdf provider", + }, + Plant { + label: "the resources are the compile argv", + root: 0, + site: "(arrayLine \"resources\" (project provider resources))", + replacement: "(arrayLine \"resources\" (project provider compileArgv))", + vector: "a pdf provider", + }, + Plant { + label: "the spec is another", + root: 1, + site: "(lit \"lexlean/project/1\")", + replacement: "(lit \"lexlean/project/2\")", + vector: "the digest of a minimal project", + }, + Plant { + label: "the digest is of the name", + root: 1, + site: "(LexLeanStd.Sha256::sha256Hex (canonicalProject config))", + replacement: "(LexLeanStd.Sha256::sha256Hex (project config projectName))", + vector: "the digest of a pdf project", + }, +]; + +fn plants(vectors: &[Vector]) { + let mut missed = Vec::new(); + for plant in PLANTS { + let vector = vectors + .iter() + .find(|vector| vector.label == plant.vector) + .unwrap_or_else(|| panic!("{}: no vector `{}`", plant.label, plant.vector)); + assert_eq!( + vector.root, plant.root, + "{}: the vector is a run of the root planted", + plant.label + ); + let planted = support::P::bootstrap(); + let source = planted.read_raw("src/ProjectText.lex.tex"); + assert_eq!( + source.matches(plant.site).count(), + 1, + "{}: the plant site occurs once", + plant.label + ); + planted.write( + "src/ProjectText.lex.tex", + &source.replacen(plant.site, plant.replacement, 1), + ); + let checked = support::checked_project(&planted); + let low = super::lower(&checked, &planted, MODULE, ROOTS[plant.root].0); + match super::evaluate(&low, std::slice::from_ref(&vector.run), 1).first() { + Some(first) => eprintln!( + "BS-33: plant detected: {}: {}", + plant.label, + first.chars().take(160).collect::() + ), + None => missed.push(plant.label), + } + } + assert!( + missed.is_empty(), + "planted defects their own vector does not detect: {missed:?}" + ); +} + +fn export(lowered: &BTreeMap<&str, super::Lowered>) { + let exported = harness::exported("bootstrap"); + let bundle = exported.bundle(); + let mut realized = 0usize; + for exported_root in &exported.output.result.roots { + let Some((member, low)) = exported_root + .root + .strip_prefix(&format!("{MODULE}.")) + .and_then(|member| lowered.get(member).map(|low| (member, low))) + else { + continue; + }; + let largest = exported_root + .outputs + .iter() + .flat_map(|output| &output.certification.certificates) + .map(|certificate| certificate.file.byte_length) + .max() + .expect("a certificate"); + eprintln!( + "BS-33: {}: largest certificate part {largest} bytes", + exported_root.root + ); + assert!( + largest <= CERTIFICATE_BUDGET, + "{}: a certificate part of {largest} bytes is beyond {CERTIFICATE_BUDGET}", + exported_root.root + ); + for output in &exported_root.outputs { + let package: Json2 = serde_json::from_slice( + &std::fs::read( + bundle + .join(&output.package.directory) + .join("package.json") + .as_std_path(), + ) + .expect("package.json"), + ) + .expect("package JSON"); + assert_eq!( + package["program"], + serde_json::to_value(&low.program).expect("a program is JSON"), + "{member}: the exported package realizes the program the differential ran" + ); + realized += 1; + } + } + assert_eq!(realized, ROOTS.len(), "one certified package per root"); + let packages = harness::tree(&bundle.join("packages")); + let committed_packages = harness::tree(&harness::fixture_dir("bootstrap").join("packages")); + assert_eq!( + packages.keys().collect::>(), + committed_packages.keys().collect::>(), + "the exported package files are the committed ones" + ); + for (relative, content) in &packages { + assert!( + committed_packages.get(relative) == Some(content), + "packages/{relative} differs from the committed package" + ); + } +} + +#[allow(clippy::too_many_lines)] +pub(super) fn case() { + let started = std::time::Instant::now(); + let project = support::P::bootstrap(); + let checked_units = project.check_ok(); + assert!( + checked_units.units.contains_key("ProjectText") + && checked_units.units.contains_key("LockText"), + "ProjectText and LockText are linked modules of the project" + ); + eligibility_of(&project, "ProjectText", MODULE, &ROOTS); + + let vectors = named(); + let labels: BTreeSet<&str> = vectors.iter().map(|vector| vector.label).collect(); + assert_eq!(labels.len(), vectors.len(), "each vector has its own label"); + let mut runs: [Vec; 2] = [Vec::new(), Vec::new()]; + for vector in &vectors { + runs[vector.root].push(vector.run.clone()); + } + + // Every committed manifest that parses, both as text and as a digest. + let (configs, found, canonical) = committed(); + assert!( + configs.iter().any(|config| config.pdf.is_some()) + && configs + .iter() + .any(|config| !config.source_packages.is_empty()) + && configs + .iter() + .any(|config| config.lexicon_sources.is_empty()), + "the committed manifests reach a pdf provider, a source package, and no lexicon source" + ); + let corpus: [Vec; 2] = [ + configs.iter().map(text_run).collect(), + configs.iter().map(digest_run).collect(), + ]; + let normative = support::normative_host(); + let (evaluator_seeded, rustc_seeded) = if normative { + ( + seeded_runs(EVALUATOR_SEEDED, 0x0B55_0033), + seeded_runs(RUSTC_SEEDED, 0x0B55_1033), + ) + } else { + ([Vec::new(), Vec::new()], [Vec::new(), Vec::new()]) + }; + eprintln!( + "BS-33: {} vectors ({} texts, {} digests), {} of {} committed manifests parse ({} are their canonical text), {} and {} seeded runs of each root ({:?})", + vectors.len(), + runs[0].len(), + runs[1].len(), + configs.len(), + found, + canonical, + EVALUATOR_SEEDED, + RUSTC_SEEDED, + started.elapsed() + ); + + let checked = support::checked_project(&project); + let lowered: BTreeMap<&str, super::Lowered> = ROOTS + .iter() + .map(|(root, _)| (*root, super::lower(&checked, &project, MODULE, root))) + .collect(); + if normative { + for (index, (root, _)) in ROOTS.iter().enumerate() { + let mut all_runs: Vec = runs[index].clone(); + all_runs.extend(corpus[index].iter().cloned()); + all_runs.extend(evaluator_seeded[index].iter().cloned()); + let disagreements = super::evaluate(&lowered[root], &all_runs, 5); + assert!( + disagreements.is_empty(), + "{root} and the Rust oracle disagree: {disagreements:?}" + ); + eprintln!( + "BS-33: {root}: the evaluator agrees on {} runs ({:?})", + all_runs.len(), + started.elapsed() + ); + } + plants(&vectors); + } else { + eprintln!("BS-33: platform-bound host: the evaluator, the seeded runs, and the planted defects run on the normative host"); + } + + let mut compiled: [Vec; 2] = [Vec::new(), Vec::new()]; + for index in 0..2 { + compiled[index].extend(runs[index].iter().cloned()); + compiled[index].extend(corpus[index].iter().cloned()); + compiled[index].extend(rustc_seeded[index].iter().cloned()); + } + let total = rustc_differential( + "BS-33", + &[ + (ROOTS[0].0, &lowered[ROOTS[0].0], compiled[0].as_slice()), + (ROOTS[1].0, &lowered[ROOTS[1].0], compiled[1].as_slice()), + ], + "oracle", + ); + eprintln!("BS-33: {total} rustc runs agree ({:?})", started.elapsed()); + + if !support::lean_backed("BS-33") { + return; + } + export(&lowered); + eprintln!("BS-33: passed in {:?}", started.elapsed()); +} diff --git a/crates/conformance/src/cases/repository.rs b/crates/conformance/src/cases/repository.rs index 10772fbae2..7719d0aea2 100644 --- a/crates/conformance/src/cases/repository.rs +++ b/crates/conformance/src/cases/repository.rs @@ -898,7 +898,7 @@ pub(crate) fn run(id: &str) { let model = repo_model::Model::load(&root.join("model").into_std_path_buf()) .expect("the model loads"); let table = spec_table(); - assert_eq!(table.len(), 479, "§31 has 479 rows"); + assert_eq!(table.len(), 480, "§31 has 480 rows"); assert_eq!(model.ids.id.len(), table.len(), "register row count"); for ((spec_id, spec_suite, spec_statement), row) in table.iter().zip(model.ids.id.iter()) diff --git a/crates/conformance/tests/conformance.rs b/crates/conformance/tests/conformance.rs index f40b14575f..187f87d1ff 100644 --- a/crates/conformance/tests/conformance.rs +++ b/crates/conformance/tests/conformance.rs @@ -2325,6 +2325,11 @@ fn conformance_bs_31() { repo_conformance::cases::run("BS-31"); } +#[test] +fn conformance_bs_33() { + repo_conformance::cases::run("BS-33"); +} + #[test] fn conformance_pk_01() { repo_conformance::cases::run("PK-01"); diff --git a/features/suites/bootstrap.feature b/features/suites/bootstrap.feature index 66ebfdc2f9..195c974717 100644 --- a/features/suites/bootstrap.feature +++ b/features/suites/bootstrap.feature @@ -126,3 +126,10 @@ Feature: bootstrap When the project is checked and built, and the lowered program and the rendered package of each root are run on those locks and texts Then each root is eligible for rust-std with exactly its declared effects, no certificate part is beyond 3,145,728 bytes, and the two roots equal `Lock::canonical_bytes` and `toml_string` on every run And every planted defect is detected by its own vector, and the exported packages are byte-identical to the committed packages and realize the programs the differential ran + + @BS-33 @build + Scenario: The canonical text of the project manifest of SPEC.md §10.1 and its digest (`config::ProjectConfig::canonical_toml`: the ten header keys in their fixed order with the pinned Lean toolchain, the lexicon sources sorted by package or the empty array when there is none, the artifact sources and the source packages in their order, the eleven limits in their fixed order, and the PDF provider; and `config_sha256`: the lower-case hex SHA-256 of that text) is language-1.2 source in the `bootstrap/` project, as two production roots of `ProjectText` over the TOML writers of `BS-31`: `canonicalProject` and `configSha256`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built configurations, which reach every kind of lexicon source, the empty source list, equal packages, every optional table, escapes, and the largest limits, on every committed `lexlean.toml` that parses, and on seeded configurations; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading and validating a manifest is outside the claim. + Given the committed `bootstrap` project, hand-built configurations, every committed `lexlean.toml` that parses, and seeded configurations + When the project is checked and built, and the lowered program and the rendered package of each root are run on those configurations + Then each root is eligible for rust-std with exactly its declared effects, no certificate part is beyond 3,145,728 bytes, and the two roots equal `ProjectConfig::canonical_toml` and the hex of `config_sha256` on every run + And every planted defect is detected by its own vector, and the exported packages are byte-identical to the committed packages and realize the programs the differential ran diff --git a/model/ids.toml b/model/ids.toml index 07b5844b94..653eb22cdf 100644 --- a/model/ids.toml +++ b/model/ids.toml @@ -2792,6 +2792,12 @@ level = "build" suite = "bootstrap" statement = "The canonical text of the lock file of SPEC.md §11.2 (`lock::Lock::canonical_bytes`: the header keys in their fixed order with the lock spec of the language and the pinned Lean toolchain, the workspace pins sorted by path, the packages sorted by identifier and version, the source packages sorted by identifier, and the PDF record with its resources sorted by path, each digest in lower-case hex; and `config::toml_string`: the TOML basic string of a text, with the quote, the backslash, and every control escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `LockText`: `canonicalLock` and `tomlString`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built locks and texts, which reach every table, both lock specs, every order of rows the sort decides, and every class of escape, on every committed `lexlean.lock` that parses and every text in them, and on seeded locks and texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading a lock, resolving its packages, and hashing its files are outside the claim." +[[id]] +id = "BS-33" +level = "build" +suite = "bootstrap" +statement = "The canonical text of the project manifest of SPEC.md §10.1 and its digest (`config::ProjectConfig::canonical_toml`: the ten header keys in their fixed order with the pinned Lean toolchain, the lexicon sources sorted by package or the empty array when there is none, the artifact sources and the source packages in their order, the eleven limits in their fixed order, and the PDF provider; and `config_sha256`: the lower-case hex SHA-256 of that text) is language-1.2 source in the `bootstrap/` project, as two production roots of `ProjectText` over the TOML writers of `BS-31`: `canonicalProject` and `configSha256`, each eligible for `rust-std` with exactly its declared effects, `allocation` and `overflow`, and each with no part of its certificate beyond 3,145,728 bytes; their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on hand-built configurations, which reach every kind of lexicon source, the empty source list, equal packages, every optional table, escapes, and the largest limits, on every committed `lexlean.toml` that parses, and on seeded configurations; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; reading and validating a manifest is outside the claim." + [[id]] id = "PK-01" level = "build" diff --git a/tests/production-export/bootstrap/roots.json b/tests/production-export/bootstrap/roots.json index fcb6c89b24..e2e0aef789 100644 --- a/tests/production-export/bootstrap/roots.json +++ b/tests/production-export/bootstrap/roots.json @@ -32,6 +32,14 @@ ], "version": "1.0.0" }, + { + "name": "canonical_project", + "root": "LexLeanBoot.ProjectText.canonicalProject", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "check_arguments", "root": "LexLeanBoot.CalculusCheck.checkArguments", @@ -56,6 +64,14 @@ ], "version": "1.0.0" }, + { + "name": "config_sha256", + "root": "LexLeanBoot.ProjectText.configSha256", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "correspond_crate", "root": "LexLeanBoot.RustValidate.correspondCrate", diff --git a/xtask/src/spec_links.rs b/xtask/src/spec_links.rs index 962afcafd8..ec77840783 100644 --- a/xtask/src/spec_links.rs +++ b/xtask/src/spec_links.rs @@ -61,8 +61,8 @@ fn parse_table(spec: &str) -> Result, Fail> { pub fn validate(root: &Path) -> Result<(), Fail> { let spec = std::fs::read_to_string(root.join("SPEC.md"))?; let rows = parse_table(&spec)?; - if rows.len() != 479 { - return Err(format!("RP-07: the §31 table has {} rows, not 479", rows.len()).into()); + if rows.len() != 480 { + return Err(format!("RP-07: the §31 table has {} rows, not 480", rows.len()).into()); } // The prose total under the table is part of the contract a reader // relies on; a row added without it would leave §31 contradicting itself.