diff --git a/.github/bdd-parts.json b/.github/bdd-parts.json index 47aa46068f..bb185b1757 100644 --- a/.github/bdd-parts.json +++ b/.github/bdd-parts.json @@ -22,6 +22,9 @@ "bootstrap-surface-lex": [ "BS-22" ], + "bootstrap-surface-doc": [ + "BS-23" + ], "bootstrap-check": [ "BS-04" ], diff --git a/.github/vv-rows.json b/.github/vv-rows.json index 00037bec15..f9c5746c8b 100644 --- a/.github/vv-rows.json +++ b/.github/vv-rows.json @@ -134,6 +134,15 @@ "memory_kib": "", "hints": "" }, + { + "part": "bdd-bootstrap-surface-doc", + "recipes": "bdd", + "examples_part": "", + "bdd_part": "bootstrap-surface-doc", + "width": "", + "memory_kib": "", + "hints": "" + }, { "part": "bdd-bootstrap-terms", "recipes": "bdd", diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b4ef27d3b..3c7da6d162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -605,6 +605,11 @@ of this tree show. `SurfaceLex` return what `surface::lex::lex` and `surface::lex::quote` return, the tokens with their byte ranges or the first lexical error, and a quoted text; they admit no `recursion`. The register is 470 IDs. +- The layout algebra of the authoring notation as language-1.2 source, third + slice of the front end (SPEC.md §17.23, §17.21, `BS-23`, #269; no new + diagnostic code, no change to the language-1.2 identity). `renderDoc` of + `SurfaceDoc` returns what `surface::doc::render` returns for a document at a + width; it admits no `recursion`. The register is 471 IDs. - Explicit shared byte literals (#68, `SM-19`, `MD-02`; languages 1.1 and 1.2): a modeled byte literal is lowered as `_root_.ByteArray.mk #[…]` with every octet written `_root_.UInt8.ofNat (nat_lit n)`, an octet occurring four or diff --git a/CONFORMANCE.md b/CONFORMANCE.md index eff53b5154..413951bdbf 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -77,6 +77,7 @@ the error registry does not sanction (R5, `ERRORS.md`). | `BS-15` | `build` | The package renderer of SPEC.md §17.16 (the checks `package::lower` makes of a manifest's own fields before it reads the program: the spec, the crate name against the Rust keywords and the reserved crates, the version, the profile, the form and the order of the sources, and the effects against the registry, with `allocation` refused on `rust-core`; the Cargo manifest over the scaffold record; and the provenance in canonical JSON with the digests of the runtime, the Cargo manifest, and the library) is language-1.2 source in the `bootstrap/` project, as two production roots of `RustPackage` that the host composes around the roots of `BS-04` to `BS-14`: `checkManifest`, which for the fields of a manifest and the registered effects returns whether the profile is `rust-std`, or the refusal of `package::lower` to the byte, and `renderPackage`, which for the scaffold record, the manifest's name, version, profile, sources, effects and exports, the program's identity, the compiler-semantics identity, the two runtime texts, and the library returns the `Cargo.toml` and the `provenance.json` of `package::package` to the byte; each is eligible for `rust-std` with exactly its declared effects and has no part of its certificate beyond 3,145,728 bytes; the oracle of `checkManifest` is `package::lower` on the manifest with its program and exports replaced by a pair every later check accepts, which agrees with each committed negative manifest's refusal, and the oracle of `renderPackage` is `package::package` on the 282 committed manifests it accepts and, on 137 hand-built inputs (names, exports, sources and effects that reach the escaper, and scaffold records that differ from the committed one) and 282 seeded mutants, `cargo_toml_of` and the canonical JSON of §21.7 by the recipe `package::package` is held to; on the normative host the lowered programs under the calculus evaluator agree with the oracle on the 301 committed manifests, 182 rule manifests and 2,000 seeded mutants, and on the 241 renders whose library is at most 16,384 bytes; the rendered packages agree with the oracle under the pinned rustc on all 3,184 runs; 21 planted defects in the source are each detected; and the certified packages the production export publishes for the project are the committed ones; the checks of the program, the exports and the crate, and the bytes of the library, which are the rows `BS-04` to `BS-14`, are outside the claim. | | `BS-21` | `build` | The fixed structural grammar of SPEC.md §15.1 and §15.2 (`grammar::structural::parse_module`: the module header, the blocks and declarations, the axiom policies, the sentences, the structured proof environments, and the closed data forms, with the coverage rows of every structural atom it consumes) is language-1.2 source in the `bootstrap/` project: `parseModule` of `StructuralGrammar`, for the atoms `scanSource` of `BS-03` returns, the path, the name the path gives the module, and `max_scope_depth`, is a production root eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; it returns the module with its rows, or the refusal's code, message, and span, and on 267 hand-built vectors, 146 of them refused, which reach every refusal of `parse_module`, and on the committed modules, whole or by the atoms of both their ends, its lowered program under the calculus evaluator on the normative host and its rendered package under the pinned rustc on every host agree with the oracle to the byte; on the normative host both also agree on 2,000 seeded edits of those texts, each of 44 planted defects in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the grammars of mathematical islands, propositions, and proof sentences, which read the lexicon's closure, and the interpretation of the module this root returns are outside the claim. | | `BS-22` | `build` | The lexing of the authoring notation of SPEC.md §17.21 (`surface::lex::lex`: a text into its delimiters, `:=` and `::`, quoted strings with their escapes decoded, and words, which end at white space in the sense of `char::is_whitespace` or at a delimiter, each with its byte range, or the offset and message of the first lexical error; and `surface::lex::quote`: a text as the notation quotes it, with the quotation mark, the backslash, and every control character escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `SurfaceLex`: `lexText` and `quoteText`, each eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and with no part of its certificate beyond 3,145,728 bytes; on every host their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on the hand-built vectors, which reach every lexical error and every escape, on windows of the committed sources, and on seeded edits of those texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; the readers of the tokens (the generic notation and the sugar of the closed schemas), the pretty-printing algebra, and canonical JSON are outside the claim. | +| `BS-23` | `build` | The layout algebra of the authoring notation of SPEC.md §17.21 (`surface::doc::render`: a document of text, concatenation, indentation, lines, soft breaks, and groups, laid out at a width, a group on one line when it and what follows it up to the next line break fit in the rest of the line and its lines broken otherwise, with the width of a text counted in scalars) is language-1.2 source in the `bootstrap/` project, as the production root `renderDoc` of `SurfaceDoc`, which takes the document, the width, and the number of nodes of the document, is eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; a concatenation is a chain of cells, so that the root pops one node at a time under loops bounded by the number of nodes; its rendered `rust-std` package under the pinned rustc, and on the normative host its lowered program under the calculus evaluator, agree with the oracle to the byte on the hand-built documents, on the documents the printer builds from the committed JSON files at three widths, and on seeded documents; each planted defect in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the printers that build documents (the generic notation and the sugar of the closed schemas) are outside the claim. | ## calculus diff --git a/README.md b/README.md index e853d3fd96..7293133159 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ just vv # the complete normative acceptance gate (SPEC.md §9.2) just release # vv, then the §30 release criterion; refused until 1.0.0 ``` -All 470 registered conformance IDs are implemented and pass; `just vv` runs clean from a checkout with the pinned toolchain installed. +All 471 registered conformance IDs are implemented and pass; `just vv` runs clean from a checkout with the pinned toolchain installed. The differential against the independent generator `lean4-prod` (SPEC.md §17.19, `OR-01`..`OR-08`) acquires it at its pinned commit: from the checkout `LEXLEAN_ORACLE_DIR` names, or by fetching the commit with `git` into the Cargo target directory (`LEXLEAN_ORACLE_CACHE` overrides), and builds it with the pinned Lean and Rust toolchains. It is validation evidence only and no input to any identity. An oracle that cannot be acquired, is not the pinned files, or fails is unavailable: that fails the gate on Linux x86-64 and is printed as `NOT VALIDATED` elsewhere, and is never reported as agreement. @@ -190,7 +190,7 @@ Every row is validated by `just vv`; the IDs link the claim to its register row, | The language-1.2 authoring notation: a readable `.lex.tex` text for a semantic module, in bijection with its canonical JSON, whose every position and variant is read from the schema, with readable forms for types and terms and a generic form for every other value, formatted canonically by `fmt` failing closed, the only form in which a language-1.2 source of the product is written (the data form is refused under language 1.2, and `lexlean::sdk` is the explicit conversion), the same for the compiler-tooling modules in the generic notation, with no compiler-semantics ID that depends on which form an embedded module is written in, with no product source owned by a generator but five named records, read and written by programs only through the authoring API (`lexlean::sdk`), and, for a native core module, a readable program over the shared table of kernel terms whose numbering is the order of its text | `AU-01`..`AU-13` | `build` | | The runtime of generated Lean as native modules: the portable, collection, model, and reasoning runtimes of language 1.2 are LexLean source that every generated module imports, migrated losslessly and audited against their exact axioms, the released runtime text of languages 1.0 and 1.1 is frozen data equal to their released output, and no other Lean runtime text remains in the compiler but the decoder of core module 2 | `RT-01`..`RT-04` | `build` | | The standard library: language-1.2 source beside the compiler whose members are certified production roots, each validated against its oracle by a seeded differential of its evaluator and rustc realizations; bytes and text over `Bytes`, an output buffer of a given capacity written in order; decimal and hexadecimal; the byte order and stable sorts over it; SHA-256 against FIPS 180-4, the NIST CAVP vectors, and the `sha2` crate; and the `std-models` and `std-reasoning` source packages, generic over types, whose laws are kernel-checked theorems and whose application is the `helpdesk` example | `SL-01`, `SL-02`, `SL-03`, `SL-04`, `SL-05`, `SL-09`, `SL-10` | `build` | -| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the structural grammar, and the lexer of the authoring notation, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-22` | `build` | +| The bootstrap project: the compiler's own components as language-1.2 source compiled by LexLean's production pipeline to certified Rust packages, the exact axiom-output parser and its closed Lean-name check, the primitive scanner of §12.2, the preservation lexer and token audit, the target calculus's checker, the Rust crate validator, the Rust printer, the runtime syntax validator, the failure typing and the types of the Rust lowering, its last-use moves, its literal values and its terms, the package renderer, the structural grammar, the lexer of the authoring notation, and its layout algebra, each held to the Rust it replaces (the stage-0 oracle) by the committed vectors and a seeded differential of its evaluator and rustc realizations, with planted defects detected and the production export committed | `BS-01`..`BS-23` | `build` | | Source packages: a language-1.2 project imports the modules of a locked, content-addressed package of its workspace: `[[source_package]]` and `lexlean-workspace.toml`, the pins in `lexlean.lock` recomputed by every command, imports closed at link, no re-export of a package's production roots, and the package's modules replayed by the importer's `verify` with their origin in the records | `PK-01`..`PK-05` | `build` | | The literal `nat-add-zero` example, the Lean-verified feature examples, and the complete negative fixture suite | `EX-01`..`EX-09` | `build` | | Language-1.2 text components: tokenizers and their round-trip laws, with the laws carried as one native core module the generated modules import rather than authored Lean text | `LM-01`, `LM-02`, `LM-03`, `LM-13` | `build` | diff --git a/SPEC.md b/SPEC.md index dd72b3e4b1..2d73249f25 100644 --- a/SPEC.md +++ b/SPEC.md @@ -8520,6 +8520,30 @@ hexadecimal digits (either case), a `\u` escape of a surrogate, an escape that i none, and a control character in a string; each is reported at the offset of the byte the oracle names, the first one standing. +The layout algebra of the authoring notation (§17.21) is the third component of +the front end (`BS-23`, issue 269), the module `SurfaceDoc` with one production +root for `rust-std`: + +| Root | Takes | Returns | Oracle | +| --- | --- | --- | --- | +| `LexLeanBoot.SurfaceDoc.renderDoc` | a document, a width, and the number of nodes of the document | the layout | `surface::doc::render` | + +A document is text, a concatenation, an indentation, a line (a space in a group +laid out flat, a newline otherwise), a soft break (nothing, or a newline), or a +group, which is laid out flat when it and what follows it up to the next break +fit in the rest of the line. The oracle keeps a stack of items, each with its +indent and its mode, and answers the width test by walking a work stack and then +the stack below it. The module keeps the same two stacks as lists and pops one +node at a time. A concatenation is a chain: `DSeq` holds a part and the rest of +the concatenation, an empty one is `DNil`, and putting the parts of a +concatenation on the stack is putting its first part and the rest of it on the +stack, in the order the oracle pops them. So no definition recurses over a +document, and the loops are bounded by the number of nodes, which the host counts +as it builds the document. The width of a text is its number of scalars, taken as +the bytes that are not continuation bytes. A break is a newline and the indent in +spaces, written as a run of 256 spaces for each 256 of the indent and the +remainder. + The bootstrap project's production export is the first project whose roots together approach `max_total_source_bytes` (§10.2): the limit counts the programs, the crates, and the certificates of every production root of the @@ -11434,6 +11458,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `BS-15` | `bootstrap` | The package renderer of SPEC.md §17.16 (the checks `package::lower` makes of a manifest's own fields before it reads the program: the spec, the crate name against the Rust keywords and the reserved crates, the version, the profile, the form and the order of the sources, and the effects against the registry, with `allocation` refused on `rust-core`; the Cargo manifest over the scaffold record; and the provenance in canonical JSON with the digests of the runtime, the Cargo manifest, and the library) is language-1.2 source in the `bootstrap/` project, as two production roots of `RustPackage` that the host composes around the roots of `BS-04` to `BS-14`: `checkManifest`, which for the fields of a manifest and the registered effects returns whether the profile is `rust-std`, or the refusal of `package::lower` to the byte, and `renderPackage`, which for the scaffold record, the manifest's name, version, profile, sources, effects and exports, the program's identity, the compiler-semantics identity, the two runtime texts, and the library returns the `Cargo.toml` and the `provenance.json` of `package::package` to the byte; each is eligible for `rust-std` with exactly its declared effects and has no part of its certificate beyond 3,145,728 bytes; the oracle of `checkManifest` is `package::lower` on the manifest with its program and exports replaced by a pair every later check accepts, which agrees with each committed negative manifest's refusal, and the oracle of `renderPackage` is `package::package` on the 282 committed manifests it accepts and, on 137 hand-built inputs (names, exports, sources and effects that reach the escaper, and scaffold records that differ from the committed one) and 282 seeded mutants, `cargo_toml_of` and the canonical JSON of §21.7 by the recipe `package::package` is held to; on the normative host the lowered programs under the calculus evaluator agree with the oracle on the 301 committed manifests, 182 rule manifests and 2,000 seeded mutants, and on the 241 renders whose library is at most 16,384 bytes; the rendered packages agree with the oracle under the pinned rustc on all 3,184 runs; 21 planted defects in the source are each detected; and the certified packages the production export publishes for the project are the committed ones; the checks of the program, the exports and the crate, and the bytes of the library, which are the rows `BS-04` to `BS-14`, are outside the claim. | §17.16, §17.23 | | `BS-21` | `bootstrap` | The fixed structural grammar of SPEC.md §15.1 and §15.2 (`grammar::structural::parse_module`: the module header, the blocks and declarations, the axiom policies, the sentences, the structured proof environments, and the closed data forms, with the coverage rows of every structural atom it consumes) is language-1.2 source in the `bootstrap/` project: `parseModule` of `StructuralGrammar`, for the atoms `scanSource` of `BS-03` returns, the path, the name the path gives the module, and `max_scope_depth`, is a production root eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; it returns the module with its rows, or the refusal's code, message, and span, and on 267 hand-built vectors, 146 of them refused, which reach every refusal of `parse_module`, and on the committed modules, whole or by the atoms of both their ends, its lowered program under the calculus evaluator on the normative host and its rendered package under the pinned rustc on every host agree with the oracle to the byte; on the normative host both also agree on 2,000 seeded edits of those texts, each of 44 planted defects in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the grammars of mathematical islands, propositions, and proof sentences, which read the lexicon's closure, and the interpretation of the module this root returns are outside the claim. | §17.23, §15.1, §15.2 | | `BS-22` | `bootstrap` | The lexing of the authoring notation of SPEC.md §17.21 (`surface::lex::lex`: a text into its delimiters, `:=` and `::`, quoted strings with their escapes decoded, and words, which end at white space in the sense of `char::is_whitespace` or at a delimiter, each with its byte range, or the offset and message of the first lexical error; and `surface::lex::quote`: a text as the notation quotes it, with the quotation mark, the backslash, and every control character escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `SurfaceLex`: `lexText` and `quoteText`, each eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and with no part of its certificate beyond 3,145,728 bytes; on every host their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on the hand-built vectors, which reach every lexical error and every escape, on windows of the committed sources, and on seeded edits of those texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; the readers of the tokens (the generic notation and the sugar of the closed schemas), the pretty-printing algebra, and canonical JSON are outside the claim. | §17.23, §17.21 | +| `BS-23` | `bootstrap` | The layout algebra of the authoring notation of SPEC.md §17.21 (`surface::doc::render`: a document of text, concatenation, indentation, lines, soft breaks, and groups, laid out at a width, a group on one line when it and what follows it up to the next line break fit in the rest of the line and its lines broken otherwise, with the width of a text counted in scalars) is language-1.2 source in the `bootstrap/` project, as the production root `renderDoc` of `SurfaceDoc`, which takes the document, the width, and the number of nodes of the document, is eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; a concatenation is a chain of cells, so that the root pops one node at a time under loops bounded by the number of nodes; its rendered `rust-std` package under the pinned rustc, and on the normative host its lowered program under the calculus evaluator, agree with the oracle to the byte on the hand-built documents, on the documents the printer builds from the committed JSON files at three widths, and on seeded documents; each planted defect in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the printers that build documents (the generic notation and the sugar of the closed schemas) are outside the claim. | §17.23, §17.21 | | `PK-01` | `packages` | A language-1.2 project that declares a source package by its identifier, version, kind `path`, and the member directory a `lexlean-workspace.toml` lists locks it by the tree digest of its manifest and source files, its manifest digest, its version, its module prefix, and its own packages, and the configuration and the lock read back to the bytes written; a project without a source package keeps its bytes, and a package of another kind, a source package in another language, a changed byte or version (LLC0105), an undeclared package (LLC0106), a module prefix claimed twice (LLC0107), a project or a package outside the workspace (LLC0108), a path that escapes it, and a symlink (LLS8001) are each refused with the named code. | §10.5, §11.1, §25.1 | | `PK-02` | `packages` | A module of a project imports only modules of its own project and modules of the packages the project declares, and a module of a package only modules of itself and of the packages its own manifest declares: a package module is a module of the importing project under its own module prefix, a module of a package the project does not declare fails with LLR3009, and an import cycle through a package fails with LLR3003. | §10.5, §15.1 | | `PK-03` | `packages` | An importing project builds and verifies under the pinned Lean offline, and its package modules are generated, elaborated, replayed with `leanchecker`, and audited for their exact axioms under the package's module prefix with the package as their origin in the records; the source ID of the project changes with the pins, and no file of the package is written by `fmt`, `build`, or `verify`. | §10.5, §21.3, §22 | @@ -11450,7 +11475,7 @@ Every row below is normative, has honesty level `build`, and MUST be copied byte | `LM-09` | `language-models` | No prompt is code: no member of the closed language maps `string` or `bytes` to a function, a term, or source — every operation of the primitive registry that takes a `string` or `bytes` operand is classified by the audit with a data result, the registry's schema enumeration and the production registry's primitive rows equal one another and equal that classified set, and the text declarations of `lexlean/semantic-module/2` state no `string`-typed member, so a prompt reaches a model only as the tokens `encodeText` makes of it; a planted operation or text member of that shape is caught by the audit with its name. | §17.12 | | `LM-13` | `language-models` | The text laws a tokenizer elaborates through are LexLean source and not authored text: the module that states them is one language-1.2 native core module of the embedded `language/text-1.2` project, the project holds no file of Lean source, no independently authored Lean file anywhere in the release tree declares the `LexLeanText` namespace or lies under its module root, and the one-time migration input is absent; the registry lists only that module, the declarations and their exact axiom policies are read from its native data, which the compiler embeds and hashes into the language-1.2 semantics, and a module that declares a text component imports the generated module rather than splicing any Lean text. | §17.10, §17.12 | -**Total required capability IDs:** 470. +**Total required capability IDs:** 471. No row may be downgraded to `some-true` or `open`. Upstream Lean facts are ledger/authority rows, not substitutions for these build behaviors. diff --git a/VERIFICATION.md b/VERIFICATION.md index e04efb015b..e82d016fee 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -10817,6 +10817,34 @@ run on the module. What it owes is its first run, to pin the counts of its corpora, and the landing regeneration, which produces the `bootstrap/lex_text` and `bootstrap/quote_text` packages. +### the layout algebra of the notation is source (BS-23) + +`BS-23` holds the production root `LexLeanBoot.SurfaceDoc.renderDoc` +(`bootstrap/src/SurfaceDoc.lex.tex`) to `surface::doc::render`: the layout of a +document of text, concatenation, indentation, lines, soft breaks, and groups at a +width. The root admits no `recursion`. A concatenation is a chain of `DSeq` +cells, so that the two stacks of the oracle (the stack of items and the work +stack of the width test) are lists popped a node at a time, and the host passes +the number of nodes, which bounds each loop. The case has the shape of `BS-21` +and `BS-22`: hand-built documents that reach each node, each branch of the width +test (the text of a scalar beyond ASCII at the exact width, a flat line, a soft +break, the rest of the stack, a break line that ends the scan, a column past the +width) and indents of 255 to 700; the documents the printer builds from the +committed JSON files, at three widths; and seeded documents. Each planted defect +names the one vector that catches it, and the case asserts that the site of each +plant occurs once. The row's statement carries no counts. + +Measured with the compiler binary of the `BS-21` tree, on a copy of the project +whose only entrypoint is `SurfaceDoc`: `lexlean check` and `lexlean build` pass, +and the eligibility report gives the root `rust-std` with exactly `allocation` +and `overflow`. The register agrees with the §31 table on 471 IDs, by the same +parse `validate-spec-links` makes. + +The case has not run. It has not been compiled either, and `lexlean verify` has +not been run on the module. It owes its first run, to pin the counts of its +corpora, and the landing regeneration, which produces the `bootstrap/render_doc` +package. + ## The `bdd` recipe as the parts of one run (Issue #131) With UOR Atlas verified as the parts of one run, the `rest` leg became the diff --git a/bootstrap/README.md b/bootstrap/README.md index 3d577f2e8d..edd040fef5 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -22,3 +22,5 @@ and the refusal it returns, and `Buffer` and `Numeral` from the standard library `src/SurfaceLex.lex.tex` imports `AxiomOutput`, whose `whiteLength` holds the list of white space the oracle's `char::is_whitespace` has, and the standard library's `Buffer` and `Text`. + +`src/SurfaceDoc.lex.tex` imports only the standard library's `Buffer` and `Text`. diff --git a/bootstrap/lexlean.lock b/bootstrap/lexlean.lock index bf0e67ad0f..eeb7a81a58 100644 --- a/bootstrap/lexlean.lock +++ b/bootstrap/lexlean.lock @@ -1,7 +1,7 @@ spec = "lexlean/lock/2" language = "1.2" compiler_semantics = "aac30cb0bda069a55aca6ce2417be48726fd578218b3e8fbaf31634694695b64" -project_config_sha256 = "80b0cdf4c2dda3d097913cf2a900054237ebf3860e9eec2924c7b1b43191b2f5" +project_config_sha256 = "b45c833c25f3453194153d55ef37fe4ac14b0c88f7f8752f9b1410350825ca43" lean_toolchain = "leanprover/lean4:v4.32.1" [[workspace_file]] diff --git a/bootstrap/lexlean.toml b/bootstrap/lexlean.toml index 01f37f55d4..358e45c8b4 100644 --- a/bootstrap/lexlean.toml +++ b/bootstrap/lexlean.toml @@ -3,7 +3,7 @@ name = "bootstrap" language = "1.2" module_prefix = "LexLeanBoot" source_roots = ["src"] -entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/PrimitiveScan.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/StructuralGrammar.lex.tex", "src/SurfaceLex.lex.tex", "src/TokenAudit.lex.tex"] +entrypoints = ["src/AxiomOutput.lex.tex", "src/CalculusCheck.lex.tex", "src/PrimitiveScan.lex.tex", "src/RuntimeValidate.lex.tex", "src/RustAssemble.lex.tex", "src/RustAst.lex.tex", "src/RustBodies.lex.tex", "src/RustLoops.lex.tex", "src/RustLower.lex.tex", "src/RustMoves.lex.tex", "src/RustPackage.lex.tex", "src/RustPrint.lex.tex", "src/RustSucceed.lex.tex", "src/RustTerms.lex.tex", "src/RustValidate.lex.tex", "src/StructuralGrammar.lex.tex", "src/SurfaceDoc.lex.tex", "src/SurfaceLex.lex.tex", "src/TokenAudit.lex.tex"] build_root = ".lexlean" lockfile = "lexlean.lock" lean_workspace = "." diff --git a/bootstrap/src/SurfaceDoc.lex.tex b/bootstrap/src/SurfaceDoc.lex.tex new file mode 100644 index 0000000000..476c89aff8 --- /dev/null +++ b/bootstrap/src/SurfaceDoc.lex.tex @@ -0,0 +1,218 @@ +\begin{lexlean}{SurfaceDoc} +\useglossary{lexlean.std.nat@1.2.0} +\importmodule{LexLeanStd.Buffer} +\importmodule{LexLeanStd.Text} +\title{Natural number addition} +\begin{semanticmodule} +inductive{ + name: Doc, + type_parameters: [], + parameters: [], + constructors: [ + {name: DNil, fields: []}, + {name: DText, fields: [Bytes]}, + {name: DSeq, fields: [Doc, Doc]}, + {name: DNest, fields: [Nat, Doc]}, + {name: DLine, fields: []}, + {name: DSoft, fields: []}, + {name: DGroup, fields: [Doc]} + ] +} + +structure{ + name: Item, + type_parameters: [], + parameters: [], + fields: [{name: indent, type: Nat}, {name: flat, type: Bool}, {name: node, type: Doc}] +} + +structure{ + name: Layout, + type_parameters: [], + parameters: [], + fields: [ + {name: out, type: (List Bytes)}, + {name: column, type: Nat}, + {name: stack, type: (List Item)} + ] +} + +structure{ + name: Fit, + type_parameters: [], + parameters: [], + fields: [ + {name: spent, type: Nat}, + {name: limit, type: Nat}, + {name: impossible, type: Bool}, + {name: work, type: (List Item)}, + {name: lower, type: (List Item)}, + {name: verdict, type: (Option Bool)} + ] +} + +definition{ + name: lit, + parameters: [literal : String], + result: Bytes, + body: (prim utf8_encode literal : Bytes), + executable: true +} + +definition{ + name: joined, + parameters: [pieces : (List Bytes)], + result: Bytes, + body: (LexLeanStd.Buffer::joinAll (LexLeanStd.Text::reverse pieces)), + executable: true +} + +definition{ + name: byteNum, + parameters: [subject : Bytes, position : Nat], + result: Nat, + body: (match (LexLeanStd.Text::byteAt subject position) + | Option.some byte => (prim to_nat byte : Nat) + | Option.none => 0), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: scalarStep, + parameters: [subject : Bytes, state : (Nat * Nat)], + result: (Option (Nat * Nat)), + body: (match (LexLeanStd.Text::byteAt subject (first state)) + | Option.some byte => (let value : Nat := (prim to_nat byte : Nat); (if (value (ctor Option.none [(Nat * Nat)])), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: scalars, + parameters: [subject : Bytes], + result: Nat, + body: (second (first (prim iterate_until (fun [subject] (state : (Nat * Nat)) => (scalarStep subject state)) (prim length subject : Nat) (0, 0) : ((Nat * Nat) * Bool)))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: spaceStep, + parameters: [count : Nat, state : (Nat * (List Bytes))], + result: (Option (Nat * (List Bytes))), + body: (if ((first state) (spaceStep count state)) (count + 1) (0, (cons (prim utf8_encode "\n" : Bytes) pieces)) : ((Nat * (List Bytes)) * Bool)))); (cons (LexLeanStd.Text::sliceOr 0x20202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020 0 (prim remainder indent 256 0 : Nat)) filled))), + executable: true +} + +definition{ + name: fitWith, + parameters: [ + f : Fit, + spent : Nat, + work : (List Item), + lower : (List Item), + verdict : (Option Bool) + ], + result: Fit, + body: (record Fit {spent := spent, limit := (project f limit), impossible := (project f impossible), work := work, lower := lower, verdict := verdict}), + executable: true +} + +definition{ + name: fitItem, + parameters: [f : Fit, item : Item, work : (List Item), lower : (List Item)], + result: Fit, + body: (let indent : Nat := (project item indent); (let flat : Bool := (project item flat); (let spent : Nat := (project f spent); (match (project item node) + | Doc.DNil => (fitWith f spent work lower (ctor Option.none [Bool])) + | Doc.DText piece => (fitWith f (spent + (scalars piece)) work lower (ctor Option.none [Bool])) + | Doc.DSeq head tail => (fitWith f spent (cons (record Item {indent := indent, flat := flat, node := head}) (cons (record Item {indent := indent, flat := flat, node := tail}) work)) lower (ctor Option.none [Bool])) + | Doc.DNest amount inner => (fitWith f spent (cons (record Item {indent := (indent + amount), flat := flat, node := inner}) work) lower (ctor Option.none [Bool])) + | Doc.DLine => (if flat then (fitWith f (spent + 1) work lower (ctor Option.none [Bool])) else (fitWith f spent work lower (ctor Option.some [Bool] true))) + | Doc.DSoft => (if flat then (fitWith f spent work lower (ctor Option.none [Bool])) else (fitWith f spent work lower (ctor Option.some [Bool] true))) + | Doc.DGroup inner => (fitWith f spent (cons (record Item {indent := indent, flat := true, node := inner}) work) lower (ctor Option.none [Bool])))))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: fitStep, + parameters: [f : Fit], + result: (Option Fit), + body: (match (project f verdict) + | Option.some decided => (ctor Option.none [Fit]) + | Option.none => (if (if (project f impossible) then true else ((project f limit) (ctor Option.some [Fit] (fitItem f item rest (project f lower))) + | List.nil => (match (project f lower) + | List.cons item rest => (ctor Option.some [Fit] (fitItem f item (nil Item) rest)) + | List.nil => (ctor Option.some [Fit] (fitWith f (project f spent) (nil Item) (nil Item) (ctor Option.some [Bool] true))))))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: fits, + parameters: [nodes : Nat, width : Nat, column : Nat, rest : (List Item), first_item : Item], + result: Bool, + body: (let final : Fit := (first (prim iterate_until (fun (f : Fit) => (fitStep f)) (nodes + 1) (record Fit {spent := 0, limit := (prim subtract width column : Nat), impossible := (width answer + | Option.none => false)), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: broken, + parameters: [l : Layout, indent : Nat, rest : (List Item)], + result: Layout, + body: (record Layout {out := (withIndent (project l out) indent), column := indent, stack := rest}), + executable: true +} + +definition{ + name: layoutItem, + parameters: [width : Nat, nodes : Nat, l : Layout, item : Item, rest : (List Item)], + result: Layout, + body: (let indent : Nat := (project item indent); (let flat : Bool := (project item flat); (match (project item node) + | Doc.DNil => (record Layout {out := (project l out), column := (project l column), stack := rest}) + | Doc.DText piece => (record Layout {out := (cons piece (project l out)), column := ((project l column) + (scalars piece)), stack := rest}) + | Doc.DSeq head tail => (record Layout {out := (project l out), column := (project l column), stack := (cons (record Item {indent := indent, flat := flat, node := head}) (cons (record Item {indent := indent, flat := flat, node := tail}) rest))}) + | Doc.DNest amount inner => (record Layout {out := (project l out), column := (project l column), stack := (cons (record Item {indent := (indent + amount), flat := flat, node := inner}) rest)}) + | Doc.DLine => (if flat then (record Layout {out := (cons (prim utf8_encode " " : Bytes) (project l out)), column := ((project l column) + 1), stack := rest}) else (broken l indent rest)) + | Doc.DSoft => (if flat then (record Layout {out := (project l out), column := (project l column), stack := rest}) else (broken l indent rest)) + | Doc.DGroup inner => (let one : Bool := (fits nodes width (project l column) rest (record Item {indent := indent, flat := true, node := inner})); (record Layout {out := (project l out), column := (project l column), stack := (cons (record Item {indent := indent, flat := one, node := inner}) rest)}))))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: layoutStep, + parameters: [width : Nat, nodes : Nat, l : Layout], + result: (Option Layout), + body: (match (project l stack) + | List.nil => (ctor Option.none [Layout]) + | List.cons item rest => (ctor Option.some [Layout] (layoutItem width nodes l item rest))), + axioms: [Quot.sound, propext], + executable: true +} + +definition{ + name: renderDoc, + parameters: [doc : Doc, width : Nat, nodes : Nat], + result: Bytes, + body: (let final : Layout := (first (prim iterate_until (fun [nodes, width] (l : Layout) => (layoutStep width nodes l)) (nodes + 1) (record Layout {out := (nil Bytes), column := 0, stack := (cons (record Item {indent := 0, flat := false, node := doc}) (nil Item))}) : (Layout * Bool))); (joined (project final out))), + axioms: [Quot.sound, propext], + executable: true, + production: {effects: [allocation, overflow], targets: [rust-std]} +} +\end{semanticmodule} +\end{lexlean} diff --git a/crates/conformance/src/cases/bootstrap.rs b/crates/conformance/src/cases/bootstrap.rs index af7190bffb..337d10edbe 100644 --- a/crates/conformance/src/cases/bootstrap.rs +++ b/crates/conformance/src/cases/bootstrap.rs @@ -34,6 +34,7 @@ mod rust_terms; mod rust_validate; mod scan; mod structural_grammar; +mod surface_doc; mod surface_lex; /// The module the first component lives in. @@ -2456,6 +2457,7 @@ fn case(id: &str) { "BS-15" => rust_package::case(), "BS-21" => structural_grammar::case(), "BS-22" => surface_lex::case(), + "BS-23" => surface_doc::case(), other => panic!("no bootstrap case is wired for {other}"), } } diff --git a/crates/conformance/src/cases/bootstrap/surface_doc.rs b/crates/conformance/src/cases/bootstrap/surface_doc.rs new file mode 100644 index 0000000000..69d7c31046 --- /dev/null +++ b/crates/conformance/src/cases/bootstrap/surface_doc.rs @@ -0,0 +1,910 @@ +//! `BS-23` (SPEC.md §17.23, §17.21): the pretty-printing algebra of the +//! authoring notation as language-1.2 source, `LexLeanBoot.SurfaceDoc`, held +//! to the byte to `lexlean::surface::doc::render`. +//! +//! `renderDoc` takes a document, a width, and the number of nodes of the +//! document, and returns the layout. The oracle keeps a stack of +//! `(indent, mode, &Doc)` and the width test `fits` walks that stack and the +//! stack below it; neither recurses over the document here. A concatenation is +//! a chain of `DSeq` cells, so that putting its parts on the stack is putting +//! two items on it, and the number of nodes is the bound of the loops, which +//! pop a node at a time. The host counts the nodes as it builds the document. +//! +//! Each planted defect names the one vector that catches it. `BS-05` and +//! `BS-11` each lost a plant when their corpus grew, because a plant caught by +//! "some run of the corpus" is only as good as that corpus. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::OnceLock; + +use lexlean::artifact::canonical_json::Json; +use lexlean::calculus::Value; +use lexlean::surface::doc::{render, Doc}; +use lexlean::surface::generic; +use serde_json::Value as Json2; + +use super::rust_validate::{eligibility_of, rustc_differential}; +use super::{nat_val, text, Run, Seeded}; +use crate::production_export as harness; +use crate::support::{self, repo_root}; + +const MODULE: &str = "LexLeanBoot.SurfaceDoc"; +const SOURCE: &str = "bootstrap/src/SurfaceDoc.lex.tex"; +const ROOTS: [(&str, &[&str]); 1] = [("renderDoc", &["allocation", "overflow"])]; + +/// The most a part of a certificate of the root may be, as the row states. +const CERTIFICATE_BUDGET: u64 = 3_145_728; +/// The documents of at most this many nodes are run under the evaluator; the +/// larger ones, up to the second bound, are run under rustc only. +const EVALUATOR_NODES: usize = 400; +const RUSTC_NODES: usize = 6_000; +/// The committed JSON files read, by size in bytes, and of those the ones the +/// evaluator and rustc are given: one in this many, by a hash of the path. +const JSON_BYTES: usize = 6_000; +const EVALUATOR_EVERY: u64 = 8; +const RUSTC_EVERY: u64 = 3; +/// The widths a document from a file is laid out at: the width the printer +/// uses, a narrow one, and one no line reaches. +const WIDTHS: [usize; 3] = [100, 40, 10_000]; +/// Seeded documents under the evaluator, and under rustc. +const EVALUATOR_MUTANTS: usize = 500; +const RUSTC_MUTANTS: usize = 2_500; + +// ---------------------------------------------------------------- the sorts + +/// The constructors of each inductive of the module, in declaration order, +/// read from its text, so that the encoders name a constructor and never +/// number it, and one moved in the module fails here. +fn sorts() -> &'static BTreeMap> { + static SORTS: OnceLock>> = OnceLock::new(); + SORTS.get_or_init(|| { + let source = + std::fs::read_to_string(repo_root().join(SOURCE).as_std_path()).expect("the module"); + let mut out = BTreeMap::new(); + for block in source.split("inductive{").skip(1) { + let block = block.split("\n}\n").next().expect("a block"); + let name = block + .split("name: ") + .nth(1) + .and_then(|rest| rest.split(',').next()) + .expect("a sort name") + .trim() + .to_owned(); + let constructors: Vec = block + .split("{name: ") + .skip(1) + .map(|rest| { + rest.split(',') + .next() + .expect("a constructor") + .trim() + .to_owned() + }) + .collect(); + out.insert(name, constructors); + } + out + }) +} + +fn adt(constructor: &str, fields: Vec) -> Value { + let found = sorts() + .get("Doc") + .expect("the module has the sort `Doc`") + .iter() + .position(|name| name == constructor) + .unwrap_or_else(|| panic!("`Doc` has no constructor `{constructor}`")); + Value::Adt { + constructor: u64::try_from(found).expect("an index"), + fields, + } +} + +// ------------------------------------------------------------- the encoders + +/// A document as the module's `Doc`, and the number of nodes it has there: a +/// concatenation is a chain of `DSeq` cells that ends in a `DNil`. +fn encode(doc: &Doc, nodes: &mut usize) -> Value { + *nodes += 1; + match doc { + Doc::Nil => adt("DNil", Vec::new()), + Doc::Text(value) => adt("DText", vec![text(value)]), + Doc::Concat(parts) => { + let mut chain = adt("DNil", Vec::new()); + *nodes += parts.len(); + for part in parts.iter().rev() { + chain = adt("DSeq", vec![encode(part, nodes), chain]); + } + // The `DNil` that ends the chain is the node counted above; an + // empty concatenation is that node alone. + chain + } + Doc::Nest(by, inner) => adt("DNest", vec![nat_val(*by), encode(inner, nodes)]), + Doc::Line => adt("DLine", Vec::new()), + Doc::Soft => adt("DSoft", Vec::new()), + Doc::Group(inner) => adt("DGroup", vec![encode(inner, nodes)]), + } +} + +/// The run of `renderDoc` on a document at a width. +fn render_run(doc: &Doc, width: usize) -> Run { + let mut nodes = 0; + let encoded = encode(doc, &mut nodes); + ( + vec![encoded, nat_val(width), nat_val(nodes)], + serde_json::to_value(text(&render(doc, width))).expect("a value is JSON"), + ) +} + +/// The nodes of a document in the module's encoding. +fn size(doc: &Doc) -> usize { + let mut nodes = 0; + let _ = encode(doc, &mut nodes); + nodes +} + +// ------------------------------------------------------------------ vectors + +/// One named vector: a document laid out at a width. +struct Vector { + label: String, + doc: Doc, + width: usize, +} + +impl Vector { + fn run(&self) -> Run { + render_run(&self.doc, self.width) + } +} + +fn t(value: &str) -> Doc { + Doc::text(value) +} + +fn seq(parts: Vec) -> Doc { + Doc::concat(parts) +} + +/// `a`, a line, and `b`, in a group. +fn pair(left: &str, right: &str) -> Doc { + seq(vec![t(left), Doc::Line, t(right)]).group() +} + +/// The list of the oracle's own tests: items between brackets, on one line +/// when they fit and one a line when they do not. +fn bracket(items: &[&str]) -> Doc { + Doc::concat(vec![ + Doc::text("["), + Doc::concat(vec![ + Doc::Soft, + Doc::join(items.iter().map(|item| Doc::text(*item)).collect(), ","), + ]) + .nest(2), + Doc::Soft, + Doc::text("]"), + ]) + .group() +} + +#[allow(clippy::too_many_lines)] +fn named() -> Vec { + let mut out: Vec = Vec::new(); + let mut add = |label: &str, doc: Doc, width: usize| { + out.push(Vector { + label: label.to_owned(), + doc, + width, + }); + }; + + // The oracle's own documents. + add("a list that fits", bracket(&["a", "bb", "ccc"]), 20); + add("a list that does not fit", bracket(&["a", "bb", "ccc"]), 8); + add( + "nested lists", + Doc::concat(vec![ + Doc::text("["), + Doc::concat(vec![ + Doc::Soft, + Doc::join(vec![bracket(&["x", "y"]), bracket(&["z"])], ","), + ]) + .nest(2), + Doc::Soft, + Doc::text("]"), + ]) + .group(), + 12, + ); + add("a list of nothing", bracket(&[]), 0); + + // Nothing, text, and the lines outside any group. + add("the empty document", Doc::Nil, 0); + add("an empty text", t(""), 0); + add("a text beyond the width", t("abcdef"), 3); + add( + "a concatenation", + seq(vec![t("a"), t("b"), t("c"), t("d")]), + 80, + ); + add("an empty concatenation", seq(Vec::new()), 80); + add("a concatenation of one", seq(vec![t("a")]), 80); + add( + "a concatenation of concatenations", + seq(vec![ + seq(vec![t("a"), t("b")]), + seq(Vec::new()), + seq(vec![t("c")]), + ]), + 80, + ); + add( + "a line outside any group", + seq(vec![t("a"), Doc::Line, t("b")]), + 80, + ); + add( + "a soft break outside any group", + seq(vec![t("a"), Doc::Soft, t("b")]), + 80, + ); + add( + "lines and soft breaks outside any group", + seq(vec![Doc::Line, Doc::Soft, Doc::Line]), + 80, + ); + add( + "joined items outside any group", + Doc::join(vec![t("a"), t("b"), t("c")], ","), + 80, + ); + add("joined items of one", Doc::join(vec![t("a")], ","), 80); + add("joined items of none", Doc::join(Vec::new(), ","), 80); + + // The width test of a group. + add("a group that fits", pair("a", "b"), 80); + add("a group that fits exactly", pair("a", "b"), 3); + add("a group one column short", pair("a", "b"), 2); + add("a group at width zero", pair("a", "b"), 0); + add( + "a group of text that fits exactly", + seq(vec![t("abc")]).group(), + 3, + ); + add( + "a group of text one column short", + seq(vec![t("abc")]).group(), + 2, + ); + add( + "a group of a two-byte scalar at its exact width", + pair("\u{e9}", "b"), + 3, + ); + add( + "a group of a three-byte scalar at its exact width", + pair("\u{2115}", "b"), + 3, + ); + add( + "a group of a four-byte scalar at its exact width", + pair("\u{1f600}", "b"), + 3, + ); + add( + "a group of the delete at its exact width", + pair("\u{7f}", "b"), + 3, + ); + add( + "a group of a soft break at its exact width", + seq(vec![t("["), Doc::Soft, t("]")]).group(), + 2, + ); + add( + "a group of a soft break one column short", + seq(vec![t("["), Doc::Soft, t("]")]).group(), + 1, + ); + add( + "a group followed by text that fits", + seq(vec![pair("a", "b"), t("cccc")]), + 7, + ); + add( + "a group followed by text that does not fit", + seq(vec![pair("a", "b"), t("cccc")]), + 5, + ); + add( + "a group followed by a break line", + seq(vec![pair("a", "b"), Doc::Line, t("ccccccccc")]), + 5, + ); + add( + "a group followed by a break soft", + seq(vec![pair("a", "b"), Doc::Soft, t("ccccccccc")]), + 5, + ); + add( + "a group followed by a group", + seq(vec![pair("a", "b"), pair("c", "d")]), + 6, + ); + add( + "a group followed by empty parts", + seq(vec![pair("a", "b"), Doc::Nil, seq(Vec::new()), t("c")]), + 4, + ); + add( + "a group after text", + seq(vec![t("xxxx"), pair("a", "b")]), + 6, + ); + add( + "a group after text of scalars beyond ASCII", + seq(vec![t("\u{e9}\u{e9}"), pair("a", "b")]), + 5, + ); + add( + "a group after text past the width", + seq(vec![t("abcdefgh"), pair("a", "b")]), + 4, + ); + add( + "a group in a group that fits", + seq(vec![t("("), pair("a", "b"), t(")")]).group(), + 80, + ); + add( + "a group in a group that breaks", + seq(vec![t("("), Doc::Soft, pair("a", "b"), Doc::Soft, t(")")]).group(), + 4, + ); + add( + "a group in a group that breaks, the inner fits", + seq(vec![t("("), Doc::Soft, pair("a", "b"), Doc::Soft, t(")")]).group(), + 5, + ); + add( + "a group of groups", + seq(vec![ + pair("a", "b"), + Doc::Line, + pair("c", "d"), + Doc::Line, + pair("e", "f"), + ]) + .group(), + 7, + ); + add( + "a group of a nest", + seq(vec![ + t("["), + seq(vec![Doc::Soft, t("x")]).nest(2), + Doc::Soft, + t("]"), + ]) + .group(), + 3, + ); + + // Nests, and the indent of a break. + add( + "a nested break", + Doc::Nest( + 2, + Box::new(Doc::Nest(3, Box::new(seq(vec![t("a"), Doc::Line, t("b")])))), + ), + 80, + ); + add("a nest of nothing", Doc::Nest(4, Box::new(Doc::Nil)), 80); + add("a nest without a break", Doc::Nest(4, Box::new(t("a"))), 80); + add( + "a break after a nest", + seq(vec![Doc::Nest(4, Box::new(t("a"))), Doc::Line, t("b")]), + 80, + ); + for indent in [1, 2, 8, 100, 255, 256, 257, 300, 511, 512, 513, 700] { + add( + &format!("a nest of {indent}"), + Doc::Nest( + indent, + Box::new(seq(vec![t("a"), Doc::Line, t("b"), Doc::Line, t("c")])), + ), + 80, + ); + } + add( + "a group after a break in a nest at its width", + Doc::Nest(2, Box::new(seq(vec![Doc::Line, pair("ab", "cd")]))), + 6, + ); + add( + "a group after a break in a nest that fits", + Doc::Nest(2, Box::new(seq(vec![Doc::Line, pair("ab", "cd")]))), + 7, + ); + add( + "a group that breaks in a nest", + Doc::Nest( + 3, + Box::new( + seq(vec![ + t("["), + Doc::Soft, + t("abc"), + Doc::Line, + t("def"), + Doc::Soft, + t("]"), + ]) + .group(), + ), + ), + 6, + ); + add( + "text with a break line in it", + seq(vec![t("a\nb"), pair("c", "d")]), + 4, + ); + out +} + +// ------------------------------------------------------------------- seeded + +/// Text of each width of scalar, and none. +const TEXTS: &[&str] = &[ + "", + "a", + "bb", + "ccc", + "abcdefgh", + "\u{e9}", + "\u{e9}\u{2115}", + "\u{1f600}", + "[", + "]", + ",", + ":", + "a longer piece of text", + "\u{2115}\u{2115}\u{2115}\u{2115}", +]; + +/// A document of at most `depth` levels. +fn drawn(seeded: &mut Seeded, depth: usize) -> Doc { + let kind = if depth == 0 { + seeded.below(4) + } else { + seeded.below(9) + }; + match kind { + 0 | 4 => Doc::text(*seeded.pick(TEXTS)), + 1 => Doc::Line, + 2 => Doc::Soft, + 3 => Doc::Nil, + 5 | 6 => { + let parts = (0..seeded.below(5)) + .map(|_| drawn(seeded, depth - 1)) + .collect(); + Doc::concat(parts) + } + 7 => { + let by = if seeded.chance(3) { + *seeded.pick(&[255, 256, 257, 300]) + } else { + seeded.below(7) + }; + drawn(seeded, depth - 1).nest(by) + } + _ => drawn(seeded, depth - 1).group(), + } +} + +/// `count` seeded documents, each at a width, with the nodes each has. +fn mutants(count: usize) -> Vec<(Doc, usize)> { + let mut seeded = Seeded(0x0B55_0023); + (0..count) + .map(|_| { + let depth = 1 + seeded.below(6); + let doc = drawn(&mut seeded, depth); + let width = *seeded.pick(&[0, 1, 2, 3, 5, 8, 12, 20, 40, 80, 100]); + (doc, width) + }) + .collect() +} + +/// A hash of a path (FNV-1a). +fn path_hash(path: &str) -> u64 { + path.bytes().fold(0xcbf2_9ce4_8422_2325_u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0100_0000_01b3) + }) +} + +/// Every committed JSON file of at most `JSON_BYTES` bytes that parses, as the +/// document the notation's printer builds from it. +fn documents() -> Vec<(String, Doc)> { + let root = repo_root(); + let walker = walkdir::WalkDir::new(root.as_std_path()) + .sort_by_file_name() + .into_iter() + .filter_entry(|entry| { + let name = entry.file_name().to_string_lossy(); + !(entry.file_type().is_dir() + && (matches!(name.as_ref(), "target" | ".git" | ".lexlean") + || support::holds_cache_tag(entry))) + }); + let mut out = Vec::new(); + for entry in walker.flatten() { + let path = entry.path(); + if !entry.file_type().is_file() || !path.to_string_lossy().ends_with(".json") { + continue; + } + let Ok(bytes) = std::fs::read(path) else { + continue; + }; + if bytes.len() > JSON_BYTES { + continue; + } + let Ok(value) = Json::parse(&bytes) else { + continue; + }; + let relative = path + .strip_prefix(root.as_std_path()) + .expect("under the root") + .to_string_lossy() + .replace('\\', "/"); + out.push((relative, generic::doc(&value))); + } + out +} + +// ------------------------------------------------------------------- plants + +/// A defect planted in the source, and the named vector that catches it. +struct Plant { + label: &'static str, + site: &'static str, + replacement: &'static str, + vector: &'static str, +} + +const PLANTS: &[Plant] = &[ + Plant { + label: "a scalar beyond ASCII counts every byte", + site: "(if (value (if flat then (fitWith f spent work lower (ctor Option.none [Bool]))", + replacement: "| Doc.DSoft => (if flat then (fitWith f (spent + 1) work lower (ctor Option.none [Bool]))", + vector: "a group of a soft break at its exact width", + }, + Plant { + label: "a break soft break does not end the scan", + site: "| Doc.DSoft => (if flat then (fitWith f spent work lower (ctor Option.none [Bool])) else (fitWith f spent work lower (ctor Option.some [Bool] true)))", + replacement: "| Doc.DSoft => (if flat then (fitWith f spent work lower (ctor Option.none [Bool])) else (fitWith f spent work lower (ctor Option.none [Bool])))", + vector: "a group followed by a break soft", + }, + Plant { + label: "a group fits only under its room", + site: "((project f limit) (ctor Option.some [Fit] (fitItem f item (nil Item) rest))", + replacement: "| List.cons item rest => (ctor Option.some [Fit] (fitWith f (project f spent) (nil Item) (nil Item) (ctor Option.some [Bool] true)))", + vector: "a group followed by text that does not fit", + }, + Plant { + label: "a group is tried at the start of the line", + site: "(fits nodes width (project l column) rest", + replacement: "(fits nodes width 0 rest", + vector: "a group after text", + }, + Plant { + label: "text advances the column by its bytes", + site: "column := ((project l column) + (scalars piece))", + replacement: "column := ((project l column) + (prim length piece : Nat))", + vector: "a group after text of scalars beyond ASCII", + }, + Plant { + label: "a group that fits is laid out broken", + site: "(record Item {indent := indent, flat := one, node := inner})", + replacement: "(record Item {indent := indent, flat := false, node := inner})", + vector: "a group that fits", + }, + Plant { + label: "a document starts flat", + site: "{indent := 0, flat := false, node := doc}", + replacement: "{indent := 0, flat := true, node := doc}", + vector: "a line outside any group", + }, + Plant { + label: "a flat line is two spaces", + site: "(prim utf8_encode \" \" : Bytes)", + replacement: "(prim utf8_encode \" \" : Bytes)", + vector: "a group that fits", + }, + Plant { + label: "a break leaves the column at zero", + site: "column := indent, stack := rest", + replacement: "column := 0, stack := rest", + vector: "a group after a break in a nest at its width", + }, + Plant { + label: "a nest sets the indent and does not add to it", + site: "stack := (cons (record Item {indent := (indent + amount), flat := flat, node := inner}) rest)})", + replacement: "stack := (cons (record Item {indent := amount, flat := flat, node := inner}) rest)})", + vector: "a nested break", + }, + Plant { + label: "an indent counts its spaces by 512", + site: "(prim quotient indent 256 0 : Nat)", + replacement: "(prim quotient indent 512 0 : Nat)", + vector: "a nest of 300", + }, + Plant { + label: "an indent's remainder is taken by 255", + site: "(prim remainder indent 256 0 : Nat)", + replacement: "(prim remainder indent 255 0 : Nat)", + vector: "a nest of 256", + }, + Plant { + label: "a concatenation's parts come last first", + site: "stack := (cons (record Item {indent := indent, flat := flat, node := head}) (cons (record Item {indent := indent, flat := flat, node := tail}) rest))", + replacement: "stack := (cons (record Item {indent := indent, flat := flat, node := tail}) (cons (record Item {indent := indent, flat := flat, node := head}) rest))", + vector: "a concatenation", + }, + Plant { + label: "the layout stops one node early", + site: "(nodes + 1) (record Layout", + replacement: "(prim subtract nodes 1 : Nat) (record Layout", + vector: "a concatenation", + }, + Plant { + label: "the width test of a group is starved of nodes", + site: "(let one : Bool := (fits nodes", + replacement: "(let one : Bool := (fits (prim subtract nodes nodes : Nat)", + vector: "a group that fits", + }, +]; + +fn plants(vectors: &[Vector]) { + let mut missed = Vec::new(); + for plant in PLANTS { + let vector = vectors + .iter() + .find(|vector| vector.label == plant.vector) + .unwrap_or_else(|| panic!("{}: no vector `{}`", plant.label, plant.vector)); + let planted = support::P::bootstrap(); + let source = planted.read_raw("src/SurfaceDoc.lex.tex"); + assert_eq!( + source.matches(plant.site).count(), + 1, + "{}: the plant site occurs once", + plant.label + ); + planted.write( + "src/SurfaceDoc.lex.tex", + &source.replacen(plant.site, plant.replacement, 1), + ); + let checked = support::checked_project(&planted); + let low = super::lower(&checked, &planted, MODULE, ROOTS[0].0); + match super::evaluate(&low, &[vector.run()], 1).first() { + Some(first) => eprintln!( + "BS-23: plant detected: {}: {}", + plant.label, + first.chars().take(160).collect::() + ), + None => missed.push(plant.label), + } + } + assert!( + missed.is_empty(), + "planted defects their own vector does not detect: {missed:?}" + ); +} + +fn export(lowered: &super::Lowered) { + let exported = harness::exported("bootstrap"); + let bundle = exported.bundle(); + let mut realized = 0usize; + for exported_root in &exported.output.result.roots { + if exported_root.root != format!("{MODULE}.{}", ROOTS[0].0) { + continue; + } + let largest = exported_root + .outputs + .iter() + .flat_map(|output| &output.certification.certificates) + .map(|certificate| certificate.file.byte_length) + .max() + .expect("a certificate"); + eprintln!( + "BS-23: {}: largest certificate part {largest} bytes", + exported_root.root + ); + assert!( + largest <= CERTIFICATE_BUDGET, + "{}: a certificate part of {largest} bytes is beyond {CERTIFICATE_BUDGET}", + exported_root.root + ); + for output in &exported_root.outputs { + let package: Json2 = serde_json::from_slice( + &std::fs::read( + bundle + .join(&output.package.directory) + .join("package.json") + .as_std_path(), + ) + .expect("package.json"), + ) + .expect("package JSON"); + assert_eq!( + package["program"], + serde_json::to_value(&lowered.program).expect("a program is JSON"), + "{}: the exported package realizes the program the differential ran", + exported_root.root + ); + realized += 1; + } + } + assert_eq!(realized, ROOTS.len(), "one certified package per root"); + let packages = harness::tree(&bundle.join("packages")); + let committed_packages = harness::tree(&harness::fixture_dir("bootstrap").join("packages")); + assert_eq!( + packages.keys().collect::>(), + committed_packages.keys().collect::>(), + "the exported package files are the committed ones" + ); + for (relative, content) in &packages { + assert!( + committed_packages.get(relative) == Some(content), + "packages/{relative} differs from the committed package" + ); + } +} + +#[allow(clippy::too_many_lines)] +pub(super) fn case() { + let started = std::time::Instant::now(); + let project = support::P::bootstrap(); + let checked_units = project.check_ok(); + assert!( + checked_units.units.contains_key("SurfaceDoc"), + "SurfaceDoc is a linked module of the project" + ); + eligibility_of(&project, "SurfaceDoc", MODULE, &ROOTS); + assert_eq!( + sorts().keys().map(String::as_str).collect::>(), + ["Doc"], + "the sorts of the module" + ); + + let vectors = named(); + let labels: BTreeSet<&str> = vectors.iter().map(|vector| vector.label.as_str()).collect(); + assert_eq!(labels.len(), vectors.len(), "each vector has its own label"); + let vector_runs: Vec = vectors.iter().map(Vector::run).collect(); + let broken = vectors + .iter() + .filter(|vector| render(&vector.doc, vector.width).contains('\n')) + .count(); + assert!( + 0 < broken && broken < vectors.len(), + "the vectors give layouts of both kinds, with a break and without" + ); + + // The documents of the committed JSON files, at each width. + let files = documents(); + let mut evaluator_files: Vec = Vec::new(); + let mut rustc_files: Vec = Vec::new(); + for (path, doc) in &files { + let nodes = size(doc); + let hash = path_hash(path); + for width in WIDTHS { + if nodes <= EVALUATOR_NODES && hash.is_multiple_of(EVALUATOR_EVERY) { + evaluator_files.push(render_run(doc, width)); + } + if nodes <= RUSTC_NODES && hash.is_multiple_of(RUSTC_EVERY) { + rustc_files.push(render_run(doc, width)); + } + } + } + let normative = support::normative_host(); + let drawn_docs = if normative { + mutants(EVALUATOR_MUTANTS + RUSTC_MUTANTS) + } else { + Vec::new() + }; + let (evaluator_drawn, rustc_drawn): (Vec, Vec) = { + let split = drawn_docs.len().min(EVALUATOR_MUTANTS); + ( + drawn_docs[..split] + .iter() + .map(|(doc, width)| render_run(doc, *width)) + .collect(), + drawn_docs[split..] + .iter() + .map(|(doc, width)| render_run(doc, *width)) + .collect(), + ) + }; + eprintln!( + "BS-23: {} vectors, {} JSON documents ({} and {} runs), {} and {} seeded runs ({:?})", + vectors.len(), + files.len(), + evaluator_files.len(), + rustc_files.len(), + evaluator_drawn.len(), + rustc_drawn.len(), + started.elapsed() + ); + + let checked = support::checked_project(&project); + let lowered = super::lower(&checked, &project, MODULE, ROOTS[0].0); + if normative { + let mut corpus: Vec = vector_runs.clone(); + corpus.extend(evaluator_files); + corpus.extend(evaluator_drawn); + let disagreements = super::evaluate(&lowered, &corpus, 5); + assert!( + disagreements.is_empty(), + "renderDoc and the Rust oracle disagree: {disagreements:?}" + ); + eprintln!( + "BS-23: the evaluator agrees on {} runs ({:?})", + corpus.len(), + started.elapsed() + ); + plants(&vectors); + } else { + eprintln!("BS-23: platform-bound host: the evaluator, the seeded runs, and the planted defects run on the normative host"); + } + + let mut compiled = vector_runs; + compiled.extend(rustc_files); + compiled.extend(rustc_drawn); + let total = rustc_differential( + "BS-23", + &[(ROOTS[0].0, &lowered, compiled.as_slice())], + "oracle", + ); + eprintln!("BS-23: {total} rustc runs agree ({:?})", started.elapsed()); + + if !support::lean_backed("BS-23") { + return; + } + export(&lowered); + eprintln!("BS-23: passed in {:?}", started.elapsed()); +} diff --git a/crates/conformance/src/cases/repository.rs b/crates/conformance/src/cases/repository.rs index 2da16b1b48..5c15ff0520 100644 --- a/crates/conformance/src/cases/repository.rs +++ b/crates/conformance/src/cases/repository.rs @@ -813,7 +813,7 @@ pub(crate) fn run(id: &str) { let model = repo_model::Model::load(&root.join("model").into_std_path_buf()) .expect("the model loads"); let table = spec_table(); - assert_eq!(table.len(), 470, "§31 has 470 rows"); + assert_eq!(table.len(), 471, "§31 has 471 rows"); assert_eq!(model.ids.id.len(), table.len(), "register row count"); for ((spec_id, spec_suite, spec_statement), row) in table.iter().zip(model.ids.id.iter()) diff --git a/crates/conformance/tests/conformance.rs b/crates/conformance/tests/conformance.rs index ef5121e0ea..31a4766b9c 100644 --- a/crates/conformance/tests/conformance.rs +++ b/crates/conformance/tests/conformance.rs @@ -2280,6 +2280,11 @@ fn conformance_bs_22() { repo_conformance::cases::run("BS-22"); } +#[test] +fn conformance_bs_23() { + repo_conformance::cases::run("BS-23"); +} + #[test] fn conformance_pk_01() { repo_conformance::cases::run("PK-01"); diff --git a/features/suites/bootstrap.feature b/features/suites/bootstrap.feature index 13048781be..d9d72d3504 100644 --- a/features/suites/bootstrap.feature +++ b/features/suites/bootstrap.feature @@ -120,3 +120,10 @@ Feature: bootstrap When the project is checked and built, and the lowered program and the rendered package of each root are run on those texts Then each root is eligible for rust-std with exactly its declared effects, no certificate part is beyond 3,145,728 bytes, and `lexText` and `quoteText` equal `lex` and `quote` on every run, every lexical error of which a vector reaches And every planted defect is detected by its own vector, and the exported packages are byte-identical to the committed packages and realize the programs the differential ran + + @BS-23 @build + Scenario: The layout algebra of the authoring notation of SPEC.md §17.21 (`surface::doc::render`: a document of text, concatenation, indentation, lines, soft breaks, and groups, laid out at a width, a group on one line when it and what follows it up to the next line break fit in the rest of the line and its lines broken otherwise, with the width of a text counted in scalars) is language-1.2 source in the `bootstrap/` project, as the production root `renderDoc` of `SurfaceDoc`, which takes the document, the width, and the number of nodes of the document, is eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; a concatenation is a chain of cells, so that the root pops one node at a time under loops bounded by the number of nodes; its rendered `rust-std` package under the pinned rustc, and on the normative host its lowered program under the calculus evaluator, agree with the oracle to the byte on the hand-built documents, on the documents the printer builds from the committed JSON files at three widths, and on seeded documents; each planted defect in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the printers that build documents (the generic notation and the sugar of the closed schemas) are outside the claim. + Given the committed `bootstrap` project, hand-built documents of every node and every width test, the documents the printer builds from the committed JSON files, and seeded documents + When the project is checked and built, and the lowered program and the rendered package of the root are run on those documents at their widths + Then the root is eligible for rust-std with exactly its declared effects, no certificate part is beyond 3,145,728 bytes, and `renderDoc` equals `render` on every run + And every planted defect is detected by its own vector, and the exported package is byte-identical to the committed package and realizes the program the differential ran diff --git a/model/ids.toml b/model/ids.toml index d7a6e2041b..ac390b3bd1 100644 --- a/model/ids.toml +++ b/model/ids.toml @@ -2738,6 +2738,12 @@ level = "build" suite = "bootstrap" statement = "The lexing of the authoring notation of SPEC.md §17.21 (`surface::lex::lex`: a text into its delimiters, `:=` and `::`, quoted strings with their escapes decoded, and words, which end at white space in the sense of `char::is_whitespace` or at a delimiter, each with its byte range, or the offset and message of the first lexical error; and `surface::lex::quote`: a text as the notation quotes it, with the quotation mark, the backslash, and every control character escaped) is language-1.2 source in the `bootstrap/` project, as two production roots of `SurfaceLex`: `lexText` and `quoteText`, each eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and with no part of its certificate beyond 3,145,728 bytes; on every host their rendered `rust-std` packages under the pinned rustc, and on the normative host their lowered programs under the calculus evaluator, agree with the oracle to the byte on the hand-built vectors, which reach every lexical error and every escape, on windows of the committed sources, and on seeded edits of those texts; each planted defect in the source is detected by the one vector it names, and the certified packages the production export publishes for the project are the committed ones; the readers of the tokens (the generic notation and the sugar of the closed schemas), the pretty-printing algebra, and canonical JSON are outside the claim." +[[id]] +id = "BS-23" +level = "build" +suite = "bootstrap" +statement = "The layout algebra of the authoring notation of SPEC.md §17.21 (`surface::doc::render`: a document of text, concatenation, indentation, lines, soft breaks, and groups, laid out at a width, a group on one line when it and what follows it up to the next line break fit in the rest of the line and its lines broken otherwise, with the width of a text counted in scalars) is language-1.2 source in the `bootstrap/` project, as the production root `renderDoc` of `SurfaceDoc`, which takes the document, the width, and the number of nodes of the document, is eligible for `rust-std` with exactly its declared effects, none of them `recursion`, and has no part of its certificate beyond 3,145,728 bytes; a concatenation is a chain of cells, so that the root pops one node at a time under loops bounded by the number of nodes; its rendered `rust-std` package under the pinned rustc, and on the normative host its lowered program under the calculus evaluator, agree with the oracle to the byte on the hand-built documents, on the documents the printer builds from the committed JSON files at three widths, and on seeded documents; each planted defect in the source is detected by the one vector it names, and the certified package the production export publishes for the project is the committed one; the printers that build documents (the generic notation and the sugar of the closed schemas) are outside the claim." + [[id]] id = "PK-01" level = "build" diff --git a/tests/production-export/bootstrap/roots.json b/tests/production-export/bootstrap/roots.json index 918e3ecfef..5af928463b 100644 --- a/tests/production-export/bootstrap/roots.json +++ b/tests/production-export/bootstrap/roots.json @@ -200,6 +200,14 @@ ], "version": "1.0.0" }, + { + "name": "render_doc", + "root": "LexLeanBoot.SurfaceDoc.renderDoc", + "targets": [ + "rust-std" + ], + "version": "1.0.0" + }, { "name": "render_package", "root": "LexLeanBoot.RustPackage.renderPackage", diff --git a/xtask/src/spec_links.rs b/xtask/src/spec_links.rs index 1957d248c8..f263b0b03c 100644 --- a/xtask/src/spec_links.rs +++ b/xtask/src/spec_links.rs @@ -61,8 +61,8 @@ fn parse_table(spec: &str) -> Result, Fail> { pub fn validate(root: &Path) -> Result<(), Fail> { let spec = std::fs::read_to_string(root.join("SPEC.md"))?; let rows = parse_table(&spec)?; - if rows.len() != 470 { - return Err(format!("RP-07: the §31 table has {} rows, not 470", rows.len()).into()); + if rows.len() != 471 { + return Err(format!("RP-07: the §31 table has {} rows, not 471", rows.len()).into()); } // The prose total under the table is part of the contract a reader // relies on; a row added without it would leave §31 contradicting itself.