diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 0b628f8a..1d22d066 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -141,11 +141,113 @@ jobs: run_tests tests + # OwnGenome's Rust code (rust/, docs/rust.md), in a job of its own: + # unit-tests is close to its 10 minutes. Format, lint, unit tests with a + # planted failure that must go red, pycompat against CPython, then the two + # musl builds as scripts/setup.sh --og makes them and the stale-binary + # guard on the real binaries. + rust: + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install the toolchain rust/rust-toolchain.toml pins, with rustfmt and clippy + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal -c rustfmt -c clippy + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + + - name: rustfmt + run: cargo fmt --all --check + + - name: clippy, warnings are errors + if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows + run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + + - name: Control (the planted failing test must fail, and be the test that failed) + run: | + log="${RUNNER_TEMP}/planted.log" + if cargo test --locked -p pycompat --features planted-failure planted_failure > "$log" 2>&1; then + cat "$log" + echo "ERROR: the planted failing test passed: the test run would report a broken test as green" + exit 1 + fi + grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } + echo "Control: the planted test failed, as it must." + + - name: cargo test + run: cargo test --workspace --locked + + - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation) + if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows + run: | + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og + ../tests/rust/pycompat-diff.sh "${PWD}/target/release/og" 100000 + + - name: Both musl builds, as scripts/setup.sh --og makes them in OG_BUILDER + run: | + ../scripts/setup.sh --og + ./og.sh check --required + file ../bin/.og/og-linux-* + ../bin/og version + + - name: The stale-binary guard refuses the real binaries once rust/ changes + run: | + echo "// edited" >> og/src/main.rs + if out=$(./og.sh check); then echo "ERROR: an edited rust/ passed the check: ${out}"; exit 1; fi + echo "Refused, as it must: ${out}" + git checkout -- og/src/main.rs + ./og.sh check --required + + # main.nf runs the same check before any task: a stale og stops + # `nextflow run` with the setup command. Without the check, this stub + # run completes and the step fails. + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'temurin' + java-version: '17' + + - name: The Nextflow version versions.env pins + id: nf + working-directory: . + run: | + # shellcheck source=/dev/null + . ./versions.env + echo "version=${NEXTFLOW_VERSION:?NEXTFLOW_VERSION is not set in versions.env}" >> "$GITHUB_OUTPUT" + + # Not a Nextflow pin: the version comes from versions.env. Quoted so the + # Renovate dry run does not count it as one. + - uses: "nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914" # v3.0.1 + with: + version: ${{ steps.nf.outputs.version }} + + - name: nextflow run stops on a stale og before any task + working-directory: . + run: | + echo "// edited" >> rust/og/src/main.rs + rc=0 + out=$(timeout 600 nextflow run main.nf -profile test -stub -ansi-log false --outdir "${RUNNER_TEMP}/og-stale" 2>&1) || rc=$? + git checkout -- rust/og/src/main.rs + if [ "$rc" -eq 0 ] || [ "$rc" -eq 124 ] || [[ "$out" != *"og is stale:"*"run scripts/setup.sh --og"* ]] || [[ "$out" == *"Submitted process"* ]]; then + echo "$out" + echo "ERROR: nextflow run did not stop on a stale og before any task (exit ${rc})" + exit 1 + fi + echo "Refused, as it must (exit ${rc}): $(grep 'og is stale' <<< "$out")" + # The one check the ruleset requires for this workflow: it runs even when a # job above failed or was skipped, and fails unless every one succeeded. ci-ok: if: always() - needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests] + needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests, rust] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index edb78f85..8ca33e0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,64 @@ on: description: 'Tag to release (e.g. v0.3.0)' required: true type: string + # A dry run: a pull request that changes this file or rust/ builds the og + # binaries and their SHA256SUMS as an artifact; nothing is released. + pull_request: + branches: [main] + paths: ['.github/workflows/release.yml', 'rust/**'] permissions: contents: read jobs: + # The og binaries every release carries (docs/rust.md): both musl builds, + # made by scripts/setup.sh --og as a checkout makes them, with SHA256SUMS. + # A checkout of the tag downloads them instead of building (rust/og.sh). + og-binaries: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + # Same rule as create-release below: the tag is untrusted text. + - name: Check the tag + if: github.event_name != 'pull_request' + env: + INPUT_TAG: ${{ inputs.tag }} + run: | + tag=${INPUT_TAG:-${GITHUB_REF#refs/tags/}} + if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: the tag is not of the form vMAJOR.MINOR.PATCH (not echoed: it is untrusted text)" >&2 + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.tag || github.ref }} + persist-credentials: false + + - name: Build og for x86_64 and aarch64 (scripts/setup.sh --og builds in OG_BUILDER) + run: | + # On a tag, setup.sh --og would download the release's own binaries, + # which this job is about to make: build them. + rust/og.sh build + rust/og.sh check --required + mkdir -p dist + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 dist/ + (cd dist && sha256sum og-linux-* > SHA256SUMS && cat SHA256SUMS) + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: og-binaries + path: dist/ + retention-days: 7 + create-release: + needs: og-binaries + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: write + id-token: write # the build attestation + attestations: write steps: # The tag comes from the dispatch form or the pushed ref, so it is # untrusted text: it reaches the shell through env only, and it must @@ -75,9 +124,25 @@ jobs: echo "All image versions are listed in [\`versions.env\`](versions.env)." echo "" echo "To set up: \`./scripts/setup.sh /path/to/data\`" + echo "" + echo "## og binaries" + echo "" + echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks them." echo "EOF" } >> "$GITHUB_OUTPUT" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: og-binaries + path: dist + + # rust/og.sh install checks a downloaded binary with + # `gh attestation verify` when gh is installed. + - name: Attest the og binaries + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: 'dist/og-linux-*' + - name: Create GitHub Release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: @@ -85,3 +150,7 @@ jobs: name: ${{ steps.tag.outputs.tag }} body: ${{ steps.notes.outputs.notes }} draft: false + files: | + dist/og-linux-x86_64 + dist/og-linux-aarch64 + dist/SHA256SUMS diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 00000000..b34d6d6f --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,120 @@ +name: Rust + +# The og binary in the images the tasks run, and the long pycompat streams. +# Guard's rust job holds the checks every pull request needs (format, lint, +# tests, pycompat at 100000 cases per operation, the musl builds and the +# stale-binary guard); this workflow holds the slow ones (docs/rust.md): +# +# images on a pull request that touches the Rust code, the shim or the +# images: builds og with scripts/setup.sh --og, cold and then +# cached (both times go to the summary), and runs it by bare name +# in PYTHON_IMAGE, BCFTOOLS_IMAGE, MANTA_IMAGE and PARASCOPY_IMAGE +# on every platform each one publishes (arm64 under QEMU), then +# once more with a wrong-arch binary planted, which must fail. The +# two binaries are kept as the artifact og-binaries. +# streams nightly: 10^7 cases per pycompat operation against CPython in +# PYTHON_IMAGE, one operation at a time, with a new seed each +# night (printed, so a failure can be replayed). On a pull request +# that touches the paths below it runs 20000 cases per operation, so the +# job is seen working. +# +# Public repo: GitHub-hosted runners only. +on: + pull_request: + branches: [main] + paths: + - 'rust/**' + - 'bin/og' + - 'tests/rust/**' + - 'versions.env' + - 'scripts/setup.sh' + - '.github/workflows/rust.yml' + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + cases: + description: 'pycompat cases per operation for the streams job' + default: '10000000' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + images: + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build og with scripts/setup.sh --og, cold and cached + run: | + . versions.env + docker pull -q "$OG_BUILDER" > /dev/null + t0=$(date +%s); scripts/setup.sh --og; t1=$(date +%s) + touch rust/og/src/main.rs + scripts/setup.sh --og; t2=$(date +%s) + rust/og.sh check --required + { + echo "| build | seconds |" + echo "|---|---|" + echo "| cold (empty cargo cache, image pulled) | $((t1 - t0)) |" + echo "| cached (one crate changed) | $((t2 - t1)) |" + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + platforms: arm64 + + # shell: bash sets pipefail, so a failing check is not masked by tee. + - name: og by bare name in the task images, amd64 and arm64 + shell: bash + run: tests/rust/og-in-images.sh | tee -a "$GITHUB_STEP_SUMMARY" + + # Control: the same check must fail when an image runs the wrong build. + # The amd64 binary planted as og-linux-aarch64 reports x86_64 on arm64. + - name: Control (a planted wrong-arch binary must fail the image check) + run: | + cp bin/.og/og-linux-aarch64 "${RUNNER_TEMP}/og-linux-aarch64" + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 + rc=0 + tests/rust/og-in-images.sh PYTHON_IMAGE || rc=$? + cp "${RUNNER_TEMP}/og-linux-aarch64" bin/.og/og-linux-aarch64 + [ "$rc" -ne 0 ] || { echo "ERROR: the image check passed a wrong-arch binary"; exit 1; } + echo "Control: the image check failed on the planted binary, as it must." + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: og-binaries + path: bin/.og/og-linux-* + retention-days: 7 + + streams: + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build og with scripts/setup.sh --og + run: | + scripts/setup.sh --og + bin/og version + + - name: pycompat against CPython, one operation at a time + env: + CASES: ${{ github.event_name == 'pull_request' && '20000' || inputs.cases || '10000000' }} + run: | + case "$CASES" in ''|*[!0-9]*) echo "ERROR: cases must be a whole number, got '${CASES}'"; exit 1 ;; esac + seed=${GITHUB_RUN_ID} + fail=0 + ops=$(python3 -I tests/rust/pycompat_oracle.py ops) + [ -n "$ops" ] || { echo "ERROR: the oracle lists no operation"; exit 1; } + for op in $ops; do + tests/rust/pycompat-diff.sh "${PWD}/bin/og" "$CASES" "$seed" "$op" || fail=1 + done + exit "$fail" diff --git a/.gitignore b/.gitignore index e09a812f..7f135ae3 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,8 @@ site/ # Scratch space of a checkout: logs, downloads, CI output /tmp/ + +# Rust: the build output, and the og binaries scripts/setup.sh --og puts +# beside the bin/og shim (rust/og.sh) +/rust/target/ +/bin/.og/ diff --git a/CLAUDE.md b/CLAUDE.md index 30d6960b..032bd266 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,7 @@ Whole genome sequencing (WGS) analysis pipeline for consumer hardware. Takes raw ## Tech Stack - Bash (pipeline scripts, `set -euo pipefail`) - Docker (all bioinformatics tools containerized) +- Python (`bin/*.py`, OwnGenome's own tools), moving tool by tool to Rust: one static binary, `og` (`rust/`, run through the `bin/og` shim; `scripts/setup.sh --og` builds it; `docs/rust.md`) - Key tools: DeepVariant, minimap2, BWA-MEM2, VEP, PharmCAT, GATK, FreeBayes, Strelka2, TIDDIT, Manta, SURVIVOR, PCGR/CPSR, pgsc_calc, haplogrep3, Yleaf ## Development @@ -71,7 +72,8 @@ owngenome/ .github/workflows/ lint.yml # ShellCheck, actionlint, gitleaks, doc links personal-data.yml # Personal-data scan of every tracked text file - guard.yml # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests; ci-ok sums them up + guard.yml # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests, Rust; ci-ok sums them up + rust.yml # og in the task images on amd64 and arm64; nightly pycompat against CPython at 10^7 cases container-test.yml # Runs each changed image on the fixture (tests/smoke/commands.tsv); container-ok sums it up e2e.yml # Real tools on a small fixture nextflow.yml # Nextflow config, lint, schema and stub runs on the pinned release and the newest 26.10.x diff --git a/bin/og b/bin/og new file mode 100755 index 00000000..30396908 --- /dev/null +++ b/bin/og @@ -0,0 +1,21 @@ +#!/bin/sh +# og: OwnGenome's Rust binary, called by bare name in a task (bin/ is on its +# PATH) or as /pgp-bin/og in a step script. It runs the static Linux build +# for this machine from bin/.og/, which scripts/setup.sh --og puts there +# (rust/og.sh). Inside an amd64 image on an Apple Silicon host, uname -m +# says x86_64, so that build runs under emulation. +case $(uname -m) in + x86_64 | amd64) arch=x86_64 ;; + aarch64 | arm64) arch=aarch64 ;; + *) arch=$(uname -m) ;; +esac +bin="$(dirname "$0")/.og/og-linux-${arch}" +if [ "$(uname -s)" != Linux ]; then + echo "ERROR: og runs inside the pipeline's Linux containers, not on $(uname -s)" >&2 + exit 127 +fi +if [ ! -x "$bin" ]; then + echo "ERROR: og is not built for ${arch}; run scripts/setup.sh --og" >&2 + exit 127 +fi +exec "$bin" "$@" diff --git a/bin/write_manifest.sh b/bin/write_manifest.sh index 72f258f6..f8d2098f 100755 --- a/bin/write_manifest.sh +++ b/bin/write_manifest.sh @@ -5,8 +5,10 @@ # # Writes ${GENOME_DIR}//run_manifest.tsv, three tab-separated columns # (section, key, value): -# run written_utc, written_by, git_commit, pipeline_version (the OwnGenome -# release: the version in nextflow.config's manifest block), declared_sex +# run written_utc, written_by, git_commit, og_tree_hash (the rust/ tree +# the og binary in bin/.og was built from, or "not built"), +# pipeline_version (the OwnGenome release: the version in +# nextflow.config's manifest block), declared_sex # version every NAME=VALUE line of versions.env # image each *_IMAGE reference and the digest Docker resolved it to # ("not pulled" when the image is not on this machine) @@ -54,6 +56,7 @@ if git -C "$PGP_ROOT" rev-parse --verify -q HEAD >/dev/null 2>&1; then fi fi row run git_commit "$commit" +row run og_tree_hash "$(bash "${PGP_ROOT}/rust/og.sh" built 2> /dev/null || echo unknown)" # The OwnGenome release: the version = '...' line inside the manifest { } block # of nextflow.config (the release tag is v). No row when there is none. release=$(awk ' diff --git a/docs/nextflow.md b/docs/nextflow.md index 06e9595e..39861401 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -24,6 +24,8 @@ Every BAM then goes through a sex check (indexcov), and the pipeline runs pharma sudo mv nextflow /usr/local/bin/ ``` +Before it starts any task, `main.nf` runs `rust/og.sh check`: when `bin/.og` holds an `og` binary built from another `rust/` than the checkout's, the run stops and names `./scripts/setup.sh --og`, which rebuilds it. No step calls `og` yet, so a checkout without the binary runs as before ([The Rust Core](rust.md)). + ### Run the Pipeline ```bash diff --git a/docs/rust.md b/docs/rust.md new file mode 100644 index 00000000..6a6cb280 --- /dev/null +++ b/docs/rust.md @@ -0,0 +1,83 @@ +# The Rust Core + +This page is for contributors. OwnGenome's own tools are moving from Python to Rust, one tool per pull request, into a single binary, `og`. This page says how the Rust code is laid out, how the binary reaches the pipeline, how it stays byte-for-byte compatible with the Python it replaces, and what CI checks. + +No pipeline step calls `og` yet. The pieces below are the base every port builds on. + +## Layout + +``` +rust/ + Cargo.toml the workspace (edition 2024); Cargo.lock pins every crate + rust-toolchain.toml the Rust release, the one OG_BUILDER ships + og.sh build, download and check the binary + crates/pycompat/ Python-exact formatting and parsing + crates/og-io/ plain, gzip and BGZF input, universal newlines, atomic writes + og/ the binary: og [args] +bin/og the shim tasks call; runs bin/.og/og-linux- +tests/rust/ the differential test against CPython, the image test +``` + +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash` and a hidden `og pycompat-probe` that the tests drive. + +The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2` so far). + +## Building + +```bash +./scripts/setup.sh --og +``` + +This puts two static musl binaries in `bin/.og/` (ignored by git): `og-linux-x86_64` and `og-linux-aarch64`. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. + +- **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. +- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's two binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. + +For day-to-day work on the Rust code a host toolchain is quicker: `cd rust && cargo test`. A plain `cargo build` stamps the binary `unknown`, which the check below refuses, so use `scripts/setup.sh --og` for a binary the pipeline will run. + +The image's Rust release (1.93.0) and `rust/rust-toolchain.toml` must agree, or the build downloads a toolchain first. `OG_BUILDER` has no `_IMAGE` suffix on purpose: no pipeline step runs it, so it has no smoke row and Renovate does not bump it. Bump the image and the toolchain file together, by hand. + +## How a task runs it + +A Nextflow task has `bin/` on its PATH, so a module calls `og ` by its bare name. A bash step calls `/pgp-bin/og ` inside the image it already uses. Either way it starts `bin/og`, a `#!/bin/sh` shim that runs `bin/.og/og-linux-$(uname -m)`, or exits 127 with `ERROR: og is not built for ; run scripts/setup.sh --og`. + +The binary is static, so it runs in any image that has `/bin/sh`: the Python image, the bcftools image, the Manta and Parascopy images. `tests/rust/og-in-images.sh` checks those four on every platform each one publishes. + +## The stale-binary check + +`bin/.og` is not in git, so after a pull or an edit it can hold a binary built from other code. Results would then come from code the checkout does not show. To stop that: + +- The build stamps the binary with the git tree hash of `rust/`: `git rev-parse HEAD:rust` on a clean checkout, or for an edited `rust/` the tree those edits would commit. Any edit, committed or not, changes the hash. +- `rust/og.sh check` reads the stamp from each binary with `grep`, without running it, and compares it with the checkout's. +- `run-all.sh` (even with `SKIP_VALIDATION=true`), `main.nf` and `validate-setup.sh` run that check. A binary built from another `rust/` stops them with `run scripts/setup.sh --og`. +- A missing binary is a note for now. The first pull request that makes a step call `og` sets `REQUIRED=true` in `rust/og.sh`, and from then on a missing binary stops a run too. +- `run_manifest.tsv` gets a row `run og_tree_hash`, so a result names the Rust code that made it (`not built` before the first build). + +## Matching Python byte for byte + +A port must write exactly what the Python tool wrote. Most differences hide in number formatting, so the shared primitives live in `pycompat`, each one written to return what CPython 3.15 returns: + +| Module | What it covers | +|---|---| +| `fmt` | `repr(float)` (also `str` and `f"{x}"`); the `f`, `e`, `g` and `%` format types at any precision; `{:,}` and `{:,.Nf}`; `{:0Nd}`; `%-Ns` and `%Ns`; `round(x)` and `round(x, n)`, both rounding the exact binary value half to even (`round(0.35, 1) == 0.3`, `round(2.675, 2) == 2.67`) | +| `num` | `float(str)` and `int(str)` with Python's whitespace, signs and underscores; `math.fsum`; the builtin `sum`, which since Python 3.12 is a compensated sum and so differs from a plain loop | +| `json` | `json.dumps` with `indent` (none or a number), `separators`, `ensure_ascii` (surrogate pairs for astral characters), `sort_keys` and `allow_nan`; `json.loads` with `NaN` and `Infinity`, where a duplicate key keeps its first place and takes the last value | + +`og-io` reads a path or `-` as plain text, gzip or BGZF (sniffed by the magic bytes; NUL padding after a gzip member is skipped, as Python's `gzip` does), splits lines as Python's text mode does (`\n`, `\r\n` and a lone `\r`), decodes UTF-8 strictly or with `replace`, and writes a file under a temporary name before renaming it into place. + +Known limits, each an error in Rust where Python would go on: ints beyond 128 bits, a lone surrogate in a JSON string, non-ASCII digits in `float()` and `int()`, a string `indent`, and JSON nested deeper than 1000 levels (`RecursionError`; CPython 3.14 reads about 100,000 on a default stack). None of them appears in what the pipeline reads. + +### The differential test + +`tests/rust/pycompat-diff.sh [N] [SEED]` checks every `pycompat` operation against CPython. `tests/rust/pycompat_oracle.py` runs with `python3 -I` in `PYTHON_IMAGE`, draws N random cases per operation and writes its result for each. `og pycompat-probe` reads the same cases and must write the same lines, byte for byte. The values include the ones that catch a formatting bug: binary ties (0.125, 2.675, 0.35), 1e-05 and 1e16 where `repr` switches layout, -0.0, NaN, subnormals, random bit patterns, numbers exactly halfway between two shortest `repr` candidates, astral characters, CRLF and lone CR (both as lines without their endings and as `readline()` results with them), and broken JSON. A failure prints the first cases that differ and the seed, so `pycompat-diff.sh ` replays it. `PYTHON=python3` runs the oracle on the host for a quick loop. + +To add a primitive: write it in `pycompat` with unit tests, add an operation to `og/src/probe.rs` and the same operation to the oracle, then run the differential test. + +## CI + +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; both musl builds through `scripts/setup.sh --og`; and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. +- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a tree hash that cannot be computed, a release download with a bad checksum or attestation, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. +- **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. +- **Release** (`release.yml`): a tag builds both binaries, writes `SHA256SUMS`, attests them and attaches all three to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. + +[Testing](testing.md) has the rest of CI. diff --git a/docs/testing.md b/docs/testing.md index e3ef3253..7564ac2a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -229,6 +229,18 @@ The **Demo pictures** workflow ([`.github/workflows/demo-pictures.yml`](https:// Look at the pictures, copy them into `docs/images/`, and update the alt text where the cards changed. A new report card that the generator does not feed yet stops the script with its name; add its input file to `make_demo_sample.py`. +## The Rust jobs + +The `rust` job of `guard.yml` checks the Rust code on every pull request: format, clippy, the unit tests with a planted failure that must fail, `pycompat` against CPython at 100,000 cases per operation, both musl builds as `scripts/setup.sh --og` makes them, and the stale-binary check on the real binaries. The **Rust** workflow (`rust.yml`) runs `og` by bare name in the task images on amd64 and arm64, and every night runs the CPython comparison at 10,000,000 cases per operation. [The Rust Core](rust.md) says what each check covers. + +How long `scripts/setup.sh --og` takes to build both binaries: + +| Where | Cold (empty cargo cache) | Cached (one crate changed) | +|---|---|---| +| Apple Silicon Mac mini, 10 cores | 25 s | 3 s | + +The cold build downloads the crates once; pulling `OG_BUILDER` the first time adds about 1 GB. The Rust workflow's `images` job prints both times for `ubuntu-latest` in its summary. + ## Telling a flaky test from a broken one The **Repeat a test** workflow (`.github/workflows/repeat-test.yml`) runs one `tests/test_*.sh` or `tests/test_*.py` many times, each run in a fresh job on its own `ubuntu-latest` runner, six at a time. Start it from the Actions tab with the test's path and the number of runs (1 to 60, default 30); it also runs, with its defaults (`tests/test_annotsv_benign_mane.sh`, 30 runs), on a pull request that changes the workflow. A flaky test fails a few runs out of many; a broken one fails them all. Each run records its exit code, and the summary job lists every run and fails when any failed. One more run, `control`, runs a test that always fails, and the summary fails when the control passed, so "0 of 30 failed" is never a count that could not see a failure. diff --git a/main.nf b/main.nf index b52aaa1a..f0ad7ff0 100644 --- a/main.nf +++ b/main.nf @@ -97,6 +97,19 @@ workflow { "decompress it (gunzip, or bgzip -d) and run 'samtools faidx' on the result." } + // og (bin/.og, the Rust binary tasks call through bin/og) built from + // another rust/ than this checkout's would run other code than the + // checkout says: rust/og.sh check stops the run (scripts/setup.sh --og). + if (file("${projectDir}/rust/og.sh").exists()) { + // stderr joins stdout, so a check that cannot even start says why. + def og_check = ['bash', '-c', 'bash "$0" check 2>&1', "${projectDir}/rust/og.sh"].execute() + def og_msg = og_check.text.trim() + og_check.waitFor() + if (og_check.exitValue() != 0) { + error(og_msg ? og_msg.replaceFirst(/^ERROR: /, '') : "rust/og.sh check failed with exit ${og_check.exitValue()} and no message") + } + } + def tools_list = params.tools ? params.tools.split(',').collect { it.trim() }.findAll { it } : [] // Every name the workflows gate on. A name outside this list would be diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..d53c8030 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,101 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "og" +version = "0.1.0" +dependencies = [ + "og-io", + "pycompat", +] + +[[package]] +name = "og-io" +version = "0.1.0" +dependencies = [ + "flate2", +] + +[[package]] +name = "pycompat" +version = "0.1.0" +dependencies = [ + "indexmap", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..1a544792 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,27 @@ +[workspace] +resolver = "3" +members = ["crates/pycompat", "crates/og-io", "og"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +license = "GPL-3.0-or-later" +publish = false + +[workspace.dependencies] +pycompat = { path = "crates/pycompat" } +og-io = { path = "crates/og-io" } +indexmap = "2" +flate2 = "1" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +all = { level = "warn", priority = -1 } + +[profile.release] +lto = true +codegen-units = 1 +panic = "abort" +strip = true diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..e6658688 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,20 @@ +# rust/ + +OwnGenome's own tools, ported from `bin/*.py` one at a time into one static binary, `og`. Each port must write what the Python wrote, byte for byte. The full guide is [docs/rust.md](../docs/rust.md). + +| Path | What it is | +|---|---| +| `crates/pycompat` | Python-exact `repr`, format types, `round`, `float()`/`int()`, `math.fsum`, `sum`, `json.dumps`/`loads` | +| `crates/og-io` | plain, gzip and BGZF input; Python's universal newlines; atomic writes | +| `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | +| `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | +| `rust-toolchain.toml` | the Rust release, the one `OG_BUILDER` (`versions.env`) ships | + +```bash +cargo test # unit tests (needs the pinned toolchain) +cargo clippy --all-targets -- -D warnings +../scripts/setup.sh --og # the binaries the pipeline runs, built in Docker +cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # pycompat against CPython +``` + +A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. diff --git a/rust/crates/og-io/Cargo.toml b/rust/crates/og-io/Cargo.toml new file mode 100644 index 00000000..0260f051 --- /dev/null +++ b/rust/crates/og-io/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "og-io" +description = "Input and output as OwnGenome's Python tools do it: plain, gzip or BGZF in, universal newlines, atomic writes" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +flate2.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs new file mode 100644 index 00000000..5ebbb18f --- /dev/null +++ b/rust/crates/og-io/src/lib.rs @@ -0,0 +1,344 @@ +//! Reading and writing files the way OwnGenome's Python tools do. +//! +//! - [`open`]: a path or `-` (stdin), plain or gzip, sniffed by its magic +//! bytes. Multi-member gzip, BGZF included, is read to the end, and NUL +//! padding after a member is skipped, as Python's `gzip` does. +//! - [`Lines`]: Python's text mode with `newline=None`: `\n`, `\r\n` and a +//! lone `\r` each end a line and read as `\n`; UTF-8 strict or `replace`. +//! - [`AtomicFile`]: written beside the target and renamed over it on +//! [`AtomicFile::commit`], so a reader never sees half a file. + +use flate2::bufread::GzDecoder; +use std::fs::{self, File}; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; + +const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; + +/// Opens `path` for reading (`-` is stdin), decompressing gzip and BGZF. +pub fn open(path: impl AsRef) -> io::Result> { + let path = path.as_ref(); + let inner: Box = if path == Path::new("-") { + Box::new(io::stdin().lock()) + } else { + Box::new(File::open(path)?) + }; + reader(inner) +} + +/// Wraps any reader: gzip when it starts with the gzip magic, plain otherwise. +pub fn reader(mut inner: impl Read + 'static) -> io::Result> { + // Read the magic byte by byte: a pipe may hand over one byte at a time. + let mut head = Vec::with_capacity(2); + let mut byte = [0u8; 1]; + while head.len() < 2 { + match inner.read(&mut byte) { + Ok(0) => break, + Ok(_) => head.push(byte[0]), + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + let gz = head == GZIP_MAGIC; + let buf = BufReader::with_capacity(1 << 16, io::Cursor::new(head).chain(inner)); + Ok(if gz { + Box::new(BufReader::with_capacity( + 1 << 16, + Members { + dec: Some(GzDecoder::new(buf)), + }, + )) + } else { + Box::new(buf) + }) +} + +/// Gzip members one after another, as Python's `gzip` reads them: NUL bytes +/// between or after members are padding, and the stream may end in them. +struct Members { + /// The member being read; `None` once the input has ended. + dec: Option>, +} + +impl Read for Members { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + while let Some(dec) = self.dec.as_mut() { + let n = dec.read(buf)?; + if n > 0 { + return Ok(n); + } + // This member has ended: skip padding, then the next one starts. + let mut inner = self.dec.take().expect("a member").into_inner(); + loop { + let rest = inner.fill_buf()?; + if rest.is_empty() { + return Ok(0); + } + let zeros = rest.iter().take_while(|&&c| c == 0).count(); + let more = zeros < rest.len(); + inner.consume(zeros); + if more { + break; + } + } + self.dec = Some(GzDecoder::new(inner)); + } + Ok(0) + } +} + +/// How invalid UTF-8 is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Utf8 { + /// `errors="strict"`: an error. + Strict, + /// `errors="replace"`: U+FFFD for each invalid sequence. + Replace, +} + +/// Python's universal-newline text reader over any [`BufRead`]. +pub struct Lines { + inner: R, + raw: Vec, + pos: usize, + utf8: Utf8, +} + +impl Lines { + pub fn new(inner: R, utf8: Utf8) -> Self { + Lines { + inner, + raw: Vec::new(), + pos: 0, + utf8, + } + } + + /// Python's `readline()`: the next line with its ending turned into + /// `\n` (none on a last line without one), or `false` at the end. + pub fn read_line(&mut self, out: &mut String) -> io::Result { + out.clear(); + if self.pos == self.raw.len() { + self.raw.clear(); + self.pos = 0; + if self.inner.read_until(b'\n', &mut self.raw)? == 0 { + return Ok(false); + } + } + let rest = &self.raw[self.pos..]; + // read_until stops after a \n or at the end, so a \r\n is never split. + let (body, used) = match rest.iter().position(|&c| c == b'\n' || c == b'\r') { + None => (rest, rest.len()), + Some(k) if rest[k] == b'\r' && rest.get(k + 1) == Some(&b'\n') => (&rest[..k], k + 2), + Some(k) => (&rest[..k], k + 1), + }; + let ended = used > body.len(); + // Past the line before decoding it, so a strict error is not raised + // again for the same line by a caller that goes on reading. (Python + // raises once too; a caller is expected to stop at the error.) + self.pos += used; + match self.utf8 { + Utf8::Strict => out.push_str( + std::str::from_utf8(body) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?, + ), + Utf8::Replace => out.push_str(&String::from_utf8_lossy(body)), + } + if ended { + out.push('\n'); + } + Ok(true) + } +} + +impl Iterator for Lines { + type Item = io::Result; + + /// Each line without its ending, as `line.rstrip("\n")` after Python's + /// universal-newline reading. + fn next(&mut self) -> Option { + let mut s = String::new(); + match self.read_line(&mut s) { + Ok(true) => { + if s.ends_with('\n') { + s.pop(); + } + Some(Ok(s)) + } + Ok(false) => None, + Err(e) => Some(Err(e)), + } + } +} + +/// A file written under a temporary name and renamed into place by +/// [`commit`](AtomicFile::commit). Dropped without a commit, the temporary +/// file is removed and the target is untouched. +pub struct AtomicFile { + tmp: PathBuf, + target: PathBuf, + out: Option>, +} + +impl AtomicFile { + pub fn create(target: impl AsRef) -> io::Result { + let target = target.as_ref().to_path_buf(); + let name = target + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no file name"))?; + let mut tmp_name = std::ffi::OsString::from("."); + tmp_name.push(name); + tmp_name.push(format!(".tmp-{}", std::process::id())); + let tmp = target.with_file_name(tmp_name); + let out = BufWriter::new(File::create(&tmp)?); + Ok(AtomicFile { + tmp, + target, + out: Some(out), + }) + } + + /// Flushes and renames over the target. On an error the temporary file + /// is removed and the target is untouched. + pub fn commit(mut self) -> io::Result<()> { + let out = self.out.take().expect("committed once"); + let done = out + .into_inner() + .map_err(io::IntoInnerError::into_error) + .and_then(|f| { + drop(f); + fs::rename(&self.tmp, &self.target) + }); + if done.is_err() { + let _ = fs::remove_file(&self.tmp); + } + done + } +} + +impl Write for AtomicFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.out.as_mut().expect("not committed").write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.out.as_mut().expect("not committed").flush() + } +} + +impl Drop for AtomicFile { + fn drop(&mut self) { + if self.out.take().is_some() { + let _ = fs::remove_file(&self.tmp); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::Compression; + use flate2::write::GzEncoder; + + fn lines(bytes: &[u8], utf8: Utf8) -> Vec { + let mut l = Lines::new(io::Cursor::new(bytes.to_vec()), utf8); + let mut out = Vec::new(); + let mut s = String::new(); + while l.read_line(&mut s).unwrap() { + out.push(s.clone()); + } + out + } + + #[test] + fn universal_newlines() { + assert_eq!( + lines(b"a\r\nb\rc\n\rd", Utf8::Strict), + ["a\n", "b\n", "c\n", "\n", "d"] + ); + assert_eq!(lines(b"x\r", Utf8::Strict), ["x\n"]); + assert_eq!(lines(b"", Utf8::Strict), Vec::::new()); + assert_eq!( + lines(b"\xff\xe2\x82\n", Utf8::Replace), + ["\u{fffd}\u{fffd}\n"] + ); + // A strict error on one line is raised once; reading goes on after it. + let mut l = Lines::new(io::Cursor::new(b"\xff\nok\n".to_vec()), Utf8::Strict); + let mut s = String::new(); + assert!(l.read_line(&mut s).is_err()); + assert!(l.read_line(&mut s).unwrap()); + assert_eq!(s, "ok\n"); + assert!(!l.read_line(&mut s).unwrap()); + let it: Vec = Lines::new(io::Cursor::new(b"a\r\nb".to_vec()), Utf8::Strict) + .map(Result::unwrap) + .collect(); + assert_eq!(it, ["a", "b"]); + } + + fn gz(data: &[u8]) -> Vec { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(data).unwrap(); + e.finish().unwrap() + } + + #[test] + fn reads_plain_gzip_and_multi_member() { + let mut two = gz(b"one\n"); + two.extend(gz(b"two\n")); + // NUL padding between and after members, as Python's gzip skips it. + let mut padded = gz(b"one\n"); + padded.extend([0u8; 16]); + padded.extend(gz(b"")); + padded.extend(gz(b"two\n")); + padded.extend([0u8; 12]); + for (input, want) in [ + (b"plain\n".to_vec(), "plain\n"), + (gz(b"zipped\n"), "zipped\n"), + (two, "one\ntwo\n"), + (padded, "one\ntwo\n"), + (vec![0x1f], "\u{1f}"), + ] { + let mut s = String::new(); + reader(io::Cursor::new(input)) + .unwrap() + .read_to_string(&mut s) + .unwrap(); + assert_eq!(s, want); + } + // Anything else after a member is an error, as in Python. + let mut junk = gz(b"one\n"); + junk.extend(b"\0\0junk"); + let mut s = String::new(); + let mut r = reader(io::Cursor::new(junk)).unwrap(); + assert!(r.read_to_string(&mut s).is_err()); + } + + #[test] + fn atomic_write() { + let dir = std::env::temp_dir().join(format!("og-io-test-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let target = dir.join("out.tsv"); + fs::write(&target, "old").unwrap(); + { + let mut f = AtomicFile::create(&target).unwrap(); + f.write_all(b"half").unwrap(); + } + assert_eq!(fs::read_to_string(&target).unwrap(), "old"); + let mut f = AtomicFile::create(&target).unwrap(); + f.write_all(b"new").unwrap(); + f.commit().unwrap(); + assert_eq!(fs::read_to_string(&target).unwrap(), "new"); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + // A rename that fails (the target is a directory that is not empty) + // leaves no temporary file behind. + let busy = dir.join("busy"); + fs::create_dir_all(busy.join("x")).unwrap(); + let mut f = AtomicFile::create(&busy).unwrap(); + f.write_all(b"data").unwrap(); + assert!(f.commit().is_err()); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 2); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/rust/crates/pycompat/Cargo.toml b/rust/crates/pycompat/Cargo.toml new file mode 100644 index 00000000..8c8128c9 --- /dev/null +++ b/rust/crates/pycompat/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "pycompat" +description = "Python-exact float, int and JSON formatting and parsing" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[features] +# A test that always fails: CI runs it and requires the run to go red. +planted-failure = [] + +[dependencies] +indexmap.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/pycompat/src/fmt.rs b/rust/crates/pycompat/src/fmt.rs new file mode 100644 index 00000000..255fe211 --- /dev/null +++ b/rust/crates/pycompat/src/fmt.rs @@ -0,0 +1,508 @@ +//! Python's float and int formatting, byte for byte. +//! +//! Rust's `{:e}` gives the shortest digits that round-trip, which are the +//! digits of Python's `repr`; Rust's `{:.N}` and `{:.Ne}` round the exact +//! binary value half to even, as CPython's dtoa does. What differs is the +//! layout (`1e-05` against `1e-5`, `nan` against `NaN`), and that is what +//! this module writes. + +use std::fmt; + +/// `nan`, `inf` and `-inf`, the spelling of every Python float format. +fn special(x: f64) -> Option<&'static str> { + if x.is_nan() { + Some("nan") + } else if x == f64::INFINITY { + Some("inf") + } else if x == f64::NEG_INFINITY { + Some("-inf") + } else { + None + } +} + +fn sign(x: f64) -> &'static str { + if x.is_sign_negative() { "-" } else { "" } +} + +/// The digits and decimal point of Rust's scientific form of a finite +/// `|x|`: `"1.25e-3"` gives `("125", -2)`, the value `0.125 * 10^-2`. +fn split_sci(s: &str) -> (String, i32) { + let (mant, exp) = s.split_once('e').expect("scientific form has an exponent"); + let digits = mant.chars().filter(|&c| c != '.').collect(); + let exp: i32 = exp.parse().expect("integer exponent"); + (digits, exp + 1) +} + +/// `d[0].d[1..]e±XX`, with at least two exponent digits, as Python writes it. +fn exp_layout(digits: &str, exp: i32) -> String { + let (first, rest) = digits.split_at(1); + let dot = if rest.is_empty() { "" } else { "." }; + let es = if exp < 0 { '-' } else { '+' }; + format!("{first}{dot}{rest}e{es}{:02}", exp.unsigned_abs()) +} + +/// The digits laid out without an exponent, `decpt` digits before the point. +fn fixed_layout(digits: &str, decpt: i32, add_dot_0: bool) -> String { + let n = digits.len() as i32; + if decpt <= 0 { + format!("0.{}{digits}", "0".repeat(decpt.unsigned_abs() as usize)) + } else if decpt >= n { + let zeros = "0".repeat((decpt - n) as usize); + let tail = if add_dot_0 { ".0" } else { "" }; + format!("{digits}{zeros}{tail}") + } else { + let (int, frac) = digits.split_at(decpt as usize); + format!("{int}.{frac}") + } +} + +/// The shortest digits that read back as `x` (finite, positive), and the +/// decimal point. Rust and CPython agree on them except when `x` lies +/// exactly halfway between the two nearest candidates: CPython's dtoa then +/// takes the one with the even last digit, Rust may take the other. +fn shortest(x: f64) -> (String, i32) { + let (digits, decpt) = split_sci(&format!("{x:e}")); + let Some(exact) = exact_digits(x) else { + return (digits, decpt); + }; + let n = digits.len(); + if exact.len() != n + 1 || !exact.ends_with('5') { + return (digits, decpt); + } + // x is the midpoint of `low` and `low + 1` in the last place. + let low = exact[..n].to_owned(); + let high = increment(&low); + let parses_back = |d: &str, decpt: i32| format!("0.{d}e{decpt}").parse::() == Ok(x); + let (high, high_decpt) = if high.len() > n { + (high[..n].to_owned(), decpt + 1) + } else { + (high, decpt) + }; + let low_even = (low.as_bytes()[n - 1] - b'0').is_multiple_of(2); + let mut picks = [(low, decpt), (high, high_decpt)]; + if !low_even { + picks.swap(0, 1); + } + for (d, p) in picks { + if parses_back(&d, p) { + let d = d.trim_end_matches('0'); + return ( + if d.is_empty() { + "0".to_owned() + } else { + d.to_owned() + }, + p, + ); + } + } + (digits, decpt) +} + +/// A decimal string of digits plus one in the last place (`"19"` -> `"20"`, `"99"` -> `"100"`). +fn increment(d: &str) -> String { + let mut b = d.as_bytes().to_vec(); + let mut i = b.len(); + loop { + if i == 0 { + b.insert(0, b'1'); + break; + } + i -= 1; + if b[i] == b'9' { + b[i] = b'0'; + } else { + b[i] += 1; + break; + } + } + String::from_utf8(b).expect("ascii digits") +} + +/// The significant digits of the exact value of a finite positive `x`, +/// when there are at most 18 of them (the only case where a tie between +/// two 17-digit candidates is possible); `None` otherwise. +fn exact_digits(x: f64) -> Option { + let bits = x.to_bits(); + let biased = ((bits >> 52) & 0x7ff) as i32; + let frac = bits & ((1 << 52) - 1); + let (mut m, mut e) = if biased == 0 { + (frac, -1074) + } else { + (frac | (1 << 52), biased - 1075) + }; + let tz = m.trailing_zeros(); + m >>= tz; + e += tz as i32; + let s = if e >= 0 { + if e < 74 { + (u128::from(m) << e).to_string() + } else { + format!("{x:.0}") + } + } else { + // x = m * 5^k / 10^k; past 5^27 there are more than 18 digits. + let k = e.unsigned_abs(); + if k > 27 { + return None; + } + (u128::from(m) * 5u128.pow(k)).to_string() + }; + let s = s.trim_end_matches('0'); + (s.len() <= 18).then(|| s.to_owned()) +} + +/// `repr(x)`, which is also `str(x)` and `f"{x}"`. +pub fn repr(x: f64) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + if x == 0.0 { + return format!("{}0.0", sign(x)); + } + let (digits, decpt) = shortest(x.abs()); + let body = if decpt <= -4 || decpt > 16 { + exp_layout(&digits, decpt - 1) + } else { + fixed_layout(&digits, decpt, true) + }; + format!("{}{body}", sign(x)) +} + +/// `f"{x:.{prec}f}"`. +pub fn fixed(x: f64, prec: usize) -> String { + match special(x) { + Some(s) => s.to_owned(), + None => format!("{x:.prec$}"), + } +} + +/// `f"{x:.{prec}e}"`. +pub fn sci(x: f64, prec: usize) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + let (digits, decpt) = split_sci(&format!("{:.prec$e}", x.abs())); + let (first, rest) = digits.split_at(1); + let es = if decpt - 1 < 0 { '-' } else { '+' }; + let dot = if prec == 0 { "" } else { "." }; + format!( + "{}{first}{dot}{rest}e{es}{:02}", + sign(x), + (decpt - 1).unsigned_abs() + ) +} + +/// `f"{x:.{prec}g}"`, also `"%.{prec}g" % x`. `f"{x:g}"` is `general(x, 6)`; +/// a precision of 0 counts as 1, as in Python. +pub fn general(x: f64, prec: usize) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + if x == 0.0 { + return format!("{}0", sign(x)); + } + let p = prec.max(1); + let (digits, decpt) = split_sci(&format!("{:.*e}", p - 1, x.abs())); + let digits = digits.trim_end_matches('0'); + let exp = decpt - 1; + let body = if exp < -4 || exp >= p as i32 { + exp_layout(digits, exp) + } else { + fixed_layout(digits, decpt, false) + }; + format!("{}{body}", sign(x)) +} + +/// `f"{x:.{prec}%}"`: `x * 100` in the `f` format, then `%`. +pub fn percent(x: f64, prec: usize) -> String { + format!("{}%", fixed(x * 100.0, prec)) +} + +/// Commas every three digits of a run of ASCII digits. +fn group(digits: &str) -> String { + let mut out = String::with_capacity(digits.len() + digits.len() / 3); + for (i, c) in digits.chars().enumerate() { + if i > 0 && (digits.len() - i).is_multiple_of(3) { + out.push(','); + } + out.push(c); + } + out +} + +/// `f"{n:,}"` of an int. +pub fn int_grouped(n: i128) -> String { + let s = if n < 0 { "-" } else { "" }; + format!("{s}{}", group(&n.unsigned_abs().to_string())) +} + +/// `f"{x:,.{prec}f}"`. +pub fn fixed_grouped(x: f64, prec: usize) -> String { + let s = fixed(x, prec); + let (neg, body) = match s.strip_prefix('-') { + Some(b) => ("-", b), + None => ("", s.as_str()), + }; + if !body.starts_with(|c: char| c.is_ascii_digit()) { + return s; + } + let (int, frac) = match body.split_once('.') { + Some((i, f)) => (i, Some(f)), + None => (body, None), + }; + match frac { + Some(f) => format!("{neg}{}.{f}", group(int)), + None => format!("{neg}{}", group(int)), + } +} + +/// `f"{n:0{width}d}"`: the sign first, then zeros up to `width` characters. +pub fn zero_pad(n: i128, width: usize) -> String { + let digits = n.unsigned_abs().to_string(); + let s = if n < 0 { "-" } else { "" }; + let pad = width.saturating_sub(digits.len() + s.len()); + format!("{s}{}{digits}", "0".repeat(pad)) +} + +/// `"%-{width}s" % s` and `s.ljust(width)`: width in code points. +pub fn ljust(s: &str, width: usize) -> String { + let pad = width.saturating_sub(s.chars().count()); + format!("{s}{}", " ".repeat(pad)) +} + +/// `"%{width}s" % s` and `s.rjust(width)`: width in code points. +pub fn rjust(s: &str, width: usize) -> String { + let pad = width.saturating_sub(s.chars().count()); + format!("{}{s}", " ".repeat(pad)) +} + +/// Why Python's `round` raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoundError { + /// `round(nan)`: ValueError. + Nan, + /// `round(inf)`: OverflowError. + Infinite, + /// `round(x, n)` whose result would not fit a float: OverflowError. + TooLarge, +} + +impl RoundError { + /// The Python exception class. + pub fn exception(self) -> &'static str { + match self { + RoundError::Nan => "ValueError", + RoundError::Infinite | RoundError::TooLarge => "OverflowError", + } + } +} + +impl fmt::Display for RoundError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + RoundError::Nan => "cannot convert float NaN to integer", + RoundError::Infinite => "cannot convert float infinity to integer", + RoundError::TooLarge => "rounded value too large to represent", + }) + } +} + +impl std::error::Error for RoundError {} + +/// `round(x)`: the nearest integer, ties to even. Python returns an int; +/// this returns it as an integral float, which [`int_str`] prints exactly. +pub fn round_int(x: f64) -> Result { + if x.is_nan() { + Err(RoundError::Nan) + } else if x.is_infinite() { + Err(RoundError::Infinite) + } else { + Ok(x.round_ties_even()) + } +} + +/// `str(int(x))` of an integral float: every digit, exactly. +pub fn int_str(x: f64) -> String { + debug_assert!(x.is_finite() && x.fract() == 0.0); + if x == 0.0 { + "0".to_owned() + } else { + format!("{x:.0}") + } +} + +/// `round(x, n)` for a float: the exact binary value rounded half to even +/// at `n` decimals (`round(0.35, 1) == 0.3`, `round(2.675, 2) == 2.67`), +/// read back as the nearest float. A negative `n` rounds to tens, hundreds. +pub fn round(x: f64, n: i32) -> Result { + // CPython's NDIGITS_MAX and NDIGITS_MIN (floatobject.c). + if !x.is_finite() || n > 323 { + return Ok(x); + } + if n < -308 { + return Ok(0.0 * x); + } + let text = if n >= 0 { + format!("{x:.0$}", n as usize) + } else { + round_integer_digits(x, n.unsigned_abs() as usize) + }; + let r: f64 = text.parse().expect("a decimal string parses"); + if r.is_infinite() { + Err(RoundError::TooLarge) + } else { + Ok(r) + } +} + +/// `x` rounded half to even to a multiple of `10^k`, as a decimal string +/// with its sign. Works on the exact integer digits so it never rounds twice. +fn round_integer_digits(x: f64, k: usize) -> String { + let a = x.abs(); + let int = a.trunc(); + let frac_nonzero = a != int; + let mut digits: Vec = int_str(int).into_bytes(); + if digits.len() < k + 1 { + let mut padded = vec![b'0'; k + 1 - digits.len()]; + padded.extend_from_slice(&digits); + digits = padded; + } + let cut = digits.len() - k; + let (kept, dropped) = digits.split_at(cut); + let mut kept = kept.to_vec(); + let up = match dropped[0].cmp(&b'5') { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Less => false, + std::cmp::Ordering::Equal => { + frac_nonzero + || dropped[1..].iter().any(|&d| d != b'0') + || (kept.last().copied().unwrap_or(b'0') - b'0') % 2 == 1 + } + }; + if up { + let mut i = kept.len(); + loop { + if i == 0 { + kept.insert(0, b'1'); + break; + } + i -= 1; + if kept[i] == b'9' { + kept[i] = b'0'; + } else { + kept[i] += 1; + break; + } + } + } + let mut s = String::from(sign(x)); + s.push_str(std::str::from_utf8(&kept).expect("ascii digits")); + s.push_str(&"0".repeat(k)); + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repr_matches_python() { + let cases = [ + (0.1, "0.1"), + (1.0, "1.0"), + (-0.0, "-0.0"), + (0.0, "0.0"), + (1e16, "1e+16"), + (1e15, "1000000000000000.0"), + (1.5e16, "1.5e+16"), + (0.0001, "0.0001"), + (0.00001, "1e-05"), + (1.2345e-7, "1.2345e-07"), + (123456.789, "123456.789"), + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + (f64::NAN, "nan"), + (f64::INFINITY, "inf"), + (f64::NEG_INFINITY, "-inf"), + (2.0f64.powi(60), "1.152921504606847e+18"), + // Exactly halfway between two 16-digit candidates: the even one. + (94614632987514.0 + 0.125, "94614632987514.12"), + (634342696773016.0 + 0.25, "634342696773016.2"), + ]; + for (x, want) in cases { + assert_eq!(repr(x), want, "repr({x:?})"); + } + } + + #[test] + fn fixed_rounds_the_exact_value_half_even() { + assert_eq!(fixed(0.125, 2), "0.12"); + assert_eq!(fixed(0.375, 2), "0.38"); + assert_eq!(fixed(2.675, 2), "2.67"); + assert_eq!(fixed(2.5, 0), "2"); + assert_eq!(fixed(-0.04, 1), "-0.0"); + assert_eq!(fixed(f64::NAN, 3), "nan"); + assert_eq!(fixed(-1.0 / 0.0, 3), "-inf"); + } + + #[test] + fn sci_and_general() { + assert_eq!(sci(123456.0, 3), "1.235e+05"); + assert_eq!(sci(0.0, 2), "0.00e+00"); + assert_eq!(sci(1.5, 0), "2e+00"); + assert_eq!(general(1234567.0, 6), "1.23457e+06"); + assert_eq!(general(123456.0, 6), "123456"); + assert_eq!(general(0.0001, 6), "0.0001"); + assert_eq!(general(0.00001, 6), "1e-05"); + assert_eq!(general(100.0, 6), "100"); + assert_eq!(general(1.5, 0), "2"); + assert_eq!(general(9.9999996, 6), "10"); + assert_eq!(general(-0.0, 6), "-0"); + assert_eq!(general(0.1 + 0.2, 10), "0.3"); + assert_eq!(general(1e16, 17), "10000000000000000"); + } + + #[test] + fn percent_grouping_padding() { + assert_eq!(percent(0.12345, 1), "12.3%"); + assert_eq!(percent(f64::NAN, 1), "nan%"); + assert_eq!(int_grouped(-1234567), "-1,234,567"); + assert_eq!(int_grouped(999), "999"); + assert_eq!(fixed_grouped(1234567.891, 2), "1,234,567.89"); + assert_eq!(fixed_grouped(-999.5, 0), "-1,000"); + assert_eq!(zero_pad(-42, 5), "-0042"); + assert_eq!(zero_pad(42, 1), "42"); + assert_eq!(ljust("é", 3), "é "); + assert_eq!(rjust("ab", 3), " ab"); + } + + #[test] + fn round_is_half_even_on_the_binary_value() { + assert_eq!(round_int(2.5), Ok(2.0)); + assert_eq!(round_int(3.5), Ok(4.0)); + assert_eq!(round_int(-0.5), Ok(-0.0)); + assert_eq!(round_int(f64::NAN), Err(RoundError::Nan)); + assert_eq!(round_int(f64::INFINITY), Err(RoundError::Infinite)); + assert_eq!(int_str(-0.0), "0"); + assert_eq!(int_str(1e22), "10000000000000000000000"); + assert_eq!(round(0.35, 1), Ok(0.3)); + assert_eq!(round(2.675, 2), Ok(2.67)); + assert_eq!(round(0.125, 2), Ok(0.12)); + assert_eq!(round(1250.0, -2), Ok(1200.0)); + assert_eq!(round(1350.0, -2), Ok(1400.0)); + assert_eq!(round(1250.4, -2), Ok(1300.0)); + assert_eq!(round(5.0, -1), Ok(0.0)); + assert_eq!(round(15.0, -1), Ok(20.0)); + assert_eq!(round(501.0, -3), Ok(1000.0)); + assert!(round(-4.0, -1).unwrap().is_sign_negative()); + assert!(round(-0.04, 1).unwrap().is_sign_negative()); + assert_eq!( + round(1.7976931348623157e308, -308), + Err(RoundError::TooLarge) + ); + assert_eq!(round(1.5, 400), Ok(1.5)); + assert!(round(f64::NAN, 2).unwrap().is_nan()); + } +} diff --git a/rust/crates/pycompat/src/json.rs b/rust/crates/pycompat/src/json.rs new file mode 100644 index 00000000..5a6bdecc --- /dev/null +++ b/rust/crates/pycompat/src/json.rs @@ -0,0 +1,565 @@ +//! Python's `json` module: `dumps` byte for byte, and `loads` with its +//! leniency (`NaN`, `Infinity`, the last of a duplicate key wins and keeps +//! the first one's place). + +use indexmap::IndexMap; +use std::fmt::{self, Write}; + +use crate::fmt::repr; + +/// A JSON value as Python holds it: ints and floats apart, objects in +/// insertion order. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + /// A Python int. Python has no limit; this port stops at 128 bits. + Int(i128), + Float(f64), + Str(String), + Array(Vec), + Object(IndexMap), +} + +/// The `json.dumps` keyword arguments this port supports. +#[derive(Debug, Clone)] +pub struct Dump<'a> { + /// `indent=N`; `None` writes one line. + pub indent: Option, + /// `separators=(item, key)`; `None` is Python's default, which depends + /// on `indent`: `(", ", ": ")` without, `(",", ": ")` with. + pub separators: Option<(&'a str, &'a str)>, + pub ensure_ascii: bool, + pub sort_keys: bool, + pub allow_nan: bool, +} + +impl Default for Dump<'_> { + /// `json.dumps(obj)` with no keywords. + fn default() -> Self { + Dump { + indent: None, + separators: None, + ensure_ascii: true, + sort_keys: false, + allow_nan: true, + } + } +} + +/// Why `dumps` or `loads` raised. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JsonError { + /// ValueError from `dumps(allow_nan=False)`. + OutOfRangeFloat, + /// JSONDecodeError: what was wrong and the byte offset. + Decode(&'static str, usize), + /// RecursionError: arrays and objects nested deeper than [`MAX_DEPTH`]. + TooDeep, +} + +/// How deep `loads` nests arrays and objects before it gives up. CPython +/// 3.14 goes deeper (its limit follows the C stack); the parser here is +/// recursive, so it stops well inside an 8 MiB stack. A known limit +/// (`docs/rust.md`). +pub const MAX_DEPTH: usize = 1000; + +impl JsonError { + /// The Python exception class. + pub fn exception(&self) -> &'static str { + match self { + JsonError::OutOfRangeFloat => "ValueError", + JsonError::Decode(..) => "JSONDecodeError", + JsonError::TooDeep => "RecursionError", + } + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::OutOfRangeFloat => { + f.write_str("Out of range float values are not JSON compliant") + } + JsonError::Decode(what, at) => write!(f, "{what}: byte {at}"), + JsonError::TooDeep => write!(f, "nested deeper than {MAX_DEPTH} levels"), + } + } +} + +impl std::error::Error for JsonError {} + +/// `json.dumps(value, **opts)`. +pub fn dumps(value: &Value, opts: &Dump<'_>) -> Result { + let mut out = String::new(); + let (item, key) = opts.separators.unwrap_or(if opts.indent.is_some() { + (",", ": ") + } else { + (", ", ": ") + }); + let w = Writer { opts, item, key }; + w.value(&mut out, value, 0)?; + Ok(out) +} + +struct Writer<'o, 'a> { + opts: &'o Dump<'a>, + item: &'o str, + key: &'o str, +} + +impl Writer<'_, '_> { + fn newline(&self, out: &mut String, level: usize) { + if let Some(n) = self.opts.indent { + out.push('\n'); + out.extend(std::iter::repeat_n(' ', n * level)); + } + } + + /// The separator between two items, and the line break after it. + fn between(&self, out: &mut String, level: usize) { + out.push_str(self.item); + self.newline(out, level); + } + + fn value(&self, out: &mut String, v: &Value, level: usize) -> Result<(), JsonError> { + match v { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + Value::Int(i) => write!(out, "{i}").expect("write to String"), + Value::Float(x) => out.push_str(&self.float(*x)?), + Value::Str(s) => quote(out, s, self.opts.ensure_ascii), + Value::Array(a) if a.is_empty() => out.push_str("[]"), + Value::Array(a) => { + out.push('['); + self.newline(out, level + 1); + for (i, x) in a.iter().enumerate() { + if i > 0 { + self.between(out, level + 1); + } + self.value(out, x, level + 1)?; + } + self.newline(out, level); + out.push(']'); + } + Value::Object(m) if m.is_empty() => out.push_str("{}"), + Value::Object(m) => { + let mut items: Vec<(&String, &Value)> = m.iter().collect(); + if self.opts.sort_keys { + items.sort_by(|a, b| a.0.cmp(b.0)); + } + out.push('{'); + self.newline(out, level + 1); + for (i, (k, x)) in items.into_iter().enumerate() { + if i > 0 { + self.between(out, level + 1); + } + quote(out, k, self.opts.ensure_ascii); + out.push_str(self.key); + self.value(out, x, level + 1)?; + } + self.newline(out, level); + out.push('}'); + } + } + Ok(()) + } + + fn float(&self, x: f64) -> Result { + if x.is_finite() { + return Ok(repr(x)); + } + if !self.opts.allow_nan { + return Err(JsonError::OutOfRangeFloat); + } + Ok(if x.is_nan() { + "NaN" + } else if x > 0.0 { + "Infinity" + } else { + "-Infinity" + } + .to_owned()) + } +} + +/// A JSON string literal as Python's encoder writes it. +pub fn quote(out: &mut String, s: &str, ensure_ascii: bool) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\x08' => out.push_str("\\b"), + '\x0c' => out.push_str("\\f"), + c if (c as u32) < 0x20 => write!(out, "\\u{:04x}", c as u32).expect("write to String"), + c if ensure_ascii && !(' '..='~').contains(&c) => { + let mut buf = [0u16; 2]; + for unit in c.encode_utf16(&mut buf) { + write!(out, "\\u{unit:04x}").expect("write to String"); + } + } + c => out.push(c), + } + } + out.push('"'); +} + +/// `json.loads(text)`. +pub fn loads(text: &str) -> Result { + let mut p = Parser { + s: text.as_bytes(), + text, + i: 0, + depth: 0, + }; + p.ws(); + let v = p.value()?; + p.ws(); + if p.i != p.s.len() { + return Err(JsonError::Decode("Extra data", p.i)); + } + Ok(v) +} + +struct Parser<'a> { + s: &'a [u8], + text: &'a str, + i: usize, + /// Arrays and objects open around `i`: a bound on the recursion. + depth: usize, +} + +impl Parser<'_> { + fn err(&self, what: &'static str) -> Result { + Err(JsonError::Decode(what, self.i)) + } + + fn ws(&mut self) { + while matches!(self.s.get(self.i), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.i += 1; + } + } + + fn eat(&mut self, lit: &str) -> bool { + if self.s[self.i..].starts_with(lit.as_bytes()) { + self.i += lit.len(); + true + } else { + false + } + } + + fn value(&mut self) -> Result { + match self.s.get(self.i) { + Some(b'{' | b'[') => { + if self.depth == MAX_DEPTH { + return Err(JsonError::TooDeep); + } + self.depth += 1; + let v = if self.s[self.i] == b'{' { + self.object() + } else { + self.array() + }; + self.depth -= 1; + v + } + Some(b'"') => Ok(Value::Str(self.string()?)), + Some(b'n') if self.eat("null") => Ok(Value::Null), + Some(b't') if self.eat("true") => Ok(Value::Bool(true)), + Some(b'f') if self.eat("false") => Ok(Value::Bool(false)), + Some(b'N') if self.eat("NaN") => Ok(Value::Float(f64::NAN)), + Some(b'I') if self.eat("Infinity") => Ok(Value::Float(f64::INFINITY)), + Some(b'-') if self.eat("-Infinity") => Ok(Value::Float(f64::NEG_INFINITY)), + Some(b'-' | b'0'..=b'9') => self.number(), + _ => self.err("Expecting value"), + } + } + + fn digits(&mut self) -> usize { + let n = self.s[self.i..] + .iter() + .take_while(|c| c.is_ascii_digit()) + .count(); + self.i += n; + n + } + + /// `-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?`, as Python's scanner. + fn number(&mut self) -> Result { + let start = self.i; + if self.s[self.i] == b'-' { + self.i += 1; + } + match self.s.get(self.i) { + Some(b'0') => self.i += 1, + Some(b'1'..=b'9') => { + self.digits(); + } + _ => { + self.i = start; + return self.err("Expecting value"); + } + } + let mut float = false; + if self.s.get(self.i) == Some(&b'.') + && self.s.get(self.i + 1).is_some_and(u8::is_ascii_digit) + { + self.i += 1; + self.digits(); + float = true; + } + if matches!(self.s.get(self.i), Some(b'e' | b'E')) { + let save = self.i; + self.i += 1; + if matches!(self.s.get(self.i), Some(b'+' | b'-')) { + self.i += 1; + } + if self.digits() == 0 { + self.i = save; + } else { + float = true; + } + } + let lit = &self.text[start..self.i]; + if float { + Ok(Value::Float(lit.parse().expect("a JSON number parses"))) + } else { + lit.parse() + .map(Value::Int) + .or_else(|_| self.err("Integer beyond 128 bits")) + } + } + + fn hex4(&mut self) -> Result { + let h = self + .s + .get(self.i..self.i + 4) + .and_then(|h| std::str::from_utf8(h).ok()); + match h.and_then(|h| { + (h.bytes().all(|c| c.is_ascii_hexdigit())).then(|| u32::from_str_radix(h, 16).ok()) + }) { + Some(Some(v)) => { + self.i += 4; + Ok(v) + } + _ => self.err("Invalid \\uXXXX escape"), + } + } + + fn string(&mut self) -> Result { + self.i += 1; + let mut out = String::new(); + loop { + let run = self.s[self.i..] + .iter() + .take_while(|&&c| c != b'"' && c != b'\\' && c >= 0x20) + .count(); + out.push_str(&self.text[self.i..self.i + run]); + self.i += run; + match self.s.get(self.i) { + Some(b'"') => { + self.i += 1; + return Ok(out); + } + Some(b'\\') => { + self.i += 1; + let c = match self.s.get(self.i) { + Some(b'"') => '"', + Some(b'\\') => '\\', + Some(b'/') => '/', + Some(b'b') => '\x08', + Some(b'f') => '\x0c', + Some(b'n') => '\n', + Some(b'r') => '\r', + Some(b't') => '\t', + Some(b'u') => { + self.i += 1; + let hi = self.hex4()?; + let cp = if (0xd800..0xdc00).contains(&hi) + && self.s[self.i..].starts_with(b"\\u") + { + let save = self.i; + self.i += 2; + let lo = self.hex4()?; + if (0xdc00..0xe000).contains(&lo) { + 0x10000 + ((hi - 0xd800) << 10) + (lo - 0xdc00) + } else { + self.i = save; + hi + } + } else { + hi + }; + match char::from_u32(cp) { + Some(c) => { + out.push(c); + continue; + } + None => { + return self + .err("Lone surrogate (Python keeps it; this port cannot)"); + } + } + } + _ => return self.err("Invalid \\escape"), + }; + out.push(c); + self.i += 1; + } + Some(_) => return self.err("Invalid control character"), + None => return self.err("Unterminated string"), + } + } + } + + fn array(&mut self) -> Result { + self.i += 1; + let mut a = Vec::new(); + self.ws(); + if self.eat("]") { + return Ok(Value::Array(a)); + } + loop { + self.ws(); + a.push(self.value()?); + self.ws(); + if self.eat("]") { + return Ok(Value::Array(a)); + } + if !self.eat(",") { + return self.err("Expecting ',' delimiter"); + } + } + } + + fn object(&mut self) -> Result { + self.i += 1; + let mut m = IndexMap::new(); + self.ws(); + if self.eat("}") { + return Ok(Value::Object(m)); + } + loop { + self.ws(); + if self.s.get(self.i) != Some(&b'"') { + return self.err("Expecting property name enclosed in double quotes"); + } + let k = self.string()?; + self.ws(); + if !self.eat(":") { + return self.err("Expecting ':' delimiter"); + } + self.ws(); + let v = self.value()?; + // A duplicate key keeps its first place and takes the last value, as a dict does. + m.insert(k, v); + self.ws(); + if self.eat("}") { + return Ok(Value::Object(m)); + } + if !self.eat(",") { + return self.err("Expecting ',' delimiter"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(text: &str, opts: &Dump<'_>) -> String { + dumps(&loads(text).unwrap(), opts).unwrap() + } + + #[test] + fn dumps_matches_python() { + let d = Dump::default(); + assert_eq!( + round_trip(r#"{"a": [1, 2.0, "é😀\u007f"], "b": {}}"#, &d), + r#"{"a": [1, 2.0, "\u00e9\ud83d\ude00\u007f"], "b": {}}"# + ); + let one = Dump { + indent: Some(1), + ..Dump::default() + }; + assert_eq!( + round_trip(r#"{"b": [1, []], "a": null}"#, &one), + "{\n \"b\": [\n 1,\n []\n ],\n \"a\": null\n}" + ); + let zero = Dump { + indent: Some(0), + ..Dump::default() + }; + assert_eq!(round_trip("[1,2]", &zero), "[\n1,\n2\n]"); + let compact = Dump { + separators: Some((",", ":")), + sort_keys: true, + ensure_ascii: false, + ..Dump::default() + }; + assert_eq!( + round_trip(r#"{"b":1,"a":"é\u0001"}"#, &compact), + "{\"a\":\"é\\u0001\",\"b\":1}" + ); + assert_eq!( + round_trip("[NaN, Infinity, -Infinity, 1e400, -0.0, 1E2]", &d), + "[NaN, Infinity, -Infinity, Infinity, -0.0, 100.0]" + ); + let strict = Dump { + allow_nan: false, + ..Dump::default() + }; + assert_eq!( + dumps(&Value::Float(f64::NAN), &strict), + Err(JsonError::OutOfRangeFloat) + ); + } + + #[test] + fn loads_keeps_python_dict_semantics() { + assert_eq!( + round_trip(r#"{"a": 1, "b": 2, "a": 3}"#, &Dump::default()), + r#"{"a": 3, "b": 2}"# + ); + assert_eq!(loads("-0").unwrap(), Value::Int(0)); + assert_eq!( + loads(" \"\\ud83d\\ude00\" ").unwrap(), + Value::Str("😀".into()) + ); + for bad in [ + "", + "[1,]", + "{\"a\" 1}", + "01", + "1.", + "\"a\u{1}\"", + "[1] x", + "{'a': 1}", + "\"\\ud800\"", + ] { + assert!(loads(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn loads_stops_deep_nesting_with_an_error() { + // 8 MiB, the main-thread stack og runs on: a test thread has 2 MiB, + // too little for MAX_DEPTH levels in an unoptimised build. + let t = std::thread::Builder::new().stack_size(8 << 20).spawn(|| { + let nested = |n: usize| format!("{}{}", "[".repeat(n), "]".repeat(n)); + assert!(loads(&nested(MAX_DEPTH)).is_ok()); + assert_eq!(loads(&nested(MAX_DEPTH + 1)), Err(JsonError::TooDeep)); + // Far past the limit: an error, never a stack overflow. + assert_eq!(loads(&"[".repeat(1_000_000)), Err(JsonError::TooDeep)); + assert_eq!(loads(&"{\"a\":".repeat(100_000)), Err(JsonError::TooDeep)); + }); + t.unwrap().join().unwrap(); + assert_eq!(JsonError::TooDeep.exception(), "RecursionError"); + } +} diff --git a/rust/crates/pycompat/src/lib.rs b/rust/crates/pycompat/src/lib.rs new file mode 100644 index 00000000..7aef3352 --- /dev/null +++ b/rust/crates/pycompat/src/lib.rs @@ -0,0 +1,26 @@ +//! Python-exact primitives for the Rust ports of OwnGenome's Python tools. +//! +//! Every function here returns the bytes CPython 3.15 returns for the same +//! input, except for the known limits in `docs/rust.md`, where it returns an +//! error instead. The differential test (`tests/rust/pycompat-diff.sh`) checks that +//! against `python3 -I` in the pinned `PYTHON_IMAGE`. +//! +//! - [`fmt`]: `repr(float)`, the `f`, `e`, `g` and `%` format types, `{:,}`, +//! zero padding, `ljust`/`rjust`, `round(x)` and `round(x, n)`. +//! - [`num`]: `float(str)`, `int(str)`, `math.fsum` and the builtin `sum`. +//! - [`json`]: `json.dumps` and a `json.loads` with Python's leniency. + +pub mod fmt; +pub mod json; +pub mod num; + +#[cfg(all(test, feature = "planted-failure"))] +mod planted { + /// Always fails. CI runs it with `--features planted-failure` and + /// requires a red result, so a test runner that reports nothing cannot + /// pass for a green one. + #[test] + fn planted_failure() { + assert_eq!(crate::fmt::repr(0.1), "0.2", "planted failure"); + } +} diff --git a/rust/crates/pycompat/src/num.rs b/rust/crates/pycompat/src/num.rs new file mode 100644 index 00000000..cb4f9e8a --- /dev/null +++ b/rust/crates/pycompat/src/num.rs @@ -0,0 +1,353 @@ +//! Python's number parsing and float sums. + +use std::fmt; + +/// `str.isspace()` for one character, the set `str.strip()` removes. +/// Wider than Rust's `char::is_whitespace`: it has `\x1c`-`\x1f`. +pub fn py_isspace(c: char) -> bool { + matches!( + c, + '\t' | '\n' + | '\x0b' + | '\x0c' + | '\r' + | '\x1c'..='\x1f' + | ' ' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} + +/// `s.strip()` with no argument. +pub fn py_strip(s: &str) -> &str { + s.trim_matches(py_isspace) +} + +/// The whitespace `float()` and `int()` skip around a number: Unicode +/// whitespace, but of ASCII only C's six (`\x1c`-`\x1f` are refused). +fn number_space(c: char) -> bool { + py_isspace(c) && !matches!(c, '\x1c'..='\x1f') +} + +/// Why `float(s)` or `int(s)` raised. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParseError { + /// ValueError: not a number Python reads. + Invalid, + /// An int beyond the 128-bit range: Python has no limit, this port has. + TooLarge, +} + +impl ParseError { + /// The Python exception class (`TooLarge` has none: Python would return the int). + pub fn exception(&self) -> &'static str { + match self { + ParseError::Invalid => "ValueError", + ParseError::TooLarge => "OverflowError", + } + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + ParseError::Invalid => "invalid literal", + ParseError::TooLarge => "integer does not fit 128 bits", + }) + } +} + +impl std::error::Error for ParseError {} + +/// Removes underscores that sit between two ASCII digits, as Python allows +/// in a number literal; any other underscore is an error. +fn strip_underscores(s: &str) -> Result { + let b = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + for (i, &c) in b.iter().enumerate() { + if c == b'_' { + let before = i > 0 && b[i - 1].is_ascii_digit(); + let after = b.get(i + 1).is_some_and(u8::is_ascii_digit); + if !(before && after) { + return Err(ParseError::Invalid); + } + } else { + out.push(c as char); + } + } + Ok(out) +} + +/// True when `s` (no sign, no underscores) is a decimal float literal: +/// digits with an optional point, at least one digit, an optional exponent. +fn is_decimal(s: &str) -> bool { + let b = s.as_bytes(); + let mut i = 0; + let int = b.iter().take_while(|c| c.is_ascii_digit()).count(); + i += int; + let mut frac = 0; + if b.get(i) == Some(&b'.') { + i += 1; + frac = b[i..].iter().take_while(|c| c.is_ascii_digit()).count(); + i += frac; + } + if int + frac == 0 { + return false; + } + if matches!(b.get(i), Some(b'e' | b'E')) { + i += 1; + if matches!(b.get(i), Some(b'+' | b'-')) { + i += 1; + } + let exp = b[i..].iter().take_while(|c| c.is_ascii_digit()).count(); + if exp == 0 { + return false; + } + i += exp; + } + i == b.len() +} + +/// `float(s)` for a str: surrounding whitespace, a sign, underscores between +/// digits, `inf`, `infinity` and `nan` in any case; `1e400` is `inf`. +/// Non-ASCII digits, which Python also reads, are refused. +pub fn parse_float(s: &str) -> Result { + let t = s.trim_matches(number_space); + if !t.is_ascii() { + return Err(ParseError::Invalid); + } + let (neg, body) = match t.as_bytes().first() { + Some(b'-') => (true, &t[1..]), + Some(b'+') => (false, &t[1..]), + _ => (false, t), + }; + let lower = body.to_ascii_lowercase(); + let v = match lower.as_str() { + "inf" | "infinity" => f64::INFINITY, + "nan" => f64::NAN, + _ => { + let clean = strip_underscores(body)?; + if !is_decimal(&clean) { + return Err(ParseError::Invalid); + } + clean.parse::().map_err(|_| ParseError::Invalid)? + } + }; + Ok(if neg { -v } else { v }) +} + +/// `int(s)` for a str in base 10: surrounding whitespace, a sign, leading +/// zeros and underscores between digits. Non-ASCII digits, which Python also +/// reads, are refused, and so is a value beyond `i128`. +pub fn parse_int(s: &str) -> Result { + let t = s.trim_matches(number_space); + let (neg, body) = match t.as_bytes().first() { + Some(b'-') => (true, &t[1..]), + Some(b'+') => (false, &t[1..]), + _ => (false, t), + }; + let clean = strip_underscores(body)?; + if clean.is_empty() || !clean.bytes().all(|c| c.is_ascii_digit()) { + return Err(ParseError::Invalid); + } + let mut v: i128 = 0; + for d in clean.bytes() { + let d = i128::from(d - b'0'); + v = v + .checked_mul(10) + .and_then(|v| { + if neg { + v.checked_sub(d) + } else { + v.checked_add(d) + } + }) + .ok_or(ParseError::TooLarge)?; + } + Ok(v) +} + +/// Why `math.fsum` raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FsumError { + /// OverflowError: finite inputs whose partial sum overflowed. + Overflow, + /// ValueError: both `inf` and `-inf` among the inputs. + InfMinusInf, +} + +impl FsumError { + /// The Python exception class. + pub fn exception(self) -> &'static str { + match self { + FsumError::Overflow => "OverflowError", + FsumError::InfMinusInf => "ValueError", + } + } +} + +impl fmt::Display for FsumError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + FsumError::Overflow => "intermediate overflow in fsum", + FsumError::InfMinusInf => "-inf + inf in fsum", + }) + } +} + +impl std::error::Error for FsumError {} + +/// `math.fsum(xs)`: the correctly rounded sum (Shewchuk's partials, with +/// CPython's half-even fix-up across partials and its inf/nan rules). +pub fn fsum>(xs: I) -> Result { + let mut p: Vec = Vec::new(); + let mut special = 0.0f64; + let mut inf_sum = 0.0f64; + for xsave in xs { + let mut x = xsave; + let mut i = 0; + for j in 0..p.len() { + let mut y = p[j]; + if x.abs() < y.abs() { + std::mem::swap(&mut x, &mut y); + } + let hi = x + y; + let lo = y - (hi - x); + if lo != 0.0 { + p[i] = lo; + i += 1; + } + x = hi; + } + p.truncate(i); + if x != 0.0 { + if !x.is_finite() { + if xsave.is_finite() { + return Err(FsumError::Overflow); + } + if xsave.is_infinite() { + inf_sum += xsave; + } + special += xsave; + p.clear(); + } else { + p.push(x); + } + } + } + if special != 0.0 { + return if inf_sum.is_nan() { + Err(FsumError::InfMinusInf) + } else { + Ok(special) + }; + } + let mut n = p.len(); + if n == 0 { + return Ok(0.0); + } + n -= 1; + let mut hi = p[n]; + let mut lo = 0.0; + while n > 0 { + let x = hi; + n -= 1; + let y = p[n]; + hi = x + y; + lo = y - (hi - x); + if lo != 0.0 { + break; + } + } + if n > 0 && ((lo < 0.0 && p[n - 1] < 0.0) || (lo > 0.0 && p[n - 1] > 0.0)) { + let y = lo * 2.0; + let x = hi + y; + if y == x - hi { + hi = x; + } + } + Ok(hi) +} + +/// The builtin `sum(xs)` of floats with start 0 (Python 3.12 and later): +/// Neumaier's compensated sum. +pub fn sum>(xs: I) -> f64 { + let mut s = 0.0f64; + let mut c = 0.0f64; + for x in xs { + let t = s + x; + if s.abs() >= x.abs() { + c += (s - t) + x; + } else { + c += (x - t) + s; + } + s = t; + } + if c != 0.0 && c.is_finite() { + s += c; + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn float_parsing() { + assert_eq!(parse_float(" 1_000.5\n"), Ok(1000.5)); + assert_eq!(parse_float("+.5"), Ok(0.5)); + assert_eq!(parse_float("1."), Ok(1.0)); + assert_eq!(parse_float("1e400"), Ok(f64::INFINITY)); + assert_eq!(parse_float("-Infinity"), Ok(f64::NEG_INFINITY)); + assert!(parse_float("nAn").unwrap().is_nan()); + assert_eq!(parse_float("\u{85}2\u{3000}"), Ok(2.0)); + assert_eq!(parse_float("\x1c2"), Err(ParseError::Invalid)); + for bad in [ + "", ".", "1_", "_1", "1__0", "1_.5", "1e", "e5", "1.5.2", "inf1", "0x10", "1 2", + ] { + assert_eq!(parse_float(bad), Err(ParseError::Invalid), "{bad:?}"); + } + } + + #[test] + fn int_parsing() { + assert_eq!(parse_int(" -0012 "), Ok(-12)); + assert_eq!(parse_int("1_000"), Ok(1000)); + assert_eq!(parse_int("+7"), Ok(7)); + for bad in ["", "-", "1.0", "1_", "1e3", " "] { + assert_eq!(parse_int(bad), Err(ParseError::Invalid), "{bad:?}"); + } + assert_eq!(parse_int(&"9".repeat(40)), Err(ParseError::TooLarge)); + assert_eq!(parse_int(&i128::MIN.to_string()), Ok(i128::MIN)); + } + + #[test] + fn fsum_is_exact() { + assert_eq!(fsum([0.1; 10]), Ok(1.0)); + assert_eq!(fsum([1e-16, 1.0, 1e16]), Ok(10000000000000002.0)); + assert_eq!(fsum([1e308, 1e308]), Err(FsumError::Overflow)); + assert_eq!( + fsum([f64::INFINITY, f64::NEG_INFINITY]), + Err(FsumError::InfMinusInf) + ); + assert_eq!(fsum([f64::INFINITY, 1.0]), Ok(f64::INFINITY)); + assert!(fsum([f64::NAN, 1.0]).unwrap().is_nan()); + assert_eq!(fsum(Vec::new()), Ok(0.0)); + } + + #[test] + fn builtin_sum_is_compensated() { + assert_eq!(sum([0.1; 10]), 1.0); + assert_eq!(sum([1e100, 1.0, -1e100, 1.0]), 2.0); + assert!(!sum([-0.0]).is_sign_negative()); + assert_eq!(sum([f64::INFINITY, 1.0]), f64::INFINITY); + } +} diff --git a/rust/og.sh b/rust/og.sh new file mode 100755 index 00000000..01804307 --- /dev/null +++ b/rust/og.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# og.sh: how the og binary reaches bin/.og, and the check that it is the one +# this checkout's rust/ makes. bin/og (the shim tasks call) runs +# bin/.og/og-linux-; both arches are kept, because an Apple Silicon +# host runs amd64 images under emulation, where uname -m says x86_64. +# +# rust/og.sh install scripts/setup.sh --og: on a release tag (vX.Y.Z, +# clean rust/) download the release's binaries and +# check them against SHA256SUMS (and the build +# attestation when gh is logged in); otherwise, or +# when the release has none, build them +# rust/og.sh build build both from rust/ in OG_BUILDER (versions.env) +# rust/og.sh check [--required] +# one line, OK, NOTE or ERROR; exit 1 on ERROR: a +# binary built from another rust/, or (with +# --required, or once REQUIRED below is true) none +# rust/og.sh tree-hash the git tree hash of rust/ as it is on disk +# rust/og.sh built the tree hash the binaries were built from, for +# run_manifest.tsv ("not built" when there are none) +# +# The tree hash is git's: `git rev-parse HEAD:rust` on a clean checkout, and +# for an edited rust/ the tree those edits would commit (a throwaway index), +# so any edit, committed or not, makes the built binary stale. The build +# stamps it into the binary as og-tree-hash:; so the check +# reads it with grep and never runs a binary the host may not be able to. +set -euo pipefail + +# True once a pipeline step calls og: until then a checkout without the +# binary runs as before, and only a stale one is refused. +REQUIRED=false + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +BIN="${ROOT}/bin/.og" +ARCHES="x86_64 aarch64" +REPO="GeiserX/OwnGenome" +ENGINE=${CONTAINER_ENGINE:-docker} + +triple() { echo "${1}-unknown-linux-musl"; } + +# tree_hash: exit 3 when this is not a git checkout, 1 when git fails. +tree_hash() { + local idx rc=0 + git -C "$ROOT" rev-parse --verify -q HEAD:rust > /dev/null 2>&1 || return 3 + if [ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all -- rust 2> /dev/null)" ]; then + git -C "$ROOT" rev-parse HEAD:rust + return + fi + idx=$(mktemp -d "${TMPDIR:-/tmp}/og-tree.XXXXXX") || return 1 + { GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" read-tree HEAD \ + && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" add -A -- rust \ + && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" write-tree --prefix=rust/; } || rc=$? + rm -rf "$idx" + [ "$rc" -eq 0 ] || return 1 +} + +# embedded FILE: the hash stamped into a binary, or "unknown". +embedded() { + local h + h=$(LC_ALL=C grep -ao 'og-tree-hash:[0-9a-f]\{40,64\};' "$1" 2> /dev/null | head -n 1 || true) + h=${h#og-tree-hash:} + echo "${h%;}" | grep . || echo unknown +} + +check() { + local required=$REQUIRED expected f h missing="" stale="" trc=0 + [ "${1:-}" != --required ] || required=true + expected=$(tree_hash 2> /dev/null) || trc=$? + if [ "$trc" -eq 3 ]; then + expected="" + elif [ "$trc" -ne 0 ]; then + echo "ERROR: the git tree hash of ${ROOT}/rust could not be computed, so og cannot be checked (is TMPDIR writable?)" + return 1 + fi + for a in $ARCHES; do + f="${BIN}/og-linux-${a}" + if [ ! -x "$f" ]; then missing+=" ${a}"; continue; fi + h=$(embedded "$f") + [ -z "$expected" ] || [ "$h" = "$expected" ] || stale+=" bin/.og/og-linux-${a} (rust/ tree ${h})" + done + if [ -n "$stale" ]; then + echo "ERROR: og is stale:${stale}, but rust/ here is tree ${expected}: run scripts/setup.sh --og" + return 1 + fi + if [ -n "$missing" ] && $required; then + echo "ERROR: og is not built for${missing}: run scripts/setup.sh --og" + return 1 + fi + if [ -n "$missing" ]; then + echo "NOTE: og is not built for${missing}; no step needs it yet (scripts/setup.sh --og builds it)" + elif [ -z "$expected" ]; then + echo "NOTE: og is built, but this is not a git checkout, so its rust/ tree cannot be checked" + else + echo "OK: og is built for ${ARCHES// /, } from this checkout's rust/ (tree ${expected})" + fi +} + +built() { + local hs="" a + for a in $ARCHES; do + if [ -x "${BIN}/og-linux-${a}" ]; then hs+="${a}=$(embedded "${BIN}/og-linux-${a}") "; fi + done + hs=${hs% } + if [ -z "$hs" ]; then echo "not built"; return; fi + # One hash when both arches agree, as they do after setup.sh --og. + local -a h + read -r -a h <<< "$hs" + if [ "${#h[@]}" -eq 2 ] && [ "${h[0]#*=}" = "${h[1]#*=}" ]; then echo "${h[0]#*=}"; else echo "$hs"; fi +} + +# put SRC...: one binary per arch of ARCHES, in that order, into bin/.og. +# All are copied beside their targets first; only when every copy worked are +# they renamed into place, so a failed copy (a full disk) leaves the old pair. +put() { + local a i=0 srcs=("$@") + mkdir -p "$BIN" || return 1 + for a in $ARCHES; do + if ! { cp "${srcs[i]}" "${BIN}/.og-linux-${a}.tmp" && chmod 755 "${BIN}/.og-linux-${a}.tmp"; }; then + echo "ERROR: could not copy og-linux-${a} into ${BIN}" >&2 + for a in $ARCHES; do rm -f "${BIN}/.og-linux-${a}.tmp"; done + return 1 + fi + i=$((i + 1)) + done + for a in $ARCHES; do + mv -f "${BIN}/.og-linux-${a}.tmp" "${BIN}/og-linux-${a}" || { echo "ERROR: could not install ${BIN}/og-linux-${a}" >&2; return 1; } + done +} + +build() { + local hash cache t0 a out targets=() outs=() + hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout: og is stamped with the git tree of rust/, so it builds only in one" >&2; return 1; } + # shellcheck source=../versions.env + . "${ROOT}/versions.env" + : "${OG_BUILDER:?OG_BUILDER is not set in versions.env}" + # Build cache and output under rust/target (ignored by git, so outside the + # tree hash); the sources go in read-only. + for a in $ARCHES; do targets+=(--target "$(triple "$a")"); done + cache="${ROOT}/rust/target/og-build" + mkdir -p "$cache" + echo "Building og ${hash} for ${ARCHES} in ${OG_BUILDER%@*} ..." + t0=$(date +%s) + "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ + -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target -e OG_TREE_HASH="$hash" \ + -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ + cargo zigbuild --release --locked -p og "${targets[@]}" + for a in $ARCHES; do + out="${cache}/target/$(triple "$a")/release/og" + [ "$(embedded "$out")" = "$hash" ] || { echo "ERROR: ${out} does not carry tree ${hash}" >&2; return 1; } + outs+=("$out") + done + put "${outs[@]}" || return 1 + echo "Built bin/.og/og-linux-{${ARCHES// /,}} in $(($(date +%s) - t0)) s" +} + +sha256() { + if command -v sha256sum > /dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi +} + +get() { # get URL DEST: 0 when downloaded, 2 when the server says 404, 1 on any other error + local code + if command -v curl > /dev/null 2>&1; then + code=$(curl -sL --retry 3 -o "$2" -w '%{http_code}' "$1") || return 1 + case "$code" in 2??) return 0 ;; 404) return 2 ;; *) return 1 ;; esac + fi + wget -q -O "$2" "$1" && return 0 + # wget exits 8 on an error response from the server; that is all it tells. + [ "$?" -eq 8 ] && return 2 || return 1 +} + +# fetch TAG: 0 when the release's binaries are installed, 2 when the release +# has none (SHA256SUMS is not there: build instead), 1 when a download fails +# or does not check out. +fetch() { + local tag=$1 tmp want hash a f rc=0 grc=0 attest=0 + hash=$(tree_hash) + # gh attestation verify needs a login: without one (gh missing, or never + # logged in) only SHA256SUMS is checked. gh auth token, not gh auth status, + # which can hang on enterprise hosts. + if command -v gh > /dev/null 2>&1 && gh auth token > /dev/null 2>&1; then attest=1; fi + tmp=$(mktemp -d "${TMPDIR:-/tmp}/og-fetch.XXXXXX") + # OG_RELEASE_BASE: another server with the same layout (tests/test_og_check.sh). + local base="${OG_RELEASE_BASE:-https://github.com/${REPO}/releases/download}/${tag}" + get "${base}/SHA256SUMS" "${tmp}/SHA256SUMS" || grc=$? + if [ "$grc" -ne 0 ]; then + rm -rf "$tmp" + [ "$grc" -eq 2 ] && return 2 + echo "ERROR: could not download ${base}/SHA256SUMS; check the network and run scripts/setup.sh --og again" >&2 + return 1 + fi + for a in $ARCHES; do + f="og-linux-${a}" + want=$(awk -v f="$f" '$2 == f || $2 == "*" f {print $1}' "${tmp}/SHA256SUMS") + if [ -z "$want" ]; then echo "ERROR: release ${tag} lists no ${f}" >&2; rc=1; break; fi + if ! get "${base}/${f}" "${tmp}/${f}"; then echo "ERROR: could not download ${f} of release ${tag}" >&2; rc=1; break; fi + if [ "$(sha256 "${tmp}/${f}")" != "$want" ]; then echo "ERROR: ${f} of release ${tag} does not match its SHA256SUMS" >&2; rc=1; break; fi + if [ "$attest" = 1 ]; then + gh attestation verify "${tmp}/${f}" --repo "$REPO" > /dev/null || { echo "ERROR: ${f} of release ${tag} has no valid build attestation" >&2; rc=1; break; } + fi + if [ "$(embedded "${tmp}/${f}")" != "$hash" ]; then echo "ERROR: ${f} of release ${tag} was not built from this rust/ (tree ${hash})" >&2; rc=1; break; fi + done + local srcs=() + for a in $ARCHES; do srcs+=("${tmp}/og-linux-${a}"); done + if [ "$rc" -eq 0 ] && ! put "${srcs[@]}"; then rc=1; fi + if [ "$rc" -eq 0 ]; then + [ "$attest" = 1 ] || echo "NOTE: gh is not installed or not logged in, so the build attestation was not checked (SHA256SUMS was)" + echo "Installed og ${hash} from release ${tag}" + fi + rm -rf "$tmp" + return "$rc" +} + +install() { + local tag rc=0 hash + hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout; clone the repository to install og" >&2; return 1; } + tag=$(git -C "$ROOT" describe --exact-match --tags --match 'v[0-9]*.[0-9]*.[0-9]*' HEAD 2> /dev/null || true) + if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(git -C "$ROOT" rev-parse HEAD:rust)" ]; then + echo "Release ${tag}: downloading its og binaries" + fetch "$tag" || rc=$? + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -eq 2 ] || return 1 + echo "Release ${tag} has no og binaries: building them" + fi + build +} + +case "${1:-}" in + install) install ;; + build) build ;; + check) shift; check "$@" ;; + tree-hash) tree_hash ;; + built) built ;; + *) sed -n '2,/^set -euo/p' "$0" | sed -e '$d' -e 's/^# \{0,1\}//' >&2; exit 2 ;; +esac diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml new file mode 100644 index 00000000..7de9ddb2 --- /dev/null +++ b/rust/og/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "og" +description = "OwnGenome's one binary: og [args]" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +pycompat.workspace = true +og-io.workspace = true + +[lints] +workspace = true diff --git a/rust/og/build.rs b/rust/og/build.rs new file mode 100644 index 00000000..fd9441ab --- /dev/null +++ b/rust/og/build.rs @@ -0,0 +1,19 @@ +//! Stamps the binary with the git tree hash of the `rust/` it was built +//! from. `rust/og.sh` computes the hash on the host (the build container +//! has no access to the repository's .git) and passes it as OG_TREE_HASH. +//! A plain `cargo build` without it stamps `unknown`, which no check accepts. + +fn main() { + println!("cargo::rerun-if-env-changed=OG_TREE_HASH"); + let hash = std::env::var("OG_TREE_HASH").unwrap_or_default(); + let ok = (40..=64).contains(&hash.len()) + && hash + .bytes() + .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()); + let hash = if ok { hash } else { "unknown".to_owned() }; + println!("cargo::rustc-env=OG_TREE_HASH={hash}"); + println!( + "cargo::rustc-env=OG_TARGET={}", + std::env::var("TARGET").expect("cargo sets TARGET") + ); +} diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs new file mode 100644 index 00000000..123c49b6 --- /dev/null +++ b/rust/og/src/main.rs @@ -0,0 +1,65 @@ +//! `og`: OwnGenome's one binary. `og [args]`; each ported Python +//! tool becomes a subcommand named after its file in `bin/`. + +use std::io::Write; +use std::process::ExitCode; + +mod probe; + +/// The tree hash between a fixed marker and a `;`, so a host check can read +/// it from the file without running it (`grep -ao 'og-tree-hash:[0-9a-f]*;'`): +/// an Apple Silicon host cannot run the x86_64 Linux binary. The `;` ends +/// it where the next string in the binary would otherwise run on. +const TREE_MARK: &str = concat!("og-tree-hash:", env!("OG_TREE_HASH"), ";"); +const TARGET: &str = env!("OG_TARGET"); + +fn tree_hash() -> &'static str { + &TREE_MARK["og-tree-hash:".len()..TREE_MARK.len() - 1] +} + +const USAGE: &str = "usage: og [args]\n og version\n og --tree-hash\n"; + +/// argparse's way to refuse a command line: usage and the error on stderr, exit 2. +fn usage_error(msg: &str) -> ExitCode { + eprint!("{USAGE}"); + eprintln!("og: error: {msg}"); + ExitCode::from(2) +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let Some(tool) = args.first() else { + return usage_error("the following arguments are required: tool"); + }; + let rest = &args[1..]; + match tool.as_str() { + "-h" | "--help" => { + print!("{USAGE}"); + ExitCode::SUCCESS + } + "version" | "--version" if rest.is_empty() => { + println!("og rust tree {} ({TARGET})", tree_hash()); + ExitCode::SUCCESS + } + "--tree-hash" if rest.is_empty() => { + println!("{}", tree_hash()); + ExitCode::SUCCESS + } + // Hidden: the differential test against CPython drives it. + "pycompat-probe" if rest.is_empty() => { + let stdin = std::io::stdin().lock(); + let stdout = std::io::stdout().lock(); + match probe::run(stdin, stdout) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + let _ = writeln!(std::io::stderr(), "og pycompat-probe: {e}"); + ExitCode::FAILURE + } + } + } + "version" | "--version" | "--tree-hash" | "pycompat-probe" => { + usage_error(&format!("unrecognized arguments: {}", rest.join(" "))) + } + other => usage_error(&format!("unknown tool '{other}'")), + } +} diff --git a/rust/og/src/probe.rs b/rust/og/src/probe.rs new file mode 100644 index 00000000..d2b859e9 --- /dev/null +++ b/rust/og/src/probe.rs @@ -0,0 +1,174 @@ +//! `og pycompat-probe`: reads one case per line and writes one result per +//! line, for `tests/rust/pycompat-diff.sh` to compare with what CPython +//! writes for the same cases (`tests/rust/pycompat_oracle.py`). +//! +//! A case is tab-separated: the operation, then its arguments. Floats travel +//! as the 16 hex digits of their bits, text as the hex of its UTF-8, so no +//! value is rounded or mangled on the way. A result is the Python result, +//! or the name of the exception Python raised, escaped by [`escape`]. + +use og_io::{Lines, Utf8}; +use pycompat::{fmt, json, num}; +use std::io::{self, BufRead, Write}; + +/// `\` `\n` `\r` `|` and NUL escaped, so every result is one line of +/// text and a list of lines can be joined with `|`. +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '|' => out.push_str("\\|"), + '\0' => out.push_str("\\0"), + c => out.push(c), + } + } + out +} + +fn bad(msg: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, msg.into()) +} + +fn float(s: &str) -> io::Result { + u64::from_str_radix(s, 16) + .map(f64::from_bits) + .map_err(|_| bad(format!("bad float bits {s:?}"))) +} + +fn bytes(s: &str) -> io::Result> { + if !s.len().is_multiple_of(2) { + return Err(bad("odd hex length")); + } + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).map_err(|_| bad("bad hex"))) + .collect() +} + +fn text(s: &str) -> io::Result { + String::from_utf8(bytes(s)?).map_err(|_| bad("hex text is not UTF-8")) +} + +fn int(s: &str) -> io::Result { + s.parse().map_err(|_| bad(format!("bad integer {s:?}"))) +} + +fn floats(s: &str) -> io::Result> { + if s.is_empty() { + Ok(Vec::new()) + } else { + s.split(',').map(float).collect() + } +} + +fn dump_opts(s: &str) -> io::Result> { + let f: Vec<&str> = s.split(',').collect(); + let [indent, sep, ascii, sort, nan] = f[..] else { + return Err(bad(format!("bad json options {s:?}"))); + }; + Ok(json::Dump { + indent: if indent == "-" { + None + } else { + Some(int(indent)?) + }, + separators: match sep { + "d" => None, + "c" => Some((",", ":")), + "p" => Some((", ", ": ")), + "i" => Some((",", ": ")), + _ => return Err(bad(format!("bad separators {sep:?}"))), + }, + ensure_ascii: ascii == "1", + sort_keys: sort == "1", + allow_nan: nan == "1", + }) +} + +/// One case's result. +fn eval(f: &[&str]) -> io::Result { + let arg = |i: usize| { + f.get(i) + .copied() + .ok_or_else(|| bad(format!("{}: missing argument {i}", f[0]))) + }; + Ok(match f[0] { + "repr" => fmt::repr(float(arg(1)?)?), + "f" => fmt::fixed(float(arg(1)?)?, int(arg(2)?)?), + "e" => fmt::sci(float(arg(1)?)?, int(arg(2)?)?), + "g" => fmt::general(float(arg(1)?)?, int(arg(2)?)?), + "pct" => fmt::percent(float(arg(1)?)?, int(arg(2)?)?), + "grpf" => fmt::fixed_grouped(float(arg(1)?)?, int(arg(2)?)?), + "grp" => fmt::int_grouped(int(arg(1)?)?), + "zpad" => fmt::zero_pad(int(arg(1)?)?, int(arg(2)?)?), + "ljust" => fmt::ljust(&text(arg(1)?)?, int(arg(2)?)?), + "rjust" => fmt::rjust(&text(arg(1)?)?, int(arg(2)?)?), + "round0" => match fmt::round_int(float(arg(1)?)?) { + Ok(v) => fmt::int_str(v), + Err(e) => e.exception().to_owned(), + }, + "round" => match fmt::round(float(arg(1)?)?, int(arg(2)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "float" => match num::parse_float(&text(arg(1)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "int" => match num::parse_int(&text(arg(1)?)?) { + Ok(v) => v.to_string(), + Err(e) => e.exception().to_owned(), + }, + "fsum" => match num::fsum(floats(arg(1)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "sum" => fmt::repr(num::sum(floats(arg(1)?)?)), + "json" => { + let opts = dump_opts(arg(2)?)?; + match json::loads(&text(arg(1)?)?).and_then(|v| json::dumps(&v, &opts)) { + Ok(s) => s, + Err(e) => e.exception().to_owned(), + } + } + "lines" => { + let lines: Vec = Lines::new(io::Cursor::new(bytes(arg(1)?)?), Utf8::Replace) + .collect::>()?; + return Ok(lines + .iter() + .map(|l| escape(l)) + .collect::>() + .join("|")); + } + // readline() keeps each line's ending, so a missing or extra "\n" shows. + "readline" => { + let mut l = Lines::new(io::Cursor::new(bytes(arg(1)?)?), Utf8::Replace); + let mut line = String::new(); + let mut all = Vec::new(); + while l.read_line(&mut line)? { + all.push(escape(&line)); + } + return Ok(all.join("|")); + } + op => return Err(bad(format!("unknown operation {op:?}"))), + }) +} + +pub fn run(input: impl BufRead, output: impl Write) -> io::Result<()> { + let mut out = io::BufWriter::new(output); + for line in input.lines() { + let line = line?; + let fields: Vec<&str> = line.split('\t').collect(); + let result = eval(&fields)?; + // A lines result is escaped line by line already. + if matches!(fields[0], "lines" | "readline") { + writeln!(out, "{result}")?; + } else { + writeln!(out, "{}", escape(&result))?; + } + } + out.flush() +} diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 00000000..bb639de6 --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,5 @@ +# The toolchain of OG_BUILDER (versions.env), so the image builds with +# what it ships and needs no download. Move both together. +[toolchain] +channel = "1.93.0" +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 3e2e37cf..5e8e153d 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -96,6 +96,8 @@ if [ -n "${FAMILIES:-}" ]; then if [ -n "$FAM" ]; then echo "[Family] ${SAMPLE} is in family ${FAM} (${FAMILIES}): steps 38 and 39 are planned after the pipeline" else echo "[Family] ${SAMPLE} is in no family of ${FAMILIES}"; fi fi +# og (bin/.og) built from another rust/ than this checkout's stops the run, SKIP_VALIDATION or not (rust/og.sh). +[ ! -f "${PGP_ROOT}/rust/og.sh" ] || OG_MSG=$(bash "${PGP_ROOT}/rust/og.sh" check 2>&1) || { echo "$OG_MSG" >&2; exit 2; } mkdir -p "${S}/nextflow" "$LOG_DIR" GENOME_DIR="$G" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" run-all.sh "$SEX" || echo "WARNING: could not write ${S}/run_manifest.tsv" STATUS="${LOG_DIR}/run_status.tsv" # bin/collect_summary.py marks results older than this run stale diff --git a/scripts/setup.sh b/scripts/setup.sh index 45d78ff3..90e85f8f 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2,6 +2,10 @@ # setup.sh — One-stop setup: download references, pull Docker images, validate # Usage: ./scripts/setup.sh # ./scripts/setup.sh --pull-only pull the Docker images and exit +# ./scripts/setup.sh --og put OwnGenome's og binary in bin/.og: +# built from rust/ in OG_BUILDER, or on +# a release tag downloaded and checked +# (rust/og.sh install; docs/rust.md) # ./scripts/setup.sh --refresh clinvar # replace ClinVar with NCBI's current # release and rebuild the files made from it @@ -103,6 +107,8 @@ REFRESH="" SAMPLE_QC_ONLY=false OPT_IN="" case "${1:-}" in + --og) + exec "$(dirname "$0")/../rust/og.sh" install ;; --cyrius|--parascopy-data|--kir-data|--ancestry-panel|--ancestry-fine-panel|--yleaf-data|--consumer-sites|--ancient-panel|--viral-data|--masked-reference|--exomiser-data|--chip-data|--strchive-data|--megane-kmers|--mocha-data|--locityper-data|--vep-cache|--annotsv-data|--archaic) OPT_IN=${1#--} shift ;; @@ -126,6 +132,7 @@ GENOME_DIR=${1:-${GENOME_DIR:-""}} if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "Usage: $0 " echo " $0 --pull-only" + echo " $0 --og" echo " $0 --refresh clinvar " echo " $0 --sample-qc-data " echo " $0 --cyrius | --parascopy-data | --kir-data | --ancestry-panel | --ancestry-fine-panel | --yleaf-data | --consumer-sites | --ancient-panel | --viral-data | --masked-reference | --exomiser-data | --chip-data | --strchive-data | --megane-kmers | --mocha-data | --locityper-data | --archaic | --vep-cache | --annotsv-data " @@ -133,6 +140,7 @@ if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo " Where to store reference data and sample outputs." echo " Needs at least 500 GB free space per sample." echo " --pull-only Pull the Docker images listed in versions.env and exit." + echo " --og Put OwnGenome's og binary in bin/.og (built from rust/, or a release's) and exit." echo " --refresh clinvar" echo " Download NCBI's current ClinVar, rebuild the files made from it" echo " and record its release date in /clinvar/RELEASE." diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index 834f1cb4..e01279bd 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -854,6 +854,20 @@ else fi fi +# --- OwnGenome's og binary (bin/.og, scripts/setup.sh --og) --- +# One built from another rust/ than this checkout's would give results from +# other code than the checkout says, so it fails; none at all is a note until +# a step calls og (rust/og.sh, REQUIRED). +header "OwnGenome binary" +# A partial copy of the scripts (some tests make one) has no rust/og.sh. +OG_MSG="NOTE: rust/og.sh is not here, so og was not checked" +[ ! -f "${PGP_ROOT}/rust/og.sh" ] || OG_MSG=$(bash "${PGP_ROOT}/rust/og.sh" check 2>&1) || true +case "$OG_MSG" in + OK:*) pass "${OG_MSG#OK: }" ;; + NOTE:*) info "${OG_MSG#NOTE: }" ;; + *) fail "${OG_MSG#ERROR: }" ;; +esac + ############################################################################### # 5. Sample Data (optional — only if $1 is provided) ############################################################################### diff --git a/tests/rust/og-in-images.sh b/tests/rust/og-in-images.sh new file mode 100755 index 00000000..f86ea4a8 --- /dev/null +++ b/tests/rust/og-in-images.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# og-in-images.sh: the og binaries in bin/.og run by bare name inside the +# images the pipeline's tasks use, as a Nextflow task calls them: bin/ mounted +# at its own path and first on PATH, the command run by sh. Each image runs on +# every platform it publishes (linux/amd64, linux/arm64; arm64 under QEMU on +# an amd64 host), and og must report the build for that platform's uname -m. +# Then the shim with no binary beside it must stop with exit 127 and the +# setup command. +# +# Usage: tests/rust/og-in-images.sh [IMAGE_VAR ...] +# default: PYTHON_IMAGE BCFTOOLS_IMAGE MANTA_IMAGE PARASCOPY_IMAGE +# Needs bin/.og built (scripts/setup.sh --og). Exit 1 when any run fails. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=../../versions.env +. "${ROOT}/versions.env" +[ $# -gt 0 ] || set -- PYTHON_IMAGE BCFTOOLS_IMAGE MANTA_IMAGE PARASCOPY_IMAGE +BIN="${ROOT}/bin" +for a in x86_64 aarch64; do + [ -x "${BIN}/.og/og-linux-${a}" ] || { echo "ERROR: ${BIN}/.og/og-linux-${a} is missing: run scripts/setup.sh --og" >&2; exit 1; } +done +fail=0 runs=0 +run_og() { # run_og IMAGE PLATFORM BINDIR: og version, as a task would call it + # -q: a first pull prints its progress on the output compared below. + docker run -q --rm --pull missing --platform "$2" -v "${3}:${3}:ro" -e "PATH=${3}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + --entrypoint sh "$1" -c 'v=$(og version) && echo "$(uname -m) $v"' 2>&1 +} +printf '%-18s %-12s %-8s %s\n' image platform result "uname -m and og version" +for var in "$@"; do + img=${!var:?$var is not set in versions.env} + for plat in linux/amd64 linux/arm64; do + rc=0 + out=$(run_og "$img" "$plat" "$BIN") || rc=$? + if [[ "$out" == *"does not provide the specified platform"* || "$out" == *"no matching manifest"* ]]; then + printf '%-18s %-12s %-8s %s\n' "$var" "$plat" skipped "the image has no ${plat} build" + continue + fi + runs=$((runs + 1)) + want=$([ "$plat" = linux/amd64 ] && echo x86_64 || echo aarch64) + # og version must also exit 0: a task that calls it fails otherwise. + if [ "$rc" -eq 0 ] && [[ "$out" == "${want} og rust tree "*" (${want}-unknown-linux-musl)" ]]; then r=ok; else r=FAIL; fail=1; out="${out} (exit ${rc})"; fi + printf '%-18s %-12s %-8s %s\n' "$var" "$plat" "$r" "$out" + done +done +[ "$runs" -gt 0 ] || { echo "ERROR: no image ran"; exit 1; } +# The shim alone: no bin/.og beside it. +# Under rust/target (ignored by git), not TMPDIR: Docker on macOS shares only some folders. +mkdir -p "${ROOT}/rust/target" +empty=$(mktemp -d "${ROOT}/rust/target/og-shim.XXXXXX") +trap 'rm -rf "$empty"' EXIT +cp "${BIN}/og" "${empty}/og" +rc=0 +out=$(docker run --rm -v "${empty}:${empty}:ro" -e "PATH=${empty}:/usr/local/bin:/usr/bin:/bin" --entrypoint sh "${!1}" -c 'og version' 2>&1) || rc=$? +if [ "$rc" -eq 127 ] && [[ "$out" == *"og is not built for "*"run scripts/setup.sh --og"* ]]; then + echo "ok: the shim without a binary exits 127: ${out}" +else + echo "FAIL: the shim without a binary exited ${rc}: ${out}"; fail=1 +fi +[ "$fail" -eq 0 ] && echo "PASS: og runs by bare name in every image and platform above" || echo "FAIL: see above" +exit "$fail" diff --git a/tests/rust/pycompat-diff.sh b/tests/rust/pycompat-diff.sh new file mode 100755 index 00000000..113698f0 --- /dev/null +++ b/tests/rust/pycompat-diff.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# pycompat-diff.sh: the pycompat differential test. CPython (python3 -I in +# PYTHON_IMAGE) draws N random cases per operation and writes its result for +# each; `og pycompat-probe` must write the same results byte for byte. +# +# Usage: tests/rust/pycompat-diff.sh OG_BINARY [N] [SEED] [OP ...] +# N defaults to 100000, SEED to a random one (printed, so a failure can be +# replayed). PYTHON=python3 runs the oracle on the host instead of in the +# image, for a quick local loop; CI always uses the image. +# Exit 0 when every result matches, 1 with the first differences otherwise. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +OG=${1:?Usage: $0 OG_BINARY [N] [SEED] [OP ...]} +N=${2:-100000} SEED=${3:-$RANDOM$RANDOM} +shift $(($# < 3 ? $# : 3)) +# shellcheck source=../../versions.env +. "${ROOT}/versions.env" +TMP=$(mktemp -d "${TMPDIR:-/tmp}/pycompat-diff.XXXXXX") +trap 'rm -rf "$TMP"' EXIT +echo "pycompat-diff: ${N} cases per operation, seed ${SEED}, oracle ${PYTHON:-${PYTHON_IMAGE}}" +t0=$(date +%s) +# The oracle goes in as an argument and its output comes back on stdout, so +# nothing is mounted (Docker on macOS shares only some folders). +ORACLE=$(cat "${ROOT}/tests/rust/pycompat_oracle.py") +if [ -n "${PYTHON:-}" ]; then + "$PYTHON" -I -c "$ORACLE" "$N" "$SEED" "$@" > "${TMP}/both.txt" +else + docker run --rm --network none "$PYTHON_IMAGE" python3 -I -c "$ORACLE" "$N" "$SEED" "$@" > "${TMP}/both.txt" +fi +awk 'NR % 2 == 1' "${TMP}/both.txt" > "${TMP}/cases.tsv" +awk 'NR % 2 == 0' "${TMP}/both.txt" > "${TMP}/expected.txt" +t1=$(date +%s) +"$OG" pycompat-probe < "${TMP}/cases.tsv" > "${TMP}/actual.txt" +t2=$(date +%s) +cases=$(wc -l < "${TMP}/cases.tsv" | tr -d ' ') +[ "$cases" -gt 0 ] || { echo "FAIL: the oracle wrote no cases"; exit 1; } +echo "pycompat-diff: ${cases} cases; CPython $((t1 - t0)) s, og $((t2 - t1)) s" +if cmp -s "${TMP}/expected.txt" "${TMP}/actual.txt"; then + echo "PASS: og pycompat-probe matches CPython on all ${cases} cases" + exit 0 +fi +# Results can hold tabs (ljust of a string with a tab), so the report pairs +# lines by number, never by splitting on tabs. +awk -v want_file="${TMP}/expected.txt" -v got_file="${TMP}/actual.txt" ' + { want = ""; got = ""; getline want < want_file; getline got < got_file + if (want == got) next + op = $0; sub(/\t.*/, "", op); bad[op]++; n++ + if (n <= 20) printf "case: %s\npython: %s\nog: %s\n\n", $0, want, got } + END { for (op in bad) printf "FAIL: %s differs on %d cases\n", op, bad[op] }' "${TMP}/cases.tsv" +echo "FAIL: og pycompat-probe differs from CPython (seed ${SEED}; rerun with that seed to replay)" +exit 1 diff --git a/tests/rust/pycompat_oracle.py b/tests/rust/pycompat_oracle.py new file mode 100644 index 00000000..894aed18 --- /dev/null +++ b/tests/rust/pycompat_oracle.py @@ -0,0 +1,287 @@ +"""The CPython side of the pycompat differential test. + + python3 -I pycompat_oracle.py N SEED [OP ...] + python3 -I pycompat_oracle.py ops the operations, one line + +Writes N random cases per operation (the hand-picked edge cases are among +the values drawn) to stdout, each line followed by the line this Python +returns for it. tests/rust/pycompat-diff.sh splits the two, feeds the cases +to `og pycompat-probe` and requires its output to equal the expected lines +byte for byte. Nothing is mounted, so the oracle runs in any image. The case format +is the one rust/og/src/probe.rs reads: tab-separated, floats as the hex of +their bits, text as the hex of its UTF-8. +""" + +import io +import json +import math +import random +import struct +import sys + +SPECIAL = [ + 0.0, -0.0, math.nan, math.inf, -math.inf, 0.125, 0.375, 2.675, 0.35, 0.5, 1.5, 2.5, -0.5, -2.5, + 1e-05, 1e-04, 1e16, 1e15, 1e17, 1e22, 1e23, 5e-324, 2.2250738585072014e-308, 2.225073858507201e-308, + 1.7976931348623157e308, 9.9999995, 999999.5, 0.1 + 0.2, 1 / 3, 2 / 3, 123456789012345678.0, 0.05, 0.15, + 0.25, 1.005, 1.115, 4.35, 8.345, 99.995, 1234.5, 2.0 ** 52 + 0.5, 2.0 ** 53, 2.0 ** 60, 1e300, -1e-300, +] +OPS = ["repr", "f", "e", "g", "pct", "grpf", "round0", "round", "grp", "zpad", "ljust", "rjust", + "float", "int", "fsum", "sum", "json", "lines", "readline"] +WHITESPACE = " \t\n\x0b\x0c\r\x1c\x1d\x1e\x1f\x85\xa0\u1680\u2000\u2005\u200a\u2028\u2029\u202f\u205f\u3000" + + +def bits(x): + return "%016x" % struct.unpack(";). Checked: no binary is a note (an error with +# --required); binaries of this rust/ pass; an edit to rust/, uncommitted, +# untracked or committed, makes them stale; a file git ignores does not; a +# binary stamped "unknown" is stale; a tree hash that cannot be computed is +# an error; built prints the hash for run_manifest.tsv; outside git the hash +# cannot be checked. Release downloads: checksum, attestation, a network +# error, a release without binaries, and a copy into bin/.og that fails. Then the callers: +# run-all.sh (with SKIP_VALIDATION=true) and validate-setup.sh refuse a stale +# binary with the setup command, and write_manifest.sh records the hash. +set -uo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +fails=0 +ok() { echo "PASS $*"; } +bad() { echo "FAIL $*"; fails=$((fails + 1)); } +expect() { # expect NAME RC WANT_RC OUTPUT PATTERN + if [ "$2" -eq "$3" ] && [[ "$4" == $5 ]]; then ok "$1"; else bad "$1: exit $2 (want $3), output: $4"; fi +} + +C="${T}/repo" +mkdir -p "$C" +cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" +rm -rf "${C}/bin/.og" "${C}/rust/target" +g() { git -C "$C" -c user.name=t -c user.email=t@example.invalid "$@"; } +g init -q && g add -A && g commit -qm base +OG="${C}/rust/og.sh" +plant() { # plant HASH [ARCH...]: fake binaries stamped with HASH + local h=$1 a; shift + [ $# -gt 0 ] || set -- x86_64 aarch64 + mkdir -p "${C}/bin/.og" + for a in "$@"; do + printf '\177ELF junk og-tree-hash:%s;attempt more junk' "$h" > "${C}/bin/.og/og-linux-${a}" + chmod 755 "${C}/bin/.og/og-linux-${a}" + done +} + +h0=$(bash "$OG" tree-hash) +[ "$h0" = "$(g rev-parse HEAD:rust)" ] && ok "tree-hash of a clean rust/ is HEAD:rust" || bad "tree-hash ${h0} != HEAD:rust" + +out=$(bash "$OG" check); rc=$? +expect "no binary: a note, exit 0" "$rc" 0 "$out" "NOTE: og is not built for x86_64 aarch64*setup.sh --og*" +out=$(bash "$OG" check --required); rc=$? +expect "no binary with --required: refused" "$rc" 1 "$out" "ERROR: og is not built for x86_64 aarch64: run scripts/setup.sh --og" +out=$(bash "$OG" built); [ "$out" = "not built" ] && ok "built without binaries: not built" || bad "built printed '${out}'" + +plant "$h0" +out=$(bash "$OG" check); rc=$? +expect "binaries of this rust/: OK" "$rc" 0 "$out" "OK: og is built for x86_64, aarch64*${h0}*" +out=$(bash "$OG" built); [ "$out" = "$h0" ] && ok "built prints the one hash" || bad "built printed '${out}'" + +plant "$h0" x86_64 +rm -f "${C}/bin/.og/og-linux-aarch64" +out=$(bash "$OG" check --required); rc=$? +expect "one arch missing with --required: refused" "$rc" 1 "$out" "ERROR: og is not built for aarch64*" + +plant "$h0" +mkdir -p "${C}/rust/target/debug" && echo junk > "${C}/rust/target/debug/x" +out=$(bash "$OG" check); rc=$? +expect "a file git ignores (rust/target) does not make it stale" "$rc" 0 "$out" "OK:*" + +echo "// edited" >> "${C}/rust/og/src/main.rs" +h1=$(bash "$OG" tree-hash) +[ "$h1" != "$h0" ] && ok "an uncommitted edit changes the tree hash" || bad "tree hash did not change after an edit" +out=$(bash "$OG" check); rc=$? +expect "uncommitted edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*${h0}*${h1}*run scripts/setup.sh --og" +g add rust/og/src/main.rs && g commit -qm edit +[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(g rev-parse HEAD:rust)" ] \ + && ok "committing the edit keeps the hash the build would stamp" || bad "hash after commit differs" +out=$(bash "$OG" check); rc=$? +expect "committed edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*" +plant "$h1" +echo "new" > "${C}/rust/new-file.txt" +out=$(bash "$OG" check); rc=$? +expect "an untracked file in rust/: stale" "$rc" 1 "$out" "ERROR: og is stale:*" +rm -f "${C}/rust/new-file.txt" +out=$(bash "$OG" check); rc=$? +expect "rebuilt from the edited rust/: OK again" "$rc" 0 "$out" "OK:*${h1}*" + +printf 'og-tree-hash:unknown;' > "${C}/bin/.og/og-linux-x86_64" +out=$(bash "$OG" check); rc=$? +expect "a binary stamped unknown (plain cargo build): stale" "$rc" 1 "$out" "ERROR: og is stale:*og-linux-x86_64 (rust/ tree unknown)*" +out=$(bash "$OG" built); [[ "$out" == "x86_64=unknown aarch64=${h1}" ]] && ok "built lists both hashes when they differ" || bad "built printed '${out}'" + +# The tree hash of an edited rust/ cannot be computed (TMPDIR not writable): +# an error, never the note of a checkout without git. +echo "// edited" >> "${C}/rust/og/src/main.rs" +out=$(TMPDIR="${T}/no-such-dir" bash "$OG" check 2> /dev/null); rc=$? +expect "a tree hash that fails is an error" "$rc" 1 "$out" "ERROR: the git tree hash of*could not be computed*" +g checkout -q -- rust/og/src/main.rs + +# The callers, on a stale binary. +plant "$h0" +mkdir -p "${T}/fake" "${T}/genome" +printf '#!/bin/sh\necho "openjdk version \\"17.0.1\\"" >&2\n' > "${T}/fake/java" +printf '#!/bin/sh\necho "fake nextflow: must not be reached" >&2\nexit 99\n' > "${T}/fake/nextflow" +chmod 755 "${T}/fake/java" "${T}/fake/nextflow" +out=$(PATH="${T}/fake:${PATH}" GENOME_DIR="${T}/genome" SKIP_VALIDATION=true bash "${C}/scripts/run-all.sh" s1 male 2>&1); rc=$? +expect "run-all.sh with SKIP_VALIDATION=true stops on a stale og" "$rc" 2 "$out" "*ERROR: og is stale:*run scripts/setup.sh --og*" +[[ "$out" != *"must not be reached"* ]] && ok "run-all.sh stopped before nextflow" || bad "run-all.sh reached nextflow" +out=$(GENOME_DIR="${T}/genome" bash "${C}/scripts/validate-setup.sh" 2>&1); rc=$? +expect "validate-setup.sh fails a stale og" "$rc" 1 "$out" "*[[]FAIL[]]*og is stale:*run scripts/setup.sh --og*" +plant "$h1" +out=$(GENOME_DIR="${T}/genome" bash "${C}/scripts/validate-setup.sh" 2>&1) +[[ "$out" == *"[OK]"*"og is built for x86_64, aarch64"* ]] && ok "validate-setup.sh passes a fresh og" || bad "validate-setup.sh: $(grep -i ' og ' <<< "$out")" +mkdir -p "${T}/genome/s1" +CONTAINER_ENGINE=true GENOME_DIR="${T}/genome" bash "${C}/bin/write_manifest.sh" s1 test > /dev/null 2>&1 +row=$(awk -F'\t' '$1 == "run" && $2 == "og_tree_hash" {print $3}' "${T}/genome/s1/run_manifest.tsv" 2> /dev/null) +[ "$row" = "$h1" ] && ok "run_manifest.tsv records og_tree_hash" || bad "run_manifest.tsv og_tree_hash row: '${row}'" + +# A release tag: the binaries come from the release, checked against +# SHA256SUMS and the attestation (a fake gh here), and are refused when +# either does not match; a gh that is not logged in skips the attestation. OG_RELEASE_BASE points at a local server. +if command -v python3 > /dev/null 2>&1 && command -v curl > /dev/null 2>&1; then + rm -rf "${C}/bin/.og" + g tag v9.9.9 + S="${T}/srv/v9.9.9" + mkdir -p "$S" + for a in x86_64 aarch64; do printf 'og-tree-hash:%s;' "$h1" > "${S}/og-linux-${a}"; done + (cd "$S" && { sha256sum og-linux-* 2> /dev/null || shasum -a 256 og-linux-*; } > SHA256SUMS) + port=$((20000 + RANDOM % 20000)) + (cd "${T}/srv" && exec python3 -m http.server "$port" --bind 127.0.0.1 > /dev/null 2>&1) & + srv=$! + for _ in $(seq 50); do curl -fs "http://127.0.0.1:${port}/v9.9.9/SHA256SUMS" > /dev/null 2>&1 && break; sleep 0.1; done + printf '#!/bin/sh +exit 0 +' > "${T}/fake/gh"; chmod 755 "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a release tag installs the release's binaries" "$rc" 0 "$out" "*Installed og ${h1} from release v9.9.9*" + [ "$(bash "$OG" built)" = "$h1" ] && ok "the installed binaries carry the tree" || bad "built after install: $(bash "$OG" built)" + rm -rf "${C}/bin/.og" + printf '#!/bin/sh +[ "$1" = auth ] && exit 0 +echo "no attestation" >&2 +exit 1 +' > "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a binary without a valid attestation is refused" "$rc" 1 "$out" "*has no valid build attestation*" + # gh installed but not logged in: attestation verify cannot run, so only + # SHA256SUMS is checked, with a note, never a refusal that reads as tampering. + printf '#!/bin/sh +echo "To get started with GitHub CLI, please run: gh auth login" >&2 +exit 4 +' > "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "gh not logged in: installed, attestation not checked" "$rc" 0 "$out" "*attestation was not checked*Installed og ${h1} from release v9.9.9*" + rm -rf "${C}/bin/.og" + printf '#!/bin/sh +exit 0 +' > "${T}/fake/gh" + printf 'tampered' >> "${S}/og-linux-aarch64" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a binary that does not match SHA256SUMS is refused" "$rc" 1 "$out" "*og-linux-aarch64 of release v9.9.9 does not match its SHA256SUMS*" + [ ! -e "${C}/bin/.og/og-linux-x86_64" ] && ok "a refused release installs nothing" || bad "a refused release left a binary" + # A copy that fails (bin/.og not writable) keeps the old pair and fails. + (cd "$S" && for a in x86_64 aarch64; do printf 'og-tree-hash:%s;' "$h1" > "og-linux-${a}"; done \ + && { sha256sum og-linux-* 2> /dev/null || shasum -a 256 og-linux-*; } > SHA256SUMS) + plant "$h0" + chmod 555 "${C}/bin/.og" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + chmod 755 "${C}/bin/.og" + expect "a copy into bin/.og that fails is an error" "$rc" 1 "$out" "*could not copy og-linux-x86_64*" + [[ "$out" != *"Installed og"* ]] && [ "$(bash "$OG" built)" = "$h0" ] && [ -z "$(ls -A "${C}/bin/.og" | grep tmp)" ] \ + && ok "a failed copy keeps the old pair and leaves no temporary file" || bad "after a failed copy: $(ls -A "${C}/bin/.og"), built $(bash "$OG" built)" + rm -rf "${C}/bin/.og" + # A network error on SHA256SUMS is an error, never a fallback to a build. + printf '#!/bin/sh\necho "fake engine: must not be reached" >&2\nexit 99\n' > "${T}/fake/engine"; chmod 755 "${T}/fake/engine" + out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=engine OG_RELEASE_BASE="http://127.0.0.1:1" bash "$OG" install 2>&1); rc=$? + expect "a release download that fails is an error, not a build" "$rc" 1 "$out" "*could not download http://127.0.0.1:1/v9.9.9/SHA256SUMS*" + [[ "$out" != *"must not be reached"* ]] && ok "no build after a failed download" || bad "a failed download started a build" + # A release without SHA256SUMS (404) has no binaries: build instead. + out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=engine OG_RELEASE_BASE="http://127.0.0.1:${port}/none" bash "$OG" install 2>&1); rc=$? + expect "a release without binaries falls back to a build" "$rc" 99 "$out" "*has no og binaries: building them*must not be reached*" + kill "$srv" 2> /dev/null; wait "$srv" 2> /dev/null + g tag -d v9.9.9 > /dev/null +else + echo "SKIP release download cases: no python3 or curl" +fi + +# Outside git the hash cannot be checked: a note, never a silent OK. +N="${T}/nogit" +plant "$h1" +cp -R "$C" "$N" && rm -rf "${N}/.git" +out=$(bash "${N}/rust/og.sh" check); rc=$? +expect "binaries outside a git checkout: a note" "$rc" 0 "$out" "NOTE: og is built, but this is not a git checkout*" + +[ "$fails" -eq 0 ] && echo "All og check cases passed" || echo "${fails} og check case(s) failed" +exit "$fails" diff --git a/versions.env b/versions.env index 18ae22e1..a40db28a 100644 --- a/versions.env +++ b/versions.env @@ -110,6 +110,16 @@ PYTHON_IMAGE="python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908f # and the build pgsc_calc uses. PLINK2_IMAGE="pgscatalog/plink2:2.00a5.10" +# --- Building OwnGenome's own Rust binary (rust/, bin/og) --- +# scripts/setup.sh --og builds bin/.og/og-linux-{x86_64,aarch64} in this +# cargo-zigbuild image. Its rustc (1.93.0) is the release rust/rust-toolchain.toml +# pins: move the two together, or the build downloads a toolchain first. The +# name has no _IMAGE suffix on purpose: no pipeline step runs it, so it has no +# row in tests/smoke/commands.tsv and Renovate does not bump it; bump it by hand +# with the toolchain. The publisher moves its tags in place, so it is pinned to +# its digest. +OG_BUILDER="ghcr.io/rust-cross/cargo-zigbuild:0.24.1@sha256:3abe26afb5537ea225c223c6790c1bf056271ef51ea52d2c4e343c569c8f129a" + # --- Polygenic scores and ancestry (steps 25 and 26) --- # pgsc_calc, the PGS Catalog's own Nextflow pipeline, runs as a pipeline of # its own (step 25 and the PRS process start it). The images below are the diff --git a/zensical.toml b/zensical.toml index 668d07b5..a1a83b39 100644 --- a/zensical.toml +++ b/zensical.toml @@ -136,6 +136,7 @@ nav = [ { "What 1.0 Keeps Stable" = "output-contract.md" }, { "Copy Number from Depth (the locus catalog)" = "cn-loci.md" }, { Testing = "testing.md" }, + { "The Rust Core" = "rust.md" }, { "Keeping versions up to date" = "updating.md" }, { Glossary = "glossary.md" }, { "Recommended Resources" = "resources.md" },