build(rust): add the og binary and the Python-compatible formatting core every port will share

.github/workflows/guard.yml

@@ -141,11 +141,113 @@ jobs:
 
           run_tests tests
 
+  # OwnGenome's Rust code (rust/, docs/rust.md), in a job of its own:
+  # unit-tests is close to its 10 minutes. Format, lint, unit tests with a
+  # planted failure that must go red, pycompat against CPython, then the two
+  # musl builds as scripts/setup.sh --og makes them and the stale-binary
+  # guard on the real binaries.
+  rust:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: rust
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install the toolchain rust/rust-toolchain.toml pins, with rustfmt and clippy
+        run: |
+          channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml)
+          rustup toolchain install "$channel" --profile minimal -c rustfmt -c clippy
+          rustc --version
+
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          workspaces: rust
+
+      - name: rustfmt
+        run: cargo fmt --all --check
+
+      - name: clippy, warnings are errors
+        if: ${{ !cancelled() }}   # runs even when a step above failed, so each result shows
+        run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
+
+      - name: Control (the planted failing test must fail, and be the test that failed)
+        run: |
+          log="${RUNNER_TEMP}/planted.log"
+          if cargo test --locked -p pycompat --features planted-failure planted_failure > "$log" 2>&1; then
+            cat "$log"
+            echo "ERROR: the planted failing test passed: the test run would report a broken test as green"
+            exit 1
+          fi
+          grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; }
+          echo "Control: the planted test failed, as it must."
+
+      - name: cargo test
+        run: cargo test --workspace --locked
+
+      - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation)
+        if: ${{ !cancelled() }}   # runs even when a step above failed, so each result shows
+        run: |
+          OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og
+          ../tests/rust/pycompat-diff.sh "${PWD}/target/release/og" 100000
+
+      - name: Both musl builds, as scripts/setup.sh --og makes them in OG_BUILDER
+        run: |
+          ../scripts/setup.sh --og
+          ./og.sh check --required
+          file ../bin/.og/og-linux-*
+          ../bin/og version
+
+      - name: The stale-binary guard refuses the real binaries once rust/ changes
+        run: |
+          echo "// edited" >> og/src/main.rs
+          if out=$(./og.sh check); then echo "ERROR: an edited rust/ passed the check: ${out}"; exit 1; fi
+          echo "Refused, as it must: ${out}"
+          git checkout -- og/src/main.rs
+          ./og.sh check --required
+
+      # main.nf runs the same check before any task: a stale og stops
+      # `nextflow run` with the setup command. Without the check, this stub
+      # run completes and the step fails.
+      - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: The Nextflow version versions.env pins
+        id: nf
+        working-directory: .
+        run: |
+          # shellcheck source=/dev/null
+          . ./versions.env
+          echo "version=${NEXTFLOW_VERSION:?NEXTFLOW_VERSION is not set in versions.env}" >> "$GITHUB_OUTPUT"
+
+      # Not a Nextflow pin: the version comes from versions.env. Quoted so the
+      # Renovate dry run does not count it as one.
+      - uses: "nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914" # v3.0.1
+        with:
+          version: ${{ steps.nf.outputs.version }}
+
+      - name: nextflow run stops on a stale og before any task
+        working-directory: .
+        run: |
+          echo "// edited" >> rust/og/src/main.rs
+          rc=0
+          out=$(timeout 600 nextflow run main.nf -profile test -stub -ansi-log false --outdir "${RUNNER_TEMP}/og-stale" 2>&1) || rc=$?
+          git checkout -- rust/og/src/main.rs
+          if [ "$rc" -eq 0 ] || [ "$rc" -eq 124 ] || [[ "$out" != *"og is stale:"*"run scripts/setup.sh --og"* ]] || [[ "$out" == *"Submitted process"* ]]; then
+            echo "$out"
+            echo "ERROR: nextflow run did not stop on a stale og before any task (exit ${rc})"
+            exit 1
+          fi
+          echo "Refused, as it must (exit ${rc}): $(grep 'og is stale' <<< "$out")"
+
   # The one check the ruleset requires for this workflow: it runs even when a
   # job above failed or was skipped, and fails unless every one succeeded.
   ci-ok:
     if: always()
-    needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests]
+    needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests, rust]
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:

.github/workflows/release.yml

@@ -9,15 +9,64 @@ on:
         description: 'Tag to release (e.g. v0.3.0)'
         required: true
         type: string
+  # A dry run: a pull request that changes this file or rust/ builds the og
+  # binaries and their SHA256SUMS as an artifact; nothing is released.
+  pull_request:
+    branches: [main]
+    paths: ['.github/workflows/release.yml', 'rust/**']
 
 permissions:
   contents: read
 
 jobs:
+  # The og binaries every release carries (docs/rust.md): both musl builds,
+  # made by scripts/setup.sh --og as a checkout makes them, with SHA256SUMS.
+  # A checkout of the tag downloads them instead of building (rust/og.sh).
+  og-binaries:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      # Same rule as create-release below: the tag is untrusted text.
+      - name: Check the tag
+        if: github.event_name != 'pull_request'
+        env:
+          INPUT_TAG: ${{ inputs.tag }}
+        run: |
+          tag=${INPUT_TAG:-${GITHUB_REF#refs/tags/}}
+          if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "ERROR: the tag is not of the form vMAJOR.MINOR.PATCH (not echoed: it is untrusted text)" >&2
+            exit 1
+          fi
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.tag || github.ref }}
+          persist-credentials: false
+
+      - name: Build og for x86_64 and aarch64 (scripts/setup.sh --og builds in OG_BUILDER)
+        run: |
+          # On a tag, setup.sh --og would download the release's own binaries,
+          # which this job is about to make: build them.
+          rust/og.sh build
+          rust/og.sh check --required
+          mkdir -p dist
+          cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 dist/
+          (cd dist && sha256sum og-linux-* > SHA256SUMS && cat SHA256SUMS)
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: og-binaries
+          path: dist/
+          retention-days: 7
+
   create-release:
+    needs: og-binaries
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     permissions:
       contents: write
+      id-token: write       # the build attestation
+      attestations: write
     steps:
       # The tag comes from the dispatch form or the pushed ref, so it is
       # untrusted text: it reaches the shell through env only, and it must
@@ -75,13 +124,33 @@ jobs:
             echo "All image versions are listed in [\`versions.env\`](versions.env)."
             echo ""
             echo "To set up: \`./scripts/setup.sh /path/to/data\`"
+            echo ""
+            echo "## og binaries"
+            echo ""
+            echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks them."
             echo "EOF"
           } >> "$GITHUB_OUTPUT"
 
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: og-binaries
+          path: dist
+
+      # rust/og.sh install checks a downloaded binary with
+      # `gh attestation verify` when gh is installed.
+      - name: Attest the og binaries
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: 'dist/og-linux-*'
+
       - name: Create GitHub Release
         uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
         with:
           tag_name: ${{ steps.tag.outputs.tag }}
           name: ${{ steps.tag.outputs.tag }}
           body: ${{ steps.notes.outputs.notes }}
           draft: false
+          files: |
+            dist/og-linux-x86_64
+            dist/og-linux-aarch64
+            dist/SHA256SUMS

.github/workflows/rust.yml

@@ -0,0 +1,120 @@
+name: Rust
+
+# The og binary in the images the tasks run, and the long pycompat streams.
+# Guard's rust job holds the checks every pull request needs (format, lint,
+# tests, pycompat at 100000 cases per operation, the musl builds and the
+# stale-binary guard); this workflow holds the slow ones (docs/rust.md):
+#
+#   images   on a pull request that touches the Rust code, the shim or the
+#            images: builds og with scripts/setup.sh --og, cold and then
+#            cached (both times go to the summary), and runs it by bare name
+#            in PYTHON_IMAGE, BCFTOOLS_IMAGE, MANTA_IMAGE and PARASCOPY_IMAGE
+#            on every platform each one publishes (arm64 under QEMU), then
+#            once more with a wrong-arch binary planted, which must fail. The
+#            two binaries are kept as the artifact og-binaries.
+#   streams  nightly: 10^7 cases per pycompat operation against CPython in
+#            PYTHON_IMAGE, one operation at a time, with a new seed each
+#            night (printed, so a failure can be replayed). On a pull request
+#            that touches the paths below it runs 20000 cases per operation, so the
+#            job is seen working.
+#
+# Public repo: GitHub-hosted runners only.
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'rust/**'
+      - 'bin/og'
+      - 'tests/rust/**'
+      - 'versions.env'
+      - 'scripts/setup.sh'
+      - '.github/workflows/rust.yml'
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+    inputs:
+      cases:
+        description: 'pycompat cases per operation for the streams job'
+        default: '10000000'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  images:
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Build og with scripts/setup.sh --og, cold and cached
+        run: |
+          . versions.env
+          docker pull -q "$OG_BUILDER" > /dev/null
+          t0=$(date +%s); scripts/setup.sh --og; t1=$(date +%s)
+          touch rust/og/src/main.rs
+          scripts/setup.sh --og; t2=$(date +%s)
+          rust/og.sh check --required
+          {
+            echo "| build | seconds |"
+            echo "|---|---|"
+            echo "| cold (empty cargo cache, image pulled) | $((t1 - t0)) |"
+            echo "| cached (one crate changed) | $((t2 - t1)) |"
+          } | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+        with:
+          platforms: arm64
+
+      # shell: bash sets pipefail, so a failing check is not masked by tee.
+      - name: og by bare name in the task images, amd64 and arm64
+        shell: bash
+        run: tests/rust/og-in-images.sh | tee -a "$GITHUB_STEP_SUMMARY"
+
+      # Control: the same check must fail when an image runs the wrong build.
+      # The amd64 binary planted as og-linux-aarch64 reports x86_64 on arm64.
+      - name: Control (a planted wrong-arch binary must fail the image check)
+        run: |
+          cp bin/.og/og-linux-aarch64 "${RUNNER_TEMP}/og-linux-aarch64"
+          cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64
+          rc=0
+          tests/rust/og-in-images.sh PYTHON_IMAGE || rc=$?
+          cp "${RUNNER_TEMP}/og-linux-aarch64" bin/.og/og-linux-aarch64
+          [ "$rc" -ne 0 ] || { echo "ERROR: the image check passed a wrong-arch binary"; exit 1; }
+          echo "Control: the image check failed on the planted binary, as it must."
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: og-binaries
+          path: bin/.og/og-linux-*
+          retention-days: 7
+
+  streams:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Build og with scripts/setup.sh --og
+        run: |
+          scripts/setup.sh --og
+          bin/og version
+
+      - name: pycompat against CPython, one operation at a time
+        env:
+          CASES: ${{ github.event_name == 'pull_request' && '20000' || inputs.cases || '10000000' }}
+        run: |
+          case "$CASES" in ''|*[!0-9]*) echo "ERROR: cases must be a whole number, got '${CASES}'"; exit 1 ;; esac
+          seed=${GITHUB_RUN_ID}
+          fail=0
+          ops=$(python3 -I tests/rust/pycompat_oracle.py ops)
+          [ -n "$ops" ] || { echo "ERROR: the oracle lists no operation"; exit 1; }
+          for op in $ops; do
+            tests/rust/pycompat-diff.sh "${PWD}/bin/og" "$CASES" "$seed" "$op" || fail=1
+          done
+          exit "$fail"

.gitignore

@@ -108,3 +108,8 @@ site/
 
 # Scratch space of a checkout: logs, downloads, CI output
 /tmp/
+
+# Rust: the build output, and the og binaries scripts/setup.sh --og puts
+# beside the bin/og shim (rust/og.sh)
+/rust/target/
+/bin/.og/

CLAUDE.md

@@ -6,6 +6,7 @@ Whole genome sequencing (WGS) analysis pipeline for consumer hardware. Takes raw
 ## Tech Stack
 - Bash (pipeline scripts, `set -euo pipefail`)
 - Docker (all bioinformatics tools containerized)
+- Python (`bin/*.py`, OwnGenome's own tools), moving tool by tool to Rust: one static binary, `og` (`rust/`, run through the `bin/og` shim; `scripts/setup.sh --og` builds it; `docs/rust.md`)
 - Key tools: DeepVariant, minimap2, BWA-MEM2, VEP, PharmCAT, GATK, FreeBayes, Strelka2, TIDDIT, Manta, SURVIVOR, PCGR/CPSR, pgsc_calc, haplogrep3, Yleaf
 
 ## Development
@@ -71,7 +72,8 @@ owngenome/
   .github/workflows/
     lint.yml                   # ShellCheck, actionlint, gitleaks, doc links
     personal-data.yml          # Personal-data scan of every tracked text file
-    guard.yml                  # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests; ci-ok sums them up
+    guard.yml                  # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests, Rust; ci-ok sums them up
+    rust.yml                   # og in the task images on amd64 and arm64; nightly pycompat against CPython at 10^7 cases
     container-test.yml         # Runs each changed image on the fixture (tests/smoke/commands.tsv); container-ok sums it up
     e2e.yml                    # Real tools on a small fixture
     nextflow.yml               # Nextflow config, lint, schema and stub runs on the pinned release and the newest 26.10.x

bin/og

@@ -0,0 +1,21 @@
+#!/bin/sh
+# og: OwnGenome's Rust binary, called by bare name in a task (bin/ is on its
+# PATH) or as /pgp-bin/og in a step script. It runs the static Linux build
+# for this machine from bin/.og/, which scripts/setup.sh --og puts there
+# (rust/og.sh). Inside an amd64 image on an Apple Silicon host, uname -m
+# says x86_64, so that build runs under emulation.
+case $(uname -m) in
+  x86_64 | amd64) arch=x86_64 ;;
+  aarch64 | arm64) arch=aarch64 ;;
+  *) arch=$(uname -m) ;;
+esac
+bin="$(dirname "$0")/.og/og-linux-${arch}"
+if [ "$(uname -s)" != Linux ]; then
+  echo "ERROR: og runs inside the pipeline's Linux containers, not on $(uname -s)" >&2
+  exit 127
+fi
+if [ ! -x "$bin" ]; then
+  echo "ERROR: og is not built for ${arch}; run scripts/setup.sh --og" >&2
+  exit 127
+fi
+exec "$bin" "$@"