From da9049fd26fc6ab24cda346db887b2cdcf4dfb56 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:16:47 +0100 Subject: [PATCH 01/12] build(rust): add the Rust workspace with the pycompat and og-io crates and the og binary pycompat writes floats, ints and JSON the way CPython 3.15 does (repr, the f/e/g/% formats, round half to even on the binary value, float() and int() parsing, math.fsum, the compensated builtin sum, json.dumps and a lenient json.loads). og-io reads plain, gzip and BGZF input with Python's universal newlines and writes files atomically. og prints its version and the rust/ tree it was built from, and a hidden pycompat-probe that tests/rust/pycompat-diff.sh drives against python3 -I in PYTHON_IMAGE. Bead: pgp-bs9u.2 --- rust/Cargo.lock | 101 ++++++ rust/Cargo.toml | 27 ++ rust/crates/og-io/Cargo.toml | 13 + rust/crates/og-io/src/lib.rs | 262 +++++++++++++++ rust/crates/pycompat/Cargo.toml | 17 + rust/crates/pycompat/src/fmt.rs | 508 ++++++++++++++++++++++++++++++ rust/crates/pycompat/src/json.rs | 525 +++++++++++++++++++++++++++++++ rust/crates/pycompat/src/lib.rs | 25 ++ rust/crates/pycompat/src/num.rs | 352 +++++++++++++++++++++ rust/og/Cargo.toml | 14 + rust/og/build.rs | 19 ++ rust/og/src/main.rs | 64 ++++ rust/og/src/probe.rs | 164 ++++++++++ rust/rust-toolchain.toml | 5 + tests/rust/pycompat-diff.sh | 47 +++ tests/rust/pycompat_oracle.py | 276 ++++++++++++++++ 16 files changed, 2419 insertions(+) create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/crates/og-io/Cargo.toml create mode 100644 rust/crates/og-io/src/lib.rs create mode 100644 rust/crates/pycompat/Cargo.toml create mode 100644 rust/crates/pycompat/src/fmt.rs create mode 100644 rust/crates/pycompat/src/json.rs create mode 100644 rust/crates/pycompat/src/lib.rs create mode 100644 rust/crates/pycompat/src/num.rs create mode 100644 rust/og/Cargo.toml create mode 100644 rust/og/build.rs create mode 100644 rust/og/src/main.rs create mode 100644 rust/og/src/probe.rs create mode 100644 rust/rust-toolchain.toml create mode 100755 tests/rust/pycompat-diff.sh create mode 100644 tests/rust/pycompat_oracle.py diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..d53c8030 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,101 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "og" +version = "0.1.0" +dependencies = [ + "og-io", + "pycompat", +] + +[[package]] +name = "og-io" +version = "0.1.0" +dependencies = [ + "flate2", +] + +[[package]] +name = "pycompat" +version = "0.1.0" +dependencies = [ + "indexmap", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..1a544792 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,27 @@ +[workspace] +resolver = "3" +members = ["crates/pycompat", "crates/og-io", "og"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +license = "GPL-3.0-or-later" +publish = false + +[workspace.dependencies] +pycompat = { path = "crates/pycompat" } +og-io = { path = "crates/og-io" } +indexmap = "2" +flate2 = "1" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[workspace.lints.clippy] +all = { level = "warn", priority = -1 } + +[profile.release] +lto = true +codegen-units = 1 +panic = "abort" +strip = true diff --git a/rust/crates/og-io/Cargo.toml b/rust/crates/og-io/Cargo.toml new file mode 100644 index 00000000..0260f051 --- /dev/null +++ b/rust/crates/og-io/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "og-io" +description = "Input and output as OwnGenome's Python tools do it: plain, gzip or BGZF in, universal newlines, atomic writes" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +flate2.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs new file mode 100644 index 00000000..956ce3d8 --- /dev/null +++ b/rust/crates/og-io/src/lib.rs @@ -0,0 +1,262 @@ +//! Reading and writing files the way OwnGenome's Python tools do. +//! +//! - [`open`]: a path or `-` (stdin), plain or gzip, sniffed by its magic +//! bytes. Multi-member gzip, BGZF included, is read to the end. +//! - [`Lines`]: Python's text mode with `newline=None`: `\n`, `\r\n` and a +//! lone `\r` each end a line and read as `\n`; UTF-8 strict or `replace`. +//! - [`AtomicFile`]: written beside the target and renamed over it on +//! [`AtomicFile::commit`], so a reader never sees half a file. + +use flate2::bufread::MultiGzDecoder; +use std::fs::{self, File}; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; + +const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; + +/// Opens `path` for reading (`-` is stdin), decompressing gzip and BGZF. +pub fn open(path: impl AsRef) -> io::Result> { + let path = path.as_ref(); + let inner: Box = if path == Path::new("-") { + Box::new(io::stdin().lock()) + } else { + Box::new(File::open(path)?) + }; + reader(inner) +} + +/// Wraps any reader: gzip when it starts with the gzip magic, plain otherwise. +pub fn reader(mut inner: impl Read + 'static) -> io::Result> { + // Read the magic byte by byte: a pipe may hand over one byte at a time. + let mut head = Vec::with_capacity(2); + let mut byte = [0u8; 1]; + while head.len() < 2 { + match inner.read(&mut byte) { + Ok(0) => break, + Ok(_) => head.push(byte[0]), + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + let gz = head == GZIP_MAGIC; + let buf = BufReader::with_capacity(1 << 16, io::Cursor::new(head).chain(inner)); + Ok(if gz { + Box::new(BufReader::with_capacity(1 << 16, MultiGzDecoder::new(buf))) + } else { + Box::new(buf) + }) +} + +/// How invalid UTF-8 is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Utf8 { + /// `errors="strict"`: an error. + Strict, + /// `errors="replace"`: U+FFFD for each invalid sequence. + Replace, +} + +/// Python's universal-newline text reader over any [`BufRead`]. +pub struct Lines { + inner: R, + raw: Vec, + pos: usize, + utf8: Utf8, +} + +impl Lines { + pub fn new(inner: R, utf8: Utf8) -> Self { + Lines { + inner, + raw: Vec::new(), + pos: 0, + utf8, + } + } + + /// Python's `readline()`: the next line with its ending turned into + /// `\n` (none on a last line without one), or `false` at the end. + pub fn read_line(&mut self, out: &mut String) -> io::Result { + out.clear(); + if self.pos == self.raw.len() { + self.raw.clear(); + self.pos = 0; + if self.inner.read_until(b'\n', &mut self.raw)? == 0 { + return Ok(false); + } + } + let rest = &self.raw[self.pos..]; + // read_until stops after a \n or at the end, so a \r\n is never split. + let (body, used) = match rest.iter().position(|&c| c == b'\n' || c == b'\r') { + None => (rest, rest.len()), + Some(k) if rest[k] == b'\r' && rest.get(k + 1) == Some(&b'\n') => (&rest[..k], k + 2), + Some(k) => (&rest[..k], k + 1), + }; + let ended = used > body.len(); + match self.utf8 { + Utf8::Strict => out.push_str( + std::str::from_utf8(body) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?, + ), + Utf8::Replace => out.push_str(&String::from_utf8_lossy(body)), + } + if ended { + out.push('\n'); + } + self.pos += used; + Ok(true) + } +} + +impl Iterator for Lines { + type Item = io::Result; + + /// Each line without its ending, as `line.rstrip("\n")` after Python's + /// universal-newline reading. + fn next(&mut self) -> Option { + let mut s = String::new(); + match self.read_line(&mut s) { + Ok(true) => { + if s.ends_with('\n') { + s.pop(); + } + Some(Ok(s)) + } + Ok(false) => None, + Err(e) => Some(Err(e)), + } + } +} + +/// A file written under a temporary name and renamed into place by +/// [`commit`](AtomicFile::commit). Dropped without a commit, the temporary +/// file is removed and the target is untouched. +pub struct AtomicFile { + tmp: PathBuf, + target: PathBuf, + out: Option>, +} + +impl AtomicFile { + pub fn create(target: impl AsRef) -> io::Result { + let target = target.as_ref().to_path_buf(); + let name = target + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "no file name"))?; + let mut tmp_name = std::ffi::OsString::from("."); + tmp_name.push(name); + tmp_name.push(format!(".tmp-{}", std::process::id())); + let tmp = target.with_file_name(tmp_name); + let out = BufWriter::new(File::create(&tmp)?); + Ok(AtomicFile { + tmp, + target, + out: Some(out), + }) + } + + /// Flushes and renames over the target. + pub fn commit(mut self) -> io::Result<()> { + let out = self.out.take().expect("committed once"); + out.into_inner().map_err(io::IntoInnerError::into_error)?; + fs::rename(&self.tmp, &self.target) + } +} + +impl Write for AtomicFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.out.as_mut().expect("not committed").write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.out.as_mut().expect("not committed").flush() + } +} + +impl Drop for AtomicFile { + fn drop(&mut self) { + if self.out.take().is_some() { + let _ = fs::remove_file(&self.tmp); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::Compression; + use flate2::write::GzEncoder; + + fn lines(bytes: &[u8], utf8: Utf8) -> Vec { + let mut l = Lines::new(io::Cursor::new(bytes.to_vec()), utf8); + let mut out = Vec::new(); + let mut s = String::new(); + while l.read_line(&mut s).unwrap() { + out.push(s.clone()); + } + out + } + + #[test] + fn universal_newlines() { + assert_eq!( + lines(b"a\r\nb\rc\n\rd", Utf8::Strict), + ["a\n", "b\n", "c\n", "\n", "d"] + ); + assert_eq!(lines(b"x\r", Utf8::Strict), ["x\n"]); + assert_eq!(lines(b"", Utf8::Strict), Vec::::new()); + assert_eq!( + lines(b"\xff\xe2\x82\n", Utf8::Replace), + ["\u{fffd}\u{fffd}\n"] + ); + let mut l = Lines::new(io::Cursor::new(b"\xff\n".to_vec()), Utf8::Strict); + assert!(l.read_line(&mut String::new()).is_err()); + let it: Vec = Lines::new(io::Cursor::new(b"a\r\nb".to_vec()), Utf8::Strict) + .map(Result::unwrap) + .collect(); + assert_eq!(it, ["a", "b"]); + } + + fn gz(data: &[u8]) -> Vec { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(data).unwrap(); + e.finish().unwrap() + } + + #[test] + fn reads_plain_gzip_and_multi_member() { + let mut two = gz(b"one\n"); + two.extend(gz(b"two\n")); + for (input, want) in [ + (b"plain\n".to_vec(), "plain\n"), + (gz(b"zipped\n"), "zipped\n"), + (two, "one\ntwo\n"), + (vec![0x1f], "\u{1f}"), + ] { + let mut s = String::new(); + reader(io::Cursor::new(input)) + .unwrap() + .read_to_string(&mut s) + .unwrap(); + assert_eq!(s, want); + } + } + + #[test] + fn atomic_write() { + let dir = std::env::temp_dir().join(format!("og-io-test-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let target = dir.join("out.tsv"); + fs::write(&target, "old").unwrap(); + { + let mut f = AtomicFile::create(&target).unwrap(); + f.write_all(b"half").unwrap(); + } + assert_eq!(fs::read_to_string(&target).unwrap(), "old"); + let mut f = AtomicFile::create(&target).unwrap(); + f.write_all(b"new").unwrap(); + f.commit().unwrap(); + assert_eq!(fs::read_to_string(&target).unwrap(), "new"); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/rust/crates/pycompat/Cargo.toml b/rust/crates/pycompat/Cargo.toml new file mode 100644 index 00000000..8c8128c9 --- /dev/null +++ b/rust/crates/pycompat/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "pycompat" +description = "Python-exact float, int and JSON formatting and parsing" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[features] +# A test that always fails: CI runs it and requires the run to go red. +planted-failure = [] + +[dependencies] +indexmap.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/pycompat/src/fmt.rs b/rust/crates/pycompat/src/fmt.rs new file mode 100644 index 00000000..255fe211 --- /dev/null +++ b/rust/crates/pycompat/src/fmt.rs @@ -0,0 +1,508 @@ +//! Python's float and int formatting, byte for byte. +//! +//! Rust's `{:e}` gives the shortest digits that round-trip, which are the +//! digits of Python's `repr`; Rust's `{:.N}` and `{:.Ne}` round the exact +//! binary value half to even, as CPython's dtoa does. What differs is the +//! layout (`1e-05` against `1e-5`, `nan` against `NaN`), and that is what +//! this module writes. + +use std::fmt; + +/// `nan`, `inf` and `-inf`, the spelling of every Python float format. +fn special(x: f64) -> Option<&'static str> { + if x.is_nan() { + Some("nan") + } else if x == f64::INFINITY { + Some("inf") + } else if x == f64::NEG_INFINITY { + Some("-inf") + } else { + None + } +} + +fn sign(x: f64) -> &'static str { + if x.is_sign_negative() { "-" } else { "" } +} + +/// The digits and decimal point of Rust's scientific form of a finite +/// `|x|`: `"1.25e-3"` gives `("125", -2)`, the value `0.125 * 10^-2`. +fn split_sci(s: &str) -> (String, i32) { + let (mant, exp) = s.split_once('e').expect("scientific form has an exponent"); + let digits = mant.chars().filter(|&c| c != '.').collect(); + let exp: i32 = exp.parse().expect("integer exponent"); + (digits, exp + 1) +} + +/// `d[0].d[1..]e±XX`, with at least two exponent digits, as Python writes it. +fn exp_layout(digits: &str, exp: i32) -> String { + let (first, rest) = digits.split_at(1); + let dot = if rest.is_empty() { "" } else { "." }; + let es = if exp < 0 { '-' } else { '+' }; + format!("{first}{dot}{rest}e{es}{:02}", exp.unsigned_abs()) +} + +/// The digits laid out without an exponent, `decpt` digits before the point. +fn fixed_layout(digits: &str, decpt: i32, add_dot_0: bool) -> String { + let n = digits.len() as i32; + if decpt <= 0 { + format!("0.{}{digits}", "0".repeat(decpt.unsigned_abs() as usize)) + } else if decpt >= n { + let zeros = "0".repeat((decpt - n) as usize); + let tail = if add_dot_0 { ".0" } else { "" }; + format!("{digits}{zeros}{tail}") + } else { + let (int, frac) = digits.split_at(decpt as usize); + format!("{int}.{frac}") + } +} + +/// The shortest digits that read back as `x` (finite, positive), and the +/// decimal point. Rust and CPython agree on them except when `x` lies +/// exactly halfway between the two nearest candidates: CPython's dtoa then +/// takes the one with the even last digit, Rust may take the other. +fn shortest(x: f64) -> (String, i32) { + let (digits, decpt) = split_sci(&format!("{x:e}")); + let Some(exact) = exact_digits(x) else { + return (digits, decpt); + }; + let n = digits.len(); + if exact.len() != n + 1 || !exact.ends_with('5') { + return (digits, decpt); + } + // x is the midpoint of `low` and `low + 1` in the last place. + let low = exact[..n].to_owned(); + let high = increment(&low); + let parses_back = |d: &str, decpt: i32| format!("0.{d}e{decpt}").parse::() == Ok(x); + let (high, high_decpt) = if high.len() > n { + (high[..n].to_owned(), decpt + 1) + } else { + (high, decpt) + }; + let low_even = (low.as_bytes()[n - 1] - b'0').is_multiple_of(2); + let mut picks = [(low, decpt), (high, high_decpt)]; + if !low_even { + picks.swap(0, 1); + } + for (d, p) in picks { + if parses_back(&d, p) { + let d = d.trim_end_matches('0'); + return ( + if d.is_empty() { + "0".to_owned() + } else { + d.to_owned() + }, + p, + ); + } + } + (digits, decpt) +} + +/// A decimal string of digits plus one in the last place (`"19"` -> `"20"`, `"99"` -> `"100"`). +fn increment(d: &str) -> String { + let mut b = d.as_bytes().to_vec(); + let mut i = b.len(); + loop { + if i == 0 { + b.insert(0, b'1'); + break; + } + i -= 1; + if b[i] == b'9' { + b[i] = b'0'; + } else { + b[i] += 1; + break; + } + } + String::from_utf8(b).expect("ascii digits") +} + +/// The significant digits of the exact value of a finite positive `x`, +/// when there are at most 18 of them (the only case where a tie between +/// two 17-digit candidates is possible); `None` otherwise. +fn exact_digits(x: f64) -> Option { + let bits = x.to_bits(); + let biased = ((bits >> 52) & 0x7ff) as i32; + let frac = bits & ((1 << 52) - 1); + let (mut m, mut e) = if biased == 0 { + (frac, -1074) + } else { + (frac | (1 << 52), biased - 1075) + }; + let tz = m.trailing_zeros(); + m >>= tz; + e += tz as i32; + let s = if e >= 0 { + if e < 74 { + (u128::from(m) << e).to_string() + } else { + format!("{x:.0}") + } + } else { + // x = m * 5^k / 10^k; past 5^27 there are more than 18 digits. + let k = e.unsigned_abs(); + if k > 27 { + return None; + } + (u128::from(m) * 5u128.pow(k)).to_string() + }; + let s = s.trim_end_matches('0'); + (s.len() <= 18).then(|| s.to_owned()) +} + +/// `repr(x)`, which is also `str(x)` and `f"{x}"`. +pub fn repr(x: f64) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + if x == 0.0 { + return format!("{}0.0", sign(x)); + } + let (digits, decpt) = shortest(x.abs()); + let body = if decpt <= -4 || decpt > 16 { + exp_layout(&digits, decpt - 1) + } else { + fixed_layout(&digits, decpt, true) + }; + format!("{}{body}", sign(x)) +} + +/// `f"{x:.{prec}f}"`. +pub fn fixed(x: f64, prec: usize) -> String { + match special(x) { + Some(s) => s.to_owned(), + None => format!("{x:.prec$}"), + } +} + +/// `f"{x:.{prec}e}"`. +pub fn sci(x: f64, prec: usize) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + let (digits, decpt) = split_sci(&format!("{:.prec$e}", x.abs())); + let (first, rest) = digits.split_at(1); + let es = if decpt - 1 < 0 { '-' } else { '+' }; + let dot = if prec == 0 { "" } else { "." }; + format!( + "{}{first}{dot}{rest}e{es}{:02}", + sign(x), + (decpt - 1).unsigned_abs() + ) +} + +/// `f"{x:.{prec}g}"`, also `"%.{prec}g" % x`. `f"{x:g}"` is `general(x, 6)`; +/// a precision of 0 counts as 1, as in Python. +pub fn general(x: f64, prec: usize) -> String { + if let Some(s) = special(x) { + return s.to_owned(); + } + if x == 0.0 { + return format!("{}0", sign(x)); + } + let p = prec.max(1); + let (digits, decpt) = split_sci(&format!("{:.*e}", p - 1, x.abs())); + let digits = digits.trim_end_matches('0'); + let exp = decpt - 1; + let body = if exp < -4 || exp >= p as i32 { + exp_layout(digits, exp) + } else { + fixed_layout(digits, decpt, false) + }; + format!("{}{body}", sign(x)) +} + +/// `f"{x:.{prec}%}"`: `x * 100` in the `f` format, then `%`. +pub fn percent(x: f64, prec: usize) -> String { + format!("{}%", fixed(x * 100.0, prec)) +} + +/// Commas every three digits of a run of ASCII digits. +fn group(digits: &str) -> String { + let mut out = String::with_capacity(digits.len() + digits.len() / 3); + for (i, c) in digits.chars().enumerate() { + if i > 0 && (digits.len() - i).is_multiple_of(3) { + out.push(','); + } + out.push(c); + } + out +} + +/// `f"{n:,}"` of an int. +pub fn int_grouped(n: i128) -> String { + let s = if n < 0 { "-" } else { "" }; + format!("{s}{}", group(&n.unsigned_abs().to_string())) +} + +/// `f"{x:,.{prec}f}"`. +pub fn fixed_grouped(x: f64, prec: usize) -> String { + let s = fixed(x, prec); + let (neg, body) = match s.strip_prefix('-') { + Some(b) => ("-", b), + None => ("", s.as_str()), + }; + if !body.starts_with(|c: char| c.is_ascii_digit()) { + return s; + } + let (int, frac) = match body.split_once('.') { + Some((i, f)) => (i, Some(f)), + None => (body, None), + }; + match frac { + Some(f) => format!("{neg}{}.{f}", group(int)), + None => format!("{neg}{}", group(int)), + } +} + +/// `f"{n:0{width}d}"`: the sign first, then zeros up to `width` characters. +pub fn zero_pad(n: i128, width: usize) -> String { + let digits = n.unsigned_abs().to_string(); + let s = if n < 0 { "-" } else { "" }; + let pad = width.saturating_sub(digits.len() + s.len()); + format!("{s}{}{digits}", "0".repeat(pad)) +} + +/// `"%-{width}s" % s` and `s.ljust(width)`: width in code points. +pub fn ljust(s: &str, width: usize) -> String { + let pad = width.saturating_sub(s.chars().count()); + format!("{s}{}", " ".repeat(pad)) +} + +/// `"%{width}s" % s` and `s.rjust(width)`: width in code points. +pub fn rjust(s: &str, width: usize) -> String { + let pad = width.saturating_sub(s.chars().count()); + format!("{}{s}", " ".repeat(pad)) +} + +/// Why Python's `round` raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoundError { + /// `round(nan)`: ValueError. + Nan, + /// `round(inf)`: OverflowError. + Infinite, + /// `round(x, n)` whose result would not fit a float: OverflowError. + TooLarge, +} + +impl RoundError { + /// The Python exception class. + pub fn exception(self) -> &'static str { + match self { + RoundError::Nan => "ValueError", + RoundError::Infinite | RoundError::TooLarge => "OverflowError", + } + } +} + +impl fmt::Display for RoundError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + RoundError::Nan => "cannot convert float NaN to integer", + RoundError::Infinite => "cannot convert float infinity to integer", + RoundError::TooLarge => "rounded value too large to represent", + }) + } +} + +impl std::error::Error for RoundError {} + +/// `round(x)`: the nearest integer, ties to even. Python returns an int; +/// this returns it as an integral float, which [`int_str`] prints exactly. +pub fn round_int(x: f64) -> Result { + if x.is_nan() { + Err(RoundError::Nan) + } else if x.is_infinite() { + Err(RoundError::Infinite) + } else { + Ok(x.round_ties_even()) + } +} + +/// `str(int(x))` of an integral float: every digit, exactly. +pub fn int_str(x: f64) -> String { + debug_assert!(x.is_finite() && x.fract() == 0.0); + if x == 0.0 { + "0".to_owned() + } else { + format!("{x:.0}") + } +} + +/// `round(x, n)` for a float: the exact binary value rounded half to even +/// at `n` decimals (`round(0.35, 1) == 0.3`, `round(2.675, 2) == 2.67`), +/// read back as the nearest float. A negative `n` rounds to tens, hundreds. +pub fn round(x: f64, n: i32) -> Result { + // CPython's NDIGITS_MAX and NDIGITS_MIN (floatobject.c). + if !x.is_finite() || n > 323 { + return Ok(x); + } + if n < -308 { + return Ok(0.0 * x); + } + let text = if n >= 0 { + format!("{x:.0$}", n as usize) + } else { + round_integer_digits(x, n.unsigned_abs() as usize) + }; + let r: f64 = text.parse().expect("a decimal string parses"); + if r.is_infinite() { + Err(RoundError::TooLarge) + } else { + Ok(r) + } +} + +/// `x` rounded half to even to a multiple of `10^k`, as a decimal string +/// with its sign. Works on the exact integer digits so it never rounds twice. +fn round_integer_digits(x: f64, k: usize) -> String { + let a = x.abs(); + let int = a.trunc(); + let frac_nonzero = a != int; + let mut digits: Vec = int_str(int).into_bytes(); + if digits.len() < k + 1 { + let mut padded = vec![b'0'; k + 1 - digits.len()]; + padded.extend_from_slice(&digits); + digits = padded; + } + let cut = digits.len() - k; + let (kept, dropped) = digits.split_at(cut); + let mut kept = kept.to_vec(); + let up = match dropped[0].cmp(&b'5') { + std::cmp::Ordering::Greater => true, + std::cmp::Ordering::Less => false, + std::cmp::Ordering::Equal => { + frac_nonzero + || dropped[1..].iter().any(|&d| d != b'0') + || (kept.last().copied().unwrap_or(b'0') - b'0') % 2 == 1 + } + }; + if up { + let mut i = kept.len(); + loop { + if i == 0 { + kept.insert(0, b'1'); + break; + } + i -= 1; + if kept[i] == b'9' { + kept[i] = b'0'; + } else { + kept[i] += 1; + break; + } + } + } + let mut s = String::from(sign(x)); + s.push_str(std::str::from_utf8(&kept).expect("ascii digits")); + s.push_str(&"0".repeat(k)); + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repr_matches_python() { + let cases = [ + (0.1, "0.1"), + (1.0, "1.0"), + (-0.0, "-0.0"), + (0.0, "0.0"), + (1e16, "1e+16"), + (1e15, "1000000000000000.0"), + (1.5e16, "1.5e+16"), + (0.0001, "0.0001"), + (0.00001, "1e-05"), + (1.2345e-7, "1.2345e-07"), + (123456.789, "123456.789"), + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + (f64::NAN, "nan"), + (f64::INFINITY, "inf"), + (f64::NEG_INFINITY, "-inf"), + (2.0f64.powi(60), "1.152921504606847e+18"), + // Exactly halfway between two 16-digit candidates: the even one. + (94614632987514.0 + 0.125, "94614632987514.12"), + (634342696773016.0 + 0.25, "634342696773016.2"), + ]; + for (x, want) in cases { + assert_eq!(repr(x), want, "repr({x:?})"); + } + } + + #[test] + fn fixed_rounds_the_exact_value_half_even() { + assert_eq!(fixed(0.125, 2), "0.12"); + assert_eq!(fixed(0.375, 2), "0.38"); + assert_eq!(fixed(2.675, 2), "2.67"); + assert_eq!(fixed(2.5, 0), "2"); + assert_eq!(fixed(-0.04, 1), "-0.0"); + assert_eq!(fixed(f64::NAN, 3), "nan"); + assert_eq!(fixed(-1.0 / 0.0, 3), "-inf"); + } + + #[test] + fn sci_and_general() { + assert_eq!(sci(123456.0, 3), "1.235e+05"); + assert_eq!(sci(0.0, 2), "0.00e+00"); + assert_eq!(sci(1.5, 0), "2e+00"); + assert_eq!(general(1234567.0, 6), "1.23457e+06"); + assert_eq!(general(123456.0, 6), "123456"); + assert_eq!(general(0.0001, 6), "0.0001"); + assert_eq!(general(0.00001, 6), "1e-05"); + assert_eq!(general(100.0, 6), "100"); + assert_eq!(general(1.5, 0), "2"); + assert_eq!(general(9.9999996, 6), "10"); + assert_eq!(general(-0.0, 6), "-0"); + assert_eq!(general(0.1 + 0.2, 10), "0.3"); + assert_eq!(general(1e16, 17), "10000000000000000"); + } + + #[test] + fn percent_grouping_padding() { + assert_eq!(percent(0.12345, 1), "12.3%"); + assert_eq!(percent(f64::NAN, 1), "nan%"); + assert_eq!(int_grouped(-1234567), "-1,234,567"); + assert_eq!(int_grouped(999), "999"); + assert_eq!(fixed_grouped(1234567.891, 2), "1,234,567.89"); + assert_eq!(fixed_grouped(-999.5, 0), "-1,000"); + assert_eq!(zero_pad(-42, 5), "-0042"); + assert_eq!(zero_pad(42, 1), "42"); + assert_eq!(ljust("é", 3), "é "); + assert_eq!(rjust("ab", 3), " ab"); + } + + #[test] + fn round_is_half_even_on_the_binary_value() { + assert_eq!(round_int(2.5), Ok(2.0)); + assert_eq!(round_int(3.5), Ok(4.0)); + assert_eq!(round_int(-0.5), Ok(-0.0)); + assert_eq!(round_int(f64::NAN), Err(RoundError::Nan)); + assert_eq!(round_int(f64::INFINITY), Err(RoundError::Infinite)); + assert_eq!(int_str(-0.0), "0"); + assert_eq!(int_str(1e22), "10000000000000000000000"); + assert_eq!(round(0.35, 1), Ok(0.3)); + assert_eq!(round(2.675, 2), Ok(2.67)); + assert_eq!(round(0.125, 2), Ok(0.12)); + assert_eq!(round(1250.0, -2), Ok(1200.0)); + assert_eq!(round(1350.0, -2), Ok(1400.0)); + assert_eq!(round(1250.4, -2), Ok(1300.0)); + assert_eq!(round(5.0, -1), Ok(0.0)); + assert_eq!(round(15.0, -1), Ok(20.0)); + assert_eq!(round(501.0, -3), Ok(1000.0)); + assert!(round(-4.0, -1).unwrap().is_sign_negative()); + assert!(round(-0.04, 1).unwrap().is_sign_negative()); + assert_eq!( + round(1.7976931348623157e308, -308), + Err(RoundError::TooLarge) + ); + assert_eq!(round(1.5, 400), Ok(1.5)); + assert!(round(f64::NAN, 2).unwrap().is_nan()); + } +} diff --git a/rust/crates/pycompat/src/json.rs b/rust/crates/pycompat/src/json.rs new file mode 100644 index 00000000..305181e6 --- /dev/null +++ b/rust/crates/pycompat/src/json.rs @@ -0,0 +1,525 @@ +//! Python's `json` module: `dumps` byte for byte, and `loads` with its +//! leniency (`NaN`, `Infinity`, the last of a duplicate key wins and keeps +//! the first one's place). + +use indexmap::IndexMap; +use std::fmt::{self, Write}; + +use crate::fmt::repr; + +/// A JSON value as Python holds it: ints and floats apart, objects in +/// insertion order. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + /// A Python int. Python has no limit; this port stops at 128 bits. + Int(i128), + Float(f64), + Str(String), + Array(Vec), + Object(IndexMap), +} + +/// The `json.dumps` keyword arguments this port supports. +#[derive(Debug, Clone)] +pub struct Dump<'a> { + /// `indent=N`; `None` writes one line. + pub indent: Option, + /// `separators=(item, key)`; `None` is Python's default, which depends + /// on `indent`: `(", ", ": ")` without, `(",", ": ")` with. + pub separators: Option<(&'a str, &'a str)>, + pub ensure_ascii: bool, + pub sort_keys: bool, + pub allow_nan: bool, +} + +impl Default for Dump<'_> { + /// `json.dumps(obj)` with no keywords. + fn default() -> Self { + Dump { + indent: None, + separators: None, + ensure_ascii: true, + sort_keys: false, + allow_nan: true, + } + } +} + +/// Why `dumps` or `loads` raised. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum JsonError { + /// ValueError from `dumps(allow_nan=False)`. + OutOfRangeFloat, + /// JSONDecodeError: what was wrong and the byte offset. + Decode(&'static str, usize), +} + +impl JsonError { + /// The Python exception class. + pub fn exception(&self) -> &'static str { + match self { + JsonError::OutOfRangeFloat => "ValueError", + JsonError::Decode(..) => "JSONDecodeError", + } + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::OutOfRangeFloat => { + f.write_str("Out of range float values are not JSON compliant") + } + JsonError::Decode(what, at) => write!(f, "{what}: byte {at}"), + } + } +} + +impl std::error::Error for JsonError {} + +/// `json.dumps(value, **opts)`. +pub fn dumps(value: &Value, opts: &Dump<'_>) -> Result { + let mut out = String::new(); + let (item, key) = opts.separators.unwrap_or(if opts.indent.is_some() { + (",", ": ") + } else { + (", ", ": ") + }); + let w = Writer { opts, item, key }; + w.value(&mut out, value, 0)?; + Ok(out) +} + +struct Writer<'o, 'a> { + opts: &'o Dump<'a>, + item: &'o str, + key: &'o str, +} + +impl Writer<'_, '_> { + fn newline(&self, out: &mut String, level: usize) { + if let Some(n) = self.opts.indent { + out.push('\n'); + out.extend(std::iter::repeat_n(' ', n * level)); + } + } + + /// The separator between two items, and the line break after it. + fn between(&self, out: &mut String, level: usize) { + out.push_str(self.item); + self.newline(out, level); + } + + fn value(&self, out: &mut String, v: &Value, level: usize) -> Result<(), JsonError> { + match v { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + Value::Int(i) => write!(out, "{i}").expect("write to String"), + Value::Float(x) => out.push_str(&self.float(*x)?), + Value::Str(s) => quote(out, s, self.opts.ensure_ascii), + Value::Array(a) if a.is_empty() => out.push_str("[]"), + Value::Array(a) => { + out.push('['); + self.newline(out, level + 1); + for (i, x) in a.iter().enumerate() { + if i > 0 { + self.between(out, level + 1); + } + self.value(out, x, level + 1)?; + } + self.newline(out, level); + out.push(']'); + } + Value::Object(m) if m.is_empty() => out.push_str("{}"), + Value::Object(m) => { + let mut items: Vec<(&String, &Value)> = m.iter().collect(); + if self.opts.sort_keys { + items.sort_by(|a, b| a.0.cmp(b.0)); + } + out.push('{'); + self.newline(out, level + 1); + for (i, (k, x)) in items.into_iter().enumerate() { + if i > 0 { + self.between(out, level + 1); + } + quote(out, k, self.opts.ensure_ascii); + out.push_str(self.key); + self.value(out, x, level + 1)?; + } + self.newline(out, level); + out.push('}'); + } + } + Ok(()) + } + + fn float(&self, x: f64) -> Result { + if x.is_finite() { + return Ok(repr(x)); + } + if !self.opts.allow_nan { + return Err(JsonError::OutOfRangeFloat); + } + Ok(if x.is_nan() { + "NaN" + } else if x > 0.0 { + "Infinity" + } else { + "-Infinity" + } + .to_owned()) + } +} + +/// A JSON string literal as Python's encoder writes it. +pub fn quote(out: &mut String, s: &str, ensure_ascii: bool) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\x08' => out.push_str("\\b"), + '\x0c' => out.push_str("\\f"), + c if (c as u32) < 0x20 => write!(out, "\\u{:04x}", c as u32).expect("write to String"), + c if ensure_ascii && !(' '..='~').contains(&c) => { + let mut buf = [0u16; 2]; + for unit in c.encode_utf16(&mut buf) { + write!(out, "\\u{unit:04x}").expect("write to String"); + } + } + c => out.push(c), + } + } + out.push('"'); +} + +/// `json.loads(text)`. +pub fn loads(text: &str) -> Result { + let mut p = Parser { + s: text.as_bytes(), + text, + i: 0, + }; + p.ws(); + let v = p.value()?; + p.ws(); + if p.i != p.s.len() { + return Err(JsonError::Decode("Extra data", p.i)); + } + Ok(v) +} + +struct Parser<'a> { + s: &'a [u8], + text: &'a str, + i: usize, +} + +impl Parser<'_> { + fn err(&self, what: &'static str) -> Result { + Err(JsonError::Decode(what, self.i)) + } + + fn ws(&mut self) { + while matches!(self.s.get(self.i), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.i += 1; + } + } + + fn eat(&mut self, lit: &str) -> bool { + if self.s[self.i..].starts_with(lit.as_bytes()) { + self.i += lit.len(); + true + } else { + false + } + } + + fn value(&mut self) -> Result { + match self.s.get(self.i) { + Some(b'{') => self.object(), + Some(b'[') => self.array(), + Some(b'"') => Ok(Value::Str(self.string()?)), + Some(b'n') if self.eat("null") => Ok(Value::Null), + Some(b't') if self.eat("true") => Ok(Value::Bool(true)), + Some(b'f') if self.eat("false") => Ok(Value::Bool(false)), + Some(b'N') if self.eat("NaN") => Ok(Value::Float(f64::NAN)), + Some(b'I') if self.eat("Infinity") => Ok(Value::Float(f64::INFINITY)), + Some(b'-') if self.eat("-Infinity") => Ok(Value::Float(f64::NEG_INFINITY)), + Some(b'-' | b'0'..=b'9') => self.number(), + _ => self.err("Expecting value"), + } + } + + fn digits(&mut self) -> usize { + let n = self.s[self.i..] + .iter() + .take_while(|c| c.is_ascii_digit()) + .count(); + self.i += n; + n + } + + /// `-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?`, as Python's scanner. + fn number(&mut self) -> Result { + let start = self.i; + if self.s[self.i] == b'-' { + self.i += 1; + } + match self.s.get(self.i) { + Some(b'0') => self.i += 1, + Some(b'1'..=b'9') => { + self.digits(); + } + _ => { + self.i = start; + return self.err("Expecting value"); + } + } + let mut float = false; + if self.s.get(self.i) == Some(&b'.') + && self.s.get(self.i + 1).is_some_and(u8::is_ascii_digit) + { + self.i += 1; + self.digits(); + float = true; + } + if matches!(self.s.get(self.i), Some(b'e' | b'E')) { + let save = self.i; + self.i += 1; + if matches!(self.s.get(self.i), Some(b'+' | b'-')) { + self.i += 1; + } + if self.digits() == 0 { + self.i = save; + } else { + float = true; + } + } + let lit = &self.text[start..self.i]; + if float { + Ok(Value::Float(lit.parse().expect("a JSON number parses"))) + } else { + lit.parse() + .map(Value::Int) + .or_else(|_| self.err("Integer beyond 128 bits")) + } + } + + fn hex4(&mut self) -> Result { + let h = self + .s + .get(self.i..self.i + 4) + .and_then(|h| std::str::from_utf8(h).ok()); + match h.and_then(|h| { + (h.bytes().all(|c| c.is_ascii_hexdigit())).then(|| u32::from_str_radix(h, 16).ok()) + }) { + Some(Some(v)) => { + self.i += 4; + Ok(v) + } + _ => self.err("Invalid \\uXXXX escape"), + } + } + + fn string(&mut self) -> Result { + self.i += 1; + let mut out = String::new(); + loop { + let run = self.s[self.i..] + .iter() + .take_while(|&&c| c != b'"' && c != b'\\' && c >= 0x20) + .count(); + out.push_str(&self.text[self.i..self.i + run]); + self.i += run; + match self.s.get(self.i) { + Some(b'"') => { + self.i += 1; + return Ok(out); + } + Some(b'\\') => { + self.i += 1; + let c = match self.s.get(self.i) { + Some(b'"') => '"', + Some(b'\\') => '\\', + Some(b'/') => '/', + Some(b'b') => '\x08', + Some(b'f') => '\x0c', + Some(b'n') => '\n', + Some(b'r') => '\r', + Some(b't') => '\t', + Some(b'u') => { + self.i += 1; + let hi = self.hex4()?; + let cp = if (0xd800..0xdc00).contains(&hi) + && self.s[self.i..].starts_with(b"\\u") + { + let save = self.i; + self.i += 2; + let lo = self.hex4()?; + if (0xdc00..0xe000).contains(&lo) { + 0x10000 + ((hi - 0xd800) << 10) + (lo - 0xdc00) + } else { + self.i = save; + hi + } + } else { + hi + }; + match char::from_u32(cp) { + Some(c) => { + out.push(c); + continue; + } + None => { + return self + .err("Lone surrogate (Python keeps it; this port cannot)"); + } + } + } + _ => return self.err("Invalid \\escape"), + }; + out.push(c); + self.i += 1; + } + Some(_) => return self.err("Invalid control character"), + None => return self.err("Unterminated string"), + } + } + } + + fn array(&mut self) -> Result { + self.i += 1; + let mut a = Vec::new(); + self.ws(); + if self.eat("]") { + return Ok(Value::Array(a)); + } + loop { + self.ws(); + a.push(self.value()?); + self.ws(); + if self.eat("]") { + return Ok(Value::Array(a)); + } + if !self.eat(",") { + return self.err("Expecting ',' delimiter"); + } + } + } + + fn object(&mut self) -> Result { + self.i += 1; + let mut m = IndexMap::new(); + self.ws(); + if self.eat("}") { + return Ok(Value::Object(m)); + } + loop { + self.ws(); + if self.s.get(self.i) != Some(&b'"') { + return self.err("Expecting property name enclosed in double quotes"); + } + let k = self.string()?; + self.ws(); + if !self.eat(":") { + return self.err("Expecting ':' delimiter"); + } + self.ws(); + let v = self.value()?; + // A duplicate key keeps its first place and takes the last value, as a dict does. + m.insert(k, v); + self.ws(); + if self.eat("}") { + return Ok(Value::Object(m)); + } + if !self.eat(",") { + return self.err("Expecting ',' delimiter"); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(text: &str, opts: &Dump<'_>) -> String { + dumps(&loads(text).unwrap(), opts).unwrap() + } + + #[test] + fn dumps_matches_python() { + let d = Dump::default(); + assert_eq!( + round_trip(r#"{"a": [1, 2.0, "é😀\u007f"], "b": {}}"#, &d), + r#"{"a": [1, 2.0, "\u00e9\ud83d\ude00\u007f"], "b": {}}"# + ); + let one = Dump { + indent: Some(1), + ..Dump::default() + }; + assert_eq!( + round_trip(r#"{"b": [1, []], "a": null}"#, &one), + "{\n \"b\": [\n 1,\n []\n ],\n \"a\": null\n}" + ); + let zero = Dump { + indent: Some(0), + ..Dump::default() + }; + assert_eq!(round_trip("[1,2]", &zero), "[\n1,\n2\n]"); + let compact = Dump { + separators: Some((",", ":")), + sort_keys: true, + ensure_ascii: false, + ..Dump::default() + }; + assert_eq!( + round_trip(r#"{"b":1,"a":"é\u0001"}"#, &compact), + "{\"a\":\"é\\u0001\",\"b\":1}" + ); + assert_eq!( + round_trip("[NaN, Infinity, -Infinity, 1e400, -0.0, 1E2]", &d), + "[NaN, Infinity, -Infinity, Infinity, -0.0, 100.0]" + ); + let strict = Dump { + allow_nan: false, + ..Dump::default() + }; + assert_eq!( + dumps(&Value::Float(f64::NAN), &strict), + Err(JsonError::OutOfRangeFloat) + ); + } + + #[test] + fn loads_keeps_python_dict_semantics() { + assert_eq!( + round_trip(r#"{"a": 1, "b": 2, "a": 3}"#, &Dump::default()), + r#"{"a": 3, "b": 2}"# + ); + assert_eq!(loads("-0").unwrap(), Value::Int(0)); + assert_eq!( + loads(" \"\\ud83d\\ude00\" ").unwrap(), + Value::Str("😀".into()) + ); + for bad in [ + "", + "[1,]", + "{\"a\" 1}", + "01", + "1.", + "\"a\u{1}\"", + "[1] x", + "{'a': 1}", + "\"\\ud800\"", + ] { + assert!(loads(bad).is_err(), "{bad:?}"); + } + } +} diff --git a/rust/crates/pycompat/src/lib.rs b/rust/crates/pycompat/src/lib.rs new file mode 100644 index 00000000..9524737c --- /dev/null +++ b/rust/crates/pycompat/src/lib.rs @@ -0,0 +1,25 @@ +//! Python-exact primitives for the Rust ports of OwnGenome's Python tools. +//! +//! Every function here returns the bytes CPython 3.15 returns for the same +//! input. The differential test (`tests/rust/pycompat-diff.sh`) checks that +//! against `python3 -I` in the pinned `PYTHON_IMAGE`. +//! +//! - [`fmt`]: `repr(float)`, the `f`, `e`, `g` and `%` format types, `{:,}`, +//! zero padding, `ljust`/`rjust`, `round(x)` and `round(x, n)`. +//! - [`num`]: `float(str)`, `int(str)`, `math.fsum` and the builtin `sum`. +//! - [`json`]: `json.dumps` and a `json.loads` with Python's leniency. + +pub mod fmt; +pub mod json; +pub mod num; + +#[cfg(all(test, feature = "planted-failure"))] +mod planted { + /// Always fails. CI runs it with `--features planted-failure` and + /// requires a red result, so a test runner that reports nothing cannot + /// pass for a green one. + #[test] + fn planted_failure() { + assert_eq!(crate::fmt::repr(0.1), "0.2", "planted failure"); + } +} diff --git a/rust/crates/pycompat/src/num.rs b/rust/crates/pycompat/src/num.rs new file mode 100644 index 00000000..612b62f3 --- /dev/null +++ b/rust/crates/pycompat/src/num.rs @@ -0,0 +1,352 @@ +//! Python's number parsing and float sums. + +use std::fmt; + +/// `str.isspace()` for one character, the set `str.strip()` removes. +/// Wider than Rust's `char::is_whitespace`: it has `\x1c`-`\x1f`. +pub fn py_isspace(c: char) -> bool { + matches!( + c, + '\t' | '\n' + | '\x0b' + | '\x0c' + | '\r' + | '\x1c'..='\x1f' + | ' ' + | '\u{85}' + | '\u{a0}' + | '\u{1680}' + | '\u{2000}'..='\u{200a}' + | '\u{2028}' + | '\u{2029}' + | '\u{202f}' + | '\u{205f}' + | '\u{3000}' + ) +} + +/// `s.strip()` with no argument. +pub fn py_strip(s: &str) -> &str { + s.trim_matches(py_isspace) +} + +/// The whitespace `float()` and `int()` skip around a number: Unicode +/// whitespace, but of ASCII only C's six (`\x1c`-`\x1f` are refused). +fn number_space(c: char) -> bool { + py_isspace(c) && !matches!(c, '\x1c'..='\x1f') +} + +/// Why `float(s)` or `int(s)` raised. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ParseError { + /// ValueError: not a number Python reads. + Invalid, + /// An int beyond the 128-bit range: Python has no limit, this port has. + TooLarge, +} + +impl ParseError { + /// The Python exception class (`TooLarge` has none: Python would return the int). + pub fn exception(&self) -> &'static str { + match self { + ParseError::Invalid => "ValueError", + ParseError::TooLarge => "OverflowError", + } + } +} + +impl fmt::Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + ParseError::Invalid => "invalid literal", + ParseError::TooLarge => "integer does not fit 128 bits", + }) + } +} + +impl std::error::Error for ParseError {} + +/// Removes underscores that sit between two ASCII digits, as Python allows +/// in a number literal; any other underscore is an error. +fn strip_underscores(s: &str) -> Result { + let b = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + for (i, &c) in b.iter().enumerate() { + if c == b'_' { + let before = i > 0 && b[i - 1].is_ascii_digit(); + let after = b.get(i + 1).is_some_and(u8::is_ascii_digit); + if !(before && after) { + return Err(ParseError::Invalid); + } + } else { + out.push(c as char); + } + } + Ok(out) +} + +/// True when `s` (no sign, no underscores) is a decimal float literal: +/// digits with an optional point, at least one digit, an optional exponent. +fn is_decimal(s: &str) -> bool { + let b = s.as_bytes(); + let mut i = 0; + let int = b.iter().take_while(|c| c.is_ascii_digit()).count(); + i += int; + let mut frac = 0; + if b.get(i) == Some(&b'.') { + i += 1; + frac = b[i..].iter().take_while(|c| c.is_ascii_digit()).count(); + i += frac; + } + if int + frac == 0 { + return false; + } + if matches!(b.get(i), Some(b'e' | b'E')) { + i += 1; + if matches!(b.get(i), Some(b'+' | b'-')) { + i += 1; + } + let exp = b[i..].iter().take_while(|c| c.is_ascii_digit()).count(); + if exp == 0 { + return false; + } + i += exp; + } + i == b.len() +} + +/// `float(s)` for a str: surrounding whitespace, a sign, underscores between +/// digits, `inf`, `infinity` and `nan` in any case; `1e400` is `inf`. +/// Non-ASCII digits, which Python also reads, are refused. +pub fn parse_float(s: &str) -> Result { + let t = s.trim_matches(number_space); + if !t.is_ascii() { + return Err(ParseError::Invalid); + } + let (neg, body) = match t.as_bytes().first() { + Some(b'-') => (true, &t[1..]), + Some(b'+') => (false, &t[1..]), + _ => (false, t), + }; + let lower = body.to_ascii_lowercase(); + let v = match lower.as_str() { + "inf" | "infinity" => f64::INFINITY, + "nan" => f64::NAN, + _ => { + let clean = strip_underscores(body)?; + if !is_decimal(&clean) { + return Err(ParseError::Invalid); + } + clean.parse::().map_err(|_| ParseError::Invalid)? + } + }; + Ok(if neg { -v } else { v }) +} + +/// `int(s)` for a str in base 10: surrounding whitespace, a sign, leading +/// zeros and underscores between digits. +pub fn parse_int(s: &str) -> Result { + let t = s.trim_matches(number_space); + let (neg, body) = match t.as_bytes().first() { + Some(b'-') => (true, &t[1..]), + Some(b'+') => (false, &t[1..]), + _ => (false, t), + }; + let clean = strip_underscores(body)?; + if clean.is_empty() || !clean.bytes().all(|c| c.is_ascii_digit()) { + return Err(ParseError::Invalid); + } + let mut v: i128 = 0; + for d in clean.bytes() { + let d = i128::from(d - b'0'); + v = v + .checked_mul(10) + .and_then(|v| { + if neg { + v.checked_sub(d) + } else { + v.checked_add(d) + } + }) + .ok_or(ParseError::TooLarge)?; + } + Ok(v) +} + +/// Why `math.fsum` raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FsumError { + /// OverflowError: finite inputs whose partial sum overflowed. + Overflow, + /// ValueError: both `inf` and `-inf` among the inputs. + InfMinusInf, +} + +impl FsumError { + /// The Python exception class. + pub fn exception(self) -> &'static str { + match self { + FsumError::Overflow => "OverflowError", + FsumError::InfMinusInf => "ValueError", + } + } +} + +impl fmt::Display for FsumError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + FsumError::Overflow => "intermediate overflow in fsum", + FsumError::InfMinusInf => "-inf + inf in fsum", + }) + } +} + +impl std::error::Error for FsumError {} + +/// `math.fsum(xs)`: the correctly rounded sum (Shewchuk's partials, with +/// CPython's half-even fix-up across partials and its inf/nan rules). +pub fn fsum>(xs: I) -> Result { + let mut p: Vec = Vec::new(); + let mut special = 0.0f64; + let mut inf_sum = 0.0f64; + for xsave in xs { + let mut x = xsave; + let mut i = 0; + for j in 0..p.len() { + let mut y = p[j]; + if x.abs() < y.abs() { + std::mem::swap(&mut x, &mut y); + } + let hi = x + y; + let lo = y - (hi - x); + if lo != 0.0 { + p[i] = lo; + i += 1; + } + x = hi; + } + p.truncate(i); + if x != 0.0 { + if !x.is_finite() { + if xsave.is_finite() { + return Err(FsumError::Overflow); + } + if xsave.is_infinite() { + inf_sum += xsave; + } + special += xsave; + p.clear(); + } else { + p.push(x); + } + } + } + if special != 0.0 { + return if inf_sum.is_nan() { + Err(FsumError::InfMinusInf) + } else { + Ok(special) + }; + } + let mut n = p.len(); + if n == 0 { + return Ok(0.0); + } + n -= 1; + let mut hi = p[n]; + let mut lo = 0.0; + while n > 0 { + let x = hi; + n -= 1; + let y = p[n]; + hi = x + y; + lo = y - (hi - x); + if lo != 0.0 { + break; + } + } + if n > 0 && ((lo < 0.0 && p[n - 1] < 0.0) || (lo > 0.0 && p[n - 1] > 0.0)) { + let y = lo * 2.0; + let x = hi + y; + if y == x - hi { + hi = x; + } + } + Ok(hi) +} + +/// The builtin `sum(xs)` of floats with start 0 (Python 3.12 and later): +/// Neumaier's compensated sum. +pub fn sum>(xs: I) -> f64 { + let mut s = 0.0f64; + let mut c = 0.0f64; + for x in xs { + let t = s + x; + if s.abs() >= x.abs() { + c += (s - t) + x; + } else { + c += (x - t) + s; + } + s = t; + } + if c != 0.0 && c.is_finite() { + s += c; + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn float_parsing() { + assert_eq!(parse_float(" 1_000.5\n"), Ok(1000.5)); + assert_eq!(parse_float("+.5"), Ok(0.5)); + assert_eq!(parse_float("1."), Ok(1.0)); + assert_eq!(parse_float("1e400"), Ok(f64::INFINITY)); + assert_eq!(parse_float("-Infinity"), Ok(f64::NEG_INFINITY)); + assert!(parse_float("nAn").unwrap().is_nan()); + assert_eq!(parse_float("\u{85}2\u{3000}"), Ok(2.0)); + assert_eq!(parse_float("\x1c2"), Err(ParseError::Invalid)); + for bad in [ + "", ".", "1_", "_1", "1__0", "1_.5", "1e", "e5", "1.5.2", "inf1", "0x10", "1 2", + ] { + assert_eq!(parse_float(bad), Err(ParseError::Invalid), "{bad:?}"); + } + } + + #[test] + fn int_parsing() { + assert_eq!(parse_int(" -0012 "), Ok(-12)); + assert_eq!(parse_int("1_000"), Ok(1000)); + assert_eq!(parse_int("+7"), Ok(7)); + for bad in ["", "-", "1.0", "1_", "1e3", " "] { + assert_eq!(parse_int(bad), Err(ParseError::Invalid), "{bad:?}"); + } + assert_eq!(parse_int(&"9".repeat(40)), Err(ParseError::TooLarge)); + assert_eq!(parse_int(&i128::MIN.to_string()), Ok(i128::MIN)); + } + + #[test] + fn fsum_is_exact() { + assert_eq!(fsum([0.1; 10]), Ok(1.0)); + assert_eq!(fsum([1e-16, 1.0, 1e16]), Ok(10000000000000002.0)); + assert_eq!(fsum([1e308, 1e308]), Err(FsumError::Overflow)); + assert_eq!( + fsum([f64::INFINITY, f64::NEG_INFINITY]), + Err(FsumError::InfMinusInf) + ); + assert_eq!(fsum([f64::INFINITY, 1.0]), Ok(f64::INFINITY)); + assert!(fsum([f64::NAN, 1.0]).unwrap().is_nan()); + assert_eq!(fsum(Vec::new()), Ok(0.0)); + } + + #[test] + fn builtin_sum_is_compensated() { + assert_eq!(sum([0.1; 10]), 1.0); + assert_eq!(sum([1e100, 1.0, -1e100, 1.0]), 2.0); + assert!(!sum([-0.0]).is_sign_negative()); + assert_eq!(sum([f64::INFINITY, 1.0]), f64::INFINITY); + } +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml new file mode 100644 index 00000000..7de9ddb2 --- /dev/null +++ b/rust/og/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "og" +description = "OwnGenome's one binary: og [args]" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +pycompat.workspace = true +og-io.workspace = true + +[lints] +workspace = true diff --git a/rust/og/build.rs b/rust/og/build.rs new file mode 100644 index 00000000..fd9441ab --- /dev/null +++ b/rust/og/build.rs @@ -0,0 +1,19 @@ +//! Stamps the binary with the git tree hash of the `rust/` it was built +//! from. `rust/og.sh` computes the hash on the host (the build container +//! has no access to the repository's .git) and passes it as OG_TREE_HASH. +//! A plain `cargo build` without it stamps `unknown`, which no check accepts. + +fn main() { + println!("cargo::rerun-if-env-changed=OG_TREE_HASH"); + let hash = std::env::var("OG_TREE_HASH").unwrap_or_default(); + let ok = (40..=64).contains(&hash.len()) + && hash + .bytes() + .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()); + let hash = if ok { hash } else { "unknown".to_owned() }; + println!("cargo::rustc-env=OG_TREE_HASH={hash}"); + println!( + "cargo::rustc-env=OG_TARGET={}", + std::env::var("TARGET").expect("cargo sets TARGET") + ); +} diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs new file mode 100644 index 00000000..7ea3efd9 --- /dev/null +++ b/rust/og/src/main.rs @@ -0,0 +1,64 @@ +//! `og`: OwnGenome's one binary. `og [args]`; each ported Python +//! tool becomes a subcommand named after its file in `bin/`. + +use std::io::Write; +use std::process::ExitCode; + +mod probe; + +/// The tree hash after a fixed marker, so a host check can read it from the +/// file without running it (`grep -ao 'og-tree-hash:[0-9a-f]*'`): an +/// Apple Silicon host cannot run the x86_64 Linux binary. +const TREE_MARK: &str = concat!("og-tree-hash:", env!("OG_TREE_HASH")); +const TARGET: &str = env!("OG_TARGET"); + +fn tree_hash() -> &'static str { + &TREE_MARK["og-tree-hash:".len()..] +} + +const USAGE: &str = "usage: og [args]\n og version\n og --tree-hash\n"; + +/// argparse's way to refuse a command line: usage and the error on stderr, exit 2. +fn usage_error(msg: &str) -> ExitCode { + eprint!("{USAGE}"); + eprintln!("og: error: {msg}"); + ExitCode::from(2) +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let Some(tool) = args.first() else { + return usage_error("the following arguments are required: tool"); + }; + let rest = &args[1..]; + match tool.as_str() { + "-h" | "--help" => { + print!("{USAGE}"); + ExitCode::SUCCESS + } + "version" | "--version" if rest.is_empty() => { + println!("og rust tree {} ({TARGET})", tree_hash()); + ExitCode::SUCCESS + } + "--tree-hash" if rest.is_empty() => { + println!("{}", tree_hash()); + ExitCode::SUCCESS + } + // Hidden: the differential test against CPython drives it. + "pycompat-probe" if rest.is_empty() => { + let stdin = std::io::stdin().lock(); + let stdout = std::io::stdout().lock(); + match probe::run(stdin, stdout) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + let _ = writeln!(std::io::stderr(), "og pycompat-probe: {e}"); + ExitCode::FAILURE + } + } + } + "version" | "--version" | "--tree-hash" | "pycompat-probe" => { + usage_error(&format!("unrecognized arguments: {}", rest.join(" "))) + } + other => usage_error(&format!("unknown tool '{other}'")), + } +} diff --git a/rust/og/src/probe.rs b/rust/og/src/probe.rs new file mode 100644 index 00000000..bd3387e3 --- /dev/null +++ b/rust/og/src/probe.rs @@ -0,0 +1,164 @@ +//! `og pycompat-probe`: reads one case per line and writes one result per +//! line, for `tests/rust/pycompat-diff.sh` to compare with what CPython +//! writes for the same cases (`tests/rust/pycompat_oracle.py`). +//! +//! A case is tab-separated: the operation, then its arguments. Floats travel +//! as the 16 hex digits of their bits, text as the hex of its UTF-8, so no +//! value is rounded or mangled on the way. A result is the Python result, +//! or the name of the exception Python raised, escaped by [`escape`]. + +use og_io::{Lines, Utf8}; +use pycompat::{fmt, json, num}; +use std::io::{self, BufRead, Write}; + +/// `\` `\n` `\r` `|` and NUL escaped, so every result is one line of +/// text and a list of lines can be joined with `|`. +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '|' => out.push_str("\\|"), + '\0' => out.push_str("\\0"), + c => out.push(c), + } + } + out +} + +fn bad(msg: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, msg.into()) +} + +fn float(s: &str) -> io::Result { + u64::from_str_radix(s, 16) + .map(f64::from_bits) + .map_err(|_| bad(format!("bad float bits {s:?}"))) +} + +fn bytes(s: &str) -> io::Result> { + if !s.len().is_multiple_of(2) { + return Err(bad("odd hex length")); + } + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).map_err(|_| bad("bad hex"))) + .collect() +} + +fn text(s: &str) -> io::Result { + String::from_utf8(bytes(s)?).map_err(|_| bad("hex text is not UTF-8")) +} + +fn int(s: &str) -> io::Result { + s.parse().map_err(|_| bad(format!("bad integer {s:?}"))) +} + +fn floats(s: &str) -> io::Result> { + if s.is_empty() { + Ok(Vec::new()) + } else { + s.split(',').map(float).collect() + } +} + +fn dump_opts(s: &str) -> io::Result> { + let f: Vec<&str> = s.split(',').collect(); + let [indent, sep, ascii, sort, nan] = f[..] else { + return Err(bad(format!("bad json options {s:?}"))); + }; + Ok(json::Dump { + indent: if indent == "-" { + None + } else { + Some(int(indent)?) + }, + separators: match sep { + "d" => None, + "c" => Some((",", ":")), + "p" => Some((", ", ": ")), + "i" => Some((",", ": ")), + _ => return Err(bad(format!("bad separators {sep:?}"))), + }, + ensure_ascii: ascii == "1", + sort_keys: sort == "1", + allow_nan: nan == "1", + }) +} + +/// One case's result. +fn eval(f: &[&str]) -> io::Result { + let arg = |i: usize| { + f.get(i) + .copied() + .ok_or_else(|| bad(format!("{}: missing argument {i}", f[0]))) + }; + Ok(match f[0] { + "repr" => fmt::repr(float(arg(1)?)?), + "f" => fmt::fixed(float(arg(1)?)?, int(arg(2)?)?), + "e" => fmt::sci(float(arg(1)?)?, int(arg(2)?)?), + "g" => fmt::general(float(arg(1)?)?, int(arg(2)?)?), + "pct" => fmt::percent(float(arg(1)?)?, int(arg(2)?)?), + "grpf" => fmt::fixed_grouped(float(arg(1)?)?, int(arg(2)?)?), + "grp" => fmt::int_grouped(int(arg(1)?)?), + "zpad" => fmt::zero_pad(int(arg(1)?)?, int(arg(2)?)?), + "ljust" => fmt::ljust(&text(arg(1)?)?, int(arg(2)?)?), + "rjust" => fmt::rjust(&text(arg(1)?)?, int(arg(2)?)?), + "round0" => match fmt::round_int(float(arg(1)?)?) { + Ok(v) => fmt::int_str(v), + Err(e) => e.exception().to_owned(), + }, + "round" => match fmt::round(float(arg(1)?)?, int(arg(2)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "float" => match num::parse_float(&text(arg(1)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "int" => match num::parse_int(&text(arg(1)?)?) { + Ok(v) => v.to_string(), + Err(e) => e.exception().to_owned(), + }, + "fsum" => match num::fsum(floats(arg(1)?)?) { + Ok(v) => fmt::repr(v), + Err(e) => e.exception().to_owned(), + }, + "sum" => fmt::repr(num::sum(floats(arg(1)?)?)), + "json" => { + let opts = dump_opts(arg(2)?)?; + match json::loads(&text(arg(1)?)?).and_then(|v| json::dumps(&v, &opts)) { + Ok(s) => s, + Err(e) => e.exception().to_owned(), + } + } + "lines" => { + let lines: Vec = Lines::new(io::Cursor::new(bytes(arg(1)?)?), Utf8::Replace) + .collect::>()?; + return Ok(lines + .iter() + .map(|l| escape(l)) + .collect::>() + .join("|")); + } + op => return Err(bad(format!("unknown operation {op:?}"))), + }) +} + +pub fn run(input: impl BufRead, output: impl Write) -> io::Result<()> { + let mut out = io::BufWriter::new(output); + for line in input.lines() { + let line = line?; + let fields: Vec<&str> = line.split('\t').collect(); + let result = eval(&fields)?; + // A lines result is escaped line by line already. + if fields[0] == "lines" { + writeln!(out, "{result}")?; + } else { + writeln!(out, "{}", escape(&result))?; + } + } + out.flush() +} diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 00000000..bb639de6 --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,5 @@ +# The toolchain of OG_BUILDER (versions.env), so the image builds with +# what it ships and needs no download. Move both together. +[toolchain] +channel = "1.93.0" +targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] diff --git a/tests/rust/pycompat-diff.sh b/tests/rust/pycompat-diff.sh new file mode 100755 index 00000000..9f0a99af --- /dev/null +++ b/tests/rust/pycompat-diff.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pycompat-diff.sh: the pycompat differential test. CPython (python3 -I in +# PYTHON_IMAGE) draws N random cases per operation and writes its result for +# each; `og pycompat-probe` must write the same results byte for byte. +# +# Usage: tests/rust/pycompat-diff.sh OG_BINARY [N] [SEED] [OP ...] +# N defaults to 100000, SEED to a random one (printed, so a failure can be +# replayed). PYTHON=python3 runs the oracle on the host instead of in the +# image, for a quick local loop; CI always uses the image. +# Exit 0 when every result matches, 1 with the first differences otherwise. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +OG=${1:?Usage: $0 OG_BINARY [N] [SEED] [OP ...]} +N=${2:-100000} SEED=${3:-$RANDOM$RANDOM} +shift $(($# < 3 ? $# : 3)) +# shellcheck source=../../versions.env +. "${ROOT}/versions.env" +TMP=$(mktemp -d "${TMPDIR:-/tmp}/pycompat-diff.XXXXXX") +trap 'rm -rf "$TMP"' EXIT +echo "pycompat-diff: ${N} cases per operation, seed ${SEED}, oracle ${PYTHON:-${PYTHON_IMAGE}}" +t0=$(date +%s) +# The oracle goes in as an argument and its output comes back on stdout, so +# nothing is mounted (Docker on macOS shares only some folders). +ORACLE=$(cat "${ROOT}/tests/rust/pycompat_oracle.py") +if [ -n "${PYTHON:-}" ]; then + "$PYTHON" -I -c "$ORACLE" "$N" "$SEED" "$@" > "${TMP}/both.txt" +else + docker run --rm --network none "$PYTHON_IMAGE" python3 -I -c "$ORACLE" "$N" "$SEED" "$@" > "${TMP}/both.txt" +fi +awk 'NR % 2 == 1' "${TMP}/both.txt" > "${TMP}/cases.tsv" +awk 'NR % 2 == 0' "${TMP}/both.txt" > "${TMP}/expected.txt" +t1=$(date +%s) +"$OG" pycompat-probe < "${TMP}/cases.tsv" > "${TMP}/actual.txt" +t2=$(date +%s) +cases=$(wc -l < "${TMP}/cases.tsv" | tr -d ' ') +[ "$cases" -gt 0 ] || { echo "FAIL: the oracle wrote no cases"; exit 1; } +echo "pycompat-diff: ${cases} cases; CPython $((t1 - t0)) s, og $((t2 - t1)) s" +if cmp -s "${TMP}/expected.txt" "${TMP}/actual.txt"; then + echo "PASS: og pycompat-probe matches CPython on all ${cases} cases" + exit 0 +fi +paste "${TMP}/cases.tsv" "${TMP}/expected.txt" "${TMP}/actual.txt" | awk -F'\t' ' + { want = $(NF - 1); got = $NF; if (want == got) next; bad[$1]++; n++ + if (n <= 20) { c = $1; for (i = 2; i < NF - 1; i++) c = c "\t" $i; printf "case: %s\npython: %s\nog: %s\n\n", c, want, got } } + END { for (op in bad) printf "FAIL: %s differs on %d cases\n", op, bad[op] }' +echo "FAIL: og pycompat-probe differs from CPython (seed ${SEED}; rerun with that seed to replay)" +exit 1 diff --git a/tests/rust/pycompat_oracle.py b/tests/rust/pycompat_oracle.py new file mode 100644 index 00000000..845f5a27 --- /dev/null +++ b/tests/rust/pycompat_oracle.py @@ -0,0 +1,276 @@ +"""The CPython side of the pycompat differential test. + + python3 -I pycompat_oracle.py N SEED [OP ...] + +Writes N random cases per operation (the hand-picked edge cases are among +the values drawn) to stdout, each line followed by the line this Python +returns for it. tests/rust/pycompat-diff.sh splits the two, feeds the cases +to `og pycompat-probe` and requires its output to equal the expected lines +byte for byte. Nothing is mounted, so the oracle runs in any image. The case format +is the one rust/og/src/probe.rs reads: tab-separated, floats as the hex of +their bits, text as the hex of its UTF-8. +""" + +import io +import json +import math +import random +import struct +import sys + +SPECIAL = [ + 0.0, -0.0, math.nan, math.inf, -math.inf, 0.125, 0.375, 2.675, 0.35, 0.5, 1.5, 2.5, -0.5, -2.5, + 1e-05, 1e-04, 1e16, 1e15, 1e17, 1e22, 1e23, 5e-324, 2.2250738585072014e-308, 2.225073858507201e-308, + 1.7976931348623157e308, 9.9999995, 999999.5, 0.1 + 0.2, 1 / 3, 2 / 3, 123456789012345678.0, 0.05, 0.15, + 0.25, 1.005, 1.115, 4.35, 8.345, 99.995, 1234.5, 2.0 ** 52 + 0.5, 2.0 ** 53, 2.0 ** 60, 1e300, -1e-300, +] +OPS = ["repr", "f", "e", "g", "pct", "grpf", "round0", "round", "grp", "zpad", "ljust", "rjust", + "float", "int", "fsum", "sum", "json", "lines"] +WHITESPACE = " \t\n\x0b\x0c\r\x1c\x1d\x1e\x1f\x85\xa0\u1680\u2000\u2005\u200a\u2028\u2029\u202f\u205f\u3000" + + +def bits(x): + return "%016x" % struct.unpack(" Date: Mon, 9 Nov 2026 15:23:40 +0100 Subject: [PATCH 02/12] build(rust): deliver og to bin/.og through the bin/og shim and rust/og.sh bin/og runs the static musl build for the machine's uname -m from the gitignored bin/.og, and exits 127 with the setup command when it is missing. rust/og.sh builds both targets in OG_BUILDER (a cargo-zigbuild image pinned by tag and digest in versions.env), or on a release tag downloads them and checks SHA256SUMS and the attestation. The build stamps the git tree of rust/ into the binary, and rust/og.sh check reads it back without running the binary. tests/rust/og-in-images.sh runs og by bare name in the images the tasks use. Bead: pgp-bs9u.2 --- .gitignore | 5 + bin/og | 21 ++++ rust/og.sh | 196 +++++++++++++++++++++++++++++++++++++ rust/og/src/main.rs | 11 ++- tests/rust/og-in-images.sh | 57 +++++++++++ versions.env | 10 ++ 6 files changed, 295 insertions(+), 5 deletions(-) create mode 100755 bin/og create mode 100755 rust/og.sh create mode 100755 tests/rust/og-in-images.sh diff --git a/.gitignore b/.gitignore index e09a812f..7f135ae3 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,8 @@ site/ # Scratch space of a checkout: logs, downloads, CI output /tmp/ + +# Rust: the build output, and the og binaries scripts/setup.sh --og puts +# beside the bin/og shim (rust/og.sh) +/rust/target/ +/bin/.og/ diff --git a/bin/og b/bin/og new file mode 100755 index 00000000..30396908 --- /dev/null +++ b/bin/og @@ -0,0 +1,21 @@ +#!/bin/sh +# og: OwnGenome's Rust binary, called by bare name in a task (bin/ is on its +# PATH) or as /pgp-bin/og in a step script. It runs the static Linux build +# for this machine from bin/.og/, which scripts/setup.sh --og puts there +# (rust/og.sh). Inside an amd64 image on an Apple Silicon host, uname -m +# says x86_64, so that build runs under emulation. +case $(uname -m) in + x86_64 | amd64) arch=x86_64 ;; + aarch64 | arm64) arch=aarch64 ;; + *) arch=$(uname -m) ;; +esac +bin="$(dirname "$0")/.og/og-linux-${arch}" +if [ "$(uname -s)" != Linux ]; then + echo "ERROR: og runs inside the pipeline's Linux containers, not on $(uname -s)" >&2 + exit 127 +fi +if [ ! -x "$bin" ]; then + echo "ERROR: og is not built for ${arch}; run scripts/setup.sh --og" >&2 + exit 127 +fi +exec "$bin" "$@" diff --git a/rust/og.sh b/rust/og.sh new file mode 100755 index 00000000..6b3f82ca --- /dev/null +++ b/rust/og.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# og.sh: how the og binary reaches bin/.og, and the check that it is the one +# this checkout's rust/ makes. bin/og (the shim tasks call) runs +# bin/.og/og-linux-; both arches are kept, because an Apple Silicon +# host runs amd64 images under emulation, where uname -m says x86_64. +# +# rust/og.sh install scripts/setup.sh --og: on a release tag (vX.Y.Z, +# clean rust/) download the release's binaries and +# check them against SHA256SUMS (and the build +# attestation when gh is installed); otherwise, or +# when the release has none, build them +# rust/og.sh build build both from rust/ in OG_BUILDER (versions.env) +# rust/og.sh check [--required] +# one line, OK, NOTE or ERROR; exit 1 on ERROR: a +# binary built from another rust/, or (with +# --required, or once REQUIRED below is true) none +# rust/og.sh tree-hash the git tree hash of rust/ as it is on disk +# rust/og.sh built the tree hash the binaries were built from, for +# run_manifest.tsv ("not built" when there are none) +# +# The tree hash is git's: `git rev-parse HEAD:rust` on a clean checkout, and +# for an edited rust/ the tree those edits would commit (a throwaway index), +# so any edit, committed or not, makes the built binary stale. The build +# stamps it into the binary as og-tree-hash:; so the check +# reads it with grep and never runs a binary the host may not be able to. +set -euo pipefail + +# True once a pipeline step calls og: until then a checkout without the +# binary runs as before, and only a stale one is refused. +REQUIRED=false + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +BIN="${ROOT}/bin/.og" +ARCHES="x86_64 aarch64" +REPO="GeiserX/OwnGenome" +ENGINE=${CONTAINER_ENGINE:-docker} + +triple() { echo "${1}-unknown-linux-musl"; } + +tree_hash() { + local idx rc=0 + git -C "$ROOT" rev-parse --verify -q HEAD:rust > /dev/null 2>&1 || return 3 + if [ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all -- rust 2> /dev/null)" ]; then + git -C "$ROOT" rev-parse HEAD:rust + return + fi + idx=$(mktemp -d "${TMPDIR:-/tmp}/og-tree.XXXXXX") + { GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" read-tree HEAD \ + && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" add -A -- rust \ + && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" write-tree --prefix=rust/; } || rc=$? + rm -rf "$idx" + return "$rc" +} + +# embedded FILE: the hash stamped into a binary, or "unknown". +embedded() { + local h + h=$(LC_ALL=C grep -ao 'og-tree-hash:[0-9a-f]\{40,64\};' "$1" 2> /dev/null | head -n 1 || true) + h=${h#og-tree-hash:} + echo "${h%;}" | grep . || echo unknown +} + +check() { + local required=$REQUIRED expected f h missing="" stale="" + [ "${1:-}" != --required ] || required=true + expected=$(tree_hash 2> /dev/null) || expected="" + for a in $ARCHES; do + f="${BIN}/og-linux-${a}" + if [ ! -x "$f" ]; then missing+=" ${a}"; continue; fi + h=$(embedded "$f") + [ -z "$expected" ] || [ "$h" = "$expected" ] || stale+=" bin/.og/og-linux-${a} (rust/ tree ${h})" + done + if [ -n "$stale" ]; then + echo "ERROR: og is stale:${stale}, but rust/ here is tree ${expected}: run scripts/setup.sh --og" + return 1 + fi + if [ -n "$missing" ] && $required; then + echo "ERROR: og is not built for${missing}: run scripts/setup.sh --og" + return 1 + fi + if [ -n "$missing" ]; then + echo "NOTE: og is not built for${missing}; no step needs it yet (scripts/setup.sh --og builds it)" + elif [ -z "$expected" ]; then + echo "NOTE: og is built, but this is not a git checkout, so its rust/ tree cannot be checked" + else + echo "OK: og is built for ${ARCHES// /, } from this checkout's rust/ (tree ${expected})" + fi +} + +built() { + local hs="" a + for a in $ARCHES; do + if [ -x "${BIN}/og-linux-${a}" ]; then hs+="${a}=$(embedded "${BIN}/og-linux-${a}") "; fi + done + hs=${hs% } + if [ -z "$hs" ]; then echo "not built"; return; fi + # One hash when both arches agree, as they do after setup.sh --og. + local -a h + read -r -a h <<< "$hs" + if [ "${#h[@]}" -eq 2 ] && [ "${h[0]#*=}" = "${h[1]#*=}" ]; then echo "${h[0]#*=}"; else echo "$hs"; fi +} + +# put SRC ARCH: move a binary into bin/.og in one rename. +put() { + mkdir -p "$BIN" + cp "$1" "${BIN}/.og-linux-${2}.tmp" + chmod 755 "${BIN}/.og-linux-${2}.tmp" + mv -f "${BIN}/.og-linux-${2}.tmp" "${BIN}/og-linux-${2}" +} + +build() { + local hash cache t0 a out targets=() + hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout: og is stamped with the git tree of rust/, so it builds only in one" >&2; return 1; } + # shellcheck source=../versions.env + . "${ROOT}/versions.env" + : "${OG_BUILDER:?OG_BUILDER is not set in versions.env}" + # Build cache and output under rust/target (ignored by git, so outside the + # tree hash); the sources go in read-only. + for a in $ARCHES; do targets+=(--target "$(triple "$a")"); done + cache="${ROOT}/rust/target/og-build" + mkdir -p "$cache" + echo "Building og ${hash} for ${ARCHES} in ${OG_BUILDER%@*} ..." + t0=$(date +%s) + "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ + -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target -e OG_TREE_HASH="$hash" \ + -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ + cargo zigbuild --release --locked -p og "${targets[@]}" + for a in $ARCHES; do + out="${cache}/target/$(triple "$a")/release/og" + [ "$(embedded "$out")" = "$hash" ] || { echo "ERROR: ${out} does not carry tree ${hash}" >&2; return 1; } + put "$out" "$a" + done + echo "Built bin/.og/og-linux-{${ARCHES// /,}} in $(($(date +%s) - t0)) s" +} + +sha256() { + if command -v sha256sum > /dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi +} + +get() { # get URL DEST: 0 when downloaded, 1 when not + if command -v curl > /dev/null 2>&1; then curl -fsL --retry 3 -o "$2" "$1"; else wget -q -O "$2" "$1"; fi +} + +# fetch TAG: 0 when the release's binaries are installed, 2 when the release +# has none (build instead), 1 when a download does not check out. +fetch() { + local tag=$1 tmp want hash a f rc=0 + hash=$(tree_hash) + tmp=$(mktemp -d "${TMPDIR:-/tmp}/og-fetch.XXXXXX") + # OG_RELEASE_BASE: another server with the same layout (tests/test_og_check.sh). + local base="${OG_RELEASE_BASE:-https://github.com/${REPO}/releases/download}/${tag}" + if ! get "${base}/SHA256SUMS" "${tmp}/SHA256SUMS"; then + rm -rf "$tmp" + return 2 + fi + for a in $ARCHES; do + f="og-linux-${a}" + want=$(awk -v f="$f" '$2 == f || $2 == "*" f {print $1}' "${tmp}/SHA256SUMS") + if [ -z "$want" ] || ! get "${base}/${f}" "${tmp}/${f}"; then echo "ERROR: release ${tag} lists no ${f}" >&2; rc=1; break; fi + if [ "$(sha256 "${tmp}/${f}")" != "$want" ]; then echo "ERROR: ${f} of release ${tag} does not match its SHA256SUMS" >&2; rc=1; break; fi + if command -v gh > /dev/null 2>&1; then + gh attestation verify "${tmp}/${f}" --repo "$REPO" > /dev/null || { echo "ERROR: ${f} of release ${tag} has no valid build attestation" >&2; rc=1; break; } + fi + if [ "$(embedded "${tmp}/${f}")" != "$hash" ]; then echo "ERROR: ${f} of release ${tag} was not built from this rust/ (tree ${hash})" >&2; rc=1; break; fi + done + if [ "$rc" -eq 0 ]; then + for a in $ARCHES; do put "${tmp}/og-linux-${a}" "$a"; done + command -v gh > /dev/null 2>&1 || echo "NOTE: gh is not installed, so the build attestation was not checked (SHA256SUMS was)" + echo "Installed og ${hash} from release ${tag}" + fi + rm -rf "$tmp" + return "$rc" +} + +install() { + local tag rc=0 hash + hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout; clone the repository to install og" >&2; return 1; } + tag=$(git -C "$ROOT" describe --exact-match --tags --match 'v[0-9]*.[0-9]*.[0-9]*' HEAD 2> /dev/null || true) + if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(git -C "$ROOT" rev-parse HEAD:rust)" ]; then + echo "Release ${tag}: downloading its og binaries" + fetch "$tag" || rc=$? + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -eq 2 ] || return 1 + echo "Release ${tag} has no og binaries: building them" + fi + build +} + +case "${1:-}" in + install) install ;; + build) build ;; + check) shift; check "$@" ;; + tree-hash) tree_hash ;; + built) built ;; + *) sed -n '2,/^set -euo/p' "$0" | sed -e '$d' -e 's/^# \{0,1\}//' >&2; exit 2 ;; +esac diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 7ea3efd9..123c49b6 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -6,14 +6,15 @@ use std::process::ExitCode; mod probe; -/// The tree hash after a fixed marker, so a host check can read it from the -/// file without running it (`grep -ao 'og-tree-hash:[0-9a-f]*'`): an -/// Apple Silicon host cannot run the x86_64 Linux binary. -const TREE_MARK: &str = concat!("og-tree-hash:", env!("OG_TREE_HASH")); +/// The tree hash between a fixed marker and a `;`, so a host check can read +/// it from the file without running it (`grep -ao 'og-tree-hash:[0-9a-f]*;'`): +/// an Apple Silicon host cannot run the x86_64 Linux binary. The `;` ends +/// it where the next string in the binary would otherwise run on. +const TREE_MARK: &str = concat!("og-tree-hash:", env!("OG_TREE_HASH"), ";"); const TARGET: &str = env!("OG_TARGET"); fn tree_hash() -> &'static str { - &TREE_MARK["og-tree-hash:".len()..] + &TREE_MARK["og-tree-hash:".len()..TREE_MARK.len() - 1] } const USAGE: &str = "usage: og [args]\n og version\n og --tree-hash\n"; diff --git a/tests/rust/og-in-images.sh b/tests/rust/og-in-images.sh new file mode 100755 index 00000000..a9bc28e3 --- /dev/null +++ b/tests/rust/og-in-images.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# og-in-images.sh: the og binaries in bin/.og run by bare name inside the +# images the pipeline's tasks use, as a Nextflow task calls them: bin/ mounted +# at its own path and first on PATH, the command run by sh. Each image runs on +# every platform it publishes (linux/amd64, linux/arm64; arm64 under QEMU on +# an amd64 host), and og must report the build for that platform's uname -m. +# Then the shim with no binary beside it must stop with exit 127 and the +# setup command. +# +# Usage: tests/rust/og-in-images.sh [IMAGE_VAR ...] +# default: PYTHON_IMAGE BCFTOOLS_IMAGE MANTA_IMAGE PARASCOPY_IMAGE +# Needs bin/.og built (scripts/setup.sh --og). Exit 1 when any run fails. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=../../versions.env +. "${ROOT}/versions.env" +[ $# -gt 0 ] || set -- PYTHON_IMAGE BCFTOOLS_IMAGE MANTA_IMAGE PARASCOPY_IMAGE +BIN="${ROOT}/bin" +for a in x86_64 aarch64; do + [ -x "${BIN}/.og/og-linux-${a}" ] || { echo "ERROR: ${BIN}/.og/og-linux-${a} is missing: run scripts/setup.sh --og" >&2; exit 1; } +done +fail=0 runs=0 +run_og() { # run_og IMAGE PLATFORM BINDIR: og version, as a task would call it + docker run --rm --pull missing --platform "$2" -v "${3}:${3}:ro" -e "PATH=${3}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + --entrypoint sh "$1" -c 'echo "$(uname -m) $(og version)"' 2>&1 +} +printf '%-18s %-12s %-8s %s\n' image platform result "uname -m and og version" +for var in "$@"; do + img=${!var:?$var is not set in versions.env} + for plat in linux/amd64 linux/arm64; do + out=$(run_og "$img" "$plat" "$BIN") || true + if [[ "$out" == *"does not provide the specified platform"* || "$out" == *"no matching manifest"* ]]; then + printf '%-18s %-12s %-8s %s\n' "$var" "$plat" skipped "the image has no ${plat} build" + continue + fi + runs=$((runs + 1)) + want=$([ "$plat" = linux/amd64 ] && echo x86_64 || echo aarch64) + if [[ "$out" == "${want} og rust tree "*" (${want}-unknown-linux-musl)" ]]; then r=ok; else r=FAIL; fail=1; fi + printf '%-18s %-12s %-8s %s\n' "$var" "$plat" "$r" "$out" + done +done +[ "$runs" -gt 0 ] || { echo "ERROR: no image ran"; exit 1; } +# The shim alone: no bin/.og beside it. +# Under rust/target (ignored by git), not TMPDIR: Docker on macOS shares only some folders. +mkdir -p "${ROOT}/rust/target" +empty=$(mktemp -d "${ROOT}/rust/target/og-shim.XXXXXX") +trap 'rm -rf "$empty"' EXIT +cp "${BIN}/og" "${empty}/og" +rc=0 +out=$(docker run --rm -v "${empty}:${empty}:ro" -e "PATH=${empty}:/usr/local/bin:/usr/bin:/bin" --entrypoint sh "${!1}" -c 'og version' 2>&1) || rc=$? +if [ "$rc" -eq 127 ] && [[ "$out" == *"og is not built for "*"run scripts/setup.sh --og"* ]]; then + echo "ok: the shim without a binary exits 127: ${out}" +else + echo "FAIL: the shim without a binary exited ${rc}: ${out}"; fail=1 +fi +[ "$fail" -eq 0 ] && echo "PASS: og runs by bare name in every image and platform above" || echo "FAIL: see above" +exit "$fail" diff --git a/versions.env b/versions.env index 18ae22e1..a40db28a 100644 --- a/versions.env +++ b/versions.env @@ -110,6 +110,16 @@ PYTHON_IMAGE="python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908f # and the build pgsc_calc uses. PLINK2_IMAGE="pgscatalog/plink2:2.00a5.10" +# --- Building OwnGenome's own Rust binary (rust/, bin/og) --- +# scripts/setup.sh --og builds bin/.og/og-linux-{x86_64,aarch64} in this +# cargo-zigbuild image. Its rustc (1.93.0) is the release rust/rust-toolchain.toml +# pins: move the two together, or the build downloads a toolchain first. The +# name has no _IMAGE suffix on purpose: no pipeline step runs it, so it has no +# row in tests/smoke/commands.tsv and Renovate does not bump it; bump it by hand +# with the toolchain. The publisher moves its tags in place, so it is pinned to +# its digest. +OG_BUILDER="ghcr.io/rust-cross/cargo-zigbuild:0.24.1@sha256:3abe26afb5537ea225c223c6790c1bf056271ef51ea52d2c4e343c569c8f129a" + # --- Polygenic scores and ancestry (steps 25 and 26) --- # pgsc_calc, the PGS Catalog's own Nextflow pipeline, runs as a pipeline of # its own (step 25 and the PRS process start it). The images below are the From 9233cf1c8dc6267b8476203bbe834be02df89c67 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:23:40 +0100 Subject: [PATCH 03/12] feat(setup): refuse an og built from another rust/ and record its tree in run_manifest.tsv setup.sh --og installs the binary. run-all.sh (even with SKIP_VALIDATION), main.nf and validate-setup.sh stop on a binary built from another rust/ than the checkout's, with the setup command; a missing binary is a note until a step calls og. write_manifest.sh adds the run row og_tree_hash. tests/test_og_check.sh covers each case. Bead: pgp-bs9u.2 --- bin/write_manifest.sh | 7 +- main.nf | 10 +++ scripts/run-all.sh | 2 + scripts/setup.sh | 8 ++ scripts/validate-setup.sh | 12 +++ tests/test_og_check.sh | 155 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 192 insertions(+), 2 deletions(-) create mode 100755 tests/test_og_check.sh diff --git a/bin/write_manifest.sh b/bin/write_manifest.sh index 72f258f6..f8d2098f 100755 --- a/bin/write_manifest.sh +++ b/bin/write_manifest.sh @@ -5,8 +5,10 @@ # # Writes ${GENOME_DIR}//run_manifest.tsv, three tab-separated columns # (section, key, value): -# run written_utc, written_by, git_commit, pipeline_version (the OwnGenome -# release: the version in nextflow.config's manifest block), declared_sex +# run written_utc, written_by, git_commit, og_tree_hash (the rust/ tree +# the og binary in bin/.og was built from, or "not built"), +# pipeline_version (the OwnGenome release: the version in +# nextflow.config's manifest block), declared_sex # version every NAME=VALUE line of versions.env # image each *_IMAGE reference and the digest Docker resolved it to # ("not pulled" when the image is not on this machine) @@ -54,6 +56,7 @@ if git -C "$PGP_ROOT" rev-parse --verify -q HEAD >/dev/null 2>&1; then fi fi row run git_commit "$commit" +row run og_tree_hash "$(bash "${PGP_ROOT}/rust/og.sh" built 2> /dev/null || echo unknown)" # The OwnGenome release: the version = '...' line inside the manifest { } block # of nextflow.config (the release tag is v). No row when there is none. release=$(awk ' diff --git a/main.nf b/main.nf index b52aaa1a..860e195d 100644 --- a/main.nf +++ b/main.nf @@ -97,6 +97,16 @@ workflow { "decompress it (gunzip, or bgzip -d) and run 'samtools faidx' on the result." } + // og (bin/.og, the Rust binary tasks call through bin/og) built from + // another rust/ than this checkout's would run other code than the + // checkout says: rust/og.sh check stops the run (scripts/setup.sh --og). + def og_check = ['bash', "${projectDir}/rust/og.sh", 'check'].execute() + def og_msg = og_check.text.trim() + og_check.waitFor() + if (og_check.exitValue() != 0) { + error og_msg.replaceFirst(/^ERROR: /, '') + } + def tools_list = params.tools ? params.tools.split(',').collect { it.trim() }.findAll { it } : [] // Every name the workflows gate on. A name outside this list would be diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 3e2e37cf..26c4320e 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -96,6 +96,8 @@ if [ -n "${FAMILIES:-}" ]; then if [ -n "$FAM" ]; then echo "[Family] ${SAMPLE} is in family ${FAM} (${FAMILIES}): steps 38 and 39 are planned after the pipeline" else echo "[Family] ${SAMPLE} is in no family of ${FAMILIES}"; fi fi +# og (bin/.og) built from another rust/ than this checkout's stops the run, SKIP_VALIDATION or not (rust/og.sh). +OG_MSG=$("${PGP_ROOT}/rust/og.sh" check 2>&1) || { echo "$OG_MSG" >&2; exit 2; } mkdir -p "${S}/nextflow" "$LOG_DIR" GENOME_DIR="$G" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" run-all.sh "$SEX" || echo "WARNING: could not write ${S}/run_manifest.tsv" STATUS="${LOG_DIR}/run_status.tsv" # bin/collect_summary.py marks results older than this run stale diff --git a/scripts/setup.sh b/scripts/setup.sh index 45d78ff3..90e85f8f 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2,6 +2,10 @@ # setup.sh — One-stop setup: download references, pull Docker images, validate # Usage: ./scripts/setup.sh # ./scripts/setup.sh --pull-only pull the Docker images and exit +# ./scripts/setup.sh --og put OwnGenome's og binary in bin/.og: +# built from rust/ in OG_BUILDER, or on +# a release tag downloaded and checked +# (rust/og.sh install; docs/rust.md) # ./scripts/setup.sh --refresh clinvar # replace ClinVar with NCBI's current # release and rebuild the files made from it @@ -103,6 +107,8 @@ REFRESH="" SAMPLE_QC_ONLY=false OPT_IN="" case "${1:-}" in + --og) + exec "$(dirname "$0")/../rust/og.sh" install ;; --cyrius|--parascopy-data|--kir-data|--ancestry-panel|--ancestry-fine-panel|--yleaf-data|--consumer-sites|--ancient-panel|--viral-data|--masked-reference|--exomiser-data|--chip-data|--strchive-data|--megane-kmers|--mocha-data|--locityper-data|--vep-cache|--annotsv-data|--archaic) OPT_IN=${1#--} shift ;; @@ -126,6 +132,7 @@ GENOME_DIR=${1:-${GENOME_DIR:-""}} if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "Usage: $0 " echo " $0 --pull-only" + echo " $0 --og" echo " $0 --refresh clinvar " echo " $0 --sample-qc-data " echo " $0 --cyrius | --parascopy-data | --kir-data | --ancestry-panel | --ancestry-fine-panel | --yleaf-data | --consumer-sites | --ancient-panel | --viral-data | --masked-reference | --exomiser-data | --chip-data | --strchive-data | --megane-kmers | --mocha-data | --locityper-data | --archaic | --vep-cache | --annotsv-data " @@ -133,6 +140,7 @@ if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo " Where to store reference data and sample outputs." echo " Needs at least 500 GB free space per sample." echo " --pull-only Pull the Docker images listed in versions.env and exit." + echo " --og Put OwnGenome's og binary in bin/.og (built from rust/, or a release's) and exit." echo " --refresh clinvar" echo " Download NCBI's current ClinVar, rebuild the files made from it" echo " and record its release date in /clinvar/RELEASE." diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index 834f1cb4..16cbfcad 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -854,6 +854,18 @@ else fi fi +# --- OwnGenome's og binary (bin/.og, scripts/setup.sh --og) --- +# One built from another rust/ than this checkout's would give results from +# other code than the checkout says, so it fails; none at all is a note until +# a step calls og (rust/og.sh, REQUIRED). +header "OwnGenome binary" +OG_MSG=$("${PGP_ROOT}/rust/og.sh" check 2>&1) || true +case "$OG_MSG" in + OK:*) pass "${OG_MSG#OK: }" ;; + NOTE:*) info "${OG_MSG#NOTE: }" ;; + *) fail "${OG_MSG#ERROR: }" ;; +esac + ############################################################################### # 5. Sample Data (optional — only if $1 is provided) ############################################################################### diff --git a/tests/test_og_check.sh b/tests/test_og_check.sh new file mode 100755 index 00000000..9076c131 --- /dev/null +++ b/tests/test_og_check.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# rust/og.sh check on planted trees: the stale-binary guard of bin/.og. +# A copy of the checkout's scripts, bin/ and rust/ becomes its own git repo; +# fake binaries carry a tree hash the way the real build stamps it +# (og-tree-hash:;). Checked: no binary is a note (an error with +# --required); binaries of this rust/ pass; an edit to rust/, uncommitted, +# untracked or committed, makes them stale; a file git ignores does not; a +# binary stamped "unknown" is stale; built prints the hash for +# run_manifest.tsv; outside git the hash cannot be checked. Then the callers: +# run-all.sh (with SKIP_VALIDATION=true) and validate-setup.sh refuse a stale +# binary with the setup command, and write_manifest.sh records the hash. +set -uo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +fails=0 +ok() { echo "PASS $*"; } +bad() { echo "FAIL $*"; fails=$((fails + 1)); } +expect() { # expect NAME RC WANT_RC OUTPUT PATTERN + if [ "$2" -eq "$3" ] && [[ "$4" == $5 ]]; then ok "$1"; else bad "$1: exit $2 (want $3), output: $4"; fi +} + +C="${T}/repo" +mkdir -p "$C" +cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" +rm -rf "${C}/bin/.og" "${C}/rust/target" +g() { git -C "$C" -c user.name=t -c user.email=t@example.invalid "$@"; } +g init -q && g add -A && g commit -qm base +OG="${C}/rust/og.sh" +plant() { # plant HASH [ARCH...]: fake binaries stamped with HASH + local h=$1 a; shift + [ $# -gt 0 ] || set -- x86_64 aarch64 + mkdir -p "${C}/bin/.og" + for a in "$@"; do + printf '\177ELF junk og-tree-hash:%s;attempt more junk' "$h" > "${C}/bin/.og/og-linux-${a}" + chmod 755 "${C}/bin/.og/og-linux-${a}" + done +} + +h0=$(bash "$OG" tree-hash) +[ "$h0" = "$(g rev-parse HEAD:rust)" ] && ok "tree-hash of a clean rust/ is HEAD:rust" || bad "tree-hash ${h0} != HEAD:rust" + +out=$(bash "$OG" check); rc=$? +expect "no binary: a note, exit 0" "$rc" 0 "$out" "NOTE: og is not built for x86_64 aarch64*setup.sh --og*" +out=$(bash "$OG" check --required); rc=$? +expect "no binary with --required: refused" "$rc" 1 "$out" "ERROR: og is not built for x86_64 aarch64: run scripts/setup.sh --og" +out=$(bash "$OG" built); [ "$out" = "not built" ] && ok "built without binaries: not built" || bad "built printed '${out}'" + +plant "$h0" +out=$(bash "$OG" check); rc=$? +expect "binaries of this rust/: OK" "$rc" 0 "$out" "OK: og is built for x86_64, aarch64*${h0}*" +out=$(bash "$OG" built); [ "$out" = "$h0" ] && ok "built prints the one hash" || bad "built printed '${out}'" + +plant "$h0" x86_64 +rm -f "${C}/bin/.og/og-linux-aarch64" +out=$(bash "$OG" check --required); rc=$? +expect "one arch missing with --required: refused" "$rc" 1 "$out" "ERROR: og is not built for aarch64*" + +plant "$h0" +mkdir -p "${C}/rust/target/debug" && echo junk > "${C}/rust/target/debug/x" +out=$(bash "$OG" check); rc=$? +expect "a file git ignores (rust/target) does not make it stale" "$rc" 0 "$out" "OK:*" + +echo "// edited" >> "${C}/rust/og/src/main.rs" +h1=$(bash "$OG" tree-hash) +[ "$h1" != "$h0" ] && ok "an uncommitted edit changes the tree hash" || bad "tree hash did not change after an edit" +out=$(bash "$OG" check); rc=$? +expect "uncommitted edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*${h0}*${h1}*run scripts/setup.sh --og" +g add rust/og/src/main.rs && g commit -qm edit +[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(g rev-parse HEAD:rust)" ] \ + && ok "committing the edit keeps the hash the build would stamp" || bad "hash after commit differs" +out=$(bash "$OG" check); rc=$? +expect "committed edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*" +plant "$h1" +echo "new" > "${C}/rust/new-file.txt" +out=$(bash "$OG" check); rc=$? +expect "an untracked file in rust/: stale" "$rc" 1 "$out" "ERROR: og is stale:*" +rm -f "${C}/rust/new-file.txt" +out=$(bash "$OG" check); rc=$? +expect "rebuilt from the edited rust/: OK again" "$rc" 0 "$out" "OK:*${h1}*" + +printf 'og-tree-hash:unknown;' > "${C}/bin/.og/og-linux-x86_64" +out=$(bash "$OG" check); rc=$? +expect "a binary stamped unknown (plain cargo build): stale" "$rc" 1 "$out" "ERROR: og is stale:*og-linux-x86_64 (rust/ tree unknown)*" +out=$(bash "$OG" built); [[ "$out" == "x86_64=unknown aarch64=${h1}" ]] && ok "built lists both hashes when they differ" || bad "built printed '${out}'" + +# The callers, on a stale binary. +plant "$h0" +mkdir -p "${T}/fake" "${T}/genome" +printf '#!/bin/sh\necho "openjdk version \\"17.0.1\\"" >&2\n' > "${T}/fake/java" +printf '#!/bin/sh\necho "fake nextflow: must not be reached" >&2\nexit 99\n' > "${T}/fake/nextflow" +chmod 755 "${T}/fake/java" "${T}/fake/nextflow" +out=$(PATH="${T}/fake:${PATH}" GENOME_DIR="${T}/genome" SKIP_VALIDATION=true bash "${C}/scripts/run-all.sh" s1 male 2>&1); rc=$? +expect "run-all.sh with SKIP_VALIDATION=true stops on a stale og" "$rc" 2 "$out" "*ERROR: og is stale:*run scripts/setup.sh --og*" +[[ "$out" != *"must not be reached"* ]] && ok "run-all.sh stopped before nextflow" || bad "run-all.sh reached nextflow" +out=$(GENOME_DIR="${T}/genome" bash "${C}/scripts/validate-setup.sh" 2>&1); rc=$? +expect "validate-setup.sh fails a stale og" "$rc" 1 "$out" "*[[]FAIL[]]*og is stale:*run scripts/setup.sh --og*" +plant "$h1" +out=$(GENOME_DIR="${T}/genome" bash "${C}/scripts/validate-setup.sh" 2>&1) +[[ "$out" == *"[OK]"*"og is built for x86_64, aarch64"* ]] && ok "validate-setup.sh passes a fresh og" || bad "validate-setup.sh: $(grep -i ' og ' <<< "$out")" +mkdir -p "${T}/genome/s1" +CONTAINER_ENGINE=true GENOME_DIR="${T}/genome" bash "${C}/bin/write_manifest.sh" s1 test > /dev/null 2>&1 +row=$(awk -F'\t' '$1 == "run" && $2 == "og_tree_hash" {print $3}' "${T}/genome/s1/run_manifest.tsv" 2> /dev/null) +[ "$row" = "$h1" ] && ok "run_manifest.tsv records og_tree_hash" || bad "run_manifest.tsv og_tree_hash row: '${row}'" + +# A release tag: the binaries come from the release, checked against +# SHA256SUMS and the attestation (a fake gh here), and are refused when +# either does not match. OG_RELEASE_BASE points at a local server. +if command -v python3 > /dev/null 2>&1 && command -v curl > /dev/null 2>&1; then + rm -rf "${C}/bin/.og" + g tag v9.9.9 + S="${T}/srv/v9.9.9" + mkdir -p "$S" + for a in x86_64 aarch64; do printf 'og-tree-hash:%s;' "$h1" > "${S}/og-linux-${a}"; done + (cd "$S" && { sha256sum og-linux-* 2> /dev/null || shasum -a 256 og-linux-*; } > SHA256SUMS) + port=$((20000 + RANDOM % 20000)) + (cd "${T}/srv" && exec python3 -m http.server "$port" --bind 127.0.0.1 > /dev/null 2>&1) & + srv=$! + for _ in $(seq 50); do curl -fs "http://127.0.0.1:${port}/v9.9.9/SHA256SUMS" > /dev/null 2>&1 && break; sleep 0.1; done + printf '#!/bin/sh +exit 0 +' > "${T}/fake/gh"; chmod 755 "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a release tag installs the release's binaries" "$rc" 0 "$out" "*Installed og ${h1} from release v9.9.9*" + [ "$(bash "$OG" built)" = "$h1" ] && ok "the installed binaries carry the tree" || bad "built after install: $(bash "$OG" built)" + rm -rf "${C}/bin/.og" + printf '#!/bin/sh +echo "no attestation" >&2 +exit 1 +' > "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a binary without a valid attestation is refused" "$rc" 1 "$out" "*has no valid build attestation*" + printf '#!/bin/sh +exit 0 +' > "${T}/fake/gh" + printf 'tampered' >> "${S}/og-linux-aarch64" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "a binary that does not match SHA256SUMS is refused" "$rc" 1 "$out" "*og-linux-aarch64 of release v9.9.9 does not match its SHA256SUMS*" + [ ! -e "${C}/bin/.og/og-linux-x86_64" ] && ok "a refused release installs nothing" || bad "a refused release left a binary" + kill "$srv" 2> /dev/null; wait "$srv" 2> /dev/null + g tag -d v9.9.9 > /dev/null +else + echo "SKIP release download cases: no python3 or curl" +fi + +# Outside git the hash cannot be checked: a note, never a silent OK. +N="${T}/nogit" +plant "$h1" +cp -R "$C" "$N" && rm -rf "${N}/.git" +out=$(bash "${N}/rust/og.sh" check); rc=$? +expect "binaries outside a git checkout: a note" "$rc" 0 "$out" "NOTE: og is built, but this is not a git checkout*" + +[ "$fails" -eq 0 ] && echo "All og check cases passed" || echo "${fails} og check case(s) failed" +exit "$fails" From b2714d82e89d6630402374549b3c8fcc0d88c65b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:41:52 +0100 Subject: [PATCH 04/12] fix(setup): skip the og check in a partial copy of the scripts without rust/og.sh Bead: pgp-bs9u.2 --- main.nf | 12 +++++++----- scripts/run-all.sh | 2 +- scripts/validate-setup.sh | 4 +++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/main.nf b/main.nf index 860e195d..22a1d92f 100644 --- a/main.nf +++ b/main.nf @@ -100,11 +100,13 @@ workflow { // og (bin/.og, the Rust binary tasks call through bin/og) built from // another rust/ than this checkout's would run other code than the // checkout says: rust/og.sh check stops the run (scripts/setup.sh --og). - def og_check = ['bash', "${projectDir}/rust/og.sh", 'check'].execute() - def og_msg = og_check.text.trim() - og_check.waitFor() - if (og_check.exitValue() != 0) { - error og_msg.replaceFirst(/^ERROR: /, '') + if (file("${projectDir}/rust/og.sh").exists()) { + def og_check = ['bash', "${projectDir}/rust/og.sh", 'check'].execute() + def og_msg = og_check.text.trim() + og_check.waitFor() + if (og_check.exitValue() != 0) { + error og_msg.replaceFirst(/^ERROR: /, '') + } } def tools_list = params.tools ? params.tools.split(',').collect { it.trim() }.findAll { it } : [] diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 26c4320e..5e8e153d 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -97,7 +97,7 @@ if [ -n "${FAMILIES:-}" ]; then else echo "[Family] ${SAMPLE} is in no family of ${FAMILIES}"; fi fi # og (bin/.og) built from another rust/ than this checkout's stops the run, SKIP_VALIDATION or not (rust/og.sh). -OG_MSG=$("${PGP_ROOT}/rust/og.sh" check 2>&1) || { echo "$OG_MSG" >&2; exit 2; } +[ ! -f "${PGP_ROOT}/rust/og.sh" ] || OG_MSG=$(bash "${PGP_ROOT}/rust/og.sh" check 2>&1) || { echo "$OG_MSG" >&2; exit 2; } mkdir -p "${S}/nextflow" "$LOG_DIR" GENOME_DIR="$G" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" run-all.sh "$SEX" || echo "WARNING: could not write ${S}/run_manifest.tsv" STATUS="${LOG_DIR}/run_status.tsv" # bin/collect_summary.py marks results older than this run stale diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index 16cbfcad..e01279bd 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -859,7 +859,9 @@ fi # other code than the checkout says, so it fails; none at all is a note until # a step calls og (rust/og.sh, REQUIRED). header "OwnGenome binary" -OG_MSG=$("${PGP_ROOT}/rust/og.sh" check 2>&1) || true +# A partial copy of the scripts (some tests make one) has no rust/og.sh. +OG_MSG="NOTE: rust/og.sh is not here, so og was not checked" +[ ! -f "${PGP_ROOT}/rust/og.sh" ] || OG_MSG=$(bash "${PGP_ROOT}/rust/og.sh" check 2>&1) || true case "$OG_MSG" in OK:*) pass "${OG_MSG#OK: }" ;; NOTE:*) info "${OG_MSG#NOTE: }" ;; From d7b7a66b43bccdb1a7e0b50f708a56f66259c482 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:41:52 +0100 Subject: [PATCH 05/12] ci(rust): check the Rust code on every pull request and ship og with each release Guard's rust job runs rustfmt, clippy, a planted failing test that must fail, cargo test, pycompat against CPython at 100000 cases per operation, both musl builds through setup.sh --og and the stale check on the real binaries; ci-ok needs it. rust.yml runs og by bare name in the task images on amd64 and arm64, times a cold and a cached build, and runs the CPython comparison nightly at 10^7 cases per operation. release.yml builds the binaries, writes SHA256SUMS, attests them and attaches them; a pull request that changes rust/ builds them as a dry run. Bead: pgp-bs9u.2 --- .github/workflows/guard.yml | 66 ++++++++++++++++++++- .github/workflows/release.yml | 69 ++++++++++++++++++++++ .github/workflows/rust.yml | 105 ++++++++++++++++++++++++++++++++++ tests/rust/pycompat_oracle.py | 4 ++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 0b628f8a..4fda48e3 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -141,11 +141,75 @@ jobs: run_tests tests + # OwnGenome's Rust code (rust/, docs/rust.md), in a job of its own: + # unit-tests is close to its 10 minutes. Format, lint, unit tests with a + # planted failure that must go red, pycompat against CPython, then the two + # musl builds as scripts/setup.sh --og makes them and the stale-binary + # guard on the real binaries. + rust: + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install the toolchain rust/rust-toolchain.toml pins, with rustfmt and clippy + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal -c rustfmt -c clippy + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + + - name: rustfmt + run: cargo fmt --all --check + + - name: clippy, warnings are errors + if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows + run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + + - name: Control (the planted failing test must fail, and be the test that failed) + run: | + if cargo test --locked -p pycompat --features planted-failure planted_failure > planted.log 2>&1; then + cat planted.log + echo "ERROR: the planted failing test passed: the test run would report a broken test as green" + exit 1 + fi + grep -E 'planted_failure.*FAILED' planted.log || { cat planted.log; echo "ERROR: the run failed, but not on the planted test"; exit 1; } + echo "Control: the planted test failed, as it must." + + - name: cargo test + run: cargo test --workspace --locked + + - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation) + run: | + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og + ../tests/rust/pycompat-diff.sh "${PWD}/target/release/og" 100000 + + - name: Both musl builds, as scripts/setup.sh --og makes them in OG_BUILDER + run: | + ../scripts/setup.sh --og + ./og.sh check --required + file ../bin/.og/og-linux-* + ../bin/og version + + - name: The stale-binary guard refuses the real binaries once rust/ changes + run: | + echo "// edited" >> og/src/main.rs + if out=$(./og.sh check); then echo "ERROR: an edited rust/ passed the check: ${out}"; exit 1; fi + echo "Refused, as it must: ${out}" + git checkout -- og/src/main.rs + ./og.sh check --required + # The one check the ruleset requires for this workflow: it runs even when a # job above failed or was skipped, and fails unless every one succeeded. ci-ok: if: always() - needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests] + needs: [image-vars, images, coupled-versions, fake-docker, helper-binaries, unit-tests, rust] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index edb78f85..8ca33e0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,64 @@ on: description: 'Tag to release (e.g. v0.3.0)' required: true type: string + # A dry run: a pull request that changes this file or rust/ builds the og + # binaries and their SHA256SUMS as an artifact; nothing is released. + pull_request: + branches: [main] + paths: ['.github/workflows/release.yml', 'rust/**'] permissions: contents: read jobs: + # The og binaries every release carries (docs/rust.md): both musl builds, + # made by scripts/setup.sh --og as a checkout makes them, with SHA256SUMS. + # A checkout of the tag downloads them instead of building (rust/og.sh). + og-binaries: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + # Same rule as create-release below: the tag is untrusted text. + - name: Check the tag + if: github.event_name != 'pull_request' + env: + INPUT_TAG: ${{ inputs.tag }} + run: | + tag=${INPUT_TAG:-${GITHUB_REF#refs/tags/}} + if ! [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: the tag is not of the form vMAJOR.MINOR.PATCH (not echoed: it is untrusted text)" >&2 + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.tag || github.ref }} + persist-credentials: false + + - name: Build og for x86_64 and aarch64 (scripts/setup.sh --og builds in OG_BUILDER) + run: | + # On a tag, setup.sh --og would download the release's own binaries, + # which this job is about to make: build them. + rust/og.sh build + rust/og.sh check --required + mkdir -p dist + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 dist/ + (cd dist && sha256sum og-linux-* > SHA256SUMS && cat SHA256SUMS) + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: og-binaries + path: dist/ + retention-days: 7 + create-release: + needs: og-binaries + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: write + id-token: write # the build attestation + attestations: write steps: # The tag comes from the dispatch form or the pushed ref, so it is # untrusted text: it reaches the shell through env only, and it must @@ -75,9 +124,25 @@ jobs: echo "All image versions are listed in [\`versions.env\`](versions.env)." echo "" echo "To set up: \`./scripts/setup.sh /path/to/data\`" + echo "" + echo "## og binaries" + echo "" + echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks them." echo "EOF" } >> "$GITHUB_OUTPUT" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: og-binaries + path: dist + + # rust/og.sh install checks a downloaded binary with + # `gh attestation verify` when gh is installed. + - name: Attest the og binaries + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: 'dist/og-linux-*' + - name: Create GitHub Release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: @@ -85,3 +150,7 @@ jobs: name: ${{ steps.tag.outputs.tag }} body: ${{ steps.notes.outputs.notes }} draft: false + files: | + dist/og-linux-x86_64 + dist/og-linux-aarch64 + dist/SHA256SUMS diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 00000000..061b3246 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,105 @@ +name: Rust + +# The og binary in the images the tasks run, and the long pycompat streams. +# Guard's rust job holds the checks every pull request needs (format, lint, +# tests, pycompat at 100000 cases per operation, the musl builds and the +# stale-binary guard); this workflow holds the slow ones (docs/rust.md): +# +# images on a pull request that touches the Rust code, the shim or the +# images: builds og with scripts/setup.sh --og, cold and then +# cached (both times go to the summary), and runs it by bare name +# in PYTHON_IMAGE, BCFTOOLS_IMAGE, MANTA_IMAGE and PARASCOPY_IMAGE +# on every platform each one publishes (arm64 under QEMU). The two +# binaries are kept as the artifact og-binaries. +# streams nightly: 10^7 cases per pycompat operation against CPython in +# PYTHON_IMAGE, one operation at a time, with a new seed each +# night (printed, so a failure can be replayed). On a pull request +# that touches the paths below it runs 20000 cases per operation, so the +# job is seen working. +# +# Public repo: GitHub-hosted runners only. +on: + pull_request: + branches: [main] + paths: + - 'rust/**' + - 'bin/og' + - 'tests/rust/**' + - 'versions.env' + - 'scripts/setup.sh' + - '.github/workflows/rust.yml' + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + cases: + description: 'pycompat cases per operation for the streams job' + default: '10000000' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + images: + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build og with scripts/setup.sh --og, cold and cached + run: | + . versions.env + docker pull -q "$OG_BUILDER" > /dev/null + t0=$(date +%s); scripts/setup.sh --og; t1=$(date +%s) + touch rust/og/src/main.rs + scripts/setup.sh --og; t2=$(date +%s) + rust/og.sh check --required + { + echo "| build | seconds |" + echo "|---|---|" + echo "| cold (empty cargo cache, image pulled) | $((t1 - t0)) |" + echo "| cached (one crate changed) | $((t2 - t1)) |" + } | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + platforms: arm64 + + - name: og by bare name in the task images, amd64 and arm64 + run: tests/rust/og-in-images.sh | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: og-binaries + path: bin/.og/og-linux-* + retention-days: 7 + + streams: + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build og with scripts/setup.sh --og + run: | + scripts/setup.sh --og + bin/og version + + - name: pycompat against CPython, one operation at a time + env: + CASES: ${{ github.event_name == 'pull_request' && '20000' || inputs.cases || '10000000' }} + run: | + case "$CASES" in ''|*[!0-9]*) echo "ERROR: cases must be a whole number, got '${CASES}'"; exit 1 ;; esac + seed=${GITHUB_RUN_ID} + fail=0 + ops=$(python3 -I tests/rust/pycompat_oracle.py ops) + [ -n "$ops" ] || { echo "ERROR: the oracle lists no operation"; exit 1; } + for op in $ops; do + tests/rust/pycompat-diff.sh "${PWD}/bin/og" "$CASES" "$seed" "$op" || fail=1 + done + exit "$fail" diff --git a/tests/rust/pycompat_oracle.py b/tests/rust/pycompat_oracle.py index 845f5a27..d20d955d 100644 --- a/tests/rust/pycompat_oracle.py +++ b/tests/rust/pycompat_oracle.py @@ -1,6 +1,7 @@ """The CPython side of the pycompat differential test. python3 -I pycompat_oracle.py N SEED [OP ...] + python3 -I pycompat_oracle.py ops the operations, one line Writes N random cases per operation (the hand-picked edge cases are among the values drawn) to stdout, each line followed by the line this Python @@ -256,6 +257,9 @@ def run(): def main(): + if sys.argv[1:] == ["ops"]: + print(" ".join(OPS)) + return n, seed = int(sys.argv[1]), int(sys.argv[2]) ops = sys.argv[3:] or OPS r = random.Random(seed) From fc6372b3f001e628cae5a57eafb983b97f3dcd97 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:41:52 +0100 Subject: [PATCH 06/12] docs(rust): describe the Rust core, the og binary and its checks for contributors Bead: pgp-bs9u.2 --- CLAUDE.md | 4 ++- docs/nextflow.md | 2 ++ docs/rust.md | 83 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/testing.md | 12 +++++++ rust/README.md | 20 ++++++++++++ zensical.toml | 1 + 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 docs/rust.md create mode 100644 rust/README.md diff --git a/CLAUDE.md b/CLAUDE.md index 30d6960b..032bd266 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,7 @@ Whole genome sequencing (WGS) analysis pipeline for consumer hardware. Takes raw ## Tech Stack - Bash (pipeline scripts, `set -euo pipefail`) - Docker (all bioinformatics tools containerized) +- Python (`bin/*.py`, OwnGenome's own tools), moving tool by tool to Rust: one static binary, `og` (`rust/`, run through the `bin/og` shim; `scripts/setup.sh --og` builds it; `docs/rust.md`) - Key tools: DeepVariant, minimap2, BWA-MEM2, VEP, PharmCAT, GATK, FreeBayes, Strelka2, TIDDIT, Manta, SURVIVOR, PCGR/CPSR, pgsc_calc, haplogrep3, Yleaf ## Development @@ -71,7 +72,8 @@ owngenome/ .github/workflows/ lint.yml # ShellCheck, actionlint, gitleaks, doc links personal-data.yml # Personal-data scan of every tracked text file - guard.yml # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests; ci-ok sums them up + guard.yml # image variables, image tags (check-images.sh), coupled versions, fake-docker suite, helper binaries, unit tests, Rust; ci-ok sums them up + rust.yml # og in the task images on amd64 and arm64; nightly pycompat against CPython at 10^7 cases container-test.yml # Runs each changed image on the fixture (tests/smoke/commands.tsv); container-ok sums it up e2e.yml # Real tools on a small fixture nextflow.yml # Nextflow config, lint, schema and stub runs on the pinned release and the newest 26.10.x diff --git a/docs/nextflow.md b/docs/nextflow.md index 06e9595e..39861401 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -24,6 +24,8 @@ Every BAM then goes through a sex check (indexcov), and the pipeline runs pharma sudo mv nextflow /usr/local/bin/ ``` +Before it starts any task, `main.nf` runs `rust/og.sh check`: when `bin/.og` holds an `og` binary built from another `rust/` than the checkout's, the run stops and names `./scripts/setup.sh --og`, which rebuilds it. No step calls `og` yet, so a checkout without the binary runs as before ([The Rust Core](rust.md)). + ### Run the Pipeline ```bash diff --git a/docs/rust.md b/docs/rust.md new file mode 100644 index 00000000..d647b5c6 --- /dev/null +++ b/docs/rust.md @@ -0,0 +1,83 @@ +# The Rust Core + +This page is for contributors. OwnGenome's own tools are moving from Python to Rust, one tool per pull request, into a single binary, `og`. This page says how the Rust code is laid out, how the binary reaches the pipeline, how it stays byte-for-byte compatible with the Python it replaces, and what CI checks. + +No pipeline step calls `og` yet. The pieces below are the base every port builds on. + +## Layout + +``` +rust/ + Cargo.toml the workspace (edition 2024); Cargo.lock pins every crate + rust-toolchain.toml the Rust release, the one OG_BUILDER ships + og.sh build, download and check the binary + crates/pycompat/ Python-exact formatting and parsing + crates/og-io/ plain, gzip and BGZF input, universal newlines, atomic writes + og/ the binary: og [args] +bin/og the shim tasks call; runs bin/.og/og-linux- +tests/rust/ the differential test against CPython, the image test +``` + +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og version`, `og --tree-hash` and a hidden `og pycompat-probe` that the tests drive. + +The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2` so far). + +## Building + +```bash +./scripts/setup.sh --og +``` + +This puts two static musl binaries in `bin/.og/` (ignored by git): `og-linux-x86_64` and `og-linux-aarch64`. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. + +- **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. +- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's two binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. + +For day-to-day work on the Rust code a host toolchain is quicker: `cd rust && cargo test`. A plain `cargo build` stamps the binary `unknown`, which the check below refuses, so use `scripts/setup.sh --og` for a binary the pipeline will run. + +The image's Rust release (1.93.0) and `rust/rust-toolchain.toml` must agree, or the build downloads a toolchain first. `OG_BUILDER` has no `_IMAGE` suffix on purpose: no pipeline step runs it, so it has no smoke row and Renovate does not bump it. Bump the image and the toolchain file together, by hand. + +## How a task runs it + +A Nextflow task has `bin/` on its PATH, so a module calls `og ` by its bare name. A bash step calls `/pgp-bin/og ` inside the image it already uses. Either way it starts `bin/og`, a `#!/bin/sh` shim that runs `bin/.og/og-linux-$(uname -m)`, or exits 127 with `ERROR: og is not built for ; run scripts/setup.sh --og`. + +The binary is static, so it runs in any image that has `/bin/sh`: the Python image, the bcftools image, the Manta and Parascopy images. `tests/rust/og-in-images.sh` checks those four on every platform each one publishes. + +## The stale-binary check + +`bin/.og` is not in git, so after a pull or an edit it can hold a binary built from other code. Results would then come from code the checkout does not show. To stop that: + +- The build stamps the binary with the git tree hash of `rust/`: `git rev-parse HEAD:rust` on a clean checkout, or for an edited `rust/` the tree those edits would commit. Any edit, committed or not, changes the hash. +- `rust/og.sh check` reads the stamp from each binary with `grep`, without running it, and compares it with the checkout's. +- `run-all.sh` (even with `SKIP_VALIDATION=true`), `main.nf` and `validate-setup.sh` run that check. A binary built from another `rust/` stops them with `run scripts/setup.sh --og`. +- A missing binary is a note for now. The first pull request that makes a step call `og` sets `REQUIRED=true` in `rust/og.sh`, and from then on a missing binary stops a run too. +- `run_manifest.tsv` gets a row `run og_tree_hash`, so a result names the Rust code that made it (`not built` before the first build). + +## Matching Python byte for byte + +A port must write exactly what the Python tool wrote. Most differences hide in number formatting, so the shared primitives live in `pycompat`, each one written to return what CPython 3.15 returns: + +| Module | What it covers | +|---|---| +| `fmt` | `repr(float)` (also `str` and `f"{x}"`); the `f`, `e`, `g` and `%` format types at any precision; `{:,}` and `{:,.Nf}`; `{:0Nd}`; `%-Ns` and `%Ns`; `round(x)` and `round(x, n)`, both rounding the exact binary value half to even (`round(0.35, 1) == 0.3`, `round(2.675, 2) == 2.67`) | +| `num` | `float(str)` and `int(str)` with Python's whitespace, signs and underscores; `math.fsum`; the builtin `sum`, which since Python 3.12 is a compensated sum and so differs from a plain loop | +| `json` | `json.dumps` with `indent` (none or a number), `separators`, `ensure_ascii` (surrogate pairs for astral characters), `sort_keys` and `allow_nan`; `json.loads` with `NaN` and `Infinity`, where a duplicate key keeps its first place and takes the last value | + +`og-io` reads a path or `-` as plain text, gzip or BGZF (sniffed by the magic bytes), splits lines as Python's text mode does (`\n`, `\r\n` and a lone `\r`), decodes UTF-8 strictly or with `replace`, and writes a file under a temporary name before renaming it into place. + +Known limits, each an error in Rust where Python would go on: ints beyond 128 bits, a lone surrogate in a JSON string, non-ASCII digits in `float()` and `int()`, and a string `indent`. None of them appears in what the pipeline reads. + +### The differential test + +`tests/rust/pycompat-diff.sh [N] [SEED]` checks every `pycompat` operation against CPython. `tests/rust/pycompat_oracle.py` runs with `python3 -I` in `PYTHON_IMAGE`, draws N random cases per operation and writes its result for each. `og pycompat-probe` reads the same cases and must write the same lines, byte for byte. The values include the ones that catch a formatting bug: binary ties (0.125, 2.675, 0.35), 1e-05 and 1e16 where `repr` switches layout, -0.0, NaN, subnormals, random bit patterns, numbers exactly halfway between two shortest `repr` candidates, astral characters, CRLF and lone CR, and broken JSON. A failure prints the first cases that differ and the seed, so `pycompat-diff.sh ` replays it. `PYTHON=python3` runs the oracle on the host for a quick loop. + +To add a primitive: write it in `pycompat` with unit tests, add an operation to `og/src/probe.rs` and the same operation to the oracle, then run the differential test. + +## CI + +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; both musl builds through `scripts/setup.sh --og`; and the stale check on the real binaries, which must refuse them after an edit to `rust/`. +- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a release download with a bad checksum or attestation), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. +- **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. +- **Release** (`release.yml`): a tag builds both binaries, writes `SHA256SUMS`, attests them and attaches all three to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. + +[Testing](testing.md) has the rest of CI. diff --git a/docs/testing.md b/docs/testing.md index e3ef3253..7564ac2a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -229,6 +229,18 @@ The **Demo pictures** workflow ([`.github/workflows/demo-pictures.yml`](https:// Look at the pictures, copy them into `docs/images/`, and update the alt text where the cards changed. A new report card that the generator does not feed yet stops the script with its name; add its input file to `make_demo_sample.py`. +## The Rust jobs + +The `rust` job of `guard.yml` checks the Rust code on every pull request: format, clippy, the unit tests with a planted failure that must fail, `pycompat` against CPython at 100,000 cases per operation, both musl builds as `scripts/setup.sh --og` makes them, and the stale-binary check on the real binaries. The **Rust** workflow (`rust.yml`) runs `og` by bare name in the task images on amd64 and arm64, and every night runs the CPython comparison at 10,000,000 cases per operation. [The Rust Core](rust.md) says what each check covers. + +How long `scripts/setup.sh --og` takes to build both binaries: + +| Where | Cold (empty cargo cache) | Cached (one crate changed) | +|---|---|---| +| Apple Silicon Mac mini, 10 cores | 25 s | 3 s | + +The cold build downloads the crates once; pulling `OG_BUILDER` the first time adds about 1 GB. The Rust workflow's `images` job prints both times for `ubuntu-latest` in its summary. + ## Telling a flaky test from a broken one The **Repeat a test** workflow (`.github/workflows/repeat-test.yml`) runs one `tests/test_*.sh` or `tests/test_*.py` many times, each run in a fresh job on its own `ubuntu-latest` runner, six at a time. Start it from the Actions tab with the test's path and the number of runs (1 to 60, default 30); it also runs, with its defaults (`tests/test_annotsv_benign_mane.sh`, 30 runs), on a pull request that changes the workflow. A flaky test fails a few runs out of many; a broken one fails them all. Each run records its exit code, and the summary job lists every run and fails when any failed. One more run, `control`, runs a test that always fails, and the summary fails when the control passed, so "0 of 30 failed" is never a count that could not see a failure. diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..e6658688 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,20 @@ +# rust/ + +OwnGenome's own tools, ported from `bin/*.py` one at a time into one static binary, `og`. Each port must write what the Python wrote, byte for byte. The full guide is [docs/rust.md](../docs/rust.md). + +| Path | What it is | +|---|---| +| `crates/pycompat` | Python-exact `repr`, format types, `round`, `float()`/`int()`, `math.fsum`, `sum`, `json.dumps`/`loads` | +| `crates/og-io` | plain, gzip and BGZF input; Python's universal newlines; atomic writes | +| `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | +| `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | +| `rust-toolchain.toml` | the Rust release, the one `OG_BUILDER` (`versions.env`) ships | + +```bash +cargo test # unit tests (needs the pinned toolchain) +cargo clippy --all-targets -- -D warnings +../scripts/setup.sh --og # the binaries the pipeline runs, built in Docker +cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # pycompat against CPython +``` + +A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. diff --git a/zensical.toml b/zensical.toml index 668d07b5..a1a83b39 100644 --- a/zensical.toml +++ b/zensical.toml @@ -136,6 +136,7 @@ nav = [ { "What 1.0 Keeps Stable" = "output-contract.md" }, { "Copy Number from Depth (the locus catalog)" = "cn-loci.md" }, { Testing = "testing.md" }, + { "The Rust Core" = "rust.md" }, { "Keeping versions up to date" = "updating.md" }, { Glossary = "glossary.md" }, { "Recommended Resources" = "resources.md" }, From 5bec1b981bd8849c4372e4ee738c2ab872fe1a57 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:42:11 +0100 Subject: [PATCH 07/12] ci(rust): keep the control log out of rust/ and show the CPython comparison after a failed test Bead: pgp-bs9u.2 --- .github/workflows/guard.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 4fda48e3..1c3713f5 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -174,18 +174,20 @@ jobs: - name: Control (the planted failing test must fail, and be the test that failed) run: | - if cargo test --locked -p pycompat --features planted-failure planted_failure > planted.log 2>&1; then - cat planted.log + log="${RUNNER_TEMP}/planted.log" + if cargo test --locked -p pycompat --features planted-failure planted_failure > "$log" 2>&1; then + cat "$log" echo "ERROR: the planted failing test passed: the test run would report a broken test as green" exit 1 fi - grep -E 'planted_failure.*FAILED' planted.log || { cat planted.log; echo "ERROR: the run failed, but not on the planted test"; exit 1; } + grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." - name: cargo test run: cargo test --workspace --locked - name: pycompat against CPython (python3 -I in PYTHON_IMAGE, 100000 cases per operation) + if: ${{ !cancelled() }} # runs even when a step above failed, so each result shows run: | OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og ../tests/rust/pycompat-diff.sh "${PWD}/target/release/og" 100000 From 6e84c46f4f394cf74889ca4ecef85d0cd76b02f4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:42:19 +0100 Subject: [PATCH 08/12] test(rust): plant round half away from zero and a stale check that passes, to see CI go red Reverted by the next commit. This head must fail the rust job (unit tests and the CPython comparison) and the og check unit test. Bead: pgp-bs9u.2 --- rust/crates/pycompat/src/fmt.rs | 2 +- rust/og.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crates/pycompat/src/fmt.rs b/rust/crates/pycompat/src/fmt.rs index 255fe211..8c93eed5 100644 --- a/rust/crates/pycompat/src/fmt.rs +++ b/rust/crates/pycompat/src/fmt.rs @@ -319,7 +319,7 @@ pub fn round_int(x: f64) -> Result { } else if x.is_infinite() { Err(RoundError::Infinite) } else { - Ok(x.round_ties_even()) + Ok(x.round()) } } diff --git a/rust/og.sh b/rust/og.sh index 6b3f82ca..df617a85 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -72,7 +72,7 @@ check() { done if [ -n "$stale" ]; then echo "ERROR: og is stale:${stale}, but rust/ here is tree ${expected}: run scripts/setup.sh --og" - return 1 + return 0 fi if [ -n "$missing" ] && $required; then echo "ERROR: og is not built for${missing}: run scripts/setup.sh --og" From bc089ff73269cab4233a288ffce95d744b0d784a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:52:25 +0100 Subject: [PATCH 09/12] test(rust): revert the planted mutations The previous commit planted round half away from zero and a stale check that passes, to see the rust job and the og check unit test go red. Bead: pgp-bs9u.2 --- rust/crates/pycompat/src/fmt.rs | 2 +- rust/og.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crates/pycompat/src/fmt.rs b/rust/crates/pycompat/src/fmt.rs index 8c93eed5..255fe211 100644 --- a/rust/crates/pycompat/src/fmt.rs +++ b/rust/crates/pycompat/src/fmt.rs @@ -319,7 +319,7 @@ pub fn round_int(x: f64) -> Result { } else if x.is_infinite() { Err(RoundError::Infinite) } else { - Ok(x.round()) + Ok(x.round_ties_even()) } } diff --git a/rust/og.sh b/rust/og.sh index df617a85..6b3f82ca 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -72,7 +72,7 @@ check() { done if [ -n "$stale" ]; then echo "ERROR: og is stale:${stale}, but rust/ here is tree ${expected}: run scripts/setup.sh --og" - return 0 + return 1 fi if [ -n "$missing" ] && $required; then echo "ERROR: og is not built for${missing}: run scripts/setup.sh --og" From 8d9bf0c7fd93dec9e6f5fbd36f36c2d00d437903 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:52:25 +0100 Subject: [PATCH 10/12] fix(rust): pair the pycompat report lines by number, not by tab A result can hold a tab (ljust of a string with a tab), so splitting paste output on tabs counted matching ljust and rjust results as differences in the failure report. The pass or fail verdict (cmp) was right; only the per-operation counts were wrong. Bead: pgp-bs9u.2 --- tests/rust/pycompat-diff.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/rust/pycompat-diff.sh b/tests/rust/pycompat-diff.sh index 9f0a99af..113698f0 100755 --- a/tests/rust/pycompat-diff.sh +++ b/tests/rust/pycompat-diff.sh @@ -39,9 +39,13 @@ if cmp -s "${TMP}/expected.txt" "${TMP}/actual.txt"; then echo "PASS: og pycompat-probe matches CPython on all ${cases} cases" exit 0 fi -paste "${TMP}/cases.tsv" "${TMP}/expected.txt" "${TMP}/actual.txt" | awk -F'\t' ' - { want = $(NF - 1); got = $NF; if (want == got) next; bad[$1]++; n++ - if (n <= 20) { c = $1; for (i = 2; i < NF - 1; i++) c = c "\t" $i; printf "case: %s\npython: %s\nog: %s\n\n", c, want, got } } - END { for (op in bad) printf "FAIL: %s differs on %d cases\n", op, bad[op] }' +# Results can hold tabs (ljust of a string with a tab), so the report pairs +# lines by number, never by splitting on tabs. +awk -v want_file="${TMP}/expected.txt" -v got_file="${TMP}/actual.txt" ' + { want = ""; got = ""; getline want < want_file; getline got < got_file + if (want == got) next + op = $0; sub(/\t.*/, "", op); bad[op]++; n++ + if (n <= 20) printf "case: %s\npython: %s\nog: %s\n\n", $0, want, got } + END { for (op in bad) printf "FAIL: %s differs on %d cases\n", op, bad[op] }' "${TMP}/cases.tsv" echo "FAIL: og pycompat-probe differs from CPython (seed ${SEED}; rerun with that seed to replay)" exit 1 From 057fe4126ccd118d92e8796f77a3d94ebaf06abf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:27:30 +0100 Subject: [PATCH 11/12] fix(rust): make the image check able to fail, skip the attestation without a gh login, test the main.nf refusal The image check compared docker's pull progress with og's output and its exit code was lost in tee. og.sh refused release binaries when gh was installed but not logged in. Guard now runs nextflow run main.nf -stub on a stale og, and pycompat's JSON parser stops at 1000 levels of nesting with RecursionError instead of overflowing the stack. --- .github/workflows/guard.yml | 36 +++++++++++++++++++++++++++ .github/workflows/rust.yml | 19 +++++++++++++-- docs/rust.md | 8 +++--- rust/crates/pycompat/src/json.rs | 42 ++++++++++++++++++++++++++++++-- rust/og.sh | 10 +++++--- tests/rust/og-in-images.sh | 3 ++- tests/test_og_check.sh | 12 ++++++++- 7 files changed, 117 insertions(+), 13 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 1c3713f5..404e0d90 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -207,6 +207,42 @@ jobs: git checkout -- og/src/main.rs ./og.sh check --required + # main.nf runs the same check before any task: a stale og stops + # `nextflow run` with the setup command. Without the check, this stub + # run completes and the step fails. + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: 'temurin' + java-version: '17' + + - name: The Nextflow version versions.env pins + id: nf + working-directory: . + run: | + # shellcheck source=/dev/null + . ./versions.env + echo "version=${NEXTFLOW_VERSION:?NEXTFLOW_VERSION is not set in versions.env}" >> "$GITHUB_OUTPUT" + + # Not a Nextflow pin: the version comes from versions.env. Quoted so the + # Renovate dry run does not count it as one. + - uses: "nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914" # v3.0.1 + with: + version: ${{ steps.nf.outputs.version }} + + - name: nextflow run stops on a stale og before any task + working-directory: . + run: | + echo "// edited" >> rust/og/src/main.rs + rc=0 + out=$(timeout 600 nextflow run main.nf -profile test -stub -ansi-log false --outdir "${RUNNER_TEMP}/og-stale" 2>&1) || rc=$? + git checkout -- rust/og/src/main.rs + if [ "$rc" -eq 0 ] || [[ "$out" != *"og is stale:"*"run scripts/setup.sh --og"* ]] || [[ "$out" == *"Submitted process"* ]]; then + echo "$out" + echo "ERROR: nextflow run did not stop on a stale og before any task (exit ${rc})" + exit 1 + fi + echo "Refused, as it must (exit ${rc}): $(grep 'og is stale' <<< "$out")" + # The one check the ruleset requires for this workflow: it runs even when a # job above failed or was skipped, and fails unless every one succeeded. ci-ok: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 061b3246..b34d6d6f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,8 +9,9 @@ name: Rust # images: builds og with scripts/setup.sh --og, cold and then # cached (both times go to the summary), and runs it by bare name # in PYTHON_IMAGE, BCFTOOLS_IMAGE, MANTA_IMAGE and PARASCOPY_IMAGE -# on every platform each one publishes (arm64 under QEMU). The two -# binaries are kept as the artifact og-binaries. +# on every platform each one publishes (arm64 under QEMU), then +# once more with a wrong-arch binary planted, which must fail. The +# two binaries are kept as the artifact og-binaries. # streams nightly: 10^7 cases per pycompat operation against CPython in # PYTHON_IMAGE, one operation at a time, with a new seed each # night (printed, so a failure can be replayed). On a pull request @@ -70,9 +71,23 @@ jobs: with: platforms: arm64 + # shell: bash sets pipefail, so a failing check is not masked by tee. - name: og by bare name in the task images, amd64 and arm64 + shell: bash run: tests/rust/og-in-images.sh | tee -a "$GITHUB_STEP_SUMMARY" + # Control: the same check must fail when an image runs the wrong build. + # The amd64 binary planted as og-linux-aarch64 reports x86_64 on arm64. + - name: Control (a planted wrong-arch binary must fail the image check) + run: | + cp bin/.og/og-linux-aarch64 "${RUNNER_TEMP}/og-linux-aarch64" + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 + rc=0 + tests/rust/og-in-images.sh PYTHON_IMAGE || rc=$? + cp "${RUNNER_TEMP}/og-linux-aarch64" bin/.og/og-linux-aarch64 + [ "$rc" -ne 0 ] || { echo "ERROR: the image check passed a wrong-arch binary"; exit 1; } + echo "Control: the image check failed on the planted binary, as it must." + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: og-binaries diff --git a/docs/rust.md b/docs/rust.md index d647b5c6..248da18d 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -31,7 +31,7 @@ The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, ` This puts two static musl binaries in `bin/.og/` (ignored by git): `og-linux-x86_64` and `og-linux-aarch64`. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. - **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. -- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's two binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. +- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's two binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. For day-to-day work on the Rust code a host toolchain is quicker: `cd rust && cargo test`. A plain `cargo build` stamps the binary `unknown`, which the check below refuses, so use `scripts/setup.sh --og` for a binary the pipeline will run. @@ -75,9 +75,9 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to ## CI -- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; both musl builds through `scripts/setup.sh --og`; and the stale check on the real binaries, which must refuse them after an edit to `rust/`. -- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a release download with a bad checksum or attestation), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. -- **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; both musl builds through `scripts/setup.sh --og`; and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. +- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a release download with a bad checksum or attestation, or with `gh` not logged in), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. +- **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. - **Release** (`release.yml`): a tag builds both binaries, writes `SHA256SUMS`, attests them and attaches all three to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. [Testing](testing.md) has the rest of CI. diff --git a/rust/crates/pycompat/src/json.rs b/rust/crates/pycompat/src/json.rs index 305181e6..ad53b7c0 100644 --- a/rust/crates/pycompat/src/json.rs +++ b/rust/crates/pycompat/src/json.rs @@ -54,14 +54,21 @@ pub enum JsonError { OutOfRangeFloat, /// JSONDecodeError: what was wrong and the byte offset. Decode(&'static str, usize), + /// RecursionError: arrays and objects nested deeper than [`MAX_DEPTH`]. + TooDeep, } +/// How deep `loads` nests arrays and objects before it gives up, about +/// where CPython's default recursion limit (1000) stops `json.loads`. +pub const MAX_DEPTH: usize = 1000; + impl JsonError { /// The Python exception class. pub fn exception(&self) -> &'static str { match self { JsonError::OutOfRangeFloat => "ValueError", JsonError::Decode(..) => "JSONDecodeError", + JsonError::TooDeep => "RecursionError", } } } @@ -73,6 +80,7 @@ impl fmt::Display for JsonError { f.write_str("Out of range float values are not JSON compliant") } JsonError::Decode(what, at) => write!(f, "{what}: byte {at}"), + JsonError::TooDeep => write!(f, "nested deeper than {MAX_DEPTH} levels"), } } } @@ -204,6 +212,7 @@ pub fn loads(text: &str) -> Result { s: text.as_bytes(), text, i: 0, + depth: 0, }; p.ws(); let v = p.value()?; @@ -218,6 +227,8 @@ struct Parser<'a> { s: &'a [u8], text: &'a str, i: usize, + /// Arrays and objects open around `i`: a bound on the recursion. + depth: usize, } impl Parser<'_> { @@ -242,8 +253,19 @@ impl Parser<'_> { fn value(&mut self) -> Result { match self.s.get(self.i) { - Some(b'{') => self.object(), - Some(b'[') => self.array(), + Some(b'{' | b'[') => { + if self.depth == MAX_DEPTH { + return Err(JsonError::TooDeep); + } + self.depth += 1; + let v = if self.s[self.i] == b'{' { + self.object() + } else { + self.array() + }; + self.depth -= 1; + v + } Some(b'"') => Ok(Value::Str(self.string()?)), Some(b'n') if self.eat("null") => Ok(Value::Null), Some(b't') if self.eat("true") => Ok(Value::Bool(true)), @@ -522,4 +544,20 @@ mod tests { assert!(loads(bad).is_err(), "{bad:?}"); } } + + #[test] + fn loads_stops_deep_nesting_with_an_error() { + // 8 MiB, the main-thread stack og runs on: a test thread has 2 MiB, + // too little for MAX_DEPTH levels in an unoptimised build. + let t = std::thread::Builder::new().stack_size(8 << 20).spawn(|| { + let nested = |n: usize| format!("{}{}", "[".repeat(n), "]".repeat(n)); + assert!(loads(&nested(MAX_DEPTH)).is_ok()); + assert_eq!(loads(&nested(MAX_DEPTH + 1)), Err(JsonError::TooDeep)); + // Far past the limit: an error, never a stack overflow. + assert_eq!(loads(&"[".repeat(1_000_000)), Err(JsonError::TooDeep)); + assert_eq!(loads(&"{\"a\":".repeat(100_000)), Err(JsonError::TooDeep)); + }); + t.unwrap().join().unwrap(); + assert_eq!(JsonError::TooDeep.exception(), "RecursionError"); + } } diff --git a/rust/og.sh b/rust/og.sh index 6b3f82ca..67aee468 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -144,8 +144,12 @@ get() { # get URL DEST: 0 when downloaded, 1 when not # fetch TAG: 0 when the release's binaries are installed, 2 when the release # has none (build instead), 1 when a download does not check out. fetch() { - local tag=$1 tmp want hash a f rc=0 + local tag=$1 tmp want hash a f rc=0 attest=0 hash=$(tree_hash) + # gh attestation verify needs a login: without one (gh missing, or never + # logged in) only SHA256SUMS is checked. gh auth token, not gh auth status, + # which can hang on enterprise hosts. + if command -v gh > /dev/null 2>&1 && gh auth token > /dev/null 2>&1; then attest=1; fi tmp=$(mktemp -d "${TMPDIR:-/tmp}/og-fetch.XXXXXX") # OG_RELEASE_BASE: another server with the same layout (tests/test_og_check.sh). local base="${OG_RELEASE_BASE:-https://github.com/${REPO}/releases/download}/${tag}" @@ -158,14 +162,14 @@ fetch() { want=$(awk -v f="$f" '$2 == f || $2 == "*" f {print $1}' "${tmp}/SHA256SUMS") if [ -z "$want" ] || ! get "${base}/${f}" "${tmp}/${f}"; then echo "ERROR: release ${tag} lists no ${f}" >&2; rc=1; break; fi if [ "$(sha256 "${tmp}/${f}")" != "$want" ]; then echo "ERROR: ${f} of release ${tag} does not match its SHA256SUMS" >&2; rc=1; break; fi - if command -v gh > /dev/null 2>&1; then + if [ "$attest" = 1 ]; then gh attestation verify "${tmp}/${f}" --repo "$REPO" > /dev/null || { echo "ERROR: ${f} of release ${tag} has no valid build attestation" >&2; rc=1; break; } fi if [ "$(embedded "${tmp}/${f}")" != "$hash" ]; then echo "ERROR: ${f} of release ${tag} was not built from this rust/ (tree ${hash})" >&2; rc=1; break; fi done if [ "$rc" -eq 0 ]; then for a in $ARCHES; do put "${tmp}/og-linux-${a}" "$a"; done - command -v gh > /dev/null 2>&1 || echo "NOTE: gh is not installed, so the build attestation was not checked (SHA256SUMS was)" + [ "$attest" = 1 ] || echo "NOTE: gh is not installed or not logged in, so the build attestation was not checked (SHA256SUMS was)" echo "Installed og ${hash} from release ${tag}" fi rm -rf "$tmp" diff --git a/tests/rust/og-in-images.sh b/tests/rust/og-in-images.sh index a9bc28e3..e374d720 100755 --- a/tests/rust/og-in-images.sh +++ b/tests/rust/og-in-images.sh @@ -21,7 +21,8 @@ for a in x86_64 aarch64; do done fail=0 runs=0 run_og() { # run_og IMAGE PLATFORM BINDIR: og version, as a task would call it - docker run --rm --pull missing --platform "$2" -v "${3}:${3}:ro" -e "PATH=${3}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + # -q: a first pull prints its progress on the output compared below. + docker run -q --rm --pull missing --platform "$2" -v "${3}:${3}:ro" -e "PATH=${3}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ --entrypoint sh "$1" -c 'echo "$(uname -m) $(og version)"' 2>&1 } printf '%-18s %-12s %-8s %s\n' image platform result "uname -m and og version" diff --git a/tests/test_og_check.sh b/tests/test_og_check.sh index 9076c131..148aa6f1 100755 --- a/tests/test_og_check.sh +++ b/tests/test_og_check.sh @@ -106,7 +106,7 @@ row=$(awk -F'\t' '$1 == "run" && $2 == "og_tree_hash" {print $3}' "${T}/genome/s # A release tag: the binaries come from the release, checked against # SHA256SUMS and the attestation (a fake gh here), and are refused when -# either does not match. OG_RELEASE_BASE points at a local server. +# either does not match; a gh that is not logged in skips the attestation. OG_RELEASE_BASE points at a local server. if command -v python3 > /dev/null 2>&1 && command -v curl > /dev/null 2>&1; then rm -rf "${C}/bin/.og" g tag v9.9.9 @@ -126,11 +126,21 @@ exit 0 [ "$(bash "$OG" built)" = "$h1" ] && ok "the installed binaries carry the tree" || bad "built after install: $(bash "$OG" built)" rm -rf "${C}/bin/.og" printf '#!/bin/sh +[ "$1" = auth ] && exit 0 echo "no attestation" >&2 exit 1 ' > "${T}/fake/gh" out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? expect "a binary without a valid attestation is refused" "$rc" 1 "$out" "*has no valid build attestation*" + # gh installed but not logged in: attestation verify cannot run, so only + # SHA256SUMS is checked, with a note, never a refusal that reads as tampering. + printf '#!/bin/sh +echo "To get started with GitHub CLI, please run: gh auth login" >&2 +exit 4 +' > "${T}/fake/gh" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + expect "gh not logged in: installed, attestation not checked" "$rc" 0 "$out" "*attestation was not checked*Installed og ${h1} from release v9.9.9*" + rm -rf "${C}/bin/.og" printf '#!/bin/sh exit 0 ' > "${T}/fake/gh" From 9db8ce5105cc14070c884ab4ad9566c54093413c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:08:39 +0100 Subject: [PATCH 12/12] fix(rust): keep the old og pair when an install fails, and read gzip padding as Python does og.sh no longer falls back to a build on a network error, keeps the old binaries when a copy fails, and refuses a tree hash it could not compute. og-io skips NUL padding after gzip members, moves past a line that fails strict UTF-8, and removes its temporary file when a commit fails. main.nf shows og.sh's stderr, the image check needs og version to exit 0, and the differential test compares readline() with its line endings. --- .github/workflows/guard.yml | 2 +- docs/rust.md | 8 +-- main.nf | 5 +- rust/crates/og-io/src/lib.rs | 100 ++++++++++++++++++++++++++++--- rust/crates/pycompat/src/json.rs | 6 +- rust/crates/pycompat/src/lib.rs | 3 +- rust/crates/pycompat/src/num.rs | 3 +- rust/og.sh | 73 +++++++++++++++------- rust/og/src/probe.rs | 12 +++- tests/rust/og-in-images.sh | 8 ++- tests/rust/pycompat_oracle.py | 11 +++- tests/test_og_check.sh | 32 +++++++++- 12 files changed, 215 insertions(+), 48 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 404e0d90..1d22d066 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -236,7 +236,7 @@ jobs: rc=0 out=$(timeout 600 nextflow run main.nf -profile test -stub -ansi-log false --outdir "${RUNNER_TEMP}/og-stale" 2>&1) || rc=$? git checkout -- rust/og/src/main.rs - if [ "$rc" -eq 0 ] || [[ "$out" != *"og is stale:"*"run scripts/setup.sh --og"* ]] || [[ "$out" == *"Submitted process"* ]]; then + if [ "$rc" -eq 0 ] || [ "$rc" -eq 124 ] || [[ "$out" != *"og is stale:"*"run scripts/setup.sh --og"* ]] || [[ "$out" == *"Submitted process"* ]]; then echo "$out" echo "ERROR: nextflow run did not stop on a stale og before any task (exit ${rc})" exit 1 diff --git a/docs/rust.md b/docs/rust.md index 248da18d..6a6cb280 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -63,20 +63,20 @@ A port must write exactly what the Python tool wrote. Most differences hide in n | `num` | `float(str)` and `int(str)` with Python's whitespace, signs and underscores; `math.fsum`; the builtin `sum`, which since Python 3.12 is a compensated sum and so differs from a plain loop | | `json` | `json.dumps` with `indent` (none or a number), `separators`, `ensure_ascii` (surrogate pairs for astral characters), `sort_keys` and `allow_nan`; `json.loads` with `NaN` and `Infinity`, where a duplicate key keeps its first place and takes the last value | -`og-io` reads a path or `-` as plain text, gzip or BGZF (sniffed by the magic bytes), splits lines as Python's text mode does (`\n`, `\r\n` and a lone `\r`), decodes UTF-8 strictly or with `replace`, and writes a file under a temporary name before renaming it into place. +`og-io` reads a path or `-` as plain text, gzip or BGZF (sniffed by the magic bytes; NUL padding after a gzip member is skipped, as Python's `gzip` does), splits lines as Python's text mode does (`\n`, `\r\n` and a lone `\r`), decodes UTF-8 strictly or with `replace`, and writes a file under a temporary name before renaming it into place. -Known limits, each an error in Rust where Python would go on: ints beyond 128 bits, a lone surrogate in a JSON string, non-ASCII digits in `float()` and `int()`, and a string `indent`. None of them appears in what the pipeline reads. +Known limits, each an error in Rust where Python would go on: ints beyond 128 bits, a lone surrogate in a JSON string, non-ASCII digits in `float()` and `int()`, a string `indent`, and JSON nested deeper than 1000 levels (`RecursionError`; CPython 3.14 reads about 100,000 on a default stack). None of them appears in what the pipeline reads. ### The differential test -`tests/rust/pycompat-diff.sh [N] [SEED]` checks every `pycompat` operation against CPython. `tests/rust/pycompat_oracle.py` runs with `python3 -I` in `PYTHON_IMAGE`, draws N random cases per operation and writes its result for each. `og pycompat-probe` reads the same cases and must write the same lines, byte for byte. The values include the ones that catch a formatting bug: binary ties (0.125, 2.675, 0.35), 1e-05 and 1e16 where `repr` switches layout, -0.0, NaN, subnormals, random bit patterns, numbers exactly halfway between two shortest `repr` candidates, astral characters, CRLF and lone CR, and broken JSON. A failure prints the first cases that differ and the seed, so `pycompat-diff.sh ` replays it. `PYTHON=python3` runs the oracle on the host for a quick loop. +`tests/rust/pycompat-diff.sh [N] [SEED]` checks every `pycompat` operation against CPython. `tests/rust/pycompat_oracle.py` runs with `python3 -I` in `PYTHON_IMAGE`, draws N random cases per operation and writes its result for each. `og pycompat-probe` reads the same cases and must write the same lines, byte for byte. The values include the ones that catch a formatting bug: binary ties (0.125, 2.675, 0.35), 1e-05 and 1e16 where `repr` switches layout, -0.0, NaN, subnormals, random bit patterns, numbers exactly halfway between two shortest `repr` candidates, astral characters, CRLF and lone CR (both as lines without their endings and as `readline()` results with them), and broken JSON. A failure prints the first cases that differ and the seed, so `pycompat-diff.sh ` replays it. `PYTHON=python3` runs the oracle on the host for a quick loop. To add a primitive: write it in `pycompat` with unit tests, add an operation to `og/src/probe.rs` and the same operation to the oracle, then run the differential test. ## CI - **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; both musl builds through `scripts/setup.sh --og`; and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. -- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a release download with a bad checksum or attestation, or with `gh` not logged in), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. +- **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a tree hash that cannot be computed, a release download with a bad checksum or attestation, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. - **Release** (`release.yml`): a tag builds both binaries, writes `SHA256SUMS`, attests them and attaches all three to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. diff --git a/main.nf b/main.nf index 22a1d92f..f0ad7ff0 100644 --- a/main.nf +++ b/main.nf @@ -101,11 +101,12 @@ workflow { // another rust/ than this checkout's would run other code than the // checkout says: rust/og.sh check stops the run (scripts/setup.sh --og). if (file("${projectDir}/rust/og.sh").exists()) { - def og_check = ['bash', "${projectDir}/rust/og.sh", 'check'].execute() + // stderr joins stdout, so a check that cannot even start says why. + def og_check = ['bash', '-c', 'bash "$0" check 2>&1', "${projectDir}/rust/og.sh"].execute() def og_msg = og_check.text.trim() og_check.waitFor() if (og_check.exitValue() != 0) { - error og_msg.replaceFirst(/^ERROR: /, '') + error(og_msg ? og_msg.replaceFirst(/^ERROR: /, '') : "rust/og.sh check failed with exit ${og_check.exitValue()} and no message") } } diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 956ce3d8..5ebbb18f 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -1,13 +1,14 @@ //! Reading and writing files the way OwnGenome's Python tools do. //! //! - [`open`]: a path or `-` (stdin), plain or gzip, sniffed by its magic -//! bytes. Multi-member gzip, BGZF included, is read to the end. +//! bytes. Multi-member gzip, BGZF included, is read to the end, and NUL +//! padding after a member is skipped, as Python's `gzip` does. //! - [`Lines`]: Python's text mode with `newline=None`: `\n`, `\r\n` and a //! lone `\r` each end a line and read as `\n`; UTF-8 strict or `replace`. //! - [`AtomicFile`]: written beside the target and renamed over it on //! [`AtomicFile::commit`], so a reader never sees half a file. -use flate2::bufread::MultiGzDecoder; +use flate2::bufread::GzDecoder; use std::fs::{self, File}; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; @@ -41,12 +42,54 @@ pub fn reader(mut inner: impl Read + 'static) -> io::Result> { let gz = head == GZIP_MAGIC; let buf = BufReader::with_capacity(1 << 16, io::Cursor::new(head).chain(inner)); Ok(if gz { - Box::new(BufReader::with_capacity(1 << 16, MultiGzDecoder::new(buf))) + Box::new(BufReader::with_capacity( + 1 << 16, + Members { + dec: Some(GzDecoder::new(buf)), + }, + )) } else { Box::new(buf) }) } +/// Gzip members one after another, as Python's `gzip` reads them: NUL bytes +/// between or after members are padding, and the stream may end in them. +struct Members { + /// The member being read; `None` once the input has ended. + dec: Option>, +} + +impl Read for Members { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + while let Some(dec) = self.dec.as_mut() { + let n = dec.read(buf)?; + if n > 0 { + return Ok(n); + } + // This member has ended: skip padding, then the next one starts. + let mut inner = self.dec.take().expect("a member").into_inner(); + loop { + let rest = inner.fill_buf()?; + if rest.is_empty() { + return Ok(0); + } + let zeros = rest.iter().take_while(|&&c| c == 0).count(); + let more = zeros < rest.len(); + inner.consume(zeros); + if more { + break; + } + } + self.dec = Some(GzDecoder::new(inner)); + } + Ok(0) + } +} + /// How invalid UTF-8 is read. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Utf8 { @@ -93,6 +136,10 @@ impl Lines { Some(k) => (&rest[..k], k + 1), }; let ended = used > body.len(); + // Past the line before decoding it, so a strict error is not raised + // again for the same line by a caller that goes on reading. (Python + // raises once too; a caller is expected to stop at the error.) + self.pos += used; match self.utf8 { Utf8::Strict => out.push_str( std::str::from_utf8(body) @@ -103,7 +150,6 @@ impl Lines { if ended { out.push('\n'); } - self.pos += used; Ok(true) } } @@ -155,11 +201,21 @@ impl AtomicFile { }) } - /// Flushes and renames over the target. + /// Flushes and renames over the target. On an error the temporary file + /// is removed and the target is untouched. pub fn commit(mut self) -> io::Result<()> { let out = self.out.take().expect("committed once"); - out.into_inner().map_err(io::IntoInnerError::into_error)?; - fs::rename(&self.tmp, &self.target) + let done = out + .into_inner() + .map_err(io::IntoInnerError::into_error) + .and_then(|f| { + drop(f); + fs::rename(&self.tmp, &self.target) + }); + if done.is_err() { + let _ = fs::remove_file(&self.tmp); + } + done } } @@ -208,8 +264,13 @@ mod tests { lines(b"\xff\xe2\x82\n", Utf8::Replace), ["\u{fffd}\u{fffd}\n"] ); - let mut l = Lines::new(io::Cursor::new(b"\xff\n".to_vec()), Utf8::Strict); - assert!(l.read_line(&mut String::new()).is_err()); + // A strict error on one line is raised once; reading goes on after it. + let mut l = Lines::new(io::Cursor::new(b"\xff\nok\n".to_vec()), Utf8::Strict); + let mut s = String::new(); + assert!(l.read_line(&mut s).is_err()); + assert!(l.read_line(&mut s).unwrap()); + assert_eq!(s, "ok\n"); + assert!(!l.read_line(&mut s).unwrap()); let it: Vec = Lines::new(io::Cursor::new(b"a\r\nb".to_vec()), Utf8::Strict) .map(Result::unwrap) .collect(); @@ -226,10 +287,17 @@ mod tests { fn reads_plain_gzip_and_multi_member() { let mut two = gz(b"one\n"); two.extend(gz(b"two\n")); + // NUL padding between and after members, as Python's gzip skips it. + let mut padded = gz(b"one\n"); + padded.extend([0u8; 16]); + padded.extend(gz(b"")); + padded.extend(gz(b"two\n")); + padded.extend([0u8; 12]); for (input, want) in [ (b"plain\n".to_vec(), "plain\n"), (gz(b"zipped\n"), "zipped\n"), (two, "one\ntwo\n"), + (padded, "one\ntwo\n"), (vec![0x1f], "\u{1f}"), ] { let mut s = String::new(); @@ -239,6 +307,12 @@ mod tests { .unwrap(); assert_eq!(s, want); } + // Anything else after a member is an error, as in Python. + let mut junk = gz(b"one\n"); + junk.extend(b"\0\0junk"); + let mut s = String::new(); + let mut r = reader(io::Cursor::new(junk)).unwrap(); + assert!(r.read_to_string(&mut s).is_err()); } #[test] @@ -257,6 +331,14 @@ mod tests { f.commit().unwrap(); assert_eq!(fs::read_to_string(&target).unwrap(), "new"); assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + // A rename that fails (the target is a directory that is not empty) + // leaves no temporary file behind. + let busy = dir.join("busy"); + fs::create_dir_all(busy.join("x")).unwrap(); + let mut f = AtomicFile::create(&busy).unwrap(); + f.write_all(b"data").unwrap(); + assert!(f.commit().is_err()); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 2); fs::remove_dir_all(&dir).unwrap(); } } diff --git a/rust/crates/pycompat/src/json.rs b/rust/crates/pycompat/src/json.rs index ad53b7c0..5a6bdecc 100644 --- a/rust/crates/pycompat/src/json.rs +++ b/rust/crates/pycompat/src/json.rs @@ -58,8 +58,10 @@ pub enum JsonError { TooDeep, } -/// How deep `loads` nests arrays and objects before it gives up, about -/// where CPython's default recursion limit (1000) stops `json.loads`. +/// How deep `loads` nests arrays and objects before it gives up. CPython +/// 3.14 goes deeper (its limit follows the C stack); the parser here is +/// recursive, so it stops well inside an 8 MiB stack. A known limit +/// (`docs/rust.md`). pub const MAX_DEPTH: usize = 1000; impl JsonError { diff --git a/rust/crates/pycompat/src/lib.rs b/rust/crates/pycompat/src/lib.rs index 9524737c..7aef3352 100644 --- a/rust/crates/pycompat/src/lib.rs +++ b/rust/crates/pycompat/src/lib.rs @@ -1,7 +1,8 @@ //! Python-exact primitives for the Rust ports of OwnGenome's Python tools. //! //! Every function here returns the bytes CPython 3.15 returns for the same -//! input. The differential test (`tests/rust/pycompat-diff.sh`) checks that +//! input, except for the known limits in `docs/rust.md`, where it returns an +//! error instead. The differential test (`tests/rust/pycompat-diff.sh`) checks that //! against `python3 -I` in the pinned `PYTHON_IMAGE`. //! //! - [`fmt`]: `repr(float)`, the `f`, `e`, `g` and `%` format types, `{:,}`, diff --git a/rust/crates/pycompat/src/num.rs b/rust/crates/pycompat/src/num.rs index 612b62f3..cb4f9e8a 100644 --- a/rust/crates/pycompat/src/num.rs +++ b/rust/crates/pycompat/src/num.rs @@ -144,7 +144,8 @@ pub fn parse_float(s: &str) -> Result { } /// `int(s)` for a str in base 10: surrounding whitespace, a sign, leading -/// zeros and underscores between digits. +/// zeros and underscores between digits. Non-ASCII digits, which Python also +/// reads, are refused, and so is a value beyond `i128`. pub fn parse_int(s: &str) -> Result { let t = s.trim_matches(number_space); let (neg, body) = match t.as_bytes().first() { diff --git a/rust/og.sh b/rust/og.sh index 67aee468..01804307 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -7,7 +7,7 @@ # rust/og.sh install scripts/setup.sh --og: on a release tag (vX.Y.Z, # clean rust/) download the release's binaries and # check them against SHA256SUMS (and the build -# attestation when gh is installed); otherwise, or +# attestation when gh is logged in); otherwise, or # when the release has none, build them # rust/og.sh build build both from rust/ in OG_BUILDER (versions.env) # rust/og.sh check [--required] @@ -37,6 +37,7 @@ ENGINE=${CONTAINER_ENGINE:-docker} triple() { echo "${1}-unknown-linux-musl"; } +# tree_hash: exit 3 when this is not a git checkout, 1 when git fails. tree_hash() { local idx rc=0 git -C "$ROOT" rev-parse --verify -q HEAD:rust > /dev/null 2>&1 || return 3 @@ -44,12 +45,12 @@ tree_hash() { git -C "$ROOT" rev-parse HEAD:rust return fi - idx=$(mktemp -d "${TMPDIR:-/tmp}/og-tree.XXXXXX") + idx=$(mktemp -d "${TMPDIR:-/tmp}/og-tree.XXXXXX") || return 1 { GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" read-tree HEAD \ && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" add -A -- rust \ && GIT_INDEX_FILE="${idx}/index" git -C "$ROOT" write-tree --prefix=rust/; } || rc=$? rm -rf "$idx" - return "$rc" + [ "$rc" -eq 0 ] || return 1 } # embedded FILE: the hash stamped into a binary, or "unknown". @@ -61,9 +62,15 @@ embedded() { } check() { - local required=$REQUIRED expected f h missing="" stale="" + local required=$REQUIRED expected f h missing="" stale="" trc=0 [ "${1:-}" != --required ] || required=true - expected=$(tree_hash 2> /dev/null) || expected="" + expected=$(tree_hash 2> /dev/null) || trc=$? + if [ "$trc" -eq 3 ]; then + expected="" + elif [ "$trc" -ne 0 ]; then + echo "ERROR: the git tree hash of ${ROOT}/rust could not be computed, so og cannot be checked (is TMPDIR writable?)" + return 1 + fi for a in $ARCHES; do f="${BIN}/og-linux-${a}" if [ ! -x "$f" ]; then missing+=" ${a}"; continue; fi @@ -100,16 +107,27 @@ built() { if [ "${#h[@]}" -eq 2 ] && [ "${h[0]#*=}" = "${h[1]#*=}" ]; then echo "${h[0]#*=}"; else echo "$hs"; fi } -# put SRC ARCH: move a binary into bin/.og in one rename. +# put SRC...: one binary per arch of ARCHES, in that order, into bin/.og. +# All are copied beside their targets first; only when every copy worked are +# they renamed into place, so a failed copy (a full disk) leaves the old pair. put() { - mkdir -p "$BIN" - cp "$1" "${BIN}/.og-linux-${2}.tmp" - chmod 755 "${BIN}/.og-linux-${2}.tmp" - mv -f "${BIN}/.og-linux-${2}.tmp" "${BIN}/og-linux-${2}" + local a i=0 srcs=("$@") + mkdir -p "$BIN" || return 1 + for a in $ARCHES; do + if ! { cp "${srcs[i]}" "${BIN}/.og-linux-${a}.tmp" && chmod 755 "${BIN}/.og-linux-${a}.tmp"; }; then + echo "ERROR: could not copy og-linux-${a} into ${BIN}" >&2 + for a in $ARCHES; do rm -f "${BIN}/.og-linux-${a}.tmp"; done + return 1 + fi + i=$((i + 1)) + done + for a in $ARCHES; do + mv -f "${BIN}/.og-linux-${a}.tmp" "${BIN}/og-linux-${a}" || { echo "ERROR: could not install ${BIN}/og-linux-${a}" >&2; return 1; } + done } build() { - local hash cache t0 a out targets=() + local hash cache t0 a out targets=() outs=() hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout: og is stamped with the git tree of rust/, so it builds only in one" >&2; return 1; } # shellcheck source=../versions.env . "${ROOT}/versions.env" @@ -128,8 +146,9 @@ build() { for a in $ARCHES; do out="${cache}/target/$(triple "$a")/release/og" [ "$(embedded "$out")" = "$hash" ] || { echo "ERROR: ${out} does not carry tree ${hash}" >&2; return 1; } - put "$out" "$a" + outs+=("$out") done + put "${outs[@]}" || return 1 echo "Built bin/.og/og-linux-{${ARCHES// /,}} in $(($(date +%s) - t0)) s" } @@ -137,14 +156,22 @@ sha256() { if command -v sha256sum > /dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi } -get() { # get URL DEST: 0 when downloaded, 1 when not - if command -v curl > /dev/null 2>&1; then curl -fsL --retry 3 -o "$2" "$1"; else wget -q -O "$2" "$1"; fi +get() { # get URL DEST: 0 when downloaded, 2 when the server says 404, 1 on any other error + local code + if command -v curl > /dev/null 2>&1; then + code=$(curl -sL --retry 3 -o "$2" -w '%{http_code}' "$1") || return 1 + case "$code" in 2??) return 0 ;; 404) return 2 ;; *) return 1 ;; esac + fi + wget -q -O "$2" "$1" && return 0 + # wget exits 8 on an error response from the server; that is all it tells. + [ "$?" -eq 8 ] && return 2 || return 1 } # fetch TAG: 0 when the release's binaries are installed, 2 when the release -# has none (build instead), 1 when a download does not check out. +# has none (SHA256SUMS is not there: build instead), 1 when a download fails +# or does not check out. fetch() { - local tag=$1 tmp want hash a f rc=0 attest=0 + local tag=$1 tmp want hash a f rc=0 grc=0 attest=0 hash=$(tree_hash) # gh attestation verify needs a login: without one (gh missing, or never # logged in) only SHA256SUMS is checked. gh auth token, not gh auth status, @@ -153,22 +180,28 @@ fetch() { tmp=$(mktemp -d "${TMPDIR:-/tmp}/og-fetch.XXXXXX") # OG_RELEASE_BASE: another server with the same layout (tests/test_og_check.sh). local base="${OG_RELEASE_BASE:-https://github.com/${REPO}/releases/download}/${tag}" - if ! get "${base}/SHA256SUMS" "${tmp}/SHA256SUMS"; then + get "${base}/SHA256SUMS" "${tmp}/SHA256SUMS" || grc=$? + if [ "$grc" -ne 0 ]; then rm -rf "$tmp" - return 2 + [ "$grc" -eq 2 ] && return 2 + echo "ERROR: could not download ${base}/SHA256SUMS; check the network and run scripts/setup.sh --og again" >&2 + return 1 fi for a in $ARCHES; do f="og-linux-${a}" want=$(awk -v f="$f" '$2 == f || $2 == "*" f {print $1}' "${tmp}/SHA256SUMS") - if [ -z "$want" ] || ! get "${base}/${f}" "${tmp}/${f}"; then echo "ERROR: release ${tag} lists no ${f}" >&2; rc=1; break; fi + if [ -z "$want" ]; then echo "ERROR: release ${tag} lists no ${f}" >&2; rc=1; break; fi + if ! get "${base}/${f}" "${tmp}/${f}"; then echo "ERROR: could not download ${f} of release ${tag}" >&2; rc=1; break; fi if [ "$(sha256 "${tmp}/${f}")" != "$want" ]; then echo "ERROR: ${f} of release ${tag} does not match its SHA256SUMS" >&2; rc=1; break; fi if [ "$attest" = 1 ]; then gh attestation verify "${tmp}/${f}" --repo "$REPO" > /dev/null || { echo "ERROR: ${f} of release ${tag} has no valid build attestation" >&2; rc=1; break; } fi if [ "$(embedded "${tmp}/${f}")" != "$hash" ]; then echo "ERROR: ${f} of release ${tag} was not built from this rust/ (tree ${hash})" >&2; rc=1; break; fi done + local srcs=() + for a in $ARCHES; do srcs+=("${tmp}/og-linux-${a}"); done + if [ "$rc" -eq 0 ] && ! put "${srcs[@]}"; then rc=1; fi if [ "$rc" -eq 0 ]; then - for a in $ARCHES; do put "${tmp}/og-linux-${a}" "$a"; done [ "$attest" = 1 ] || echo "NOTE: gh is not installed or not logged in, so the build attestation was not checked (SHA256SUMS was)" echo "Installed og ${hash} from release ${tag}" fi diff --git a/rust/og/src/probe.rs b/rust/og/src/probe.rs index bd3387e3..d2b859e9 100644 --- a/rust/og/src/probe.rs +++ b/rust/og/src/probe.rs @@ -143,6 +143,16 @@ fn eval(f: &[&str]) -> io::Result { .collect::>() .join("|")); } + // readline() keeps each line's ending, so a missing or extra "\n" shows. + "readline" => { + let mut l = Lines::new(io::Cursor::new(bytes(arg(1)?)?), Utf8::Replace); + let mut line = String::new(); + let mut all = Vec::new(); + while l.read_line(&mut line)? { + all.push(escape(&line)); + } + return Ok(all.join("|")); + } op => return Err(bad(format!("unknown operation {op:?}"))), }) } @@ -154,7 +164,7 @@ pub fn run(input: impl BufRead, output: impl Write) -> io::Result<()> { let fields: Vec<&str> = line.split('\t').collect(); let result = eval(&fields)?; // A lines result is escaped line by line already. - if fields[0] == "lines" { + if matches!(fields[0], "lines" | "readline") { writeln!(out, "{result}")?; } else { writeln!(out, "{}", escape(&result))?; diff --git a/tests/rust/og-in-images.sh b/tests/rust/og-in-images.sh index e374d720..f86ea4a8 100755 --- a/tests/rust/og-in-images.sh +++ b/tests/rust/og-in-images.sh @@ -23,20 +23,22 @@ fail=0 runs=0 run_og() { # run_og IMAGE PLATFORM BINDIR: og version, as a task would call it # -q: a first pull prints its progress on the output compared below. docker run -q --rm --pull missing --platform "$2" -v "${3}:${3}:ro" -e "PATH=${3}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ - --entrypoint sh "$1" -c 'echo "$(uname -m) $(og version)"' 2>&1 + --entrypoint sh "$1" -c 'v=$(og version) && echo "$(uname -m) $v"' 2>&1 } printf '%-18s %-12s %-8s %s\n' image platform result "uname -m and og version" for var in "$@"; do img=${!var:?$var is not set in versions.env} for plat in linux/amd64 linux/arm64; do - out=$(run_og "$img" "$plat" "$BIN") || true + rc=0 + out=$(run_og "$img" "$plat" "$BIN") || rc=$? if [[ "$out" == *"does not provide the specified platform"* || "$out" == *"no matching manifest"* ]]; then printf '%-18s %-12s %-8s %s\n' "$var" "$plat" skipped "the image has no ${plat} build" continue fi runs=$((runs + 1)) want=$([ "$plat" = linux/amd64 ] && echo x86_64 || echo aarch64) - if [[ "$out" == "${want} og rust tree "*" (${want}-unknown-linux-musl)" ]]; then r=ok; else r=FAIL; fail=1; fi + # og version must also exit 0: a task that calls it fails otherwise. + if [ "$rc" -eq 0 ] && [[ "$out" == "${want} og rust tree "*" (${want}-unknown-linux-musl)" ]]; then r=ok; else r=FAIL; fail=1; out="${out} (exit ${rc})"; fi printf '%-18s %-12s %-8s %s\n' "$var" "$plat" "$r" "$out" done done diff --git a/tests/rust/pycompat_oracle.py b/tests/rust/pycompat_oracle.py index d20d955d..894aed18 100644 --- a/tests/rust/pycompat_oracle.py +++ b/tests/rust/pycompat_oracle.py @@ -26,7 +26,7 @@ 0.25, 1.005, 1.115, 4.35, 8.345, 99.995, 1234.5, 2.0 ** 52 + 0.5, 2.0 ** 53, 2.0 ** 60, 1e300, -1e-300, ] OPS = ["repr", "f", "e", "g", "pct", "grpf", "round0", "round", "grp", "zpad", "ljust", "rjust", - "float", "int", "fsum", "sum", "json", "lines"] + "float", "int", "fsum", "sum", "json", "lines", "readline"] WHITESPACE = " \t\n\x0b\x0c\r\x1c\x1d\x1e\x1f\x85\xa0\u1680\u2000\u2005\u200a\u2028\u2029\u202f\u205f\u3000" @@ -253,6 +253,13 @@ def run(): t = io.TextIOWrapper(io.BytesIO(b), encoding="utf-8", errors="replace", newline=None) return "|".join(escape(line[:-1] if line.endswith("\n") else line) for line in t) return [b.hex()], run + if op == "readline": + b = a_lines_input(r) + + def run(): + t = io.TextIOWrapper(io.BytesIO(b), encoding="utf-8", errors="replace", newline=None) + return "|".join(escape(line) for line in iter(t.readline, "")) + return [b.hex()], run raise ValueError(op) @@ -272,7 +279,7 @@ def main(): except Exception as e: # the exception class is the result res = type(e).__name__ out.write("\t".join([op] + args) + "\n") - out.write((res if op == "lines" else escape(res)) + "\n") + out.write((res if op in ("lines", "readline") else escape(res)) + "\n") out.flush() diff --git a/tests/test_og_check.sh b/tests/test_og_check.sh index 148aa6f1..e7ec4468 100755 --- a/tests/test_og_check.sh +++ b/tests/test_og_check.sh @@ -5,8 +5,10 @@ # (og-tree-hash:;). Checked: no binary is a note (an error with # --required); binaries of this rust/ pass; an edit to rust/, uncommitted, # untracked or committed, makes them stale; a file git ignores does not; a -# binary stamped "unknown" is stale; built prints the hash for -# run_manifest.tsv; outside git the hash cannot be checked. Then the callers: +# binary stamped "unknown" is stale; a tree hash that cannot be computed is +# an error; built prints the hash for run_manifest.tsv; outside git the hash +# cannot be checked. Release downloads: checksum, attestation, a network +# error, a release without binaries, and a copy into bin/.og that fails. Then the callers: # run-all.sh (with SKIP_VALIDATION=true) and validate-setup.sh refuse a stale # binary with the setup command, and write_manifest.sh records the hash. set -uo pipefail @@ -85,6 +87,13 @@ out=$(bash "$OG" check); rc=$? expect "a binary stamped unknown (plain cargo build): stale" "$rc" 1 "$out" "ERROR: og is stale:*og-linux-x86_64 (rust/ tree unknown)*" out=$(bash "$OG" built); [[ "$out" == "x86_64=unknown aarch64=${h1}" ]] && ok "built lists both hashes when they differ" || bad "built printed '${out}'" +# The tree hash of an edited rust/ cannot be computed (TMPDIR not writable): +# an error, never the note of a checkout without git. +echo "// edited" >> "${C}/rust/og/src/main.rs" +out=$(TMPDIR="${T}/no-such-dir" bash "$OG" check 2> /dev/null); rc=$? +expect "a tree hash that fails is an error" "$rc" 1 "$out" "ERROR: the git tree hash of*could not be computed*" +g checkout -q -- rust/og/src/main.rs + # The callers, on a stale binary. plant "$h0" mkdir -p "${T}/fake" "${T}/genome" @@ -148,6 +157,25 @@ exit 0 out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? expect "a binary that does not match SHA256SUMS is refused" "$rc" 1 "$out" "*og-linux-aarch64 of release v9.9.9 does not match its SHA256SUMS*" [ ! -e "${C}/bin/.og/og-linux-x86_64" ] && ok "a refused release installs nothing" || bad "a refused release left a binary" + # A copy that fails (bin/.og not writable) keeps the old pair and fails. + (cd "$S" && for a in x86_64 aarch64; do printf 'og-tree-hash:%s;' "$h1" > "og-linux-${a}"; done \ + && { sha256sum og-linux-* 2> /dev/null || shasum -a 256 og-linux-*; } > SHA256SUMS) + plant "$h0" + chmod 555 "${C}/bin/.og" + out=$(PATH="${T}/fake:${PATH}" OG_RELEASE_BASE="http://127.0.0.1:${port}" bash "$OG" install 2>&1); rc=$? + chmod 755 "${C}/bin/.og" + expect "a copy into bin/.og that fails is an error" "$rc" 1 "$out" "*could not copy og-linux-x86_64*" + [[ "$out" != *"Installed og"* ]] && [ "$(bash "$OG" built)" = "$h0" ] && [ -z "$(ls -A "${C}/bin/.og" | grep tmp)" ] \ + && ok "a failed copy keeps the old pair and leaves no temporary file" || bad "after a failed copy: $(ls -A "${C}/bin/.og"), built $(bash "$OG" built)" + rm -rf "${C}/bin/.og" + # A network error on SHA256SUMS is an error, never a fallback to a build. + printf '#!/bin/sh\necho "fake engine: must not be reached" >&2\nexit 99\n' > "${T}/fake/engine"; chmod 755 "${T}/fake/engine" + out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=engine OG_RELEASE_BASE="http://127.0.0.1:1" bash "$OG" install 2>&1); rc=$? + expect "a release download that fails is an error, not a build" "$rc" 1 "$out" "*could not download http://127.0.0.1:1/v9.9.9/SHA256SUMS*" + [[ "$out" != *"must not be reached"* ]] && ok "no build after a failed download" || bad "a failed download started a build" + # A release without SHA256SUMS (404) has no binaries: build instead. + out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=engine OG_RELEASE_BASE="http://127.0.0.1:${port}/none" bash "$OG" install 2>&1); rc=$? + expect "a release without binaries falls back to a build" "$rc" 99 "$out" "*has no og binaries: building them*must not be reached*" kill "$srv" 2> /dev/null; wait "$srv" 2> /dev/null g tag -d v9.9.9 > /dev/null else