diff --git a/conformance/canonicalize/float_exact_decimal_tie/expected.yaml b/conformance/canonicalize/float_exact_decimal_tie/expected.yaml new file mode 100644 index 0000000..8a54a14 --- /dev/null +++ b/conformance/canonicalize/float_exact_decimal_tie/expected.yaml @@ -0,0 +1,3 @@ +outcome: accepted +canonical: '{"a":912968984149762.2,"b":-912968984149762.2,"c":730721488272018.2,"d":126065375359190.62,"e":0.5}' +why: A3: when the double's exact value lies exactly halfway between two equally short decimals, Go's strconv rounds half to even on the shortest digit string (912968984149762.25 -> ...762.2). Rust's Display chose the odd neighbour (...762.3) until #57; the two languages produced different digests for the same number. Found by the generated cross-language corpus (#54); values checked against real Go output. diff --git a/conformance/canonicalize/float_exact_decimal_tie/input.yaml b/conformance/canonicalize/float_exact_decimal_tie/input.yaml new file mode 100644 index 0000000..b70e48d --- /dev/null +++ b/conformance/canonicalize/float_exact_decimal_tie/input.yaml @@ -0,0 +1,5 @@ +a: 912968984149762.3 +b: -912968984149762.25 +c: 730721488272018.3 +d: 126065375359190.63 +e: 0.5 diff --git a/engine/src/canonical.rs b/engine/src/canonical.rs index 1fb9e44..33b3b83 100644 --- a/engine/src/canonical.rs +++ b/engine/src/canonical.rs @@ -31,6 +31,9 @@ use crate::error::{code, Error, Result, Stage}; use crate::reader::{Map, Value}; +use num_bigint::BigInt; +use num_rational::BigRational; +use num_traits::{FromPrimitive as _, Pow as _}; use sha2::{Digest as _, Sha256}; /// Canonical JSON bytes for `v` (A1–A6): compact, no inter-token whitespace, @@ -106,16 +109,111 @@ fn write_value(out: &mut Vec, v: &Value, path: &str) -> Result<()> { /// A3, ported from `cic-canonical`'s own `canonical_float` (not imported as a /// dependency — see module docs): shortest round-trip plain-decimal form, -/// `-0.0` folded to `0`. Rust's `f64` `Display` is shortest round-trip and -/// never uses an exponent, matching Go's `FormatFloat(f, 'f', -1, 64)` — -/// verified empirically in this session (`4.0` -> `"4"`, `1000.0` -> -/// `"1000"`, not `"1e3"`). +/// `-0.0` folded to `0`, never an exponent, matching Go's +/// `FormatFloat(f, 'f', -1, 64)` — verified empirically in this session +/// (`4.0` -> `"4"`, `1000.0` -> `"1000"`, not `"1e3"`). The digits come from +/// [`shortest_digits`], which follows Go's tie rule where Rust's own +/// `Display` does not (#57). pub(crate) fn canonical_float(f: f64) -> String { - let s = format!("{f}"); - if s == "-0" { - "0".to_string() + if f == 0.0 { + return "0".to_string(); // both zeros, `-0.0` included + } + let (digits, exp) = shortest_digits(f.abs()); + let mut out = String::new(); + if f < 0.0 { + out.push('-'); + } + if exp >= 0 { + let int_len = (exp + 1) as usize; + if digits.len() <= int_len { + out.push_str(&digits); + out.push_str(&"0".repeat(int_len - digits.len())); + } else { + out.push_str(&digits[..int_len]); + out.push('.'); + out.push_str(&digits[int_len..]); + } + } else { + out.push_str("0."); + out.push_str(&"0".repeat((-exp - 1) as usize)); + out.push_str(&digits); + } + out +} + +/// The shortest decimal digits that round-trip `a` (finite, positive), as +/// Go's `strconv` produces them: `(digits, exp)` with no trailing zeros and +/// `exp` the decimal exponent of the leading digit (so `a ~ d.ddd * 10^exp`, +/// the shape Rust's `{:e}` prints). +/// +/// Rust's `{:e}` and Go agree on the digit COUNT and on which candidate is +/// closest to the exact value. They differ when the exact binary value lies +/// exactly halfway between two equally short candidates (e.g. +/// `912968984149762.25` between `...762.2` and `...762.3`): Go rounds half to +/// even on the shortest digit string (`strconv/ftoaryu.go`, `ryuDigits32`), +/// Rust may pick the odd neighbour. A tie needs the exact expansion to be one +/// digit longer than the shortest form and end in 5, so a single exact-mode +/// format is the cheap pre-check; only then is the tie confirmed with exact +/// rational arithmetic. +pub(crate) fn shortest_digits(a: f64) -> (String, i32) { + debug_assert!(a.is_finite() && a > 0.0); + let (digits, exp) = split_sci(&format!("{a:e}")); + let n = digits.len(); + let (exact, exact_exp) = split_sci(&format!("{a:.n$e}")); + // A carry in the exact rendering (9.99..5 -> 10.0) means it is not a tie + // of `n`-digit neighbours at this exponent. + if exact_exp != exp || exact.len() != n + 1 || !exact.ends_with('5') { + return (digits, exp); + } + let (Ok(x), Ok(shown)) = (exact[..n].parse::(), digits.parse::()) else { + return (digits, exp); + }; + // exact == (10x + 5) * 10^(exp - n) exactly? + let Some(exact_value) = BigRational::from_f64(a) else { + return (digits, exp); + }; + let tie = BigRational::new(BigInt::from(10 * x + 5), BigInt::from(1)) * pow10(exp - n as i32); + if exact_value != tie { + return (digits, exp); + } + // A genuine tie between `x` and `x + 1`; Go takes the even one, if it + // is inside the rounding interval (i.e. parses back to `a`). + let other = if shown == x { x + 1 } else { x }; + let go_picks_other = shown % 2 == 1 + && other % 2 == 0 + && format!("{other}e{}", exp - (n as i32 - 1)) + .parse::() + .is_ok_and(|back| back == a); + if !go_picks_other { + return (digits, exp); + } + // `other` can be 10^n (carry from x = 99..9): one digit, exponent + 1. + let text = other.to_string(); + let carry = text.len() > n; + ( + text.trim_end_matches('0').to_string(), + if carry { exp + 1 } else { exp }, + ) +} + +/// `"d.ddde-5"` -> `("ddd", -5)`. +fn split_sci(sci: &str) -> (String, i32) { + let (mantissa, exp) = sci + .split_once('e') + .expect("Rust's `{:e}` formatting always contains 'e'"); + ( + mantissa.chars().filter(char::is_ascii_digit).collect(), + exp.parse() + .expect("the exponent after 'e' is always a valid integer"), + ) +} + +fn pow10(e: i32) -> BigRational { + let p = BigInt::from(10).pow(e.unsigned_abs()); + if e >= 0 { + BigRational::from_integer(p) } else { - s + BigRational::new(BigInt::from(1), p) } } @@ -305,6 +403,62 @@ mod tests { // reachable path, verified empirically against saphyr, not a // hypothetical API misuse) must not silently become an invalid JSON // token (`NaN`, `inf`) in what claims to be canonical JSON output. + // #57. Every expected string below is real Go output + // (`strconv.FormatFloat(f, 'f', -1, 64)`), taken from a 1.5M-float + // comparison; the doubles are exact decimal ties, where Go rounds half + // to even on the shortest digit string and Rust's `Display` did not. + #[test] + fn exact_decimal_ties_round_half_even_like_go() { + for (bits, want) in [ + (0x4309_f2b9_3fed_f812_u64, "912968984149762.2"), // ...762.25, odd neighbour .3 rejected + (0x4304_c4b2_d5e7_d492, "730721488272018.2"), + (0x42dc_a9f8_7aac_35a8, "126065375359190.62"), + (0x430a_60df_f0e3_7f42, "928108041236456.2"), + ] { + let f = f64::from_bits(bits); + assert_eq!(canonical_float(f), want, "{bits:#x}"); + assert_eq!(canonical_float(-f), format!("-{want}"), "-{bits:#x}"); + } + } + + #[test] + fn ties_that_already_end_even_and_non_ties_are_unchanged() { + // 912968984149762.75 (a tie whose even neighbour is .8; Go: .8). + assert_eq!( + canonical_float(f64::from_bits(0x4309_f2b9_3fed_f816)), + "912968984149762.8" + ); + // Not ties: the shortest digits are unambiguous. + for (f, want) in [ + (0.1, "0.1"), + (0.5, "0.5"), + (1.5, "1.5"), + (2.5, "2.5"), + (1e21, "1000000000000000000000"), + (123456.789, "123456.789"), + (5e-324, &*format!("0.{}5", "0".repeat(323))), + ] { + assert_eq!(canonical_float(f), want, "{f:e}"); + } + } + + #[test] + fn shortest_digits_only_changes_on_a_true_tie() { + // The same value as the first tie above, spelled via its neighbours: + // every spelling of the double canonicalizes identically. + let tie = f64::from_bits(0x4309_f2b9_3fed_f812); + for text in [ + "912968984149762.2", + "912968984149762.25", + "912968984149762.3", + ] { + let f: f64 = text.parse().unwrap(); + assert_eq!(f.to_bits(), tie.to_bits(), "{text}"); + assert_eq!(canonical_float(f), "912968984149762.2", "{text}"); + } + assert_eq!(shortest_digits(tie), ("9129689841497622".to_string(), 14)); + } + #[test] fn non_finite_floats_are_rejected_not_written() { for v in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { diff --git a/engine/src/collection.rs b/engine/src/collection.rs index d4e8dfc..f79d9fe 100644 --- a/engine/src/collection.rs +++ b/engine/src/collection.rs @@ -200,23 +200,12 @@ fn go_float_display(f: f64) -> Option { "0".to_string() }); } - // Rust's `{:e}` is exactly the normalized form this needs: one - // nonzero digit before the point, the shortest round-trip digit - // string after it, and the decimal exponent of that leading digit -- - // the same quantity Go's algorithm thresholds on. - let sci = format!("{f:e}"); - let (mantissa_part, exp_part) = sci - .split_once('e') - .expect("Rust's `{:e}` formatting always contains 'e'"); - let exp: i32 = exp_part - .parse() - .expect("the exponent after 'e' is always a valid integer"); - let neg = mantissa_part.starts_with('-'); - let digits: String = mantissa_part - .trim_start_matches('-') - .chars() - .filter(|c| *c != '.') - .collect(); + // One nonzero digit before the point, the shortest round-trip digit + // string after it, and the decimal exponent of that leading digit -- the + // same quantity Go's algorithm thresholds on. `shortest_digits` follows + // Go's tie rule where Rust's `{:e}` does not (#57). + let neg = f.is_sign_negative(); + let (digits, exp) = crate::canonical::shortest_digits(f.abs()); let mut out = String::new(); if neg { diff --git a/go/canonical/canonical_test.go b/go/canonical/canonical_test.go index 0eb7264..a186b81 100644 --- a/go/canonical/canonical_test.go +++ b/go/canonical/canonical_test.go @@ -319,3 +319,35 @@ func TestMalformedNumberIsAnErrorNotAPanic(t *testing.T) { t.Error("nested malformed json.Number was accepted") } } + +// #57: where a double lies exactly halfway between two equally short decimals, +// Go's strconv rounds half to even on the shortest digit string, and the Rust +// engine must agree byte for byte (see canonical.rs, shortest_digits). These +// pin the reference side so a change in the Go toolchain's behaviour is noticed +// instead of silently turning the Rust port into the odd one out. +func TestFloatExactDecimalTiesRoundHalfEven(t *testing.T) { + for _, c := range []struct { + bits uint64 + want string + }{ + {0x4309f2b93fedf812, "912968984149762.2"}, + {0x4304c4b2d5e7d492, "730721488272018.2"}, + {0x42dca9f87aac35a8, "126065375359190.62"}, + {0x430a60dff0e37f42, "928108041236456.2"}, + } { + f := math.Float64frombits(c.bits) + if got := CanonicalFloat(f); got != c.want { + t.Errorf("%#x: CanonicalFloat = %s, want %s", c.bits, got, c.want) + } + if got := CanonicalFloat(-f); got != "-"+c.want { + t.Errorf("-%#x: CanonicalFloat = %s, want -%s", c.bits, got, c.want) + } + } + // The literal that exposed it: three spellings, one double. + for _, lit := range []string{"912968984149762.2", "912968984149762.25", "912968984149762.3"} { + got, err := ToCanonicalJSON(json.Number(lit)) + if err != nil || string(got) != "912968984149762.2" { + t.Errorf("%s -> %s, %v", lit, got, err) + } + } +}