From 85a42af914b8ffb70254f186a26b7b0bc3fcb118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 17 Oct 2026 13:35:42 +0200 Subject: [PATCH 1/2] fix: follow Go's round-half-even on exact decimal ties in float formatting (#57) When a double's exact value lies exactly halfway between two equally short decimals, Go's strconv rounds half to even on the shortest digit string (ftoaryu.go, ryuDigits32); Rust's {:e}/Display could pick the odd neighbour. 912968984149762.3 is the double 912968984149762.25: Go prints ...762.2, Rust printed ...762.3, so the same document produced different canonical bytes and digests in the two languages. Found by the generated cross-language corpus added for #54. - canonical: shortest_digits applies Go's tie rule. A tie needs the exact expansion to be one digit longer than the shortest form and end in 5, so one exact-mode format is the cheap pre-check; the tie itself is confirmed with exact rational arithmetic, and the even neighbour is used only if it parses back to the same double. canonical_float renders from it. - collection: go_float_display (the %v port) uses the same routine. Measured against real Go on 1.5M floats (j * 2^-q, large values with eighth fractions, random bit patterns, decimal-looking values): 44,029 mismatches before (about 3%), 0 after, for both the canonical form and the %v form. Tests: unit tests with Go-verified values, a canonicalize corpus vector (float_exact_decimal_tie), and a Go test pinning the reference behaviour. Disabling the tie rule fails the unit tests and the corpus vector. Closes #57 --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCIQCG0PK/2i5B3O8SH5yQmdU/n+2CszFgQA9MyEDdQd7MIAIgFF/+QUYv+cs+sOdD6Tf+AiEWcMVGpfUUwglnGpOsOcE= hash-algorithm = sha256 digest = vOrwqFncAtxzlHXC2jKAaTdMNvqSOoxBhVloXAxfOC4= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- .../float_exact_decimal_tie/expected.yaml | 3 + .../float_exact_decimal_tie/input.yaml | 5 + engine/src/canonical.rs | 167 +++++++++++++++++- engine/src/collection.rs | 23 +-- go/canonical/canonical_test.go | 32 ++++ 5 files changed, 205 insertions(+), 25 deletions(-) create mode 100644 conformance/canonicalize/float_exact_decimal_tie/expected.yaml create mode 100644 conformance/canonicalize/float_exact_decimal_tie/input.yaml diff --git a/conformance/canonicalize/float_exact_decimal_tie/expected.yaml b/conformance/canonicalize/float_exact_decimal_tie/expected.yaml new file mode 100644 index 0000000..8a54a14 --- /dev/null +++ b/conformance/canonicalize/float_exact_decimal_tie/expected.yaml @@ -0,0 +1,3 @@ +outcome: accepted +canonical: '{"a":912968984149762.2,"b":-912968984149762.2,"c":730721488272018.2,"d":126065375359190.62,"e":0.5}' +why: A3: when the double's exact value lies exactly halfway between two equally short decimals, Go's strconv rounds half to even on the shortest digit string (912968984149762.25 -> ...762.2). Rust's Display chose the odd neighbour (...762.3) until #57; the two languages produced different digests for the same number. Found by the generated cross-language corpus (#54); values checked against real Go output. diff --git a/conformance/canonicalize/float_exact_decimal_tie/input.yaml b/conformance/canonicalize/float_exact_decimal_tie/input.yaml new file mode 100644 index 0000000..b70e48d --- /dev/null +++ b/conformance/canonicalize/float_exact_decimal_tie/input.yaml @@ -0,0 +1,5 @@ +a: 912968984149762.3 +b: -912968984149762.25 +c: 730721488272018.3 +d: 126065375359190.63 +e: 0.5 diff --git a/engine/src/canonical.rs b/engine/src/canonical.rs index 1fb9e44..2b62e91 100644 --- a/engine/src/canonical.rs +++ b/engine/src/canonical.rs @@ -31,6 +31,9 @@ use crate::error::{code, Error, Result, Stage}; use crate::reader::{Map, Value}; +use num_bigint::BigInt; +use num_rational::BigRational; +use num_traits::{FromPrimitive as _, Pow as _}; use sha2::{Digest as _, Sha256}; /// Canonical JSON bytes for `v` (A1–A6): compact, no inter-token whitespace, @@ -106,16 +109,111 @@ fn write_value(out: &mut Vec, v: &Value, path: &str) -> Result<()> { /// A3, ported from `cic-canonical`'s own `canonical_float` (not imported as a /// dependency — see module docs): shortest round-trip plain-decimal form, -/// `-0.0` folded to `0`. Rust's `f64` `Display` is shortest round-trip and -/// never uses an exponent, matching Go's `FormatFloat(f, 'f', -1, 64)` — -/// verified empirically in this session (`4.0` -> `"4"`, `1000.0` -> -/// `"1000"`, not `"1e3"`). +/// `-0.0` folded to `0`, never an exponent, matching Go's +/// `FormatFloat(f, 'f', -1, 64)` — verified empirically in this session +/// (`4.0` -> `"4"`, `1000.0` -> `"1000"`, not `"1e3"`). The digits come from +/// [`shortest_digits`], which follows Go's tie rule where Rust's own +/// `Display` does not (#57). pub(crate) fn canonical_float(f: f64) -> String { - let s = format!("{f}"); - if s == "-0" { - "0".to_string() + if f == 0.0 { + return "0".to_string(); // both zeros, `-0.0` included + } + let (digits, exp) = shortest_digits(f.abs()); + let mut out = String::new(); + if f < 0.0 { + out.push('-'); + } + if exp >= 0 { + let int_len = (exp + 1) as usize; + if digits.len() <= int_len { + out.push_str(&digits); + out.push_str(&"0".repeat(int_len - digits.len())); + } else { + out.push_str(&digits[..int_len]); + out.push('.'); + out.push_str(&digits[int_len..]); + } } else { - s + out.push_str("0."); + out.push_str(&"0".repeat((-exp - 1) as usize)); + out.push_str(&digits); + } + out +} + +/// The shortest decimal digits that round-trip `a` (finite, positive), as +/// Go's `strconv` produces them: `(digits, exp)` with no trailing zeros and +/// `exp` the decimal exponent of the leading digit (so `a ~ d.ddd * 10^exp`, +/// the shape Rust's `{:e}` prints). +/// +/// Rust's `{:e}` and Go agree on the digit COUNT and on which candidate is +/// closest to the exact value. They differ when the exact binary value lies +/// exactly halfway between two equally short candidates (e.g. +/// `912968984149762.25` between `...762.2` and `...762.3`): Go rounds half to +/// even on the shortest digit string (`strconv/ftoaryu.go`, `ryuDigits32`), +/// Rust may pick the odd neighbour. A tie needs the exact expansion to be one +/// digit longer than the shortest form and end in 5, so a single exact-mode +/// format is the cheap pre-check; only then is the tie confirmed with exact +/// rational arithmetic. +pub(crate) fn shortest_digits(a: f64) -> (String, i32) { + debug_assert!(a.is_finite() && a > 0.0); + let (digits, exp) = split_sci(&format!("{a:e}")); + let n = digits.len(); + let (exact, exact_exp) = split_sci(&format!("{a:.n$e}")); + // A carry in the exact rendering (9.99..5 -> 10.0) means it is not a tie + // of `n`-digit neighbours at this exponent. + if exact_exp != exp || exact.len() != n + 1 || !exact.ends_with('5') { + return (digits, exp); + } + let (Ok(x), Ok(shown)) = (exact[..n].parse::(), digits.parse::()) else { + return (digits, exp); + }; + // exact == (10x + 5) * 10^(exp - n) exactly? + let Some(exact_value) = BigRational::from_f64(a) else { + return (digits, exp); + }; + let tie = BigRational::new(BigInt::from(10 * x + 5), BigInt::from(1)) * pow10(exp - n as i32); + if exact_value != tie { + return (digits, exp); + } + // A genuine tie between `x` and `x + 1`; Go takes the even one, if it + // is inside the rounding interval (i.e. parses back to `a`). + let other = if shown == x { x + 1 } else { x }; + let go_picks_other = shown % 2 == 1 + && other % 2 == 0 + && format!("{other}e{}", exp - (n as i32 - 1)) + .parse::() + .is_ok_and(|back| back == a); + if !go_picks_other { + return (digits, exp); + } + // `other` can be 10^n (carry from x = 99..9): one digit, exponent + 1. + let text = other.to_string(); + let carry = text.len() > n; + ( + text.trim_end_matches('0').to_string(), + if carry { exp + 1 } else { exp }, + ) +} + +/// `"d.ddde-5"` -> `("ddd", -5)`. +fn split_sci(sci: &str) -> (String, i32) { + let (mantissa, exp) = sci + .split_once('e') + .expect("Rust's `{:e}` formatting always contains 'e'"); + ( + mantissa.chars().filter(char::is_ascii_digit).collect(), + exp.parse() + .expect("the exponent after 'e' is always a valid integer"), + ) +} + +fn pow10(e: i32) -> BigRational { + let p = BigInt::from(10).pow(e.unsigned_abs()); + if e >= 0 { + BigRational::from_integer(p) + } else { + BigRational::new(BigInt::from(1), p) } } @@ -305,6 +403,59 @@ mod tests { // reachable path, verified empirically against saphyr, not a // hypothetical API misuse) must not silently become an invalid JSON // token (`NaN`, `inf`) in what claims to be canonical JSON output. + // #57. Every expected string below is real Go output + // (`strconv.FormatFloat(f, 'f', -1, 64)`), taken from a 1.5M-float + // comparison; the doubles are exact decimal ties, where Go rounds half + // to even on the shortest digit string and Rust's `Display` did not. + #[test] + fn exact_decimal_ties_round_half_even_like_go() { + for (bits, want) in [ + (0x4309_f2b9_3fed_f812_u64, "912968984149762.2"), // ...762.25, odd neighbour .3 rejected + (0x4304_c4b2_d5e7_d492, "730721488272018.2"), + (0x42dc_a9f8_7aac_35a8, "126065375359190.62"), + (0x430a_60df_f0e3_7f42, "928108041236456.2"), + ] { + let f = f64::from_bits(bits); + assert_eq!(canonical_float(f), want, "{bits:#x}"); + assert_eq!(canonical_float(-f), format!("-{want}"), "-{bits:#x}"); + } + } + + #[test] + fn ties_that_already_end_even_and_non_ties_are_unchanged() { + // 912968984149762.75 is a tie whose even neighbour is .8 (Go: .8). + assert_eq!(canonical_float(912_968_984_149_762.75), "912968984149762.8"); + // Not ties: the shortest digits are unambiguous. + for (f, want) in [ + (0.1, "0.1"), + (0.5, "0.5"), + (1.5, "1.5"), + (2.5, "2.5"), + (1e21, "1000000000000000000000"), + (123456.789, "123456.789"), + (5e-324, &*format!("0.{}5", "0".repeat(323))), + ] { + assert_eq!(canonical_float(f), want, "{f:e}"); + } + } + + #[test] + fn shortest_digits_only_changes_on_a_true_tie() { + // The same value as the first tie above, spelled via its neighbours: + // every spelling of the double canonicalizes identically. + let tie = f64::from_bits(0x4309_f2b9_3fed_f812); + for text in [ + "912968984149762.2", + "912968984149762.25", + "912968984149762.3", + ] { + let f: f64 = text.parse().unwrap(); + assert_eq!(f.to_bits(), tie.to_bits(), "{text}"); + assert_eq!(canonical_float(f), "912968984149762.2", "{text}"); + } + assert_eq!(shortest_digits(tie), ("9129689841497622".to_string(), 14)); + } + #[test] fn non_finite_floats_are_rejected_not_written() { for v in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { diff --git a/engine/src/collection.rs b/engine/src/collection.rs index d4e8dfc..f79d9fe 100644 --- a/engine/src/collection.rs +++ b/engine/src/collection.rs @@ -200,23 +200,12 @@ fn go_float_display(f: f64) -> Option { "0".to_string() }); } - // Rust's `{:e}` is exactly the normalized form this needs: one - // nonzero digit before the point, the shortest round-trip digit - // string after it, and the decimal exponent of that leading digit -- - // the same quantity Go's algorithm thresholds on. - let sci = format!("{f:e}"); - let (mantissa_part, exp_part) = sci - .split_once('e') - .expect("Rust's `{:e}` formatting always contains 'e'"); - let exp: i32 = exp_part - .parse() - .expect("the exponent after 'e' is always a valid integer"); - let neg = mantissa_part.starts_with('-'); - let digits: String = mantissa_part - .trim_start_matches('-') - .chars() - .filter(|c| *c != '.') - .collect(); + // One nonzero digit before the point, the shortest round-trip digit + // string after it, and the decimal exponent of that leading digit -- the + // same quantity Go's algorithm thresholds on. `shortest_digits` follows + // Go's tie rule where Rust's `{:e}` does not (#57). + let neg = f.is_sign_negative(); + let (digits, exp) = crate::canonical::shortest_digits(f.abs()); let mut out = String::new(); if neg { diff --git a/go/canonical/canonical_test.go b/go/canonical/canonical_test.go index 0eb7264..a186b81 100644 --- a/go/canonical/canonical_test.go +++ b/go/canonical/canonical_test.go @@ -319,3 +319,35 @@ func TestMalformedNumberIsAnErrorNotAPanic(t *testing.T) { t.Error("nested malformed json.Number was accepted") } } + +// #57: where a double lies exactly halfway between two equally short decimals, +// Go's strconv rounds half to even on the shortest digit string, and the Rust +// engine must agree byte for byte (see canonical.rs, shortest_digits). These +// pin the reference side so a change in the Go toolchain's behaviour is noticed +// instead of silently turning the Rust port into the odd one out. +func TestFloatExactDecimalTiesRoundHalfEven(t *testing.T) { + for _, c := range []struct { + bits uint64 + want string + }{ + {0x4309f2b93fedf812, "912968984149762.2"}, + {0x4304c4b2d5e7d492, "730721488272018.2"}, + {0x42dca9f87aac35a8, "126065375359190.62"}, + {0x430a60dff0e37f42, "928108041236456.2"}, + } { + f := math.Float64frombits(c.bits) + if got := CanonicalFloat(f); got != c.want { + t.Errorf("%#x: CanonicalFloat = %s, want %s", c.bits, got, c.want) + } + if got := CanonicalFloat(-f); got != "-"+c.want { + t.Errorf("-%#x: CanonicalFloat = %s, want -%s", c.bits, got, c.want) + } + } + // The literal that exposed it: three spellings, one double. + for _, lit := range []string{"912968984149762.2", "912968984149762.25", "912968984149762.3"} { + got, err := ToCanonicalJSON(json.Number(lit)) + if err != nil || string(got) != "912968984149762.2" { + t.Errorf("%s -> %s, %v", lit, got, err) + } + } +} From 05da76e04f82a69000248c3ddb4350d096314003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 17 Oct 2026 13:36:09 +0200 Subject: [PATCH 2/2] test: build the .75 tie from its bits (clippy excessive_precision) --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEYCIQDWqbAwKjkvzjsD2RFlqcrnLwEbs1/qmImWlpBHu7ch6wIhANm14+usP3HC8e9zOdnnJGlEW8OXL8YOF9Hk5lXFdJDt hash-algorithm = sha256 digest = 2X9ZfbCwR7J1ys3SJyVYuHNMa/n3dgiN5/hDFoxlPRA= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- engine/src/canonical.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/engine/src/canonical.rs b/engine/src/canonical.rs index 2b62e91..33b3b83 100644 --- a/engine/src/canonical.rs +++ b/engine/src/canonical.rs @@ -423,8 +423,11 @@ mod tests { #[test] fn ties_that_already_end_even_and_non_ties_are_unchanged() { - // 912968984149762.75 is a tie whose even neighbour is .8 (Go: .8). - assert_eq!(canonical_float(912_968_984_149_762.75), "912968984149762.8"); + // 912968984149762.75 (a tie whose even neighbour is .8; Go: .8). + assert_eq!( + canonical_float(f64::from_bits(0x4309_f2b9_3fed_f816)), + "912968984149762.8" + ); // Not ties: the shortest digits are unambiguous. for (f, want) in [ (0.1, "0.1"),