From 5e7e8ff2b1a4282efc87539f4775df6ed4411d73 Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 14:26:45 +0800 Subject: [PATCH 1/3] Improve Snappy CRC32 with crc-fast Replace the optional crc32fast dependency with crc-fast 1.10.0 and use its CRC-32/ISO-HDLC helper for Snappy block checksums. This keeps the Avro trailer format unchanged while allowing the checksum path to use crc-fast's SIMD-backed implementation. Bump the workspace and CI MSRV to 1.89.0 because crc-fast 1.10.0 requires Rust 1.89. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --- .github/workflows/test-lang-rust-ci.yml | 2 +- Cargo.lock | 54 ++++++++++++++++++++----- Cargo.toml | 2 +- avro/Cargo.toml | 4 +- avro/src/codec.rs | 19 ++++++--- 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-lang-rust-ci.yml b/.github/workflows/test-lang-rust-ci.yml index b9c7f226..6befde6e 100644 --- a/.github/workflows/test-lang-rust-ci.yml +++ b/.github/workflows/test-lang-rust-ci.yml @@ -54,7 +54,7 @@ jobs: rust: - "stable" - "nightly" - - "1.88.0" # MSRV + - "1.89.0" # MSRV runner: - name: ubuntu-24.04 target: x86_64-unknown-linux-gnu diff --git a/Cargo.lock b/Cargo.lock index b91f0005..20b2b92c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,9 +53,9 @@ dependencies = [ "bigdecimal", "bon", "bzip2", - "crc32fast", + "crc-fast", "criterion", - "digest", + "digest 0.11.3", "hex-literal", "liblzma", "log", @@ -299,12 +299,13 @@ dependencies = [ ] [[package]] -name = "crc32fast" -version = "1.5.1" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "cfg-if", + "digest 0.10.7", + "spin", ] [[package]] @@ -346,6 +347,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -404,6 +415,15 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "crypto-common 0.1.7", +] + [[package]] name = "digest" version = "0.11.3" @@ -411,7 +431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", - "crypto-common", + "crypto-common 0.2.2", ] [[package]] @@ -491,6 +511,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -723,7 +753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -1179,7 +1209,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.11.3", ] [[package]] @@ -1200,6 +1230,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "static_assertions" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 519c737e..83324900 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ version = "0.23.0" license = "Apache-2.0" repository = "https://github.com/apache/avro-rs" edition = "2024" -rust-version = "1.88.0" +rust-version = "1.89.0" keywords = ["avro", "data", "serialization"] categories = ["encoding"] documentation = "https://docs.rs/apache-avro" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index 6bba447c..aff68486 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -31,7 +31,7 @@ documentation.workspace = true [features] bzip = ["dep:bzip2"] derive = ["dep:apache-avro-derive"] -snappy = ["dep:crc32fast", "dep:snap"] +snappy = ["dep:crc-fast", "dep:snap"] xz = ["dep:liblzma"] zstandard = ["dep:zstd"] @@ -53,7 +53,7 @@ apache-avro-derive = { default-features = false, version = "0.23.0", path = "../ bigdecimal = { default-features = false, version = "0.4.10", features = ["std", "serde"] } bon = { default-features = false, version = "3.9.1" } bzip2 = { version = "0.6.1", optional = true } -crc32fast = { default-features = false, version = "1.5.0", optional = true } +crc-fast = { default-features = false, features = ["std"], version = "1.10.0", optional = true } digest = { default-features = false, version = "0.11.3", features = ["block-api"] } miniz_oxide = { default-features = false, version = "0.9.1", features = ["with-alloc"] } log = { workspace = true } diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 661558e6..0749687c 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -96,9 +96,7 @@ impl Codec { .compress(&stream[..], &mut encoded[..]) .map_err(Details::SnappyCompress)?; - let mut hasher = crc32fast::Hasher::new(); - hasher.update(&stream[..]); - let checksum = hasher.finalize(); + let checksum = crc_fast::crc32_iso_hdlc(&stream[..]); let checksum_as_bytes = checksum.to_be_bytes(); let checksum_len = checksum_as_bytes.len(); encoded.truncate(compressed_size + checksum_len); @@ -191,9 +189,7 @@ impl Codec { last_four.copy_from_slice(&stream[data_end..]); let expected: u32 = u32::from_be_bytes(last_four); - let mut hasher = crc32fast::Hasher::new(); - hasher.update(&decoded); - let actual = hasher.finalize(); + let actual = crc_fast::crc32_iso_hdlc(&decoded); if expected != actual { return Err(Details::SnappyCrc32{expected, actual}.into()); @@ -354,6 +350,17 @@ mod tests { compress_and_decompress(Codec::Snappy) } + #[cfg(feature = "snappy")] + #[test] + fn snappy_compress_writes_crc32_iso_hdlc_trailer() -> TestResult { + let mut stream = b"123456789".to_vec(); + Codec::Snappy.compress(&mut stream)?; + + assert_eq!(stream[stream.len() - 4..], 0xcbf4_3926_u32.to_be_bytes()); + + Ok(()) + } + #[cfg(feature = "snappy")] #[test] fn snappy_decompress_short_block_errors_without_panicking() { From 6bf492672bddd4f98ec690f43a9b49e352a9b45c Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 5 Sep 2026 19:27:19 +0800 Subject: [PATCH 2/3] Add Snappy CRC32 benchmark results Add a Criterion benchmark that compares crc32fast and crc-fast for raw CRC-32/ISO-HDLC, Snappy compression plus CRC, and Snappy decompression plus CRC verification. Document local aarch64 and remote x86_64 benchmark results, including the relevant SIMD instruction-set differences and the remaining MSRV caveat. Co-Authored-By: heihutu Co-Authored-By: zhi22915 --- Cargo.lock | 10 ++ avro/Cargo.toml | 5 + avro/benches/crc32_snappy.rs | 153 +++++++++++++++++++++++++++ avro/benches/crc32_snappy_results.md | 111 +++++++++++++++++++ 4 files changed, 279 insertions(+) create mode 100644 avro/benches/crc32_snappy.rs create mode 100644 avro/benches/crc32_snappy_results.md diff --git a/Cargo.lock b/Cargo.lock index 20b2b92c..8422f407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,6 +54,7 @@ dependencies = [ "bon", "bzip2", "crc-fast", + "crc32fast", "criterion", "digest 0.11.3", "hex-literal", @@ -308,6 +309,15 @@ dependencies = [ "spin", ] +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.8.2" diff --git a/avro/Cargo.toml b/avro/Cargo.toml index aff68486..fb1fed94 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -48,6 +48,10 @@ name = "serde" harness = false name = "single" +[[bench]] +harness = false +name = "crc32_snappy" + [dependencies] apache-avro-derive = { default-features = false, version = "0.23.0", path = "../avro_derive", optional = true } bigdecimal = { default-features = false, version = "0.4.10", features = ["std", "serde"] } @@ -81,6 +85,7 @@ rand = { default-features = false, version = "0.10.1", features = ["default"] } apache-avro-derive = { default-features = false, version = "0.23.0", path = "../avro_derive" } apache-avro-test-helper = { default-features = false, version = "0.23.0", path = "../avro_test_helper" } criterion = { default-features = false, version = "0.8.2" } +crc32fast = { default-features = false, version = "1.5.0" } hex-literal = { default-features = false, version = "1.1.0" } md-5 = { default-features = false, version = "0.11.0" } pretty_assertions = { workspace = true } diff --git a/avro/benches/crc32_snappy.rs b/avro/benches/crc32_snappy.rs new file mode 100644 index 00000000..f0d19246 --- /dev/null +++ b/avro/benches/crc32_snappy.rs @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use criterion::{BatchSize, Criterion, Throughput, criterion_group, criterion_main}; +use std::{hint::black_box, time::Duration}; + +const BLOCK_SIZES: &[usize] = &[128, 1024, 16 * 1024, 1024 * 1024]; + +fn input(size: usize) -> Vec { + (0..size) + .map(|i| { + let word = (i as u64) + .wrapping_mul(0x9e37_79b9_7f4a_7c15) + .rotate_left((i % 63) as u32); + (word ^ (word >> 32)) as u8 + }) + .collect() +} + +fn crc32fast_checksum(data: &[u8]) -> u32 { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(data); + hasher.finalize() +} + +fn crc_fast_checksum(data: &[u8]) -> u32 { + crc_fast::crc32_iso_hdlc(data) +} + +fn compress_with_crc32fast(data: &[u8]) -> Vec { + compress_with(data, crc32fast_checksum) +} + +fn compress_with_crc_fast(data: &[u8]) -> Vec { + compress_with(data, crc_fast_checksum) +} + +fn compress_with(data: &[u8], checksum: fn(&[u8]) -> u32) -> Vec { + let mut encoded = vec![0; snap::raw::max_compress_len(data.len())]; + let compressed_size = snap::raw::Encoder::new() + .compress(data, &mut encoded) + .expect("snappy compression should succeed"); + + encoded.truncate(compressed_size + 4); + encoded[compressed_size..].copy_from_slice(&checksum(data).to_be_bytes()); + encoded +} + +fn decompress_with_crc32fast(compressed: &[u8]) -> Vec { + decompress_with(compressed, crc32fast_checksum) +} + +fn decompress_with_crc_fast(compressed: &[u8]) -> Vec { + decompress_with(compressed, crc_fast_checksum) +} + +fn decompress_with(compressed: &[u8], checksum: fn(&[u8]) -> u32) -> Vec { + let data_end = compressed.len() - 4; + let decompressed_size = snap::raw::decompress_len(&compressed[..data_end]) + .expect("snappy decompressed length should be readable"); + let mut decoded = vec![0; decompressed_size]; + + snap::raw::Decoder::new() + .decompress(&compressed[..data_end], &mut decoded) + .expect("snappy decompression should succeed"); + + let expected = u32::from_be_bytes(compressed[data_end..].try_into().unwrap()); + assert_eq!(expected, checksum(&decoded)); + decoded +} + +fn bench_crc32(c: &mut Criterion) { + let mut group = c.benchmark_group("crc32_iso_hdlc"); + for &size in BLOCK_SIZES { + let data = input(size); + assert_eq!(crc32fast_checksum(&data), crc_fast_checksum(&data)); + group.throughput(Throughput::Bytes(size as u64)); + + group.bench_with_input(format!("crc32fast/{size}"), &data, |b, data| { + b.iter(|| crc32fast_checksum(black_box(data))); + }); + group.bench_with_input(format!("crc-fast/{size}"), &data, |b, data| { + b.iter(|| crc_fast_checksum(black_box(data))); + }); + } + group.finish(); +} + +fn bench_snappy_compress(c: &mut Criterion) { + let mut group = c.benchmark_group("snappy_compress_with_crc32"); + for &size in BLOCK_SIZES { + let data = input(size); + assert_eq!( + compress_with_crc32fast(&data)[..], + compress_with_crc_fast(&data)[..] + ); + group.throughput(Throughput::Bytes(size as u64)); + + group.bench_with_input(format!("crc32fast/{size}"), &data, |b, data| { + b.iter(|| compress_with_crc32fast(black_box(data))); + }); + group.bench_with_input(format!("crc-fast/{size}"), &data, |b, data| { + b.iter(|| compress_with_crc_fast(black_box(data))); + }); + } + group.finish(); +} + +fn bench_snappy_decompress(c: &mut Criterion) { + let mut group = c.benchmark_group("snappy_decompress_with_crc32"); + for &size in BLOCK_SIZES { + let data = input(size); + let compressed = compress_with_crc_fast(&data); + group.throughput(Throughput::Bytes(size as u64)); + + group.bench_with_input(format!("crc32fast/{size}"), &compressed, |b, compressed| { + b.iter_batched( + || compressed.clone(), + |compressed| decompress_with_crc32fast(black_box(&compressed)), + BatchSize::SmallInput, + ); + }); + group.bench_with_input(format!("crc-fast/{size}"), &compressed, |b, compressed| { + b.iter_batched( + || compressed.clone(), + |compressed| decompress_with_crc_fast(black_box(&compressed)), + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(20).measurement_time(Duration::from_secs(5)); + targets = bench_crc32, bench_snappy_compress, bench_snappy_decompress +); +criterion_main!(benches); diff --git a/avro/benches/crc32_snappy_results.md b/avro/benches/crc32_snappy_results.md new file mode 100644 index 00000000..862b5da0 --- /dev/null +++ b/avro/benches/crc32_snappy_results.md @@ -0,0 +1,111 @@ + + +# CRC32 Snappy Benchmark Results + +This benchmark compares the current Snappy checksum replacement against the old +`crc32fast` call pattern. It exercises three paths: + +- raw CRC-32/ISO-HDLC over contiguous blocks; +- Snappy compression plus Avro's trailing CRC32; +- Snappy decompression plus Avro's trailing CRC32 verification. + +Run command: + +```bash +cargo bench -p apache-avro --bench crc32_snappy --features snappy,derive +``` + +The benchmark asserts that `crc32fast` and `crc-fast` produce identical CRC32 +values before timing the cases. The Snappy compression benchmark also asserts +that both variants produce identical compressed blocks, including the big-endian +CRC trailer. + +## Environments + +| Label | Host | CPU / arch | Rust | +| --- | --- | --- | --- | +| aarch64 | local | Darwin arm64, Mac17,6 | rustc 1.98.1 | +| x86_64 | azure-4074205200 | AMD EPYC 9V45, x86_64, `pclmulqdq`, `sse4_1`, `sse4_2`, `avx2`, `avx512f`, `avx512vl`, `vpclmulqdq` | rustc 1.98.0 | + +## aarch64 Results + +Throughput values are Criterion midpoint estimates. + +| Path | Block size | crc32fast | crc-fast | Speedup | +| --- | ---: | ---: | ---: | ---: | +| CRC only | 128 B | 22.95 GiB/s | 29.00 GiB/s | 1.26x | +| CRC only | 1 KiB | 12.68 GiB/s | 66.72 GiB/s | 5.26x | +| CRC only | 16 KiB | 22.34 GiB/s | 78.37 GiB/s | 3.51x | +| CRC only | 1 MiB | 33.64 GiB/s | 100.17 GiB/s | 2.98x | +| Snappy compress + CRC | 128 B | 1.64 GiB/s | 1.68 GiB/s | 1.03x | +| Snappy compress + CRC | 1 KiB | 4.47 GiB/s | 6.18 GiB/s | 1.38x | +| Snappy compress + CRC | 16 KiB | 7.27 GiB/s | 9.54 GiB/s | 1.31x | +| Snappy compress + CRC | 1 MiB | 16.55 GiB/s | 26.11 GiB/s | 1.58x | +| Snappy decompress + CRC | 128 B | 4.68 GiB/s | 4.69 GiB/s | 1.00x | +| Snappy decompress + CRC | 1 KiB | 8.53 GiB/s | 19.51 GiB/s | 2.29x | +| Snappy decompress + CRC | 16 KiB | 12.15 GiB/s | 21.99 GiB/s | 1.81x | +| Snappy decompress + CRC | 1 MiB | 17.42 GiB/s | 25.64 GiB/s | 1.47x | + +On aarch64, `crc32fast` uses the ARM CRC extension instructions such as +`__crc32d` and `__crc32b`, with multi-stream combining for large buffers to +reduce dependency-chain stalls. `crc-fast` can use PMULL/AES folding for +CRC-32/ISO-HDLC, and can select a SHA3/EOR3-assisted variant on newer aarch64 +CPUs. That explains why the raw CRC win is strongest once the input is large +enough to amortize the wider folding setup, and why the win still appears in +Snappy encode/decode once block sizes grow. + +## x86_64 Results + +Throughput values are Criterion midpoint estimates. + +| Path | Block size | crc32fast | crc-fast | Speedup | +| --- | ---: | ---: | ---: | ---: | +| CRC only | 128 B | 6.74 GiB/s | 5.92 GiB/s | 0.88x | +| CRC only | 1 KiB | 6.18 GiB/s | 26.30 GiB/s | 4.26x | +| CRC only | 16 KiB | 6.01 GiB/s | 64.49 GiB/s | 10.74x | +| CRC only | 1 MiB | 5.98 GiB/s | 66.72 GiB/s | 11.15x | +| Snappy compress + CRC | 128 B | 1.19 GiB/s | 1.38 GiB/s | 1.16x | +| Snappy compress + CRC | 1 KiB | 3.37 GiB/s | 6.04 GiB/s | 1.79x | +| Snappy compress + CRC | 16 KiB | 4.50 GiB/s | 15.32 GiB/s | 3.40x | +| Snappy compress + CRC | 1 MiB | 5.04 GiB/s | 21.76 GiB/s | 4.32x | +| Snappy decompress + CRC | 128 B | 2.60 GiB/s | 3.20 GiB/s | 1.23x | +| Snappy decompress + CRC | 1 KiB | 4.07 GiB/s | 10.20 GiB/s | 2.51x | +| Snappy decompress + CRC | 16 KiB | 4.75 GiB/s | 16.31 GiB/s | 3.44x | +| Snappy decompress + CRC | 1 MiB | 4.83 GiB/s | 17.94 GiB/s | 3.72x | + +On x86/x86_64, the SSE4.2 CRC32 instruction is CRC32C/ISCSI, not +CRC-32/ISO-HDLC, so it is not the direct instruction for Avro's Snappy +checksum. The relevant accelerated family for this checksum is carry-less +multiply: PCLMULQDQ and, where available, wider VPCLMULQDQ/AVX-512 paths. +On the EPYC 9V45 test host, `crc-fast` scales much better for 1 KiB and larger +blocks. The 128 B raw CRC case is the one exception, where fixed overhead makes +`crc32fast` slightly faster, but the Snappy encode/decode paths still favor +`crc-fast` even at 128 B. + +## Conclusion + +The replacement is supported by both isolated checksum measurements and +Snappy-shaped encode/decode measurements. The improvement is not just a raw +microbenchmark artifact: it remains visible when the checksum is placed next to +Snappy compression/decompression, especially for 1 KiB and larger Avro blocks. + +The main caveat remains MSRV. `crc-fast` 1.10.0 requires Rust 1.89, so this +replacement should be accepted only together with the workspace and CI MSRV bump +from 1.88.0 to 1.89.0. From 5f984045a87094a425f33b5c338bda2df39c9362 Mon Sep 17 00:00:00 2001 From: Kriskras99 Date: Sat, 5 Sep 2026 18:58:31 +0200 Subject: [PATCH 3/3] chore: Remove benchmark and adjust test name The benchmark was useful for comparison between `crc32fast` and `crc-fast` but we're not going to run those benchmarks again. --- Cargo.lock | 208 +++++++++++++-------------- avro/Cargo.toml | 5 - avro/benches/crc32_snappy.rs | 153 -------------------- avro/benches/crc32_snappy_results.md | 111 -------------- avro/src/codec.rs | 2 +- 5 files changed, 100 insertions(+), 379 deletions(-) delete mode 100644 avro/benches/crc32_snappy.rs delete mode 100644 avro/benches/crc32_snappy_results.md diff --git a/Cargo.lock b/Cargo.lock index 8422f407..d2f78ba5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -54,7 +54,6 @@ dependencies = [ "bon", "bzip2", "crc-fast", - "crc32fast", "criterion", "digest 0.11.3", "hex-literal", @@ -95,7 +94,7 @@ dependencies = [ "rustversion", "serde", "serde_json", - "syn 3.0.4", + "syn 3.0.5", "trybuild", "uuid", ] @@ -111,13 +110,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -142,9 +141,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block-buffer" @@ -175,7 +174,7 @@ dependencies = [ "prettyplease 0.3.0", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -201,9 +200,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.66" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -219,9 +218,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures", @@ -257,18 +256,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstyle", "clap_lex", @@ -292,9 +291,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -309,15 +308,6 @@ dependencies = [ "spin", ] -[[package]] -name = "crc32fast" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" -dependencies = [ - "cfg-if", -] - [[package]] name = "criterion" version = "0.8.2" @@ -405,7 +395,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -416,7 +406,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -455,9 +445,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "env_filter" @@ -487,33 +477,33 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-task", @@ -557,9 +547,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "half" @@ -609,9 +599,9 @@ checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -624,9 +614,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -659,9 +649,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -676,9 +666,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "liblzma" @@ -768,9 +758,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minicov" @@ -813,9 +803,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -890,9 +880,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "ppv-lite86" @@ -930,7 +920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" dependencies = [ "proc-macro2", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -963,7 +953,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bitflags", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha", "rand_xorshift", "regex-syntax", @@ -999,9 +989,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -1054,9 +1044,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1066,9 +1056,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1186,7 +1176,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1292,9 +1282,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -1303,9 +1293,9 @@ dependencies = [ [[package]] name = "target-triple" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "591ef38edfb78ca4771ee32cf494cb8771944bee237a9b91fc9c1424ac4b777b" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" [[package]] name = "termcolor" @@ -1333,7 +1323,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1348,9 +1338,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" dependencies = [ "indexmap", "serde_core", @@ -1372,18 +1362,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "trybuild" @@ -1456,9 +1446,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -1469,9 +1459,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.77" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -1479,9 +1469,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1489,31 +1479,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895a2607575412a4eda1df892084a375ea10dfeadc4d7d2ab87b854e4ddc7ba1" +checksum = "45863ef0bef521c12124eb39d9a38513c47db75f22e503c06beacd40afeb35db" dependencies = [ "async-trait", "cast", @@ -1533,20 +1523,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288cb0ebe215033bf949ae1fd046726daa4c32a157f24b9dc6ac387a52aa759" +checksum = "8c89dcab8b516b6b603baca9d550b7282d68fcc7f367e3956cff7ebf406a3f12" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "wasm-bindgen-test-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff1c1b360982e93b6d8ea9c04836f71dba0817a16f91e229cf3a51bdd9d987" +checksum = "f37b4f992cebe528ef34964ae69681ac0fe7080071e7298e46008f9d380302af" [[package]] name = "winapi" @@ -1596,9 +1586,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "wit-bindgen" @@ -1614,18 +1604,18 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -1634,9 +1624,9 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" @@ -1649,18 +1639,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/avro/Cargo.toml b/avro/Cargo.toml index fb1fed94..aff68486 100644 --- a/avro/Cargo.toml +++ b/avro/Cargo.toml @@ -48,10 +48,6 @@ name = "serde" harness = false name = "single" -[[bench]] -harness = false -name = "crc32_snappy" - [dependencies] apache-avro-derive = { default-features = false, version = "0.23.0", path = "../avro_derive", optional = true } bigdecimal = { default-features = false, version = "0.4.10", features = ["std", "serde"] } @@ -85,7 +81,6 @@ rand = { default-features = false, version = "0.10.1", features = ["default"] } apache-avro-derive = { default-features = false, version = "0.23.0", path = "../avro_derive" } apache-avro-test-helper = { default-features = false, version = "0.23.0", path = "../avro_test_helper" } criterion = { default-features = false, version = "0.8.2" } -crc32fast = { default-features = false, version = "1.5.0" } hex-literal = { default-features = false, version = "1.1.0" } md-5 = { default-features = false, version = "0.11.0" } pretty_assertions = { workspace = true } diff --git a/avro/benches/crc32_snappy.rs b/avro/benches/crc32_snappy.rs deleted file mode 100644 index f0d19246..00000000 --- a/avro/benches/crc32_snappy.rs +++ /dev/null @@ -1,153 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use criterion::{BatchSize, Criterion, Throughput, criterion_group, criterion_main}; -use std::{hint::black_box, time::Duration}; - -const BLOCK_SIZES: &[usize] = &[128, 1024, 16 * 1024, 1024 * 1024]; - -fn input(size: usize) -> Vec { - (0..size) - .map(|i| { - let word = (i as u64) - .wrapping_mul(0x9e37_79b9_7f4a_7c15) - .rotate_left((i % 63) as u32); - (word ^ (word >> 32)) as u8 - }) - .collect() -} - -fn crc32fast_checksum(data: &[u8]) -> u32 { - let mut hasher = crc32fast::Hasher::new(); - hasher.update(data); - hasher.finalize() -} - -fn crc_fast_checksum(data: &[u8]) -> u32 { - crc_fast::crc32_iso_hdlc(data) -} - -fn compress_with_crc32fast(data: &[u8]) -> Vec { - compress_with(data, crc32fast_checksum) -} - -fn compress_with_crc_fast(data: &[u8]) -> Vec { - compress_with(data, crc_fast_checksum) -} - -fn compress_with(data: &[u8], checksum: fn(&[u8]) -> u32) -> Vec { - let mut encoded = vec![0; snap::raw::max_compress_len(data.len())]; - let compressed_size = snap::raw::Encoder::new() - .compress(data, &mut encoded) - .expect("snappy compression should succeed"); - - encoded.truncate(compressed_size + 4); - encoded[compressed_size..].copy_from_slice(&checksum(data).to_be_bytes()); - encoded -} - -fn decompress_with_crc32fast(compressed: &[u8]) -> Vec { - decompress_with(compressed, crc32fast_checksum) -} - -fn decompress_with_crc_fast(compressed: &[u8]) -> Vec { - decompress_with(compressed, crc_fast_checksum) -} - -fn decompress_with(compressed: &[u8], checksum: fn(&[u8]) -> u32) -> Vec { - let data_end = compressed.len() - 4; - let decompressed_size = snap::raw::decompress_len(&compressed[..data_end]) - .expect("snappy decompressed length should be readable"); - let mut decoded = vec![0; decompressed_size]; - - snap::raw::Decoder::new() - .decompress(&compressed[..data_end], &mut decoded) - .expect("snappy decompression should succeed"); - - let expected = u32::from_be_bytes(compressed[data_end..].try_into().unwrap()); - assert_eq!(expected, checksum(&decoded)); - decoded -} - -fn bench_crc32(c: &mut Criterion) { - let mut group = c.benchmark_group("crc32_iso_hdlc"); - for &size in BLOCK_SIZES { - let data = input(size); - assert_eq!(crc32fast_checksum(&data), crc_fast_checksum(&data)); - group.throughput(Throughput::Bytes(size as u64)); - - group.bench_with_input(format!("crc32fast/{size}"), &data, |b, data| { - b.iter(|| crc32fast_checksum(black_box(data))); - }); - group.bench_with_input(format!("crc-fast/{size}"), &data, |b, data| { - b.iter(|| crc_fast_checksum(black_box(data))); - }); - } - group.finish(); -} - -fn bench_snappy_compress(c: &mut Criterion) { - let mut group = c.benchmark_group("snappy_compress_with_crc32"); - for &size in BLOCK_SIZES { - let data = input(size); - assert_eq!( - compress_with_crc32fast(&data)[..], - compress_with_crc_fast(&data)[..] - ); - group.throughput(Throughput::Bytes(size as u64)); - - group.bench_with_input(format!("crc32fast/{size}"), &data, |b, data| { - b.iter(|| compress_with_crc32fast(black_box(data))); - }); - group.bench_with_input(format!("crc-fast/{size}"), &data, |b, data| { - b.iter(|| compress_with_crc_fast(black_box(data))); - }); - } - group.finish(); -} - -fn bench_snappy_decompress(c: &mut Criterion) { - let mut group = c.benchmark_group("snappy_decompress_with_crc32"); - for &size in BLOCK_SIZES { - let data = input(size); - let compressed = compress_with_crc_fast(&data); - group.throughput(Throughput::Bytes(size as u64)); - - group.bench_with_input(format!("crc32fast/{size}"), &compressed, |b, compressed| { - b.iter_batched( - || compressed.clone(), - |compressed| decompress_with_crc32fast(black_box(&compressed)), - BatchSize::SmallInput, - ); - }); - group.bench_with_input(format!("crc-fast/{size}"), &compressed, |b, compressed| { - b.iter_batched( - || compressed.clone(), - |compressed| decompress_with_crc_fast(black_box(&compressed)), - BatchSize::SmallInput, - ); - }); - } - group.finish(); -} - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(20).measurement_time(Duration::from_secs(5)); - targets = bench_crc32, bench_snappy_compress, bench_snappy_decompress -); -criterion_main!(benches); diff --git a/avro/benches/crc32_snappy_results.md b/avro/benches/crc32_snappy_results.md deleted file mode 100644 index 862b5da0..00000000 --- a/avro/benches/crc32_snappy_results.md +++ /dev/null @@ -1,111 +0,0 @@ - - -# CRC32 Snappy Benchmark Results - -This benchmark compares the current Snappy checksum replacement against the old -`crc32fast` call pattern. It exercises three paths: - -- raw CRC-32/ISO-HDLC over contiguous blocks; -- Snappy compression plus Avro's trailing CRC32; -- Snappy decompression plus Avro's trailing CRC32 verification. - -Run command: - -```bash -cargo bench -p apache-avro --bench crc32_snappy --features snappy,derive -``` - -The benchmark asserts that `crc32fast` and `crc-fast` produce identical CRC32 -values before timing the cases. The Snappy compression benchmark also asserts -that both variants produce identical compressed blocks, including the big-endian -CRC trailer. - -## Environments - -| Label | Host | CPU / arch | Rust | -| --- | --- | --- | --- | -| aarch64 | local | Darwin arm64, Mac17,6 | rustc 1.98.1 | -| x86_64 | azure-4074205200 | AMD EPYC 9V45, x86_64, `pclmulqdq`, `sse4_1`, `sse4_2`, `avx2`, `avx512f`, `avx512vl`, `vpclmulqdq` | rustc 1.98.0 | - -## aarch64 Results - -Throughput values are Criterion midpoint estimates. - -| Path | Block size | crc32fast | crc-fast | Speedup | -| --- | ---: | ---: | ---: | ---: | -| CRC only | 128 B | 22.95 GiB/s | 29.00 GiB/s | 1.26x | -| CRC only | 1 KiB | 12.68 GiB/s | 66.72 GiB/s | 5.26x | -| CRC only | 16 KiB | 22.34 GiB/s | 78.37 GiB/s | 3.51x | -| CRC only | 1 MiB | 33.64 GiB/s | 100.17 GiB/s | 2.98x | -| Snappy compress + CRC | 128 B | 1.64 GiB/s | 1.68 GiB/s | 1.03x | -| Snappy compress + CRC | 1 KiB | 4.47 GiB/s | 6.18 GiB/s | 1.38x | -| Snappy compress + CRC | 16 KiB | 7.27 GiB/s | 9.54 GiB/s | 1.31x | -| Snappy compress + CRC | 1 MiB | 16.55 GiB/s | 26.11 GiB/s | 1.58x | -| Snappy decompress + CRC | 128 B | 4.68 GiB/s | 4.69 GiB/s | 1.00x | -| Snappy decompress + CRC | 1 KiB | 8.53 GiB/s | 19.51 GiB/s | 2.29x | -| Snappy decompress + CRC | 16 KiB | 12.15 GiB/s | 21.99 GiB/s | 1.81x | -| Snappy decompress + CRC | 1 MiB | 17.42 GiB/s | 25.64 GiB/s | 1.47x | - -On aarch64, `crc32fast` uses the ARM CRC extension instructions such as -`__crc32d` and `__crc32b`, with multi-stream combining for large buffers to -reduce dependency-chain stalls. `crc-fast` can use PMULL/AES folding for -CRC-32/ISO-HDLC, and can select a SHA3/EOR3-assisted variant on newer aarch64 -CPUs. That explains why the raw CRC win is strongest once the input is large -enough to amortize the wider folding setup, and why the win still appears in -Snappy encode/decode once block sizes grow. - -## x86_64 Results - -Throughput values are Criterion midpoint estimates. - -| Path | Block size | crc32fast | crc-fast | Speedup | -| --- | ---: | ---: | ---: | ---: | -| CRC only | 128 B | 6.74 GiB/s | 5.92 GiB/s | 0.88x | -| CRC only | 1 KiB | 6.18 GiB/s | 26.30 GiB/s | 4.26x | -| CRC only | 16 KiB | 6.01 GiB/s | 64.49 GiB/s | 10.74x | -| CRC only | 1 MiB | 5.98 GiB/s | 66.72 GiB/s | 11.15x | -| Snappy compress + CRC | 128 B | 1.19 GiB/s | 1.38 GiB/s | 1.16x | -| Snappy compress + CRC | 1 KiB | 3.37 GiB/s | 6.04 GiB/s | 1.79x | -| Snappy compress + CRC | 16 KiB | 4.50 GiB/s | 15.32 GiB/s | 3.40x | -| Snappy compress + CRC | 1 MiB | 5.04 GiB/s | 21.76 GiB/s | 4.32x | -| Snappy decompress + CRC | 128 B | 2.60 GiB/s | 3.20 GiB/s | 1.23x | -| Snappy decompress + CRC | 1 KiB | 4.07 GiB/s | 10.20 GiB/s | 2.51x | -| Snappy decompress + CRC | 16 KiB | 4.75 GiB/s | 16.31 GiB/s | 3.44x | -| Snappy decompress + CRC | 1 MiB | 4.83 GiB/s | 17.94 GiB/s | 3.72x | - -On x86/x86_64, the SSE4.2 CRC32 instruction is CRC32C/ISCSI, not -CRC-32/ISO-HDLC, so it is not the direct instruction for Avro's Snappy -checksum. The relevant accelerated family for this checksum is carry-less -multiply: PCLMULQDQ and, where available, wider VPCLMULQDQ/AVX-512 paths. -On the EPYC 9V45 test host, `crc-fast` scales much better for 1 KiB and larger -blocks. The 128 B raw CRC case is the one exception, where fixed overhead makes -`crc32fast` slightly faster, but the Snappy encode/decode paths still favor -`crc-fast` even at 128 B. - -## Conclusion - -The replacement is supported by both isolated checksum measurements and -Snappy-shaped encode/decode measurements. The improvement is not just a raw -microbenchmark artifact: it remains visible when the checksum is placed next to -Snappy compression/decompression, especially for 1 KiB and larger Avro blocks. - -The main caveat remains MSRV. `crc-fast` 1.10.0 requires Rust 1.89, so this -replacement should be accepted only together with the workspace and CI MSRV bump -from 1.88.0 to 1.89.0. diff --git a/avro/src/codec.rs b/avro/src/codec.rs index 0749687c..77d275ee 100644 --- a/avro/src/codec.rs +++ b/avro/src/codec.rs @@ -352,7 +352,7 @@ mod tests { #[cfg(feature = "snappy")] #[test] - fn snappy_compress_writes_crc32_iso_hdlc_trailer() -> TestResult { + fn avro_rs_660_snappy_compress_writes_crc32_iso_hdlc_trailer() -> TestResult { let mut stream = b"123456789".to_vec(); Codec::Snappy.compress(&mut stream)?;