Replace the nightly simd feature with runtime-dispatched SIMD, and vectorize more of the encoder

.github/workflows/ci.yml

@@ -33,6 +33,12 @@ jobs:
         run: just ci-test
       - name: Check if changes break public API and need a new version. Use `just semver-checks` to run locally.
         uses: obi1kenobi/cargo-semver-checks-action@v2
+        with:
+          # The default heuristic enables every feature, including `benchmark`. The published
+          # baseline cannot be documented with it: `benchmark` pulls in a module that
+          # `include_bytes!`es from `/testdata`, which the `include` list above excludes from the
+          # packaged crate. Check the default feature set instead.
+          feature-group: default-features
   msrv:
     name: Test MSRV
     runs-on: ubuntu-latest

.travis.yml

@@ -9,7 +9,6 @@ os:
   - osx
 
 script:
-  - rustc --version | grep nightly && cargo test --features=simd || ( echo skip && rustc --version | grep -v nightly )
   - cargo test --no-default-features
   - cargo test --no-default-features --features=std
   - cargo test --no-default-features --features=std --release

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli"
-version = "8.0.4"
+version = "9.0.0"
 authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause AND MIT"
@@ -11,8 +11,9 @@ keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"]
 categories = ["compression", "no-std"]
 readme = "README.md"
 autobins = false
-edition = "2015"
-rust-version = "1.59.0"
+edition = "2024"
+# Bounded by `fearless_simd`, which the encoder's vectorized paths use unconditionally.
+rust-version = "1.89.0"
 include = [
   "/src/**/*.rs",
   "/examples/**/*.rs",
@@ -35,11 +36,21 @@ lto = true
 incremental = false
 
 [dependencies]
+# Pinned to 2.x/0.2.x by `brotli-decompressor`, which requires `alloc-no-stdlib >=2.0.4, <3` as of
+# 5.0.3. Our allocators are handed straight to its decompressor types, so moving to 3.x/0.3.x here
+# would link both majors and leave `HeapAllocator` failing `brotli_decompressor::Allocator`.
+# Bump both together once brotli-decompressor releases against 3.x.
 "alloc-no-stdlib" = { version = ">=2.0.4, <3" }
 "alloc-stdlib" = { version = "~0.2", optional = true }
 "brotli-decompressor" = { version = "~5.0", default-features = false }
+# `libm` is what makes the no-stdlib build possible; the `std` feature below overrides it.
+"fearless_simd" = { version = "~0.7", default-features = false, features = ["libm"] }
 
-"sha2" = { version = "~0.10", optional = true }
+"sha2" = { version = "~0.11", optional = true }
+
+# Profiling instrumentation. Inert unless the `hotpath` feature is on: every call site is
+# behind `cfg_attr`, so a default build never links it.
+"hotpath" = { version = "~0.23", optional = true }
 
 [dev-dependencies]
 # The test suite (src/enc/test.rs) builds calloc-backed memory pools, which on
@@ -57,10 +68,16 @@ external-literal-probability = []
 ffi-api = ["brotli-decompressor/ffi-api"]
 float64 = []
 floating_point_context_mixing = []
+# Wall-clock profiling of the encoder pipeline. Requires `std`; run the `brotli` binary with
+# `--features hotpath` and it prints a per-stage table on exit.
+hotpath = ["dep:hotpath", "hotpath/hotpath", "std"]
+# Same, but measuring allocation counts/bytes instead of time (installs a counting allocator).
+hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
+# Same, but measuring CPU time instead of wall-clock.
+hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
 no-stdlib-ffi-binding = []
 pass-through-ffi-panics = []
 seccomp = ["brotli-decompressor/seccomp"]
-simd = []
-std = ["alloc-stdlib", "brotli-decompressor/std"]
+std = ["alloc-stdlib", "brotli-decompressor/std", "fearless_simd/std"]
 validation = ["sha2"]
 vector_scratch_space = []

README.md

@@ -318,3 +318,34 @@ params.catable = true;        // Sets catable=true, appendable=true, use_diction
 // All parameter dependencies are handled automatically by the library.
 // No manual fixups required - just set the primary flags you want.
 ```
+
+## Profiling the encoder
+
+The encoder pipeline is instrumented with [hotpath](https://docs.rs/hotpath/). The
+instrumentation is behind `cfg_attr`, so a default build neither links `hotpath` nor pays any
+runtime cost; only `--features hotpath` turns it on.
+
+```bash
+# wall-clock per pipeline stage
+cargo run --release --features hotpath --bin brotli -- -c -q11 input.bin /dev/null
+
+# CPU time instead of wall-clock
+cargo run --release --features hotpath-cpu --bin brotli -- -c -q11 input.bin /dev/null
+
+# allocation counts/bytes instead of time
+cargo run --release --features hotpath-alloc --bin brotli -- -c -q11 input.bin /dev/null
+```
+
+The report prints on exit. `HOTPATH_OUTPUT_FORMAT=json-pretty` emits the full table as JSON
+(the default table view truncates to fit the terminal).
+
+Measured stages: `encode_data`, `copy_input_to_ring_buffer`, `WriteMetaBlockInternal`,
+`ChooseContextMap`, `DecideOverLiteralContextModeling`, `compress_stream_fast`, the three
+`store_meta_block*` writers, `LogMetaBlock`, `BrotliCreateBackwardReferences` and the Zopfli
+entry points, `BrotliBuildMetaBlock`/`Greedy`/`BrotliOptimizeHistograms`, `BrotliSplitBlock`
+and its internals, the `cluster.rs` histogram-clustering functions,
+`BrotliEstimateBitCostsForLiterals`, and the two `compress_fragment` fast paths.
+
+Instrumentation sits at metablock granularity, not per-byte, so overhead is under measurement
+noise (q9/q10/q11 on a 3.9 MB corpus timed within 1% of an uninstrumented build). Leaf-level
+attribution inside a stage needs a sampling profiler (`sample` on macOS, `perf` on Linux).

c/Cargo.toml

@@ -11,6 +11,7 @@ keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"]
 categories = ["compression", "no-std", "external-ffi-bindings"]
 readme = "README.md"
 autobins = false
+edition = "2024"
 
 [lib]
 path = "src/lib.rs"
@@ -27,7 +28,6 @@ default = ["std"]
 benchmark = ["brotli/benchmark"]
 disable-timer = ["brotli/disable-timer"]
 seccomp = ["brotli/seccomp"]
-simd = ["brotli/simd"]
 std = ["brotli/std"]
 validation = ["brotli/validation"]
 vector_scratch_space = ["brotli/vector_scratch_space"]

c/src/lib.rs

@@ -16,104 +16,112 @@ pub use brotli::*;
 
 #[cfg(feature = "std")]
 unsafe fn std_only_functions() {
-    let _ =
-        brotli::ffi::decompressor::CBrotliDecoderDecompress(0, null_mut(), null_mut(), null_mut());
+    unsafe {
+        let _ = brotli::ffi::decompressor::CBrotliDecoderDecompress(
+            0,
+            null_mut(),
+            null_mut(),
+            null_mut(),
+        );
+    }
 }
 #[cfg(not(feature = "std"))]
 unsafe fn std_only_functions() {}
 
-#[no_mangle]
+#[unsafe(no_mangle)]
 pub unsafe extern "C" fn instantiate_functions(must_be_null: *const u8) {
-    if !must_be_null.is_null() {
-        let _ = brotli::ffi::compressor::BrotliEncoderVersion();
-        let _ = brotli::ffi::decompressor::CBrotliDecoderCreateInstance(None, None, null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderSetParameter(null_mut(), brotli::ffi::decompressor::ffi::interface::BrotliDecoderParameter::BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, 0);
-        let _ = brotli::ffi::decompressor::CBrotliDecoderDecompressStream(
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            null_mut(),
-        );
-        std_only_functions();
-        let _ = brotli::ffi::decompressor::CBrotliDecoderMallocU8(null_mut(), 0);
-        let _ = brotli::ffi::decompressor::CBrotliDecoderMallocUsize(null_mut(), 0);
-        let _ = brotli::ffi::decompressor::CBrotliDecoderFreeU8(null_mut(), null_mut(), 0);
-        let _ = brotli::ffi::decompressor::CBrotliDecoderFreeUsize(null_mut(), null_mut(), 0);
-        let _ = brotli::ffi::decompressor::CBrotliDecoderDestroyInstance(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderHasMoreOutput(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderTakeOutput(null_mut(), null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderIsUsed(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderIsFinished(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorCode(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorString(null_mut());
-        let _ = brotli::ffi::decompressor::CBrotliDecoderErrorString(
+    unsafe {
+        if !must_be_null.is_null() {
+            let _ = brotli::ffi::compressor::BrotliEncoderVersion();
+            let _ = brotli::ffi::decompressor::CBrotliDecoderCreateInstance(None, None, null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderSetParameter(null_mut(), brotli::ffi::decompressor::ffi::interface::BrotliDecoderParameter::BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, 0);
+            let _ = brotli::ffi::decompressor::CBrotliDecoderDecompressStream(
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                null_mut(),
+            );
+            std_only_functions();
+            let _ = brotli::ffi::decompressor::CBrotliDecoderMallocU8(null_mut(), 0);
+            let _ = brotli::ffi::decompressor::CBrotliDecoderMallocUsize(null_mut(), 0);
+            let _ = brotli::ffi::decompressor::CBrotliDecoderFreeU8(null_mut(), null_mut(), 0);
+            let _ = brotli::ffi::decompressor::CBrotliDecoderFreeUsize(null_mut(), null_mut(), 0);
+            let _ = brotli::ffi::decompressor::CBrotliDecoderDestroyInstance(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderHasMoreOutput(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderTakeOutput(null_mut(), null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderIsUsed(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderIsFinished(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorCode(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorString(null_mut());
+            let _ = brotli::ffi::decompressor::CBrotliDecoderErrorString(
             brotli::ffi::decompressor::ffi::BrotliDecoderErrorCode::BROTLI_DECODER_ERROR_UNREACHABLE);
-        let _ = BrotliEncoderCreateInstance(None, None, null_mut());
-        let _ = BrotliEncoderSetParameter(
-            null_mut(),
-            brotli::enc::encode::BrotliEncoderParameter::BROTLI_PARAM_MODE,
-            0,
-        );
-        let _ = BrotliEncoderDestroyInstance(null_mut());
-        let _ = BrotliEncoderIsFinished(null_mut());
-        let _ = BrotliEncoderHasMoreOutput(null_mut());
-        let _ = BrotliEncoderTakeOutput(null_mut(), null_mut());
-        let _ = BrotliEncoderMaxCompressedSize(0);
-        let _ = BrotliEncoderSetCustomDictionary(null_mut(), 0, null_mut());
-        let _ = BrotliEncoderCompress(
-            0,
-            0,
-            BrotliEncoderMode::BROTLI_MODE_GENERIC,
-            0,
-            null_mut(),
-            null_mut(),
-            null_mut(),
-        );
-        let _ = BrotliEncoderCompressStream(
-            null_mut(),
-            BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            null_mut(),
-        );
-        let _ = BrotliEncoderMallocU8(null_mut(), 0);
-        let _ = BrotliEncoderFreeU8(null_mut(), null_mut(), 0);
-        let _ = BrotliEncoderMallocUsize(null_mut(), 0);
-        let _ = BrotliEncoderFreeUsize(null_mut(), null_mut(), 0);
-        let _ = BrotliEncoderMaxCompressedSizeMulti(0, 0);
-        let _ = BrotliEncoderCompressMulti(
-            0,
-            null_mut(),
-            null_mut(),
-            0,
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            0,
-            None,
-            None,
-            null_mut(),
-        );
-        let _ = BrotliEncoderCreateWorkPool(0, None, None, null_mut());
-        let _ = BrotliEncoderDestroyWorkPool(null_mut());
-        let _ = BrotliEncoderCompressWorkPool(
-            null_mut(),
-            0,
-            null_mut(),
-            null_mut(),
-            0,
-            null_mut(),
-            null_mut(),
-            null_mut(),
-            0,
-            None,
-            None,
-            null_mut(),
-        );
+            let _ = BrotliEncoderCreateInstance(None, None, null_mut());
+            let _ = BrotliEncoderSetParameter(
+                null_mut(),
+                brotli::enc::encode::BrotliEncoderParameter::BROTLI_PARAM_MODE,
+                0,
+            );
+            let _ = BrotliEncoderDestroyInstance(null_mut());
+            let _ = BrotliEncoderIsFinished(null_mut());
+            let _ = BrotliEncoderHasMoreOutput(null_mut());
+            let _ = BrotliEncoderTakeOutput(null_mut(), null_mut());
+            let _ = BrotliEncoderMaxCompressedSize(0);
+            let _ = BrotliEncoderSetCustomDictionary(null_mut(), 0, null_mut());
+            let _ = BrotliEncoderCompress(
+                0,
+                0,
+                BrotliEncoderMode::BROTLI_MODE_GENERIC,
+                0,
+                null_mut(),
+                null_mut(),
+                null_mut(),
+            );
+            let _ = BrotliEncoderCompressStream(
+                null_mut(),
+                BrotliEncoderOperation::BROTLI_OPERATION_FINISH,
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                null_mut(),
+            );
+            let _ = BrotliEncoderMallocU8(null_mut(), 0);
+            let _ = BrotliEncoderFreeU8(null_mut(), null_mut(), 0);
+            let _ = BrotliEncoderMallocUsize(null_mut(), 0);
+            let _ = BrotliEncoderFreeUsize(null_mut(), null_mut(), 0);
+            let _ = BrotliEncoderMaxCompressedSizeMulti(0, 0);
+            let _ = BrotliEncoderCompressMulti(
+                0,
+                null_mut(),
+                null_mut(),
+                0,
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                0,
+                None,
+                None,
+                null_mut(),
+            );
+            let _ = BrotliEncoderCreateWorkPool(0, None, None, null_mut());
+            let _ = BrotliEncoderDestroyWorkPool(null_mut());
+            let _ = BrotliEncoderCompressWorkPool(
+                null_mut(),
+                0,
+                null_mut(),
+                null_mut(),
+                0,
+                null_mut(),
+                null_mut(),
+                null_mut(),
+                0,
+                None,
+                None,
+                null_mut(),
+            );
+        }
     }
 }
 

justfile

@@ -8,16 +8,12 @@ clean:
     cargo clean
 
 # Build everything
-build: build-brotli build-simd build-ffi
+build: build-brotli build-ffi
 
 # Build the main crate
 build-brotli:
     RUSTFLAGS='-D warnings' cargo build --workspace --all-targets --bins --tests --lib --benches --examples
 
-# Build simd with nightly
-build-simd:
-    RUSTFLAGS='-D warnings' cargo +nightly build --features simd
-
 # Build the brotli-ffi crate (in ./c dir)
 build-ffi:
     # TODO: The c/Cargo.toml does not depend on the **unpublished** main crate, so its build never actually gets tested
@@ -77,5 +73,7 @@ ci-test: sys-info (fmt "--check") build test test-doc
 ci-test-msrv: sys-info build-brotli build-ffi test
 
 # Test if changes are backwards compatible (patch), or need a new minor/major version
+# `--default-features` matches CI: the default heuristic also enables `benchmark`, which the
+# published baseline cannot build because `/testdata` is not in the packaged crate.
 semver-checks:
-    cargo semver-checks
+    cargo semver-checks --default-features