Features/optimize simd

.github/workflows/ci.yml

@@ -33,6 +33,12 @@ jobs:
         run: just ci-test
       - name: Check if changes break public API and need a new version. Use `just semver-checks` to run locally.
         uses: obi1kenobi/cargo-semver-checks-action@v2
+        with:
+          # The default heuristic enables every feature, including `benchmark`. The published
+          # baseline cannot be documented with it: `benchmark` pulls in a module that
+          # `include_bytes!`es from `/testdata`, which the `include` list above excludes from the
+          # packaged crate. Check the default feature set instead.
+          feature-group: default-features
   msrv:
     name: Test MSRV
     runs-on: ubuntu-latest

.travis.yml

@@ -9,7 +9,6 @@ os:
   - osx
 
 script:
-  - rustc --version | grep nightly && cargo test --features=simd || ( echo skip && rustc --version | grep -v nightly )
   - cargo test --no-default-features
   - cargo test --no-default-features --features=std
   - cargo test --no-default-features --features=std --release

Cargo.toml

@@ -1,27 +1,40 @@
 [package]
-name = "brotli"
-version = "8.0.4"
-authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
-description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
+name = "simd-brotli"
+version = "9.1.1"
+authors = [
+  "Mikhail Panfilov <mihan@panfilov.biz>",
+  "Daniel Reiter Horn <danielrh@dropbox.com>",
+  "The Brotli Authors",
+]
+description = "A brotli compressor and decompressor with an interface avoiding the rust stdlib, so it suits embedded devices and kernels. A fork of `brotli` whose encoder hot paths are vectorized with `fearless_simd`, giving runtime-dispatched SIMD on stable Rust with no unsafe code. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause AND MIT"
-documentation = "https://docs.rs/brotli/"
-homepage = "https://github.com/dropbox/rust-brotli"
-repository = "https://github.com/dropbox/rust-brotli"
-keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"]
+documentation = "https://docs.rs/simd-brotli/"
+homepage = "https://github.com/Mnwa/rust-brotli"
+repository = "https://github.com/Mnwa/rust-brotli"
+keywords = ["brotli", "compression", "simd", "lz77", "nostd"]
 categories = ["compression", "no-std"]
 readme = "README.md"
 autobins = false
-edition = "2015"
-rust-version = "1.59.0"
+edition = "2024"
+# Bounded by `fearless_simd`, which the encoder's vectorized paths use unconditionally.
+rust-version = "1.89.0"
 include = [
   "/src/**/*.rs",
   "/examples/**/*.rs",
   "/Cargo.toml",
   "/README.md",
+  "/CHANGELOG.md",
   "/LICENSE.BSD-3-Clause",
   "/LICENSE.MIT",
 ]
 
+# The library target takes the package's own name: `use simd_brotli::...`. Upstream `brotli` can
+# therefore sit in the same dependency graph without a symbol or import collision.
+[lib]
+name = "simd_brotli"
+path = "src/lib.rs"
+
+# The binaries keep the upstream names so command lines and scripts are unchanged.
 [[bin]]
 doc = false
 name = "brotli"
@@ -35,11 +48,21 @@ lto = true
 incremental = false
 
 [dependencies]
+# Pinned to 2.x/0.2.x by `brotli-decompressor`, which requires `alloc-no-stdlib >=2.0.4, <3` as of
+# 5.0.3. Our allocators are handed straight to its decompressor types, so moving to 3.x/0.3.x here
+# would link both majors and leave `HeapAllocator` failing `brotli_decompressor::Allocator`.
+# Bump both together once brotli-decompressor releases against 3.x.
 "alloc-no-stdlib" = { version = ">=2.0.4, <3" }
 "alloc-stdlib" = { version = "~0.2", optional = true }
 "brotli-decompressor" = { version = "~5.0", default-features = false }
+# `libm` is what makes the no-stdlib build possible; the `std` feature below overrides it.
+"fearless_simd" = { version = "~0.7", default-features = false, features = ["libm"] }
+
+"sha2" = { version = "~0.11", optional = true }
 
-"sha2" = { version = "~0.10", optional = true }
+# Profiling instrumentation. Inert unless the `hotpath` feature is on: every call site is
+# behind `cfg_attr`, so a default build never links it.
+"hotpath" = { version = "~0.23", optional = true }
 
 [dev-dependencies]
 # The test suite (src/enc/test.rs) builds calloc-backed memory pools, which on
@@ -57,10 +80,16 @@ external-literal-probability = []
 ffi-api = ["brotli-decompressor/ffi-api"]
 float64 = []
 floating_point_context_mixing = []
+# Wall-clock profiling of the encoder pipeline. Requires `std`; run the `brotli` binary with
+# `--features hotpath` and it prints a per-stage table on exit.
+hotpath = ["dep:hotpath", "hotpath/hotpath", "std"]
+# Same, but measuring allocation counts/bytes instead of time (installs a counting allocator).
+hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
+# Same, but measuring CPU time instead of wall-clock.
+hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
 no-stdlib-ffi-binding = []
 pass-through-ffi-panics = []
 seccomp = ["brotli-decompressor/seccomp"]
-simd = []
-std = ["alloc-stdlib", "brotli-decompressor/std"]
+std = ["alloc-stdlib", "brotli-decompressor/std", "fearless_simd/std"]
 validation = ["sha2"]
 vector_scratch_space = []

README.md

@@ -1,7 +1,54 @@
-# rust-brotli
+# simd-brotli
 
-[![crates.io](https://img.shields.io/crates/v/brotli.svg)](https://crates.io/crates/brotli)
-[![Build Status](https://travis-ci.org/dropbox/rust-brotli.svg?branch=master)](https://travis-ci.org/dropbox/rust-brotli)
+[![crates.io](https://img.shields.io/crates/v/simd-brotli.svg)](https://crates.io/crates/simd-brotli)
+[![docs.rs](https://img.shields.io/docsrs/simd-brotli)](https://docs.rs/simd-brotli/)
+
+A brotli compressor and decompressor, forked from
+[`brotli`](https://crates.io/crates/brotli) with the encoder's hot paths rewritten to run on
+SIMD. It produces byte-identical output to the crate it forks; the difference is how fast it
+gets there.
+
+```toml
+[dependencies]
+simd-brotli = "9.0"
+```
+
+```rust
+use simd_brotli::CompressorWriter;
+```
+
+## What this fork changes
+
+* **Vectorized with [`fearless_simd`](https://crates.io/crates/fearless_simd)** instead of the
+  old `packed_simd`/`stdsimd` shims, so the wide paths build on **stable** Rust with **no
+  `unsafe`** and no nightly-only features. The instruction set is picked at runtime — an AVX2
+  machine takes the AVX2 path, an Apple Silicon machine takes the NEON path, from the same
+  binary — and `no_std` builds still work, falling back to the level the crate was compiled
+  for.
+* **More of the encoder is vectorized than upstream.** Upstream only vectorized a couple of
+  cost loops; this fork also runs the H10 match finder, the Zopfli node update, the static
+  dictionary's match-length probe, block splitting's per-histogram cost scan, and the
+  population-cost walk on wide lanes.
+* **Hot-path algorithmic work**, most visibly histogram clustering, which now costs the sum of
+  two histograms without materializing it.
+* **A built-in profiler.** `--features hotpath` instruments the encoder pipeline stage by stage;
+  a default build neither links it nor pays for it. See
+  [Profiling the encoder](#profiling-the-encoder).
+
+Compressed output is unchanged: every optimization here is bit-identical to the scalar code it
+replaces, so streams stay byte-for-byte the same as upstream's and the format guarantees below
+still hold. This is checked, not assumed — output is diffed against the upstream base across
+qualities 0–11 (including 9.5, 9.5x and 9.5y) on four corpora, and all 60 pairs match byte for
+byte.
+
+On a 3.1 MB varied corpus (Apple M5 Pro, NEON, release + LTO, best of three) the fork encodes
+about **11% faster at q9, 15% at q10 and 8% at q11**. The win depends on your CPU's instruction
+set and on how much of your input reaches the slow paths, so measure your own workload. See
+[CHANGELOG.md](CHANGELOG.md) for the full list of changes.
+
+The library is named `simd_brotli`, not `brotli`, so it can coexist with the upstream crate in
+one dependency graph. Migrating from `brotli` is a rename of the import; the API is otherwise
+untouched.
 
 # What's new in 8.0.4
 Fix: adjust versions of rust-decompressor and rust-alloc-no-stdlib and
@@ -96,15 +143,15 @@ Recommended lg_window_size is between 20 and 22
 
 ### With the io::Read abstraction
 ```rust
-let mut input = brotli::CompressorReader::new(&mut io::stdin(), 4096 /* buffer size */,
+let mut input = simd_brotli::CompressorReader::new(&mut io::stdin(), 4096 /* buffer size */,
                                               quality as u32, lg_window_size as u32);
 ```
 then you can simply read input as you would any other io::Read class
 
 ### With the io::Write abstraction
 
 ```rust
-let mut writer = brotli::Compressor::new(&mut io::stdout(), 4096 /* buffer size */,
+let mut writer = simd_brotli::Compressor::new(&mut io::stdout(), 4096 /* buffer size */,
                                          quality as u32, lg_window_size as u32);
 ```
 
@@ -114,22 +161,22 @@ eg:
 ```rust
 let params = BrotliEncoderParams::default();
 // modify params to fit the application needs
-let mut writer = brotli::Compressor::with_params(&mut io::stdout(), 4096 /* buffer size */,
+let mut writer = simd_brotli::Compressor::with_params(&mut io::stdout(), 4096 /* buffer size */,
                                          params);
 ```
 or for the reader
 ```rust
 let params = BrotliEncoderParams::default();
 // modify params to fit the application needs
-let mut writer = brotli::CompressorReader::with_params(&mut io::stdin(), 4096 /* buffer size */,
+let mut writer = simd_brotli::CompressorReader::with_params(&mut io::stdin(), 4096 /* buffer size */,
                                                        params);
 ```
 
 
 ### With the Stream Copy abstraction
 
 ```rust
-match brotli::BrotliCompress(&mut io::stdin(), &mut io::stdout(), &brotli_encoder_params) {
+match simd_brotli::BrotliCompress(&mut io::stdin(), &mut io::stdout(), &brotli_encoder_params) {
     Ok(_) => {},
     Err(e) => panic!("Error {:?}", e),
 }
@@ -140,20 +187,20 @@ match brotli::BrotliCompress(&mut io::stdin(), &mut io::stdout(), &brotli_encode
 ### With the io::Read abstraction
 
 ```rust
-let mut input = brotli::Decompressor::new(&mut io::stdin(), 4096 /* buffer size */);
+let mut input = simd_brotli::Decompressor::new(&mut io::stdin(), 4096 /* buffer size */);
 ```
 then you can simply read input as you would any other io::Read class
 
 ### With the io::Write abstraction
 
 ```rust
-let mut writer = brotli::DecompressorWriter::new(&mut io::stdout(), 4096 /* buffer size */);
+let mut writer = simd_brotli::DecompressorWriter::new(&mut io::stdout(), 4096 /* buffer size */);
 ```
 
 ### With the Stream Copy abstraction
 
 ```rust
-match brotli::BrotliDecompress(&mut io::stdin(), &mut io::stdout()) {
+match simd_brotli::BrotliDecompress(&mut io::stdin(), &mut io::stdout()) {
     Ok(_) => {},
     Err(e) => panic!("Error {:?}", e),
 }
@@ -318,3 +365,38 @@ params.catable = true;        // Sets catable=true, appendable=true, use_diction
 // All parameter dependencies are handled automatically by the library.
 // No manual fixups required - just set the primary flags you want.
 ```
+
+## Profiling the encoder
+
+The encoder pipeline is instrumented with [hotpath](https://docs.rs/hotpath/). The
+instrumentation is behind `cfg_attr`, so a default build neither links `hotpath` nor pays any
+runtime cost; only `--features hotpath` turns it on.
+
+```bash
+# wall-clock per pipeline stage
+cargo run --release --features hotpath --bin brotli -- -c -q11 input.bin /dev/null
+
+# CPU time instead of wall-clock
+cargo run --release --features hotpath-cpu --bin brotli -- -c -q11 input.bin /dev/null
+
+# allocation counts/bytes instead of time
+cargo run --release --features hotpath-alloc --bin brotli -- -c -q11 input.bin /dev/null
+```
+
+The report prints on exit. `HOTPATH_OUTPUT_FORMAT=json-pretty` emits the full table as JSON
+(the default table view truncates to fit the terminal).
+
+Measured stages: `encode_data`, `copy_input_to_ring_buffer`, `WriteMetaBlockInternal`,
+`ChooseContextMap`, `DecideOverLiteralContextModeling`, `compress_stream_fast`, the three
+`store_meta_block*` writers, `LogMetaBlock`, `BrotliCreateBackwardReferences` and the Zopfli
+entry points, the HQ match finder, binary-tree walk, Zopfli node update and shortest-path walk,
+the q5/q6 scalar and tagged match finders and the SIMD tag filter,
+`BrotliBuildMetaBlock`/`Greedy`/`BrotliOptimizeHistograms`, `BrotliSplitBlock` and its internals,
+the `cluster.rs` histogram-clustering functions, `BrotliEstimateBitCostsForLiterals`, and the two
+`compress_fragment` fast paths.
+
+Most instrumentation sits at metablock granularity. The scalar, tagged and HQ match finders, tag
+filter, binary-tree walk and node update are deliberately measured per position so their inclusive
+totals and call counts can be compared directly; that extra detail adds profiler overhead. Use the
+report for attribution and an uninstrumented release build for end-to-end benchmarks.
+Instruction-level attribution still needs a sampling profiler (`sample` on macOS, `perf` on Linux).

c/Cargo.toml

@@ -11,6 +11,7 @@ keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"]
 categories = ["compression", "no-std", "external-ffi-bindings"]
 readme = "README.md"
 autobins = false
+edition = "2024"
 
 [lib]
 path = "src/lib.rs"
@@ -27,7 +28,6 @@ default = ["std"]
 benchmark = ["brotli/benchmark"]
 disable-timer = ["brotli/disable-timer"]
 seccomp = ["brotli/seccomp"]
-simd = ["brotli/simd"]
 std = ["brotli/std"]
 validation = ["brotli/validation"]
 vector_scratch_space = ["brotli/vector_scratch_space"]