From 480210a0d58caefe634f8c66c907bb432ff51481 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 12:51:48 +0300 Subject: [PATCH 1/6] Replace simd with fearless simd --- .travis.yml | 1 - Cargo.toml | 8 +- c/Cargo.toml | 1 - justfile | 6 +- src/bin/test_broccoli.rs | 3 - src/enc/block_splitter.rs | 75 ++++--- src/enc/compat.rs | 392 ----------------------------------- src/enc/mod.rs | 19 +- src/enc/pdf.rs | 10 +- src/enc/prior_eval.rs | 128 +++++++----- src/enc/vectorization.rs | 138 ++++++------ src/ffi/alloc_util.rs | 2 - src/ffi/compressor.rs | 2 - src/ffi/multicompress/mod.rs | 1 - src/lib.rs | 3 +- 15 files changed, 223 insertions(+), 566 deletions(-) delete mode 100644 src/enc/compat.rs diff --git a/.travis.yml b/.travis.yml index 9c214bba..47a557c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ os: - osx script: - - rustc --version | grep nightly && cargo test --features=simd || ( echo skip && rustc --version | grep -v nightly ) - cargo test --no-default-features - cargo test --no-default-features --features=std - cargo test --no-default-features --features=std --release diff --git a/Cargo.toml b/Cargo.toml index 0cc105a6..b850f422 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,8 @@ categories = ["compression", "no-std"] readme = "README.md" autobins = false edition = "2015" -rust-version = "1.59.0" +# Bounded by `fearless_simd`, which the encoder's vectorized paths use unconditionally. +rust-version = "1.89.0" include = [ "/src/**/*.rs", "/examples/**/*.rs", @@ -38,6 +39,8 @@ incremental = false "alloc-no-stdlib" = { version = ">=2.0.4, <3" } "alloc-stdlib" = { version = "~0.2", optional = true } "brotli-decompressor" = { version = "~5.0", default-features = false } +# `libm` is what makes the no-stdlib build possible; the `std` feature below overrides it. +"fearless_simd" = { version = "~0.6", default-features = false, features = ["libm"] } "sha2" = { version = "~0.10", optional = true } @@ -60,7 +63,6 @@ floating_point_context_mixing = [] no-stdlib-ffi-binding = [] pass-through-ffi-panics = [] seccomp = ["brotli-decompressor/seccomp"] -simd = [] -std = ["alloc-stdlib", "brotli-decompressor/std"] +std = ["alloc-stdlib", "brotli-decompressor/std", "fearless_simd/std"] validation = ["sha2"] vector_scratch_space = [] diff --git a/c/Cargo.toml b/c/Cargo.toml index 7c729a4c..70ce85f6 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -27,7 +27,6 @@ default = ["std"] benchmark = ["brotli/benchmark"] disable-timer = ["brotli/disable-timer"] seccomp = ["brotli/seccomp"] -simd = ["brotli/simd"] std = ["brotli/std"] validation = ["brotli/validation"] vector_scratch_space = ["brotli/vector_scratch_space"] diff --git a/justfile b/justfile index dd619c0a..c1301e63 100644 --- a/justfile +++ b/justfile @@ -8,16 +8,12 @@ clean: cargo clean # Build everything -build: build-brotli build-simd build-ffi +build: build-brotli build-ffi # Build the main crate build-brotli: RUSTFLAGS='-D warnings' cargo build --workspace --all-targets --bins --tests --lib --benches --examples -# Build simd with nightly -build-simd: - RUSTFLAGS='-D warnings' cargo +nightly build --features simd - # Build the brotli-ffi crate (in ./c dir) build-ffi: # TODO: The c/Cargo.toml does not depend on the **unpublished** main crate, so its build never actually gets tested diff --git a/src/bin/test_broccoli.rs b/src/bin/test_broccoli.rs index 4b21b2a0..4025e9ee 100644 --- a/src/bin/test_broccoli.rs +++ b/src/bin/test_broccoli.rs @@ -335,9 +335,6 @@ fn test_concat() { params3.lgwin = 16; params3.magic_number = true; let mut params4 = params0.clone(); - params4.quality = 7; - params4.lgwin = 14; - let mut params4 = params0.clone(); params4.quality = 1; params4.lgwin = 10; let mut params5 = params0.clone(); diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index 3a852a41..413622e5 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -1,7 +1,7 @@ use core; use core::cmp::{max, min}; -#[cfg(feature = "simd")] -use core::simd::prelude::{SimdFloat, SimdPartialOrd}; + +use fearless_simd::{f32x8, Simd, SimdBase, SimdFloat, SimdMask}; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; @@ -14,7 +14,7 @@ use super::histogram::{ HistogramClear, HistogramCommand, HistogramDistance, HistogramLiteral, }; use super::util::FastLog2; -use super::vectorization::{sum8i, v256, v256i, Mem256f}; +use super::vectorization::{detect_level, Mem256f}; use crate::enc::combined_alloc::allocate; use crate::enc::floatX; @@ -45,7 +45,8 @@ static kIterMulForRefining: usize = 2usize; static kMinItersForRefining: usize = 100usize; #[inline(always)] -fn update_cost_and_signal( +fn update_cost_and_signal( + simd: S, num_histograms32: u32, ix: usize, min_cost: floatX, @@ -53,34 +54,19 @@ fn update_cost_and_signal( cost: &mut [Mem256f], switch_signal: &mut [u8], ) { - let ymm_min_cost = v256::splat(min_cost); - let ymm_block_switch_cost = v256::splat(block_switch_cost); - let ymm_and_mask = v256i::from([ - 1 << 0, - 1 << 1, - 1 << 2, - 1 << 3, - 1 << 4, - 1 << 5, - 1 << 6, - 1 << 7, - ]); + let ymm_min_cost = f32x8::splat(simd, min_cost); + let ymm_block_switch_cost = f32x8::splat(simd, block_switch_cost); for (index, cost_it) in cost[..((num_histograms32 as usize + 7) >> 3)] .iter_mut() .enumerate() { - let mut ymm_cost = *cost_it; - let costk_minus_min_cost = ymm_cost - ymm_min_cost; - let ymm_cmpge: v256i = costk_minus_min_cost + let costk_minus_min_cost = cost_it.to_simd(simd) - ymm_min_cost; + // One bit per lane that is at least a block switch away from the cheapest histogram. + switch_signal[ix + index] |= costk_minus_min_cost .simd_ge(ymm_block_switch_cost) - .to_simd(); - let ymm_bits = ymm_cmpge & ymm_and_mask; - let result = sum8i(ymm_bits); - //super::vectorization::sum8(ymm_bits) as u8; - switch_signal[ix + index] |= result as u8; - ymm_cost = costk_minus_min_cost.simd_min(ymm_block_switch_cost); - *cost_it = Mem256f::from(ymm_cost); + .to_bitmask() as u8; + *cost_it = Mem256f::from_simd(costk_minus_min_cost.min(ymm_block_switch_cost)); //println_stderr!("{:} ss {:} c {:?}", (index << 3) + 7, switch_signal[ix + index],*cost_it); } } @@ -222,6 +208,8 @@ fn BitCost(count: usize) -> floatX { } } +/// Entry point into the vectorized cost loop: picks the best instruction set available +/// and runs [`FindBlocksSimd`] with it. fn FindBlocks< HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors, IntegerType: Sized + Clone, @@ -236,6 +224,40 @@ fn FindBlocks< switch_signal: &mut [u8], block_id: &mut [u8], ) -> usize +where + u64: core::convert::From, +{ + dispatch!(detect_level(), simd => FindBlocksSimd( + simd, + data, + length, + block_switch_bitcost, + num_histograms, + histograms, + insert_cost, + cost, + switch_signal, + block_id, + )) +} + +#[inline(always)] +fn FindBlocksSimd< + S: Simd, + HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors, + IntegerType: Sized + Clone, +>( + simd: S, + data: &[IntegerType], + length: usize, + block_switch_bitcost: floatX, + num_histograms: usize, + histograms: &[HistogramType], + insert_cost: &mut [floatX], + cost: &mut [Mem256f], + switch_signal: &mut [u8], + block_id: &mut [u8], +) -> usize where u64: core::convert::From, { @@ -322,6 +344,7 @@ where block_switch_cost *= (0.77 + 0.07 * (byte_ix as floatX) / 2000.0); } update_cost_and_signal( + simd, num_histograms as u32, ix, min_cost, diff --git a/src/enc/compat.rs b/src/enc/compat.rs deleted file mode 100644 index 284dcd82..00000000 --- a/src/enc/compat.rs +++ /dev/null @@ -1,392 +0,0 @@ -#![cfg_attr(feature = "simd", allow(unused))] - -use core::ops::{Add, AddAssign, BitAnd, Index, IndexMut, Mul, Shr, Sub}; - -#[derive(Default, Copy, Clone, Debug)] -pub struct Compat16x16([i16; 16]); -impl Compat16x16 { - #[inline(always)] - pub fn splat(a: i16) -> Compat16x16 { - Compat16x16([a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a]) - } - #[inline(always)] - pub fn to_simd(&self) -> Self { - *self - } - #[inline(always)] - pub fn simd_gt(&self, rhs: Compat16x16) -> Compat16x16 { - Self([ - -((self[0] > rhs[0]) as i16), - -((self[1] > rhs[1]) as i16), - -((self[2] > rhs[2]) as i16), - -((self[3] > rhs[3]) as i16), - -((self[4] > rhs[4]) as i16), - -((self[5] > rhs[5]) as i16), - -((self[6] > rhs[6]) as i16), - -((self[7] > rhs[7]) as i16), - -((self[8] > rhs[8]) as i16), - -((self[9] > rhs[9]) as i16), - -((self[10] > rhs[10]) as i16), - -((self[11] > rhs[11]) as i16), - -((self[12] > rhs[12]) as i16), - -((self[13] > rhs[13]) as i16), - -((self[14] > rhs[14]) as i16), - -((self[15] > rhs[15]) as i16), - ]) - } -} - -macro_rules! op16 { - ($a: expr, $b: expr, $op: expr) => { - Compat16x16([ - $op($a[0], $b[0]), - $op($a[1], $b[1]), - $op($a[2], $b[2]), - $op($a[3], $b[3]), - $op($a[4], $b[4]), - $op($a[5], $b[5]), - $op($a[6], $b[6]), - $op($a[7], $b[7]), - $op($a[8], $b[8]), - $op($a[9], $b[9]), - $op($a[10], $b[10]), - $op($a[11], $b[11]), - $op($a[12], $b[12]), - $op($a[13], $b[13]), - $op($a[14], $b[14]), - $op($a[15], $b[15]), - ]) - }; -} -macro_rules! scalar_op16 { - ($a: expr, $b: expr, $op: expr) => { - Compat16x16([ - $op($a[0], $b), - $op($a[1], $b), - $op($a[2], $b), - $op($a[3], $b), - $op($a[4], $b), - $op($a[5], $b), - $op($a[6], $b), - $op($a[7], $b), - $op($a[8], $b), - $op($a[9], $b), - $op($a[10], $b), - $op($a[11], $b), - $op($a[12], $b), - $op($a[13], $b), - $op($a[14], $b), - $op($a[15], $b), - ]) - }; -} -#[inline(always)] -fn wrapping_i16_add(a: i16, b: i16) -> i16 { - a.wrapping_add(b) -} -#[inline(always)] -fn wrapping_i16_sub(a: i16, b: i16) -> i16 { - a.wrapping_sub(b) -} -#[inline(always)] -fn i16_bitand(a: i16, b: i16) -> i16 { - a & b -} -#[inline(always)] -fn shift16(a: i16, b: Scalar) -> i16 -where - i64: From, -{ - a >> i64::from(b) -} -impl Add for Compat16x16 { - type Output = Compat16x16; - #[inline(always)] - fn add(self, other: Compat16x16) -> Compat16x16 { - op16!(self.0, other.0, wrapping_i16_add) - } -} -impl Sub for Compat16x16 { - type Output = Compat16x16; - #[inline(always)] - fn sub(self, other: Compat16x16) -> Compat16x16 { - op16!(self.0, other.0, wrapping_i16_sub) - } -} -impl BitAnd for Compat16x16 { - type Output = Compat16x16; - #[inline(always)] - fn bitand(self, other: Compat16x16) -> Compat16x16 { - op16!(self.0, other.0, i16_bitand) - } -} -impl From<[i16; 16]> for Compat16x16 { - fn from(value: [i16; 16]) -> Self { - Self(value) - } -} -impl Index for Compat16x16 -where - I: core::slice::SliceIndex<[i16]>, -{ - type Output = I::Output; - - fn index(&self, index: I) -> &Self::Output { - &self.0[index] - } -} -impl IndexMut for Compat16x16 -where - I: core::slice::SliceIndex<[i16]>, -{ - fn index_mut(&mut self, index: I) -> &mut Self::Output { - &mut self.0[index] - } -} -impl Shr for Compat16x16 -where - i64: From, -{ - type Output = Compat16x16; - #[inline(always)] - fn shr(self, other: Scalar) -> Compat16x16 { - scalar_op16!(self.0, other.clone(), shift16) - } -} - -#[derive(Default, Copy, Clone, Debug)] -pub struct Compat32x8([i32; 8]); -impl Compat32x8 { - #[inline(always)] - pub fn splat(a: i32) -> Compat32x8 { - Compat32x8([a, a, a, a, a, a, a, a]) - } - #[inline(always)] - pub fn simd_gt(&self, rhs: Compat32x8) -> Compat32x8 { - Self([ - -((self[0] > rhs[0]) as i32), - -((self[1] > rhs[1]) as i32), - -((self[2] > rhs[2]) as i32), - -((self[3] > rhs[3]) as i32), - -((self[4] > rhs[4]) as i32), - -((self[5] > rhs[5]) as i32), - -((self[6] > rhs[6]) as i32), - -((self[7] > rhs[7]) as i32), - ]) - } - #[inline(always)] - pub fn simd_ge(&self, rhs: Compat32x8) -> Compat32x8 { - Self([ - -((self[0] >= rhs[0]) as i32), - -((self[1] >= rhs[1]) as i32), - -((self[2] >= rhs[2]) as i32), - -((self[3] >= rhs[3]) as i32), - -((self[4] >= rhs[4]) as i32), - -((self[5] >= rhs[5]) as i32), - -((self[6] >= rhs[6]) as i32), - -((self[7] >= rhs[7]) as i32), - ]) - } - pub fn to_simd(&self) -> Self { - *self - } -} - -#[inline(always)] -fn fmin(a: f32, b: f32) -> f32 { - if a < b { - a - } else { - b - } -} -#[derive(Default, Copy, Clone, Debug)] -pub struct CompatF8([f32; 8]); -impl CompatF8 { - #[inline(always)] - pub fn splat(a: f32) -> CompatF8 { - CompatF8([a, a, a, a, a, a, a, a]) - } - #[inline(always)] - pub fn simd_ge(&self, rhs: CompatF8) -> Compat32x8 { - Compat32x8([ - -((self[0] >= rhs[0]) as i32), - -((self[1] >= rhs[1]) as i32), - -((self[2] >= rhs[2]) as i32), - -((self[3] >= rhs[3]) as i32), - -((self[4] >= rhs[4]) as i32), - -((self[5] >= rhs[5]) as i32), - -((self[6] >= rhs[6]) as i32), - -((self[7] >= rhs[7]) as i32), - ]) - } - #[inline(always)] - pub fn simd_min(&self, rhs: CompatF8) -> CompatF8 { - Self([ - fmin(self[0], rhs[0]), - fmin(self[1], rhs[1]), - fmin(self[2], rhs[2]), - fmin(self[3], rhs[3]), - fmin(self[4], rhs[4]), - fmin(self[5], rhs[5]), - fmin(self[6], rhs[6]), - fmin(self[7], rhs[7]), - ]) - } -} -impl Add for Compat32x8 { - type Output = Compat32x8; - #[inline(always)] - fn add(self, other: Compat32x8) -> Compat32x8 { - Compat32x8([ - self.0[0].wrapping_add(other.0[0]), - self.0[1].wrapping_add(other.0[1]), - self.0[2].wrapping_add(other.0[2]), - self.0[3].wrapping_add(other.0[3]), - self.0[4].wrapping_add(other.0[4]), - self.0[5].wrapping_add(other.0[5]), - self.0[6].wrapping_add(other.0[6]), - self.0[7].wrapping_add(other.0[7]), - ]) - } -} - -impl BitAnd for Compat32x8 { - type Output = Compat32x8; - #[inline(always)] - fn bitand(self, other: Compat32x8) -> Compat32x8 { - Compat32x8([ - self.0[0] & other.0[0], - self.0[1] & other.0[1], - self.0[2] & other.0[2], - self.0[3] & other.0[3], - self.0[4] & other.0[4], - self.0[5] & other.0[5], - self.0[6] & other.0[6], - self.0[7] & other.0[7], - ]) - } -} -impl Mul for Compat32x8 { - type Output = Compat32x8; - #[inline(always)] - fn mul(self, other: Compat32x8) -> Compat32x8 { - Compat32x8([ - self.0[0].wrapping_mul(other.0[0]), - self.0[1].wrapping_mul(other.0[1]), - self.0[2].wrapping_mul(other.0[2]), - self.0[3].wrapping_mul(other.0[3]), - self.0[4].wrapping_mul(other.0[4]), - self.0[5].wrapping_mul(other.0[5]), - self.0[6].wrapping_mul(other.0[6]), - self.0[7].wrapping_mul(other.0[7]), - ]) - } -} -impl From<[i32; 8]> for Compat32x8 { - fn from(value: [i32; 8]) -> Self { - Self(value) - } -} -impl Index for Compat32x8 -where - I: core::slice::SliceIndex<[i32]>, -{ - type Output = I::Output; - - fn index(&self, index: I) -> &Self::Output { - &self.0[index] - } -} -impl IndexMut for Compat32x8 -where - I: core::slice::SliceIndex<[i32]>, -{ - fn index_mut(&mut self, index: I) -> &mut Self::Output { - &mut self.0[index] - } -} -impl Add for CompatF8 { - type Output = CompatF8; - #[inline(always)] - fn add(self, other: CompatF8) -> CompatF8 { - CompatF8([ - self.0[0] + other.0[0], - self.0[1] + other.0[1], - self.0[2] + other.0[2], - self.0[3] + other.0[3], - self.0[4] + other.0[4], - self.0[5] + other.0[5], - self.0[6] + other.0[6], - self.0[7] + other.0[7], - ]) - } -} -impl Sub for CompatF8 { - type Output = CompatF8; - #[inline(always)] - fn sub(self, other: CompatF8) -> CompatF8 { - CompatF8([ - self.0[0] - other.0[0], - self.0[1] - other.0[1], - self.0[2] - other.0[2], - self.0[3] - other.0[3], - self.0[4] - other.0[4], - self.0[5] - other.0[5], - self.0[6] - other.0[6], - self.0[7] - other.0[7], - ]) - } -} -impl Mul for CompatF8 { - type Output = CompatF8; - #[inline(always)] - fn mul(self, other: CompatF8) -> CompatF8 { - CompatF8([ - self.0[0] * other.0[0], - self.0[1] * other.0[1], - self.0[2] * other.0[2], - self.0[3] * other.0[3], - self.0[4] * other.0[4], - self.0[5] * other.0[5], - self.0[6] * other.0[6], - self.0[7] * other.0[7], - ]) - } -} -impl AddAssign for CompatF8 { - #[inline(always)] - fn add_assign(&mut self, other: CompatF8) { - self.0[0] += other.0[0]; - self.0[1] += other.0[1]; - self.0[2] += other.0[2]; - self.0[3] += other.0[3]; - self.0[4] += other.0[4]; - self.0[5] += other.0[5]; - self.0[6] += other.0[6]; - self.0[7] += other.0[7]; - } -} -impl From<[f32; 8]> for CompatF8 { - fn from(value: [f32; 8]) -> Self { - Self(value) - } -} -impl Index for CompatF8 -where - I: core::slice::SliceIndex<[f32]>, -{ - type Output = I::Output; - - fn index(&self, index: I) -> &Self::Output { - &self.0[index] - } -} -impl IndexMut for CompatF8 -where - I: core::slice::SliceIndex<[f32]>, -{ - fn index_mut(&mut self, index: I) -> &mut Self::Output { - &mut self.0[index] - } -} diff --git a/src/enc/mod.rs b/src/enc/mod.rs index 4962bd23..7060c43d 100644 --- a/src/enc/mod.rs +++ b/src/enc/mod.rs @@ -1,5 +1,3 @@ -#[macro_use] -pub mod vectorization; pub mod backward_references; pub mod bit_cost; pub mod block_split; @@ -8,7 +6,6 @@ pub mod brotli_bit_stream; pub mod cluster; pub mod combined_alloc; pub mod command; -mod compat; pub mod compress_fragment; pub mod compress_fragment_two_pass; pub mod constants; @@ -39,6 +36,7 @@ mod test; pub mod threading; pub mod utf8_util; pub mod util; +pub mod vectorization; mod weights; pub mod worker_pool; pub mod writer; @@ -78,18 +76,9 @@ pub use self::vectorization::{v256, v256i, Mem256f}; pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool}; use crate::enc::encode::BrotliEncoderStateStruct; -#[cfg(feature = "simd")] -pub type s16 = core::simd::i16x16; -#[cfg(feature = "simd")] -pub type v8 = core::simd::f32x8; -#[cfg(feature = "simd")] -pub type s8 = core::simd::i32x8; -#[cfg(not(feature = "simd"))] -pub type s16 = compat::Compat16x16; -#[cfg(not(feature = "simd"))] -pub type v8 = compat::CompatF8; -#[cfg(not(feature = "simd"))] -pub type s8 = compat::Compat32x8; +pub type s16 = vectorization::Mem16x16; +pub type v8 = vectorization::Mem256f; +pub type s8 = vectorization::Mem256i; #[cfg(feature = "std")] pub fn compress_multi< diff --git a/src/enc/pdf.rs b/src/enc/pdf.rs index 60dadd44..5fa5bb41 100644 --- a/src/enc/pdf.rs +++ b/src/enc/pdf.rs @@ -1,6 +1,6 @@ -//TODO: replace with builtin SIMD type +//! The probability distribution vector. +//! +//! `PDF` is one of the encoder's fixed-width vector storage types, so it is defined +//! alongside the others in [`super::vectorization`] and re-exported here. -// FIXME!!! -#[allow(dead_code)] -#[derive(Copy, Clone, Default, Debug)] -pub struct PDF([i16; 16]); +pub use super::vectorization::PDF; diff --git a/src/enc/prior_eval.rs b/src/enc/prior_eval.rs index d6608793..5f4d1173 100644 --- a/src/enc/prior_eval.rs +++ b/src/enc/prior_eval.rs @@ -1,7 +1,7 @@ use core; use core::cmp::min; -#[cfg(feature = "simd")] -use core::simd::prelude::SimdPartialOrd; + +use fearless_simd::{f32x8, i16x16, Level, Select, Simd, SimdBase, SimdInt}; use super::super::alloc; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; @@ -9,6 +9,7 @@ use super::backward_references::BrotliEncoderParams; use super::input_pair::{InputPair, InputReference, InputReferenceMut}; use super::ir_interpret::{push_base, IRInterpreter}; use super::util::{floatX, FastLog2u16}; +use super::vectorization::detect_level; use super::{find_stride, interface, s16, v8}; use crate::enc::combined_alloc::{alloc_default, alloc_if}; @@ -40,15 +41,16 @@ pub trait Prior { high_nibble: Option, ) -> usize; #[inline(always)] - fn lookup_mut( + fn lookup_mut( + simd: S, data: &mut [s16], stride_byte: u8, selected_context: u8, actual_context: usize, high_nibble: Option, - ) -> CDF<'_> { + ) -> CDF<'_, S> { let index = Self::lookup_lin(stride_byte, selected_context, actual_context, high_nibble); - CDF::from(&mut data[index]) + CDF::new(simd, &mut data[index]) } #[inline(always)] fn lookup( @@ -327,11 +329,18 @@ impl Prior for AdvPrior { } } -pub struct CDF<'a> { +const ONE_TO_16: [i16; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + +pub struct CDF<'a, S: Simd> { cdf: &'a mut s16, + simd: S, } -impl<'a> CDF<'a> { +impl<'a, S: Simd> CDF<'a, S> { + #[inline(always)] + pub fn new(simd: S, cdf: &'a mut s16) -> Self { + CDF { cdf, simd } + } #[inline(always)] pub fn cost(&self, nibble_u8: u8) -> floatX { let nibble = nibble_u8 as usize & 0xf; @@ -343,25 +352,19 @@ impl<'a> CDF<'a> { } #[inline(always)] pub fn update(&mut self, nibble_u8: u8, speed: (u16, u16)) { - let mut cdf = *self.cdf; - let increment_v = s16::splat(speed.0 as i16); - let one_to_16 = s16::from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); - let mask_v: s16 = one_to_16 - .simd_gt(s16::splat(i16::from(nibble_u8))) - .to_simd(); - cdf = cdf + (increment_v & mask_v); - if cdf[15] >= speed.1 as i16 { - let cdf_bias = one_to_16; - cdf = cdf + cdf_bias - ((cdf + cdf_bias) >> 2); - } - *self.cdf = cdf; - } -} - -impl<'a> From<&'a mut s16> for CDF<'a> { - #[inline(always)] - fn from(cdf: &'a mut s16) -> CDF<'a> { - CDF { cdf } + let simd = self.simd; + let one_to_16 = i16x16::from_slice(simd, &ONE_TO_16); + let increment = i16x16::splat(simd, speed.0 as i16); + // Bump every bucket at or above the coded nibble. + let above_nibble = one_to_16.simd_gt(i16x16::splat(simd, i16::from(nibble_u8))); + let mut cdf = + self.cdf.to_simd(simd) + above_nibble.select(increment, i16x16::splat(simd, 0)); + if cdf.as_slice()[15] >= speed.1 as i16 { + // Renormalize: scale the whole cdf down by 3/4, biased to keep it monotonic. + let biased = cdf + one_to_16; + cdf = biased - (biased >> 2); + } + *self.cdf = s16::from_simd(cdf); } } @@ -390,6 +393,8 @@ pub struct PriorEval< cm_speed: [(u16, u16); 2], stride_speed: [(u16, u16); 2], cur_stride: u8, + /// Detected once, so the per-literal cost update doesn't have to probe the CPU. + level: Level, } impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator> @@ -452,6 +457,7 @@ impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator score: alloc_if::(do_alloc, alloc, 8192), cm_speed, stride_speed, + level: detect_level(), }; init_cdfs(ret.cm_priors.slice_mut()); init_cdfs(ret.slow_cm_priors.slice_mut()); @@ -566,16 +572,17 @@ impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator }, ) } - fn update_cost_base( + fn update_cost_base( &mut self, + simd: S, stride_prior: [u8; 8], stride_prior_offset: usize, selected_bits: u8, cm_prior: usize, literal: u8, ) { - let mut l_score = v8::splat(0.0); - let mut h_score = v8::splat(0.0); + let mut l_score = f32x8::splat(simd, 0.0); + let mut h_score = f32x8::splat(simd, 0.0); let base_stride_prior = stride_prior[stride_prior_offset.wrapping_sub(self.cur_stride as usize) & 7]; let hscore_index = upper_score_index(base_stride_prior, selected_bits, cm_prior); @@ -584,176 +591,190 @@ impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator { type CurPrior = CMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.cm_speed[1]); } { type CurPrior = CMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.cm_speed[0]); } { type CurPrior = SlowCMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.slow_cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, (0, 1024)); } { type CurPrior = SlowCMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.slow_cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, (0, 1024)); } { type CurPrior = FastCMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.fast_cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.cm_speed[0]); } { type CurPrior = FastCMPrior; let mut cdf = CurPrior::lookup_mut( + simd, self.fast_cm_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.cm_speed[0]); } { type CurPrior = Stride1Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[0].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.stride_speed[1]); } { type CurPrior = Stride1Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[0].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.stride_speed[0]); } { type CurPrior = Stride2Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[1].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.stride_speed[1]); } { type CurPrior = Stride2Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[1].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.stride_speed[0]); } { type CurPrior = Stride3Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[2].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.stride_speed[1]); } { type CurPrior = Stride3Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[2].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.stride_speed[0]); } { type CurPrior = Stride4Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[3].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.stride_speed[1]); } { type CurPrior = Stride4Prior; let mut cdf = CurPrior::lookup_mut( + simd, self.stride_priors[3].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset()) & 7], selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.stride_speed[0]); } /* { type CurPrior = Stride8Prior; let mut cdf = CurPrior::lookup_mut(self.stride_priors[4].slice_mut(), stride_prior[stride_prior_offset.wrapping_sub(CurPrior::offset())&7], selected_bits, cm_prior, None); - h_score[CurPrior::which()] = cdf.cost(literal>>4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal>>4); cdf.update(literal >> 4, self.stride_speed[1]); } { @@ -763,35 +784,38 @@ impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator selected_bits, cm_prior, Some(literal >> 4)); - l_score[CurPrior::which()] = cdf.cost(literal&0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal&0xf); cdf.update(literal&0xf, self.stride_speed[0]); } */ type CurPrior = AdvPrior; { let mut cdf = CurPrior::lookup_mut( + simd, self.adv_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, None, ); - h_score[CurPrior::which()] = cdf.cost(literal >> 4); + h_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal >> 4); cdf.update(literal >> 4, self.stride_speed[1]); } { let mut cdf = CurPrior::lookup_mut( + simd, self.adv_priors.slice_mut(), base_stride_prior, selected_bits, cm_prior, Some(literal >> 4), ); - l_score[CurPrior::which()] = cdf.cost(literal & 0xf); + l_score.as_mut_slice()[CurPrior::which()] = cdf.cost(literal & 0xf); cdf.update(literal & 0xf, self.stride_speed[0]); } - self.score.slice_mut()[lscore_index] += l_score; - self.score.slice_mut()[hscore_index] += h_score; + let score = self.score.slice_mut(); + score[lscore_index] = v8::from_simd(score[lscore_index].to_simd(simd) + l_score); + score[hscore_index] = v8::from_simd(score[hscore_index].to_simd(simd) + h_score); } } impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator> IRInterpreter @@ -836,13 +860,15 @@ impl<'a, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator literal: u8, ) { //let stride = self.cur_stride as usize; - self.update_cost_base( + let level = self.level; + dispatch!(level, simd => self.update_cost_base( + simd, stride_prior, stride_prior_offset, selected_bits, cm_prior, literal, - ) + )) } } diff --git a/src/enc/vectorization.rs b/src/enc/vectorization.rs index a7778b5a..968fe592 100644 --- a/src/enc/vectorization.rs +++ b/src/enc/vectorization.rs @@ -1,62 +1,84 @@ -#![allow(unknown_lints)] -#![allow(unused_macros)] - -use crate::enc::util::FastLog2; -use crate::enc::{s8, v8}; -pub type Mem256f = v8; -pub type Mem256i = s8; -pub type v256 = v8; -pub type v256i = s8; -pub fn sum8(x: v256) -> f32 { - x[0] + x[1] + x[2] + x[3] + x[4] + x[5] + x[6] + x[7] -} +//! Fixed-width vector storage for the encoder. +//! +//! The types here are plain arrays: `Default` + `Copy`, so they can live in the +//! encoder's allocator-backed slices and be handed to `Allocator`. They carry no +//! arithmetic of their own. Math is done on [`fearless_simd`] vectors instead: inside a +//! `dispatch!` region, [`Mem256f::to_simd`] (and friends) loads a register and +//! [`Mem256f::from_simd`] stores it back. -pub fn sum8i(x: v256i) -> i32 { - x[0].wrapping_add(x[1]) - .wrapping_add(x[2]) - .wrapping_add(x[3]) - .wrapping_add(x[4]) - .wrapping_add(x[5]) - .wrapping_add(x[6]) - .wrapping_add(x[7]) -} +use core::ops::{Index, IndexMut}; +use core::slice::SliceIndex; -pub fn log2i(x: v256i) -> v256 { - [ - FastLog2(x[0] as u64), - FastLog2(x[1] as u64), - FastLog2(x[2] as u64), - FastLog2(x[3] as u64), - FastLog2(x[4] as u64), - FastLog2(x[5] as u64), - FastLog2(x[6] as u64), - FastLog2(x[7] as u64), - ] - .into() -} -pub fn cast_i32_to_f32(x: v256i) -> v256 { - [ - x[0] as f32, - x[1] as f32, - x[2] as f32, - x[3] as f32, - x[4] as f32, - x[5] as f32, - x[6] as f32, - x[7] as f32, - ] - .into() +use fearless_simd::{f32x8, i16x16, i32x8, Level, Simd, SimdInto}; + +/// The instruction set the vectorized encoder paths run on. +/// +/// Detected at runtime where the platform allows it (`std` builds, wasm), otherwise the +/// best level this crate was compiled for. +#[inline] +pub fn detect_level() -> Level { + Level::try_detect().unwrap_or_else(Level::baseline) } -pub fn cast_f32_to_i32(x: v256) -> v256i { - [ - x[0] as i32, - x[1] as i32, - x[2] as i32, - x[3] as i32, - x[4] as i32, - x[5] as i32, - x[6] as i32, - x[7] as i32, - ] - .into() + +macro_rules! define_vector { + ($(#[$attr:meta])* $name:ident, $elem:ty, $lanes:literal, $simd:ident) => { + $(#[$attr])* + #[derive(Default, Copy, Clone, Debug)] + pub struct $name([$elem; $lanes]); + + impl $name { + /// Load the lanes into a SIMD register. + #[inline(always)] + pub fn to_simd(self, simd: S) -> $simd { + self.0.simd_into(simd) + } + + /// Store a SIMD register back into plain memory. + #[inline(always)] + pub fn from_simd(value: $simd) -> Self { + Self(value.into()) + } + } + + impl From<[$elem; $lanes]> for $name { + #[inline(always)] + fn from(value: [$elem; $lanes]) -> Self { + Self(value) + } + } + + impl> Index for $name { + type Output = I::Output; + + #[inline(always)] + fn index(&self, index: I) -> &Self::Output { + &self.0[index] + } + } + + impl> IndexMut for $name { + #[inline(always)] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + &mut self.0[index] + } + } + }; } + +define_vector!(Mem256f, f32, 8, f32x8); +define_vector!(Mem256i, i32, 8, i32x8); +define_vector!(Mem16x16, i16, 16, i16x16); +define_vector!( + /// A 16-bucket probability distribution. + /// + /// Same shape as [`Mem16x16`], but deliberately a separate type: `BrotliAlloc` + /// requires `Allocator` and `Allocator` as distinct bounds, so the two + /// cannot be aliases of each other. Re-exported as [`crate::enc::pdf::PDF`]. + PDF, + i16, + 16, + i16x16 +); + +pub type v256 = Mem256f; +pub type v256i = Mem256i; diff --git a/src/ffi/alloc_util.rs b/src/ffi/alloc_util.rs index 9ccc2771..92353948 100644 --- a/src/ffi/alloc_util.rs +++ b/src/ffi/alloc_util.rs @@ -37,10 +37,8 @@ impl Allocator for BrotliSubclassableAllocator { } impl BrotliAlloc for BrotliSubclassableAllocator {} -#[cfg(not(feature = "safe"))] unsafe impl Send for BrotliSubclassableAllocator {} -#[cfg(not(feature = "safe"))] unsafe impl Send for SendableMemoryBlock {} #[cfg(not(feature = "std"))] diff --git a/src/ffi/compressor.rs b/src/ffi/compressor.rs index a96b99fa..fdd4245b 100644 --- a/src/ffi/compressor.rs +++ b/src/ffi/compressor.rs @@ -1,5 +1,3 @@ -#![cfg(not(feature = "safe"))] - use core; #[cfg(feature = "std")] use std::io::Write; diff --git a/src/ffi/multicompress/mod.rs b/src/ffi/multicompress/mod.rs index 86fe9dab..df348010 100755 --- a/src/ffi/multicompress/mod.rs +++ b/src/ffi/multicompress/mod.rs @@ -1,4 +1,3 @@ -#![cfg(not(feature = "safe"))] mod test; use alloc::SliceWrapper; diff --git a/src/lib.rs b/src/lib.rs index ed0cae39..aa16a3a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ #![allow(non_snake_case)] #![allow(non_upper_case_globals)] #![cfg_attr(feature = "benchmark", feature(test))] -#![cfg_attr(feature = "simd", feature(portable_simd))] #![cfg_attr( feature = "no-stdlib-ffi-binding", cfg_attr(not(feature = "std"), feature(lang_items)) @@ -19,6 +18,8 @@ extern crate alloc_no_stdlib as alloc; #[cfg(feature = "std")] extern crate alloc_stdlib; extern crate brotli_decompressor; +#[macro_use] +extern crate fearless_simd; pub mod concat; pub mod enc; From ec1d7ffc3c8e7e80c3aa5a5f6e4ce529930dc038 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 14:02:40 +0300 Subject: [PATCH 2/6] Bump brotli edition --- .github/workflows/ci.yml | 6 + Cargo.toml | 10 +- c/Cargo.toml | 1 + c/src/lib.rs | 192 ++++--- justfile | 4 +- src/bin/brotli.rs | 67 ++- src/bin/test_broccoli.rs | 6 +- src/bin/test_custom_dict.rs | 2 +- src/bin/test_threading.rs | 6 +- src/bin/util.rs | 59 +- src/bin/validate.rs | 2 +- .../hash_to_binary_tree.rs | 44 +- src/enc/backward_references/hq.rs | 22 +- src/enc/backward_references/mod.rs | 42 +- src/enc/bit_cost.rs | 2 +- src/enc/block_splitter.rs | 4 +- src/enc/brotli_bit_stream.rs | 49 +- src/enc/cluster.rs | 4 +- src/enc/combined_alloc.rs | 232 ++++---- src/enc/compress_fragment.rs | 6 +- src/enc/compress_fragment_two_pass.rs | 8 +- src/enc/context_map_entropy.rs | 6 +- src/enc/encode.rs | 28 +- src/enc/interface.rs | 48 +- src/enc/literal_cost.rs | 2 +- src/enc/mod.rs | 14 +- src/enc/multithreading.rs | 22 +- src/enc/prior_eval.rs | 6 +- src/enc/reader.rs | 2 +- src/enc/singlethreading.rs | 22 +- src/enc/static_dict.rs | 6 +- src/enc/stride_eval.rs | 4 +- src/enc/test.rs | 12 +- src/enc/threading/mod.rs | 67 +-- src/enc/threading/test.rs | 2 +- src/enc/util.rs | 2 +- src/enc/vectorization.rs | 2 +- src/enc/worker_pool.rs | 84 +-- src/enc/writer.rs | 13 +- src/ffi/broccoli.rs | 147 ++--- src/ffi/compressor.rs | 506 +++++++++-------- src/ffi/decompressor.rs | 120 ++-- src/ffi/multicompress/mod.rs | 527 +++++++++--------- src/lib.rs | 20 +- 44 files changed, 1278 insertions(+), 1152 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 844771f9..406a2786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,12 @@ jobs: run: just ci-test - name: Check if changes break public API and need a new version. Use `just semver-checks` to run locally. uses: obi1kenobi/cargo-semver-checks-action@v2 + with: + # The default heuristic enables every feature, including `benchmark`. The published + # baseline cannot be documented with it: `benchmark` pulls in a module that + # `include_bytes!`es from `/testdata`, which the `include` list above excludes from the + # packaged crate. Check the default feature set instead. + feature-group: default-features msrv: name: Test MSRV runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index b850f422..29dfe558 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "brotli" -version = "8.0.4" +version = "9.0.0" authors = ["Daniel Reiter Horn ", "The Brotli Authors"] description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe." license = "BSD-3-Clause AND MIT" @@ -11,7 +11,7 @@ keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"] categories = ["compression", "no-std"] readme = "README.md" autobins = false -edition = "2015" +edition = "2024" # Bounded by `fearless_simd`, which the encoder's vectorized paths use unconditionally. rust-version = "1.89.0" include = [ @@ -36,13 +36,17 @@ lto = true incremental = false [dependencies] +# Pinned to 2.x/0.2.x by `brotli-decompressor`, which requires `alloc-no-stdlib >=2.0.4, <3` as of +# 5.0.3. Our allocators are handed straight to its decompressor types, so moving to 3.x/0.3.x here +# would link both majors and leave `HeapAllocator` failing `brotli_decompressor::Allocator`. +# Bump both together once brotli-decompressor releases against 3.x. "alloc-no-stdlib" = { version = ">=2.0.4, <3" } "alloc-stdlib" = { version = "~0.2", optional = true } "brotli-decompressor" = { version = "~5.0", default-features = false } # `libm` is what makes the no-stdlib build possible; the `std` feature below overrides it. "fearless_simd" = { version = "~0.6", default-features = false, features = ["libm"] } -"sha2" = { version = "~0.10", optional = true } +"sha2" = { version = "~0.11", optional = true } [dev-dependencies] # The test suite (src/enc/test.rs) builds calloc-backed memory pools, which on diff --git a/c/Cargo.toml b/c/Cargo.toml index 70ce85f6..f050c0f0 100644 --- a/c/Cargo.toml +++ b/c/Cargo.toml @@ -11,6 +11,7 @@ keywords = ["brotli", "decompression", "lz77", "huffman", "nostd"] categories = ["compression", "no-std", "external-ffi-bindings"] readme = "README.md" autobins = false +edition = "2024" [lib] path = "src/lib.rs" diff --git a/c/src/lib.rs b/c/src/lib.rs index 9e44b3af..cef33afb 100644 --- a/c/src/lib.rs +++ b/c/src/lib.rs @@ -16,104 +16,112 @@ pub use brotli::*; #[cfg(feature = "std")] unsafe fn std_only_functions() { - let _ = - brotli::ffi::decompressor::CBrotliDecoderDecompress(0, null_mut(), null_mut(), null_mut()); + unsafe { + let _ = brotli::ffi::decompressor::CBrotliDecoderDecompress( + 0, + null_mut(), + null_mut(), + null_mut(), + ); + } } #[cfg(not(feature = "std"))] unsafe fn std_only_functions() {} -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn instantiate_functions(must_be_null: *const u8) { - if !must_be_null.is_null() { - let _ = brotli::ffi::compressor::BrotliEncoderVersion(); - let _ = brotli::ffi::decompressor::CBrotliDecoderCreateInstance(None, None, null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderSetParameter(null_mut(), brotli::ffi::decompressor::ffi::interface::BrotliDecoderParameter::BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, 0); - let _ = brotli::ffi::decompressor::CBrotliDecoderDecompressStream( - null_mut(), - null_mut(), - null_mut(), - null_mut(), - null_mut(), - null_mut(), - ); - std_only_functions(); - let _ = brotli::ffi::decompressor::CBrotliDecoderMallocU8(null_mut(), 0); - let _ = brotli::ffi::decompressor::CBrotliDecoderMallocUsize(null_mut(), 0); - let _ = brotli::ffi::decompressor::CBrotliDecoderFreeU8(null_mut(), null_mut(), 0); - let _ = brotli::ffi::decompressor::CBrotliDecoderFreeUsize(null_mut(), null_mut(), 0); - let _ = brotli::ffi::decompressor::CBrotliDecoderDestroyInstance(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderHasMoreOutput(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderTakeOutput(null_mut(), null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderIsUsed(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderIsFinished(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorCode(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorString(null_mut()); - let _ = brotli::ffi::decompressor::CBrotliDecoderErrorString( + unsafe { + if !must_be_null.is_null() { + let _ = brotli::ffi::compressor::BrotliEncoderVersion(); + let _ = brotli::ffi::decompressor::CBrotliDecoderCreateInstance(None, None, null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderSetParameter(null_mut(), brotli::ffi::decompressor::ffi::interface::BrotliDecoderParameter::BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION, 0); + let _ = brotli::ffi::decompressor::CBrotliDecoderDecompressStream( + null_mut(), + null_mut(), + null_mut(), + null_mut(), + null_mut(), + null_mut(), + ); + std_only_functions(); + let _ = brotli::ffi::decompressor::CBrotliDecoderMallocU8(null_mut(), 0); + let _ = brotli::ffi::decompressor::CBrotliDecoderMallocUsize(null_mut(), 0); + let _ = brotli::ffi::decompressor::CBrotliDecoderFreeU8(null_mut(), null_mut(), 0); + let _ = brotli::ffi::decompressor::CBrotliDecoderFreeUsize(null_mut(), null_mut(), 0); + let _ = brotli::ffi::decompressor::CBrotliDecoderDestroyInstance(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderHasMoreOutput(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderTakeOutput(null_mut(), null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderIsUsed(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderIsFinished(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorCode(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderGetErrorString(null_mut()); + let _ = brotli::ffi::decompressor::CBrotliDecoderErrorString( brotli::ffi::decompressor::ffi::BrotliDecoderErrorCode::BROTLI_DECODER_ERROR_UNREACHABLE); - let _ = BrotliEncoderCreateInstance(None, None, null_mut()); - let _ = BrotliEncoderSetParameter( - null_mut(), - brotli::enc::encode::BrotliEncoderParameter::BROTLI_PARAM_MODE, - 0, - ); - let _ = BrotliEncoderDestroyInstance(null_mut()); - let _ = BrotliEncoderIsFinished(null_mut()); - let _ = BrotliEncoderHasMoreOutput(null_mut()); - let _ = BrotliEncoderTakeOutput(null_mut(), null_mut()); - let _ = BrotliEncoderMaxCompressedSize(0); - let _ = BrotliEncoderSetCustomDictionary(null_mut(), 0, null_mut()); - let _ = BrotliEncoderCompress( - 0, - 0, - BrotliEncoderMode::BROTLI_MODE_GENERIC, - 0, - null_mut(), - null_mut(), - null_mut(), - ); - let _ = BrotliEncoderCompressStream( - null_mut(), - BrotliEncoderOperation::BROTLI_OPERATION_FINISH, - null_mut(), - null_mut(), - null_mut(), - null_mut(), - null_mut(), - ); - let _ = BrotliEncoderMallocU8(null_mut(), 0); - let _ = BrotliEncoderFreeU8(null_mut(), null_mut(), 0); - let _ = BrotliEncoderMallocUsize(null_mut(), 0); - let _ = BrotliEncoderFreeUsize(null_mut(), null_mut(), 0); - let _ = BrotliEncoderMaxCompressedSizeMulti(0, 0); - let _ = BrotliEncoderCompressMulti( - 0, - null_mut(), - null_mut(), - 0, - null_mut(), - null_mut(), - null_mut(), - 0, - None, - None, - null_mut(), - ); - let _ = BrotliEncoderCreateWorkPool(0, None, None, null_mut()); - let _ = BrotliEncoderDestroyWorkPool(null_mut()); - let _ = BrotliEncoderCompressWorkPool( - null_mut(), - 0, - null_mut(), - null_mut(), - 0, - null_mut(), - null_mut(), - null_mut(), - 0, - None, - None, - null_mut(), - ); + let _ = BrotliEncoderCreateInstance(None, None, null_mut()); + let _ = BrotliEncoderSetParameter( + null_mut(), + brotli::enc::encode::BrotliEncoderParameter::BROTLI_PARAM_MODE, + 0, + ); + let _ = BrotliEncoderDestroyInstance(null_mut()); + let _ = BrotliEncoderIsFinished(null_mut()); + let _ = BrotliEncoderHasMoreOutput(null_mut()); + let _ = BrotliEncoderTakeOutput(null_mut(), null_mut()); + let _ = BrotliEncoderMaxCompressedSize(0); + let _ = BrotliEncoderSetCustomDictionary(null_mut(), 0, null_mut()); + let _ = BrotliEncoderCompress( + 0, + 0, + BrotliEncoderMode::BROTLI_MODE_GENERIC, + 0, + null_mut(), + null_mut(), + null_mut(), + ); + let _ = BrotliEncoderCompressStream( + null_mut(), + BrotliEncoderOperation::BROTLI_OPERATION_FINISH, + null_mut(), + null_mut(), + null_mut(), + null_mut(), + null_mut(), + ); + let _ = BrotliEncoderMallocU8(null_mut(), 0); + let _ = BrotliEncoderFreeU8(null_mut(), null_mut(), 0); + let _ = BrotliEncoderMallocUsize(null_mut(), 0); + let _ = BrotliEncoderFreeUsize(null_mut(), null_mut(), 0); + let _ = BrotliEncoderMaxCompressedSizeMulti(0, 0); + let _ = BrotliEncoderCompressMulti( + 0, + null_mut(), + null_mut(), + 0, + null_mut(), + null_mut(), + null_mut(), + 0, + None, + None, + null_mut(), + ); + let _ = BrotliEncoderCreateWorkPool(0, None, None, null_mut()); + let _ = BrotliEncoderDestroyWorkPool(null_mut()); + let _ = BrotliEncoderCompressWorkPool( + null_mut(), + 0, + null_mut(), + null_mut(), + 0, + null_mut(), + null_mut(), + null_mut(), + 0, + None, + None, + null_mut(), + ); + } } } diff --git a/justfile b/justfile index c1301e63..eac0145c 100644 --- a/justfile +++ b/justfile @@ -73,5 +73,7 @@ ci-test: sys-info (fmt "--check") build test test-doc ci-test-msrv: sys-info build-brotli build-ffi test # Test if changes are backwards compatible (patch), or need a new minor/major version +# `--default-features` matches CI: the default heuristic also enables `benchmark`, which the +# published baseline cannot build because `/testdata` is not in the packaged crate. semver-checks: - cargo semver-checks + cargo semver-checks --default-features diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs index 79f25bae..9bdb794b 100644 --- a/src/bin/brotli.rs +++ b/src/bin/brotli.rs @@ -22,15 +22,15 @@ use std::fs::File; use std::io::{self, Error, ErrorKind, Read, Seek, SeekFrom, Write}; use alloc_no_stdlib::{Allocator, SliceWrapper, SliceWrapperMut}; +use brotli::CustomRead; use brotli::enc::backward_references::BrotliEncoderMode; use brotli::enc::threading::{ BrotliEncoderThreadError, CompressMulti, CompressionThreadResult, Owned, SendAlloc, }; use brotli::enc::{ - compress_worker_pool, new_work_pool, BrotliEncoderMaxCompressedSizeMulti, BrotliEncoderParams, - UnionHasher, WorkerPool, + BrotliEncoderMaxCompressedSizeMulti, BrotliEncoderParams, UnionHasher, WorkerPool, + compress_worker_pool, new_work_pool, }; -use brotli::CustomRead; const MAX_THREADS: usize = 16; @@ -830,7 +830,9 @@ fn main() { continue; } if argument == "-h" || argument == "-help" || argument == "--help" && !double_dash { - println_stderr!("Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s to direct better compression\n\nStream concatenation options:\n-catable Create stream that can be concatenated with other catable streams\n-appendable Create stream that can have catable streams appended to it\n-bytealign Align output to byte boundaries (requires -catable or -appendable)\n-bare Output bare stream without wrapper (requires -catable or -appendable)\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed="); + println_stderr!( + "Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s to direct better compression\n\nStream concatenation options:\n-catable Create stream that can be concatenated with other catable streams\n-appendable Create stream that can have catable streams appended to it\n-bytealign Align output to byte boundaries (requires -catable or -appendable)\n-bare Output bare stream without wrapper (requires -catable or -appendable)\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed=" + ); return; } if filenames[0].is_empty() { @@ -929,9 +931,12 @@ fn main() { custom_dictionary = dict.clone(); } match decompress(&mut input, &mut output, buffer_size, dict.into()) { - Ok(_) => {} - Err(e) => panic!("Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", e), - } + Ok(_) => {} + Err(e) => panic!( + "Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", + e + ), + } } if i + 1 != num_benchmarks { input.seek(SeekFrom::Start(0)).unwrap(); @@ -980,10 +985,18 @@ fn main() { Err(e) => panic!("Error {:?}", e), } } else { - match decompress(&mut input, &mut io::stdout(), buffer_size, custom_dictionary.into()) { - Ok(_) => {} - Err(e) => panic!("Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", e), - } + match decompress( + &mut input, + &mut io::stdout(), + buffer_size, + custom_dictionary.into(), + ) { + Ok(_) => {} + Err(e) => panic!( + "Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", + e + ), + } } } drop(input); @@ -1028,17 +1041,33 @@ fn main() { Err(e) => panic!("Error {:?}", e), } } else { - match decompress(&mut io::stdin(), &mut io::stdout(), buffer_size, custom_dictionary.into()) { - Ok(_) => return, - Err(e) => panic!("Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", e), - } + match decompress( + &mut io::stdin(), + &mut io::stdout(), + buffer_size, + custom_dictionary.into(), + ) { + Ok(_) => return, + Err(e) => panic!( + "Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", + e + ), + } } } } else { assert_eq!(num_benchmarks, 1); - match decompress(&mut io::stdin(), &mut io::stdout(), buffer_size, custom_dictionary.into()) { - Ok(_) => (), - Err(e) => panic!("Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", e), - } + match decompress( + &mut io::stdin(), + &mut io::stdout(), + buffer_size, + custom_dictionary.into(), + ) { + Ok(_) => (), + Err(e) => panic!( + "Error: {:} during brotli decompress\nTo compress with Brotli, specify the -c flag.", + e + ), + } } } diff --git a/src/bin/test_broccoli.rs b/src/bin/test_broccoli.rs index 4025e9ee..ade533f3 100644 --- a/src/bin/test_broccoli.rs +++ b/src/bin/test_broccoli.rs @@ -9,10 +9,10 @@ use core::cmp::{max, min}; use brotli_decompressor::{CustomRead, CustomWrite}; +use super::Rebox; use super::brotli::concat::{BroCatli, BroCatliResult}; use super::brotli::enc::BrotliEncoderParams; use super::integration_tests::UnlimitedBuffer; -use super::Rebox; static RANDOM_THEN_UNICODE: &[u8] = include_bytes!("../../testdata/random_then_unicode"); static ALICE: &[u8] = include_bytes!("../../testdata/alice29.txt"); @@ -74,7 +74,9 @@ fn concat( break; } BroCatliResult::Success => { - panic!("Unexpected state: Success when streaming before finish"); + panic!( + "Unexpected state: Success when streaming before finish" + ); } failure => { panic!("{:?}", failure); diff --git a/src/bin/test_custom_dict.rs b/src/bin/test_custom_dict.rs index 0654ca15..e9552aa5 100644 --- a/src/bin/test_custom_dict.rs +++ b/src/bin/test_custom_dict.rs @@ -7,10 +7,10 @@ extern crate core; use std::io::{Read, Write}; +use super::Rebox; use super::brotli::concat::{BroCatli, BroCatliResult}; use super::brotli::enc::BrotliEncoderParams; use super::integration_tests::UnlimitedBuffer; -use super::Rebox; static RANDOM_THEN_UNICODE: &[u8] = include_bytes!("../../testdata/random_then_unicode"); static ALICE: &[u8] = include_bytes!("../../testdata/alice29.txt"); diff --git a/src/bin/test_threading.rs b/src/bin/test_threading.rs index ea1add34..4fdbdcef 100644 --- a/src/bin/test_threading.rs +++ b/src/bin/test_threading.rs @@ -9,11 +9,11 @@ use brotli::enc::threading::{Owned, SendAlloc}; use brotli_decompressor::{SliceWrapper, SliceWrapperMut}; use super::brotli::enc::{ - compress_multi, compress_multi_no_threadpool, BrotliEncoderMaxCompressedSizeMulti, - BrotliEncoderParams, UnionHasher, + BrotliEncoderMaxCompressedSizeMulti, BrotliEncoderParams, UnionHasher, compress_multi, + compress_multi_no_threadpool, }; use super::integration_tests::UnlimitedBuffer; -use super::{new_brotli_heap_alloc, Rebox}; +use super::{Rebox, new_brotli_heap_alloc}; static RANDOM_THEN_UNICODE: &[u8] = include_bytes!("../../testdata/random_then_unicode"); static ALICE: &[u8] = include_bytes!("../../testdata/alice29.txt"); diff --git a/src/bin/util.rs b/src/bin/util.rs index 352510b9..f54795f6 100644 --- a/src/bin/util.rs +++ b/src/bin/util.rs @@ -9,11 +9,11 @@ use alloc_no_stdlib::{Allocator, SliceWrapper}; use brotli::dictionary::{ kBrotliDictionary, kBrotliDictionaryOffsetsByLength, kBrotliDictionarySizeBitsByLength, }; +use brotli::enc::BrotliAlloc; use brotli::enc::threading::{ AnyBoxConstructor, BatchSpawnable, BatchSpawnableLite, BrotliEncoderThreadError, InternalOwned, InternalSendAlloc, Joinable, Owned, OwnedRetriever, PoisonedThreadError, SendAlloc, }; -use brotli::enc::BrotliAlloc; use brotli::interface; use brotli::transform::TransformDictionaryWord; @@ -116,24 +116,25 @@ pub fn write_one>(cmd: &interface::Command) { res + " " + &val.to_string() }); if prediction.has_context_speeds() { - println_stderr!("prediction {} lcontextmap{} dcontextmap{} mixingvalues{} cmspeedinc {} {} cmspeedmax {} {} stspeedinc {} {} stspeedmax {} {} mxspeedinc {} {} mxspeedmax {} {}", - prediction_mode, - lit_cm, - dist_cm, - mixing_values, - prediction.context_map_speed()[0].0, - prediction.context_map_speed()[1].0, - prediction.context_map_speed()[0].1, - prediction.context_map_speed()[1].1, - prediction.stride_context_speed()[0].0, - prediction.stride_context_speed()[1].0, - prediction.stride_context_speed()[0].1, - prediction.stride_context_speed()[1].1, - prediction.combined_stride_context_speed()[0].0, - prediction.combined_stride_context_speed()[1].0, - prediction.combined_stride_context_speed()[0].1, - prediction.combined_stride_context_speed()[0].1, - ); + println_stderr!( + "prediction {} lcontextmap{} dcontextmap{} mixingvalues{} cmspeedinc {} {} cmspeedmax {} {} stspeedinc {} {} stspeedmax {} {} mxspeedinc {} {} mxspeedmax {} {}", + prediction_mode, + lit_cm, + dist_cm, + mixing_values, + prediction.context_map_speed()[0].0, + prediction.context_map_speed()[1].0, + prediction.context_map_speed()[0].1, + prediction.context_map_speed()[1].1, + prediction.stride_context_speed()[0].0, + prediction.stride_context_speed()[1].0, + prediction.stride_context_speed()[0].1, + prediction.stride_context_speed()[1].1, + prediction.combined_stride_context_speed()[0].0, + prediction.combined_stride_context_speed()[1].0, + prediction.combined_stride_context_speed()[0].1, + prediction.combined_stride_context_speed()[0].1, + ); } else { println_stderr!( "prediction {} lcontextmap{} dcontextmap{} mixingvalues{}", @@ -256,11 +257,11 @@ where } impl< - T: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnable for MTSpawner + T: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnable for MTSpawner where >::AllocatedMemory: Send + 'static, { @@ -292,11 +293,11 @@ where } } impl< - T: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnableLite for MTSpawner + T: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnableLite for MTSpawner where >::AllocatedMemory: Send + 'static, { diff --git a/src/bin/validate.rs b/src/bin/validate.rs index 46e7ddb9..03bfd381 100644 --- a/src/bin/validate.rs +++ b/src/bin/validate.rs @@ -91,7 +91,7 @@ impl<'a, InputType: Read + 'a> Read for ShaReader<'a, InputType> { } } #[cfg(feature = "validation")] -fn make_sha_reader(r: &mut InputType) -> ShaReader { +fn make_sha_reader(r: &mut InputType) -> ShaReader<'_, InputType> { ShaReader { reader: r, checksum: Checksum::default(), diff --git a/src/enc/backward_references/hash_to_binary_tree.rs b/src/enc/backward_references/hash_to_binary_tree.rs index 8275ea02..464a719f 100644 --- a/src/enc/backward_references/hash_to_binary_tree.rs +++ b/src/enc/backward_references/hash_to_binary_tree.rs @@ -1,14 +1,14 @@ -use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core; use core::cmp::min; use super::{ - fix_unbroken_len, kHashMul32, AnyHasher, BrotliEncoderParams, CloneWithAlloc, H9Opts, - HasherSearchResult, HowPrepared, Struct1, + AnyHasher, BrotliEncoderParams, CloneWithAlloc, H9Opts, HasherSearchResult, HowPrepared, + Struct1, fix_unbroken_len, kHashMul32, }; use crate::enc::combined_alloc::allocate; use crate::enc::static_dict::{ - BrotliDictionary, FindMatchLengthWithLimit, BROTLI_UNALIGNED_LOAD32, + BROTLI_UNALIGNED_LOAD32, BrotliDictionary, FindMatchLengthWithLimit, }; use crate::enc::util::floatX; @@ -120,10 +120,10 @@ pub struct H10< } impl< - AllocU32: Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, - Params: H10Params, - > PartialEq> for H10 + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Params: H10Params, +> PartialEq> for H10 where Buckets: PartialEq, { @@ -190,10 +190,10 @@ where } impl< - AllocU32: Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, - Params: H10Params, - > H10 + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Params: H10Params, +> H10 where Buckets: PartialEq, { @@ -203,10 +203,10 @@ where } } impl< - Alloc: Allocator + Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, - Params: H10Params, - > CloneWithAlloc for H10 + Alloc: Allocator + Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Params: H10Params, +> CloneWithAlloc for H10 where Buckets: PartialEq, { @@ -229,10 +229,10 @@ where } impl< - AllocU32: Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, - Params: H10Params, - > AnyHasher for H10 + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Params: H10Params, +> AnyHasher for H10 where Buckets: PartialEq, { @@ -401,12 +401,12 @@ impl<'a> BackwardMatchMut<'a> { } macro_rules! LeftChildIndexH10 { - ($xself: expr, $pos: expr) => { + ($xself: expr_2021, $pos: expr_2021) => { (2usize).wrapping_mul($pos & (*$xself).window_mask_) }; } macro_rules! RightChildIndexH10 { - ($xself: expr, $pos: expr) => { + ($xself: expr_2021, $pos: expr_2021) => { (2usize) .wrapping_mul($pos & (*$xself).window_mask_) .wrapping_add(1) diff --git a/src/enc/backward_references/hq.rs b/src/enc/backward_references/hq.rs index a22badc8..3c967f2f 100644 --- a/src/enc/backward_references/hq.rs +++ b/src/enc/backward_references/hq.rs @@ -1,19 +1,19 @@ -use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core; use core::cmp::{max, min}; use super::hash_to_binary_tree::{ - kInfinity, Allocable, BackwardMatch, BackwardMatchMut, H10Params, StoreAndFindMatchesH10, - Union1, ZopfliNode, H10, + Allocable, BackwardMatch, BackwardMatchMut, H10, H10Params, StoreAndFindMatchesH10, Union1, + ZopfliNode, kInfinity, }; use super::{ - fix_unbroken_len, kDistanceCacheIndex, kDistanceCacheOffset, kInvalidMatch, AnyHasher, - BrotliEncoderParams, + AnyHasher, BrotliEncoderParams, fix_unbroken_len, kDistanceCacheIndex, kDistanceCacheOffset, + kInvalidMatch, }; use crate::enc::combined_alloc::{alloc_if, alloc_or_default}; use crate::enc::command::{ - combine_length_codes, BrotliDistanceParams, Command, GetCopyLengthCode, GetInsertLengthCode, - PrefixEncodeCopyDistance, + BrotliDistanceParams, Command, GetCopyLengthCode, GetInsertLengthCode, + PrefixEncodeCopyDistance, combine_length_codes, }; use crate::enc::constants::{kCopyExtra, kInsExtra}; use crate::enc::encode; @@ -21,7 +21,7 @@ use crate::enc::literal_cost::BrotliEstimateBitCostsForLiterals; use crate::enc::static_dict::{ BrotliDictionary, BrotliFindAllStaticDictionaryMatches, FindMatchLengthWithLimit, }; -use crate::enc::util::{floatX, FastLog2, FastLog2f64}; +use crate::enc::util::{FastLog2, FastLog2f64, floatX}; const BROTLI_WINDOW_GAP: usize = 16; const BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN: usize = 37; @@ -633,11 +633,7 @@ impl BackwardMatch { #[inline(always)] fn length_code(&self) -> usize { let code = (self.length_and_code() & 31u32) as usize; - if code != 0 { - code - } else { - self.length() - } + if code != 0 { code } else { self.length() } } } diff --git a/src/enc/backward_references/mod.rs b/src/enc/backward_references/mod.rs index fb172d4e..398ac94b 100644 --- a/src/enc/backward_references/mod.rs +++ b/src/enc/backward_references/mod.rs @@ -8,12 +8,12 @@ use core::cmp::{max, min}; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::command::{BrotliDistanceParams, Command, ComputeDistanceCode}; use super::dictionary_hash::kStaticDictionaryHash; -use super::hash_to_binary_tree::{H10Buckets, H10DefaultParams, ZopfliNode, H10}; +use super::hash_to_binary_tree::{H10, H10Buckets, H10DefaultParams, ZopfliNode}; use super::static_dict::{ - BrotliDictionary, FindMatchLengthWithLimit, FindMatchLengthWithLimitMin4, - BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, + BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, BrotliDictionary, FindMatchLengthWithLimit, + FindMatchLengthWithLimitMin4, }; -use super::util::{floatX, Log2FloorNonZero}; +use super::util::{Log2FloorNonZero, floatX}; use crate::enc::combined_alloc::allocate; pub static kInvalidMatch: u32 = 0x0fff_ffff; @@ -941,9 +941,9 @@ pub struct AdvHasher< } impl< - Specialization: AdvHashSpecialization + Sized + Clone, - Alloc: alloc::Allocator + alloc::Allocator, - > PartialEq> for AdvHasher + Specialization: AdvHashSpecialization + Sized + Clone, + Alloc: alloc::Allocator + alloc::Allocator, +> PartialEq> for AdvHasher { fn eq(&self, other: &Self) -> bool { self.GetHasherCommon == other.GetHasherCommon @@ -1154,9 +1154,9 @@ fn BackwardReferencePenaltyUsingLastDistance(distance_short_code: usize) -> u64 } impl< - Specialization: AdvHashSpecialization + Clone, - Alloc: alloc::Allocator + alloc::Allocator, - > AdvHasher + Specialization: AdvHashSpecialization + Clone, + Alloc: alloc::Allocator + alloc::Allocator, +> AdvHasher { // 7 opt // returns a new ix_start @@ -1468,9 +1468,9 @@ impl< } impl< - Specialization: AdvHashSpecialization + Clone, - Alloc: alloc::Allocator + alloc::Allocator, - > AnyHasher for AdvHasher + Specialization: AdvHashSpecialization + Clone, + Alloc: alloc::Allocator + alloc::Allocator, +> AnyHasher for AdvHasher { fn Opts(&self) -> H9Opts { self.h9_opts @@ -2074,9 +2074,9 @@ impl + alloc::Allocator> CloneWithAlloc } } impl< - Alloc: alloc::Allocator + alloc::Allocator, - Special: AdvHashSpecialization + Sized + Clone, - > CloneWithAlloc for AdvHasher + Alloc: alloc::Allocator + alloc::Allocator, + Special: AdvHashSpecialization + Sized + Clone, +> CloneWithAlloc for AdvHasher { fn clone_with_alloc(&self, m: &mut Alloc) -> Self { let mut num = allocate::(m, self.num.len()); @@ -2178,7 +2178,7 @@ impl + alloc::Allocator> CloneWithAlloc } } macro_rules! match_all_hashers_mut { - ($xself : expr, $func_call : ident, $( $args:expr),*) => { + ($xself : expr_2021, $func_call : ident, $( $args:expr_2021),*) => { match $xself { &mut UnionHasher::H2(ref mut hasher) => hasher.$func_call($($args),*), &mut UnionHasher::H3(ref mut hasher) => hasher.$func_call($($args),*), @@ -2195,7 +2195,7 @@ macro_rules! match_all_hashers_mut { }; } macro_rules! match_all_hashers { - ($xself : expr, $func_call : ident, $( $args:expr),*) => { + ($xself : expr_2021, $func_call : ident, $( $args:expr_2021),*) => { match $xself { &UnionHasher::H2(ref hasher) => hasher.$func_call($($args),*), &UnionHasher::H3(ref hasher) => hasher.$func_call($($args),*), @@ -2218,9 +2218,9 @@ impl + alloc::Allocator> AnyHasher for UnionHa fn GetHasherCommon(&mut self) -> &mut Struct1 { match_all_hashers_mut!(self, GetHasherCommon,) } /* - fn GetH10Tree(&mut self) -> Option<&mut H10> { - return match_all_hashers_mut!(self, GetH10Tree,); - }*/ + fn GetH10Tree(&mut self) -> Option<&mut H10> { + return match_all_hashers_mut!(self, GetH10Tree,); + }*/ fn Prepare(&mut self, one_shot: bool, input_size: usize, data: &[u8]) -> HowPrepared { match_all_hashers_mut!(self, Prepare, one_shot, input_size, data) } diff --git a/src/enc/bit_cost.rs b/src/enc/bit_cost.rs index 948f9ca9..a79937c4 100644 --- a/src/enc/bit_cost.rs +++ b/src/enc/bit_cost.rs @@ -1,4 +1,4 @@ -use alloc::SliceWrapperMut; +use crate::alloc::SliceWrapperMut; use core::cmp::{max, min}; use super::super::alloc::SliceWrapper; diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index 413622e5..9cadd7a8 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -1,7 +1,7 @@ use core; use core::cmp::{max, min}; -use fearless_simd::{f32x8, Simd, SimdBase, SimdFloat, SimdMask}; +use fearless_simd::{Simd, SimdBase, SimdFloat, SimdMask, f32x8}; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; @@ -14,7 +14,7 @@ use super::histogram::{ HistogramClear, HistogramCommand, HistogramDistance, HistogramLiteral, }; use super::util::FastLog2; -use super::vectorization::{detect_level, Mem256f}; +use super::vectorization::{Mem256f, detect_level}; use crate::enc::combined_alloc::allocate; use crate::enc::floatX; diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs index c7317958..9b93c123 100755 --- a/src/enc/brotli_bit_stream.rs +++ b/src/enc/brotli_bit_stream.rs @@ -5,23 +5,24 @@ use core::cmp::{max, min}; #[cfg(feature = "std")] use std::io::Write; +use super::super::alloc; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::super::dictionary::{ kBrotliDictionary, kBrotliDictionaryOffsetsByLength, kBrotliDictionarySizeBitsByLength, }; use super::super::transform::TransformDictionaryWord; -use super::super::{alloc, core}; use super::block_split::BlockSplit; use super::combined_alloc::BrotliAlloc; use super::command::{Command, GetCopyLengthCode, GetInsertLengthCode}; use super::constants::{ - kCodeLengthBits, kCodeLengthDepth, kCopyBase, kCopyExtra, kInsBase, kInsExtra, - kNonZeroRepsBits, kNonZeroRepsDepth, kSigned3BitContextLookup, kStaticCommandCodeBits, - kStaticCommandCodeDepth, kStaticDistanceCodeBits, kStaticDistanceCodeDepth, kUTF8ContextLookup, - kZeroRepsBits, kZeroRepsDepth, BROTLI_CONTEXT_LUT, BROTLI_NUM_BLOCK_LEN_SYMBOLS, - BROTLI_NUM_COMMAND_SYMBOLS, BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS, BROTLI_NUM_LITERAL_SYMBOLS, + BROTLI_CONTEXT_LUT, BROTLI_NUM_BLOCK_LEN_SYMBOLS, BROTLI_NUM_COMMAND_SYMBOLS, + BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS, BROTLI_NUM_LITERAL_SYMBOLS, kCodeLengthBits, + kCodeLengthDepth, kCopyBase, kCopyExtra, kInsBase, kInsExtra, kNonZeroRepsBits, + kNonZeroRepsDepth, kSigned3BitContextLookup, kStaticCommandCodeBits, kStaticCommandCodeDepth, + kStaticDistanceCodeBits, kStaticDistanceCodeDepth, kUTF8ContextLookup, kZeroRepsBits, + kZeroRepsDepth, }; -use super::context_map_entropy::{speed_to_tuple, ContextMapEntropy, SpeedAndMax}; +use super::context_map_entropy::{ContextMapEntropy, SpeedAndMax, speed_to_tuple}; use super::entropy_encode::{ BrotliConvertBitDepthsToSymbols, BrotliCreateHuffmanTree, BrotliSetDepth, BrotliWriteHuffmanTree, HuffmanComparator, HuffmanTree, SortHuffmanTreeItems, @@ -34,9 +35,9 @@ use super::interface::StaticCommand; use super::static_dict::kNumDistanceCacheEntries; use super::util::floatX; use super::{find_stride, interface, prior_eval, stride_eval}; +use crate::VERSION; use crate::enc::backward_references::BrotliEncoderParams; use crate::enc::combined_alloc::{alloc_default, alloc_or_default, allocate}; -use crate::VERSION; pub struct PrefixCodeRange { pub offset: u32, @@ -1153,12 +1154,12 @@ pub struct MetaBlockSplit< pub distance_histograms_size: usize, } impl< - Alloc: alloc::Allocator - + alloc::Allocator - + alloc::Allocator - + alloc::Allocator - + alloc::Allocator, - > Default for MetaBlockSplit + Alloc: alloc::Allocator + + alloc::Allocator + + alloc::Allocator + + alloc::Allocator + + alloc::Allocator, +> Default for MetaBlockSplit { fn default() -> Self { Self { @@ -1180,12 +1181,12 @@ impl< } impl< - Alloc: alloc::Allocator - + alloc::Allocator - + alloc::Allocator - + alloc::Allocator - + alloc::Allocator, - > MetaBlockSplit + Alloc: alloc::Allocator + + alloc::Allocator + + alloc::Allocator + + alloc::Allocator + + alloc::Allocator, +> MetaBlockSplit { pub fn new() -> Self { Self::default() @@ -1369,11 +1370,7 @@ fn NextBlockTypeCode(calculator: &mut BlockTypeCodeCalculator, type_: u8) -> usi fn BlockLengthPrefixCode(len: u32) -> u32 { let mut code: u32 = (if len >= 177u32 { - if len >= 753u32 { - 20i32 - } else { - 14i32 - } + if len >= 753u32 { 20i32 } else { 14i32 } } else if len >= 41u32 { 7i32 } else { @@ -2897,7 +2894,7 @@ pub fn BrotliWriteMetadataMetaBlock( #[cfg(test)] mod test { - use crate::enc::brotli_bit_stream::{encode_base_128, MAX_SIZE_ENCODING}; + use crate::enc::brotli_bit_stream::{MAX_SIZE_ENCODING, encode_base_128}; #[test] fn test_encode_base_128() { diff --git a/src/enc/cluster.rs b/src/enc/cluster.rs index bb86acdf..244786b2 100644 --- a/src/enc/cluster.rs +++ b/src/enc/cluster.rs @@ -1,7 +1,7 @@ -use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core::cmp::min; -use {alloc, core}; +use crate::alloc; use super::bit_cost::BrotliPopulationCost; use super::histogram::{ diff --git a/src/enc/combined_alloc.rs b/src/enc/combined_alloc.rs index 17fa9d70..5ed65178 100644 --- a/src/enc/combined_alloc.rs +++ b/src/enc/combined_alloc.rs @@ -1,4 +1,4 @@ -pub use alloc::Allocator; +pub use crate::alloc::Allocator; #[cfg(feature = "std")] use alloc_stdlib::StandardAlloc; @@ -10,7 +10,7 @@ use super::hash_to_binary_tree::ZopfliNode; use super::histogram::{ContextType, HistogramCommand, HistogramDistance, HistogramLiteral}; use super::interface::StaticCommand; use super::util::floatX; -use super::{s16, v8, PDF}; +use super::{PDF, s16, v8}; /* struct CombiningAllocator, AllocT2:Allocator>(AllocT1, AllocT2); @@ -96,25 +96,25 @@ pub struct CombiningAllocator< } impl< - AllocU8: Allocator, - AllocU16: Allocator, - AllocI32: Allocator, - AllocU32: Allocator, - AllocU64: Allocator, - AllocCommand: Allocator, - AllocFloatX: Allocator, - AllocV8: Allocator, - AllocS16: Allocator, - AllocPDF: Allocator, - AllocStaticCommand: Allocator, - AllocHistogramLiteral: Allocator, - AllocHistogramCommand: Allocator, - AllocHistogramDistance: Allocator, - AllocHistogramPair: Allocator, - AllocContextType: Allocator, - AllocHuffmanTree: Allocator, - AllocZopfliNode: Allocator, - > + AllocU8: Allocator, + AllocU16: Allocator, + AllocI32: Allocator, + AllocU32: Allocator, + AllocU64: Allocator, + AllocCommand: Allocator, + AllocFloatX: Allocator, + AllocV8: Allocator, + AllocS16: Allocator, + AllocPDF: Allocator, + AllocStaticCommand: Allocator, + AllocHistogramLiteral: Allocator, + AllocHistogramCommand: Allocator, + AllocHistogramDistance: Allocator, + AllocHistogramPair: Allocator, + AllocContextType: Allocator, + AllocHuffmanTree: Allocator, + AllocZopfliNode: Allocator, +> CombiningAllocator< AllocU8, AllocU16, @@ -180,25 +180,25 @@ impl< } impl< - AllocU8: Allocator, - AllocU16: Allocator, - AllocI32: Allocator, - AllocU32: Allocator, - AllocU64: Allocator, - AllocCommand: Allocator, - AllocFloatX: Allocator, - AllocV8: Allocator, - AllocS16: Allocator, - AllocPDF: Allocator, - AllocStaticCommand: Allocator, - AllocHistogramLiteral: Allocator, - AllocHistogramCommand: Allocator, - AllocHistogramDistance: Allocator, - AllocHistogramPair: Allocator, - AllocContextType: Allocator, - AllocHuffmanTree: Allocator, - AllocZopfliNode: Allocator, - > BrotliAlloc + AllocU8: Allocator, + AllocU16: Allocator, + AllocI32: Allocator, + AllocU32: Allocator, + AllocU64: Allocator, + AllocCommand: Allocator, + AllocFloatX: Allocator, + AllocV8: Allocator, + AllocS16: Allocator, + AllocPDF: Allocator, + AllocStaticCommand: Allocator, + AllocHistogramLiteral: Allocator, + AllocHistogramCommand: Allocator, + AllocHistogramDistance: Allocator, + AllocHistogramPair: Allocator, + AllocContextType: Allocator, + AllocHuffmanTree: Allocator, + AllocZopfliNode: Allocator, +> BrotliAlloc for CombiningAllocator< AllocU8, AllocU16, @@ -223,25 +223,25 @@ impl< } impl< - AllocU8: Allocator + Default, - AllocU16: Allocator + Default, - AllocI32: Allocator + Default, - AllocU32: Allocator + Default, - AllocU64: Allocator + Default, - AllocCommand: Allocator + Default, - AllocFloatX: Allocator + Default, - AllocV8: Allocator + Default, - AllocS16: Allocator + Default, - AllocPDF: Allocator + Default, - AllocStaticCommand: Allocator + Default, - AllocHistogramLiteral: Allocator + Default, - AllocHistogramCommand: Allocator + Default, - AllocHistogramDistance: Allocator + Default, - AllocHistogramPair: Allocator + Default, - AllocContextType: Allocator + Default, - AllocHuffmanTree: Allocator + Default, - AllocZopfliNode: Allocator + Default, - > Default + AllocU8: Allocator + Default, + AllocU16: Allocator + Default, + AllocI32: Allocator + Default, + AllocU32: Allocator + Default, + AllocU64: Allocator + Default, + AllocCommand: Allocator + Default, + AllocFloatX: Allocator + Default, + AllocV8: Allocator + Default, + AllocS16: Allocator + Default, + AllocPDF: Allocator + Default, + AllocStaticCommand: Allocator + Default, + AllocHistogramLiteral: Allocator + Default, + AllocHistogramCommand: Allocator + Default, + AllocHistogramDistance: Allocator + Default, + AllocHistogramPair: Allocator + Default, + AllocContextType: Allocator + Default, + AllocHuffmanTree: Allocator + Default, + AllocZopfliNode: Allocator + Default, +> Default for CombiningAllocator< AllocU8, AllocU16, @@ -288,25 +288,25 @@ impl< } impl< - AllocU8: Allocator + Clone, - AllocU16: Allocator + Clone, - AllocI32: Allocator + Clone, - AllocU32: Allocator + Clone, - AllocU64: Allocator + Clone, - AllocCommand: Allocator + Clone, - AllocFloatX: Allocator + Clone, - AllocV8: Allocator + Clone, - AllocS16: Allocator + Clone, - AllocPDF: Allocator + Clone, - AllocStaticCommand: Allocator + Clone, - AllocHistogramLiteral: Allocator + Clone, - AllocHistogramCommand: Allocator + Clone, - AllocHistogramDistance: Allocator + Clone, - AllocHistogramPair: Allocator + Clone, - AllocContextType: Allocator + Clone, - AllocHuffmanTree: Allocator + Clone, - AllocZopfliNode: Allocator + Clone, - > Clone + AllocU8: Allocator + Clone, + AllocU16: Allocator + Clone, + AllocI32: Allocator + Clone, + AllocU32: Allocator + Clone, + AllocU64: Allocator + Clone, + AllocCommand: Allocator + Clone, + AllocFloatX: Allocator + Clone, + AllocV8: Allocator + Clone, + AllocS16: Allocator + Clone, + AllocPDF: Allocator + Clone, + AllocStaticCommand: Allocator + Clone, + AllocHistogramLiteral: Allocator + Clone, + AllocHistogramCommand: Allocator + Clone, + AllocHistogramDistance: Allocator + Clone, + AllocHistogramPair: Allocator + Clone, + AllocContextType: Allocator + Clone, + AllocHuffmanTree: Allocator + Clone, + AllocZopfliNode: Allocator + Clone, +> Clone for CombiningAllocator< AllocU8, AllocU16, @@ -353,25 +353,25 @@ impl< } impl< - AllocU8: Allocator + Copy, - AllocU16: Allocator + Copy, - AllocI32: Allocator + Copy, - AllocU32: Allocator + Copy, - AllocU64: Allocator + Copy, - AllocCommand: Allocator + Copy, - AllocFloatX: Allocator + Copy, - AllocV8: Allocator + Copy, - AllocS16: Allocator + Copy, - AllocPDF: Allocator + Copy, - AllocStaticCommand: Allocator + Copy, - AllocHistogramLiteral: Allocator + Copy, - AllocHistogramCommand: Allocator + Copy, - AllocHistogramDistance: Allocator + Copy, - AllocHistogramPair: Allocator + Copy, - AllocContextType: Allocator + Copy, - AllocHuffmanTree: Allocator + Copy, - AllocZopfliNode: Allocator + Copy, - > Copy + AllocU8: Allocator + Copy, + AllocU16: Allocator + Copy, + AllocI32: Allocator + Copy, + AllocU32: Allocator + Copy, + AllocU64: Allocator + Copy, + AllocCommand: Allocator + Copy, + AllocFloatX: Allocator + Copy, + AllocV8: Allocator + Copy, + AllocS16: Allocator + Copy, + AllocPDF: Allocator + Copy, + AllocStaticCommand: Allocator + Copy, + AllocHistogramLiteral: Allocator + Copy, + AllocHistogramCommand: Allocator + Copy, + AllocHistogramDistance: Allocator + Copy, + AllocHistogramPair: Allocator + Copy, + AllocContextType: Allocator + Copy, + AllocHuffmanTree: Allocator + Copy, + AllocZopfliNode: Allocator + Copy, +> Copy for CombiningAllocator< AllocU8, AllocU16, @@ -401,25 +401,25 @@ macro_rules! implement_allocator { $sub_type_name: ty, $local_name: ident) => { impl< - AllocU8: Allocator, - AllocU16: Allocator, - AllocI32: Allocator, - AllocU32: Allocator, - AllocU64: Allocator, - AllocCommand: Allocator, - AllocFloatX: Allocator, - AllocV8: Allocator, - AllocS16: Allocator, - AllocPDF: Allocator, - AllocStaticCommand: Allocator, - AllocHistogramLiteral: Allocator, - AllocHistogramCommand: Allocator, - AllocHistogramDistance: Allocator, - AllocHistogramPair: Allocator, - AllocContextType: Allocator, - AllocHuffmanTree: Allocator, - AllocZopfliNode: Allocator, - > Allocator<$type_name> + AllocU8: Allocator, + AllocU16: Allocator, + AllocI32: Allocator, + AllocU32: Allocator, + AllocU64: Allocator, + AllocCommand: Allocator, + AllocFloatX: Allocator, + AllocV8: Allocator, + AllocS16: Allocator, + AllocPDF: Allocator, + AllocStaticCommand: Allocator, + AllocHistogramLiteral: Allocator, + AllocHistogramCommand: Allocator, + AllocHistogramDistance: Allocator, + AllocHistogramPair: Allocator, + AllocContextType: Allocator, + AllocHuffmanTree: Allocator, + AllocZopfliNode: Allocator, + > Allocator<$type_name> for CombiningAllocator< AllocU8, AllocU16, diff --git a/src/enc/compress_fragment.rs b/src/enc/compress_fragment.rs index 56bb8cf3..ab6616e7 100644 --- a/src/enc/compress_fragment.rs +++ b/src/enc/compress_fragment.rs @@ -9,12 +9,12 @@ use core::cmp::min; use super::super::alloc; use super::backward_references::kHashMul32; use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree}; -use super::compress_fragment_two_pass::{memcpy, BrotliWriteBits}; +use super::compress_fragment_two_pass::{BrotliWriteBits, memcpy}; use super::entropy_encode::{ BrotliConvertBitDepthsToSymbols, BrotliCreateHuffmanTree, HuffmanTree, }; use super::static_dict::{ - FindMatchLengthWithLimit, BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, + BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, FindMatchLengthWithLimit, }; use super::util::{FastLog2, Log2FloorNonZero}; use crate::enc::compress_fragment_two_pass::store_meta_block_header; @@ -1049,7 +1049,7 @@ fn compress_fragment_fast_impl>( } macro_rules! compress_specialization { - ($table_bits : expr, $fname: ident) => { + ($table_bits : expr_2021, $fname: ident) => { fn $fname>( mht: &mut AllocHT, input: &[u8], diff --git a/src/enc/compress_fragment_two_pass.rs b/src/enc/compress_fragment_two_pass.rs index 8b8bbd06..e0217175 100644 --- a/src/enc/compress_fragment_two_pass.rs +++ b/src/enc/compress_fragment_two_pass.rs @@ -9,10 +9,10 @@ use super::entropy_encode::{ BrotliConvertBitDepthsToSymbols, BrotliCreateHuffmanTree, HuffmanTree, }; use super::static_dict::{ - FindMatchLengthWithLimit, BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, - BROTLI_UNALIGNED_STORE64, + BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64, BROTLI_UNALIGNED_STORE64, + FindMatchLengthWithLimit, }; -use super::util::{floatX, Log2FloorNonZero}; +use super::util::{Log2FloorNonZero, floatX}; static kCompressFragmentTwoPassBlockSize: usize = (1i32 << 17) as usize; // returns number of commands inserted @@ -698,7 +698,7 @@ fn compress_fragment_two_pass_impl>( } } macro_rules! compress_specialization { - ($table_bits : expr, $fname: ident) => { + ($table_bits : expr_2021, $fname: ident) => { fn $fname>( mht: &mut AllocHT, input: &[u8], diff --git a/src/enc/context_map_entropy.rs b/src/enc/context_map_entropy.rs index 9adaf65f..5161a4e5 100644 --- a/src/enc/context_map_entropy.rs +++ b/src/enc/context_map_entropy.rs @@ -3,9 +3,9 @@ use core; use super::super::alloc; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::input_pair::{InputPair, InputReference, InputReferenceMut}; -pub use super::ir_interpret::{push_base, Context, IRInterpreter}; -use super::util::{floatX, FastLog2u16}; -use super::weights::{Weights, BLEND_FIXED_POINT_PRECISION}; +pub use super::ir_interpret::{Context, IRInterpreter, push_base}; +use super::util::{FastLog2u16, floatX}; +use super::weights::{BLEND_FIXED_POINT_PRECISION, Weights}; use super::{find_stride, interface}; use crate::enc::combined_alloc::alloc_if; diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 52129451..7e1abf14 100644 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -1,4 +1,4 @@ -use alloc::Allocator; +use crate::alloc::Allocator; use core; use core::cmp::{max, min}; @@ -6,20 +6,20 @@ use super::super::alloc; use super::super::alloc::{SliceWrapper, SliceWrapperMut}; use super::backward_references::{ AdvHashSpecialization, AdvHasher, AnyHasher, BasicHasher, BrotliCreateBackwardReferences, - BrotliEncoderMode, BrotliEncoderParams, BrotliHasherParams, H2Sub, H3Sub, H4Sub, H54Sub, H5Sub, - H6Sub, HQ5Sub, HQ7Sub, HowPrepared, StoreLookaheadThenStore, Struct1, UnionHasher, H9, - H9_BLOCK_BITS, H9_BLOCK_SIZE, H9_BUCKET_BITS, H9_NUM_LAST_DISTANCES_TO_CHECK, + BrotliEncoderMode, BrotliEncoderParams, BrotliHasherParams, H2Sub, H3Sub, H4Sub, H5Sub, H6Sub, + H9, H9_BLOCK_BITS, H9_BLOCK_SIZE, H9_BUCKET_BITS, H9_NUM_LAST_DISTANCES_TO_CHECK, H54Sub, + HQ5Sub, HQ7Sub, HowPrepared, StoreLookaheadThenStore, Struct1, UnionHasher, }; -use super::bit_cost::{shannon_entropy, BitsEntropy}; +use super::bit_cost::{BitsEntropy, shannon_entropy}; use super::brotli_bit_stream::{ - store_meta_block, store_meta_block_fast, store_meta_block_trivial, - store_uncompressed_meta_block, BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock, - BrotliWritePaddingMetaBlock, MetaBlockSplit, RecoderState, + BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock, BrotliWritePaddingMetaBlock, + MetaBlockSplit, RecoderState, store_meta_block, store_meta_block_fast, + store_meta_block_trivial, store_uncompressed_meta_block, }; use super::combined_alloc::BrotliAlloc; -use super::command::{get_length_code, BrotliDistanceParams, Command}; +use super::command::{BrotliDistanceParams, Command, get_length_code}; use super::compress_fragment::compress_fragment_fast; -use super::compress_fragment_two_pass::{compress_fragment_two_pass, BrotliWriteBits}; +use super::compress_fragment_two_pass::{BrotliWriteBits, compress_fragment_two_pass}; use super::constants::{ BROTLI_CONTEXT, BROTLI_CONTEXT_LUT, BROTLI_MAX_NDIRECT, BROTLI_MAX_NPOSTFIX, BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS, BROTLI_WINDOW_GAP, @@ -34,8 +34,8 @@ use super::metablock::{ BrotliOptimizeHistograms, }; pub use super::parameters::BrotliEncoderParameter; -use super::static_dict::{kNumDistanceCacheEntries, BrotliGetDictionary}; -use super::util::{floatX, Log2FloorNonZero}; +use super::static_dict::{BrotliGetDictionary, kNumDistanceCacheEntries}; +use super::util::{Log2FloorNonZero, floatX}; use crate::enc::combined_alloc::{alloc_default, allocate}; use crate::enc::input_pair::InputReferenceMut; use crate::enc::utf8_util::is_mostly_utf8; @@ -119,7 +119,7 @@ fn GetNextOutInternal<'a>( } } macro_rules! GetNextOut { - ($s : expr) => { + ($s : expr_2021) => { GetNextOutInternal(&$s.next_out_, $s.storage_.slice_mut(), &mut $s.tiny_buf_) }; } @@ -1657,7 +1657,7 @@ fn HashTableSize(max_table_size: usize, input_size: usize) -> usize { } macro_rules! GetHashTable { - ($s : expr, $quality: expr, $input_size : expr, $table_size : expr) => { + ($s : expr_2021, $quality: expr_2021, $input_size : expr_2021, $table_size : expr_2021) => { GetHashTableInternal( &mut $s.m8, &mut $s.small_table_, diff --git a/src/enc/interface.rs b/src/enc/interface.rs index 79f2bd16..e6e21113 100644 --- a/src/enc/interface.rs +++ b/src/enc/interface.rs @@ -1,4 +1,4 @@ -use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core; use super::histogram; @@ -481,8 +481,8 @@ impl + Default> Command { F: FnMut(SliceType), { match self { - Command::Literal(ref mut lit) => apply_func(core::mem::take(&mut lit.data)), - Command::PredictionMode(ref mut pm) => { + Command::Literal(lit) => apply_func(core::mem::take(&mut lit.data)), + Command::PredictionMode(pm) => { apply_func(core::mem::take(&mut pm.literal_context_map)); apply_func(core::mem::take( &mut pm.predmode_speed_and_distance_context_map, @@ -674,44 +674,44 @@ pub trait CommandProcessor<'a> { impl> Command { pub fn thaw_pair<'a>(&self, data: &InputPair<'a>) -> Command> { match self { - Command::Literal(ref lit) => Command::Literal(LiteralCommand { + Command::Literal(lit) => Command::Literal(LiteralCommand { data: lit.data.thaw_pair(data).unwrap(), prob: FeatureFlagSliceType::default(), high_entropy: lit.high_entropy, }), - Command::PredictionMode(ref pm) => Command::PredictionMode(PredictionModeContextMap { + Command::PredictionMode(pm) => Command::PredictionMode(PredictionModeContextMap { literal_context_map: pm.literal_context_map.thaw_pair(data).unwrap(), predmode_speed_and_distance_context_map: pm .predmode_speed_and_distance_context_map .thaw_pair(data) .unwrap(), }), - Command::Dict(ref d) => Command::Dict(*d), - Command::Copy(ref c) => Command::Copy(*c), - Command::BlockSwitchCommand(ref c) => Command::BlockSwitchCommand(*c), - Command::BlockSwitchLiteral(ref c) => Command::BlockSwitchLiteral(*c), - Command::BlockSwitchDistance(ref c) => Command::BlockSwitchDistance(*c), + Command::Dict(d) => Command::Dict(*d), + Command::Copy(c) => Command::Copy(*c), + Command::BlockSwitchCommand(c) => Command::BlockSwitchCommand(*c), + Command::BlockSwitchLiteral(c) => Command::BlockSwitchLiteral(*c), + Command::BlockSwitchDistance(c) => Command::BlockSwitchDistance(*c), } } pub fn thaw<'a>(&self, data: &'a [u8]) -> Command> { match self { - Command::Literal(ref lit) => Command::Literal(LiteralCommand { + Command::Literal(lit) => Command::Literal(LiteralCommand { data: lit.data.thaw(data), prob: FeatureFlagSliceType::default(), high_entropy: lit.high_entropy, }), - Command::PredictionMode(ref pm) => Command::PredictionMode(PredictionModeContextMap { + Command::PredictionMode(pm) => Command::PredictionMode(PredictionModeContextMap { literal_context_map: pm.literal_context_map.thaw(data), predmode_speed_and_distance_context_map: pm .predmode_speed_and_distance_context_map .thaw(data), }), - Command::Dict(ref d) => Command::Dict(*d), - Command::Copy(ref c) => Command::Copy(*c), - Command::BlockSwitchCommand(ref c) => Command::BlockSwitchCommand(*c), - Command::BlockSwitchLiteral(ref c) => Command::BlockSwitchLiteral(*c), - Command::BlockSwitchDistance(ref c) => Command::BlockSwitchDistance(*c), + Command::Dict(d) => Command::Dict(*d), + Command::Copy(c) => Command::Copy(*c), + Command::BlockSwitchCommand(c) => Command::BlockSwitchCommand(*c), + Command::BlockSwitchLiteral(c) => Command::BlockSwitchLiteral(*c), + Command::BlockSwitchDistance(c) => Command::BlockSwitchDistance(*c), } } } @@ -719,22 +719,22 @@ impl> Command { impl + Freezable> Command { pub fn freeze(&self) -> Command { match self { - Command::Literal(ref lit) => Command::Literal(LiteralCommand { + Command::Literal(lit) => Command::Literal(LiteralCommand { data: lit.data.freeze(), prob: FeatureFlagSliceType::default(), high_entropy: lit.high_entropy, }), - Command::PredictionMode(ref pm) => Command::PredictionMode(PredictionModeContextMap { + Command::PredictionMode(pm) => Command::PredictionMode(PredictionModeContextMap { literal_context_map: pm.literal_context_map.freeze(), predmode_speed_and_distance_context_map: pm .predmode_speed_and_distance_context_map .freeze(), }), - Command::Dict(ref d) => Command::Dict(*d), - Command::Copy(ref c) => Command::Copy(*c), - Command::BlockSwitchCommand(ref c) => Command::BlockSwitchCommand(*c), - Command::BlockSwitchLiteral(ref c) => Command::BlockSwitchLiteral(*c), - Command::BlockSwitchDistance(ref c) => Command::BlockSwitchDistance(*c), + Command::Dict(d) => Command::Dict(*d), + Command::Copy(c) => Command::Copy(*c), + Command::BlockSwitchCommand(c) => Command::BlockSwitchCommand(*c), + Command::BlockSwitchLiteral(c) => Command::BlockSwitchLiteral(*c), + Command::BlockSwitchDistance(c) => Command::BlockSwitchDistance(*c), } } } diff --git a/src/enc/literal_cost.rs b/src/enc/literal_cost.rs index df1d8ba6..9634ae4a 100644 --- a/src/enc/literal_cost.rs +++ b/src/enc/literal_cost.rs @@ -1,6 +1,6 @@ use core::cmp::min; -use super::util::{floatX, FastLog2f64}; +use super::util::{FastLog2f64, floatX}; use crate::enc::utf8_util::is_mostly_utf8; static kMinUTF8Ratio: floatX = 0.75; diff --git a/src/enc/mod.rs b/src/enc/mod.rs index 7060c43d..a8d2577a 100644 --- a/src/enc/mod.rs +++ b/src/enc/mod.rs @@ -41,21 +41,23 @@ mod weights; pub mod worker_pool; pub mod writer; -pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator}; +pub use crate::alloc::{ + AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator, +}; #[cfg(feature = "std")] use std::io; #[cfg(feature = "std")] use std::io::{Error, ErrorKind, Read, Write}; +pub use crate::interface::{InputPair, InputReference, InputReferenceMut}; #[cfg(feature = "std")] pub use alloc_stdlib::StandardAlloc; use brotli_decompressor::{CustomRead, CustomWrite}; #[cfg(feature = "std")] pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper}; -pub use interface::{InputPair, InputReference, InputReferenceMut}; pub use self::backward_references::{ - hash_to_binary_tree, hq as backward_references_hq, BrotliEncoderParams, UnionHasher, + BrotliEncoderParams, UnionHasher, hash_to_binary_tree, hq as backward_references_hq, }; pub use self::combined_alloc::{BrotliAlloc, CombiningAllocator}; use self::encode::{BrotliEncoderDestroyInstance, BrotliEncoderOperation}; @@ -66,14 +68,14 @@ pub use self::hash_to_binary_tree::ZopfliNode; pub use self::interface::StaticCommand; pub use self::pdf::PDF; #[cfg(not(feature = "std"))] -pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool}; +pub use self::singlethreading::{WorkerPool, compress_worker_pool, new_work_pool}; pub use self::threading::{ BatchSpawnableLite, BrotliEncoderThreadError, CompressionThreadResult, Owned, SendAlloc, }; pub use self::util::floatX; -pub use self::vectorization::{v256, v256i, Mem256f}; +pub use self::vectorization::{Mem256f, v256, v256i}; #[cfg(feature = "std")] -pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool}; +pub use self::worker_pool::{WorkerPool, compress_worker_pool, new_work_pool}; use crate::enc::encode::BrotliEncoderStateStruct; pub type s16 = vectorization::Mem16x16; diff --git a/src/enc/multithreading.rs b/src/enc/multithreading.rs index ce52f649..ddfb8cac 100644 --- a/src/enc/multithreading.rs +++ b/src/enc/multithreading.rs @@ -1,6 +1,6 @@ #![cfg(feature = "std")] -use alloc::{Allocator, SliceWrapper}; +use crate::alloc::{Allocator, SliceWrapper}; use core::marker::PhantomData; use core::mem; use std; @@ -88,11 +88,11 @@ where } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnable for MultiThreadedSpawner + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnable for MultiThreadedSpawner where >::AllocatedMemory: Send + 'static, { @@ -120,11 +120,11 @@ where } } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnableLite for MultiThreadedSpawner + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnableLite for MultiThreadedSpawner where >::AllocatedMemory: Send + 'static, >::AllocatedMemory: Send + Sync, diff --git a/src/enc/prior_eval.rs b/src/enc/prior_eval.rs index 5f4d1173..0a86d071 100644 --- a/src/enc/prior_eval.rs +++ b/src/enc/prior_eval.rs @@ -1,14 +1,14 @@ use core; use core::cmp::min; -use fearless_simd::{f32x8, i16x16, Level, Select, Simd, SimdBase, SimdInt}; +use fearless_simd::{Level, Select, Simd, SimdBase, SimdInt, f32x8, i16x16}; use super::super::alloc; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; use super::input_pair::{InputPair, InputReference, InputReferenceMut}; -use super::ir_interpret::{push_base, IRInterpreter}; -use super::util::{floatX, FastLog2u16}; +use super::ir_interpret::{IRInterpreter, push_base}; +use super::util::{FastLog2u16, floatX}; use super::vectorization::detect_level; use super::{find_stride, interface, s16, v8}; use crate::enc::combined_alloc::{alloc_default, alloc_if}; diff --git a/src/enc/reader.rs b/src/enc/reader.rs index 746c3961..38c538c9 100644 --- a/src/enc/reader.rs +++ b/src/enc/reader.rs @@ -1,4 +1,4 @@ -use alloc::{Allocator, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapperMut}; #[cfg(feature = "std")] use std::io; #[cfg(feature = "std")] diff --git a/src/enc/singlethreading.rs b/src/enc/singlethreading.rs index 92455afd..df1dca2f 100644 --- a/src/enc/singlethreading.rs +++ b/src/enc/singlethreading.rs @@ -1,4 +1,4 @@ -use alloc::{Allocator, SliceWrapper}; +use crate::alloc::{Allocator, SliceWrapper}; use core::marker::PhantomData; use core::mem; #[cfg(feature = "std")] @@ -60,11 +60,11 @@ impl OwnedRetriever for SingleThreadedOwnedRetriever { pub struct SingleThreadedSpawner {} impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnable for SingleThreadedSpawner + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnable for SingleThreadedSpawner where >::AllocatedMemory: Send + 'static, { @@ -92,11 +92,11 @@ where } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnableLite for SingleThreadedSpawner + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnableLite for SingleThreadedSpawner where >::AllocatedMemory: Send + 'static, { diff --git a/src/enc/static_dict.rs b/src/enc/static_dict.rs index fa8f7a36..58d31894 100644 --- a/src/enc/static_dict.rs +++ b/src/enc/static_dict.rs @@ -5,7 +5,7 @@ use super::super::dictionary::{ kBrotliDictionary, kBrotliDictionaryOffsetsByLength, kBrotliDictionarySizeBitsByLength, }; use super::static_dict_lut::{ - kDictHashMul32, kDictNumBits, kStaticDictionaryBuckets, kStaticDictionaryWords, DictWord, + DictWord, kDictHashMul32, kDictNumBits, kStaticDictionaryBuckets, kStaticDictionaryWords, }; #[allow(unused)] static kUppercaseFirst: u8 = 10u8; @@ -69,7 +69,7 @@ pub fn BROTLI_UNALIGNED_STORE64(outp: &mut [u8], v: u64) { } macro_rules! sub_match { - ($s1 : expr, $s2 : expr, $limit : expr, $matched : expr, $split_pair1 : expr, $split_pair2 : expr, $s1_lo : expr, $s2_lo : expr, $s1_as_64 : expr, $s2_as_64 : expr, $vec_len: expr) => { + ($s1 : expr_2021, $s2 : expr_2021, $limit : expr_2021, $matched : expr_2021, $split_pair1 : expr_2021, $split_pair2 : expr_2021, $s1_lo : expr_2021, $s2_lo : expr_2021, $s1_as_64 : expr_2021, $s2_as_64 : expr_2021, $vec_len: expr_2021) => { $split_pair1 = $s1.split_at($vec_len); $s1_lo[..$vec_len].clone_from_slice($split_pair1.0); $s1 = $split_pair1.1; @@ -93,7 +93,7 @@ macro_rules! sub_match { } macro_rules! sub_match8 { - ($s1 : expr, $s2 : expr, $limit : expr, $matched : expr, $s1_as_64 : expr, $s2_as_64 : expr) => { + ($s1 : expr_2021, $s2 : expr_2021, $limit : expr_2021, $matched : expr_2021, $s1_as_64 : expr_2021, $s2_as_64 : expr_2021) => { $limit -= 8; $s1_as_64 = BROTLI_UNALIGNED_LOAD64($s1); $s1 = $s1.split_at(8).1; diff --git a/src/enc/stride_eval.rs b/src/enc/stride_eval.rs index 511730ff..0ef129cd 100644 --- a/src/enc/stride_eval.rs +++ b/src/enc/stride_eval.rs @@ -5,9 +5,9 @@ use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; use super::input_pair::{InputPair, InputReference, InputReferenceMut}; use super::interface; -use super::ir_interpret::{push_base, IRInterpreter}; +use super::ir_interpret::{IRInterpreter, push_base}; use super::prior_eval::DEFAULT_SPEED; -use super::util::{floatX, FastLog2u16}; +use super::util::{FastLog2u16, floatX}; use crate::enc::combined_alloc::{alloc_default, allocate}; const NIBBLE_PRIOR_SIZE: usize = 16; pub const STRIDE_PRIOR_SIZE: usize = 256 * 256 * NIBBLE_PRIOR_SIZE * 2; diff --git a/src/enc/test.rs b/src/enc/test.rs index 18709323..187888bd 100644 --- a/src/enc/test.rs +++ b/src/enc/test.rs @@ -2,15 +2,15 @@ extern crate alloc_no_stdlib; extern crate brotli_decompressor; -extern "C" { +unsafe extern "C" { fn calloc(n_elem: usize, el_size: usize) -> *mut u8; } -extern "C" { +unsafe extern "C" { fn free(ptr: *mut u8); } // FIXME: Remove this after https://github.com/dropbox/rust-alloc-no-stdlib/issues/19 is fixed -use alloc::{ +use crate::alloc::{ declare_stack_allocator_struct, define_allocator_memory_pool, define_stack_allocator_traits, static_array, }; @@ -21,7 +21,7 @@ use core::ops; use brotli_decompressor::HuffmanCode; use super::super::alloc::{ - bzero, AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator, + AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator, bzero, }; pub use super::super::{BrotliDecompressStream, BrotliResult, BrotliState}; use super::cluster::HistogramPair; @@ -31,7 +31,7 @@ use super::encode::{BrotliEncoderOperation, BrotliEncoderParameter}; use super::entropy_encode::HuffmanTree; use super::histogram::{ContextType, HistogramCommand, HistogramDistance, HistogramLiteral}; use super::pdf::PDF; -use super::{interface, s16, v8, StaticCommand, ZopfliNode}; +use super::{StaticCommand, ZopfliNode, interface, s16, v8}; use crate::enc::encode::BrotliEncoderStateStruct; declare_stack_allocator_struct!(MemPool, 128, stack); @@ -345,7 +345,7 @@ fn test_roundtrip_10x10y() { } macro_rules! test_roundtrip_file { - ($filedata : expr, $bufsize: expr, $quality: expr, $lgwin: expr, $magic: expr, $in_buf:expr, $out_buf:expr) => {{ + ($filedata : expr_2021, $bufsize: expr_2021, $quality: expr_2021, $lgwin: expr_2021, $magic: expr_2021, $in_buf:expr_2021, $out_buf:expr_2021) => {{ let stack_u8_buffer = unsafe { alloc::define_allocator_memory_pool!(4096, u8, [0; 18 * 1024 * 1024], calloc) }; diff --git a/src/enc/threading/mod.rs b/src/enc/threading/mod.rs index ec98b218..c6eebaaa 100644 --- a/src/enc/threading/mod.rs +++ b/src/enc/threading/mod.rs @@ -1,16 +1,16 @@ -use alloc::{Allocator, SliceWrapper, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core::marker::PhantomData; use core::ops::Range; use core::{any, mem}; #[cfg(feature = "std")] use std; +use super::BrotliAlloc; use super::backward_references::{AnyHasher, BrotliEncoderParams, CloneWithAlloc, UnionHasher}; use super::encode::{ - hasher_setup, BrotliEncoderDestroyInstance, BrotliEncoderMaxCompressedSize, - BrotliEncoderOperation, SanitizeParams, + BrotliEncoderDestroyInstance, BrotliEncoderMaxCompressedSize, BrotliEncoderOperation, + SanitizeParams, hasher_setup, }; -use super::BrotliAlloc; use crate::concat::{BroCatli, BroCatliResult}; use crate::enc::combined_alloc::{alloc_default, allocate}; use crate::enc::encode::BrotliEncoderStateStruct; @@ -81,11 +81,11 @@ pub enum InternalSendAlloc< SpawningOrJoining(PhantomData), } impl< - ReturnVal: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - Join: Joinable, - > InternalSendAlloc + ReturnVal: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + Join: Joinable, +> InternalSendAlloc where >::AllocatedMemory: Send, { @@ -108,11 +108,11 @@ where >::AllocatedMemory: Send; impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - Join: Joinable, - > SendAlloc + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + Join: Joinable, +> SendAlloc where >::AllocatedMemory: Send, { @@ -284,14 +284,14 @@ impl, - UnionHasher, - Alloc, - ( - >::AllocatedMemory, - BrotliEncoderParams, - ), - >, + CompressionThreadResult, + UnionHasher, + Alloc, + ( + >::AllocatedMemory, + BrotliEncoderParams, + ), + >, >( params: &BrotliEncoderParams, input_slice: &[u8], @@ -414,11 +414,11 @@ pub fn CompressMulti< Alloc: BrotliAlloc + Send + 'static, SliceW: SliceWrapper + Send + 'static + Sync, Spawner: BatchSpawnableLite< - CompressionThreadResult, - UnionHasher, - Alloc, - (SliceW, BrotliEncoderParams), - >, + CompressionThreadResult, + UnionHasher, + Alloc, + (SliceW, BrotliEncoderParams), + >, >( params: &BrotliEncoderParams, owned_input: &mut Owned, @@ -652,10 +652,15 @@ where } } } - if let Ok(retrieved_owned_input) = spawner_and_input.unwrap() { - *owned_input = Owned::new(retrieved_owned_input.0); // return the input to its rightful owner before returning - } else if compression_result.is_ok() { - compression_result = Err(BrotliEncoderThreadError::OtherThreadPanic); + match spawner_and_input.unwrap() { + Ok(retrieved_owned_input) => { + *owned_input = Owned::new(retrieved_owned_input.0); // return the input to its rightful owner before returning + } + _ => { + if compression_result.is_ok() { + compression_result = Err(BrotliEncoderThreadError::OtherThreadPanic); + } + } } compression_result } diff --git a/src/enc/threading/test.rs b/src/enc/threading/test.rs index 937f9963..0ba0be52 100755 --- a/src/enc/threading/test.rs +++ b/src/enc/threading/test.rs @@ -3,7 +3,7 @@ // Unit tests for the parent `threading` module. These exercise CompressMulti's // error-draining behavior use super::*; -use alloc::SliceWrapper; +use crate::alloc::SliceWrapper; use alloc_stdlib::StandardAlloc; use core::sync::atomic::{AtomicUsize, Ordering}; diff --git a/src/enc/util.rs b/src/enc/util.rs index 5738ff07..77f7f784 100644 --- a/src/enc/util.rs +++ b/src/enc/util.rs @@ -1,7 +1,7 @@ #![allow(clippy::excessive_precision)] -use crate::enc::log_table_16::logs_16; use crate::enc::log_table_8::logs_8; +use crate::enc::log_table_16::logs_16; #[cfg(feature = "float64")] pub type floatX = f64; diff --git a/src/enc/vectorization.rs b/src/enc/vectorization.rs index 968fe592..abfdc7b5 100644 --- a/src/enc/vectorization.rs +++ b/src/enc/vectorization.rs @@ -9,7 +9,7 @@ use core::ops::{Index, IndexMut}; use core::slice::SliceIndex; -use fearless_simd::{f32x8, i16x16, i32x8, Level, Simd, SimdInto}; +use fearless_simd::{Level, Simd, SimdInto, f32x8, i16x16, i32x8}; /// The instruction set the vectorized encoder paths run on. /// diff --git a/src/enc/worker_pool.rs b/src/enc/worker_pool.rs index b307b27e..cce2d66d 100644 --- a/src/enc/worker_pool.rs +++ b/src/enc/worker_pool.rs @@ -1,6 +1,6 @@ #![cfg(feature = "std")] -use alloc::{Allocator, SliceWrapper}; +use crate::alloc::{Allocator, SliceWrapper}; use core::mem; use std; // in-place thread create @@ -49,11 +49,11 @@ struct WorkQueue< cur_work_id: u64, } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > Default for WorkQueue + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> Default for WorkQueue { fn default() -> Self { WorkQueue { @@ -84,11 +84,11 @@ pub struct WorkerPool< } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > Drop for WorkerPool + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> Drop for WorkerPool { fn drop(&mut self) { { @@ -105,11 +105,11 @@ impl< } } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > WorkerPool + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> WorkerPool { fn do_work(queue: Arc<(Mutex>, Condvar)>) { loop { @@ -126,19 +126,24 @@ impl< if local_queue.immediate_shutdown { break; } - possible_job = if let Some(res) = local_queue.jobs.pop() { - cvar.notify_all(); - local_queue.num_in_progress += 1; - res - } else if local_queue.shutdown { - break; - } else { - let _lock = cvar.wait(local_queue); // unlock immediately, unfortunately - continue; + possible_job = match local_queue.jobs.pop() { + Some(res) => { + cvar.notify_all(); + local_queue.num_in_progress += 1; + res + } + _ => { + if local_queue.shutdown { + break; + } else { + let _lock = cvar.wait(local_queue); // unlock immediately, unfortunately + continue; + } + } }; } - ret = if let Ok(job_data) = possible_job.data.read() { - JobReply { + ret = match possible_job.data.read() { + Ok(job_data) => JobReply { result: (possible_job.func)( possible_job.extra_input, possible_job.index, @@ -147,9 +152,10 @@ impl< possible_job.alloc, ), work_id: possible_job.work_id, + }, + _ => { + break; // poisoned lock } - } else { - break; // poisoned lock }; } { @@ -311,11 +317,11 @@ pub struct WorkerJoinable< work_id: u64, } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > Joinable + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> Joinable for WorkerJoinable { fn join(self) -> Result { @@ -339,11 +345,11 @@ impl< } impl< - ReturnValue: Send + 'static, - ExtraInput: Send + 'static, - Alloc: BrotliAlloc + Send + 'static, - U: Send + 'static + Sync, - > BatchSpawnableLite + ReturnValue: Send + 'static, + ExtraInput: Send + 'static, + Alloc: BrotliAlloc + Send + 'static, + U: Send + 'static + Sync, +> BatchSpawnableLite for WorkerPool where >::AllocatedMemory: Send + 'static, diff --git a/src/enc/writer.rs b/src/enc/writer.rs index b18c976d..338f919e 100644 --- a/src/enc/writer.rs +++ b/src/enc/writer.rs @@ -1,4 +1,4 @@ -use alloc::{Allocator, SliceWrapperMut}; +use crate::alloc::{Allocator, SliceWrapperMut}; #[cfg(feature = "std")] use std::io; #[cfg(feature = "std")] @@ -141,10 +141,13 @@ pub fn write_all, ErrMaker: FnMut() -> Option { + return Err(err); + } + _ => { + return Ok(()); + } } } } diff --git a/src/ffi/broccoli.rs b/src/ffi/broccoli.rs index 76eb19e4..c8b89dad 100644 --- a/src/ffi/broccoli.rs +++ b/src/ffi/broccoli.rs @@ -52,33 +52,35 @@ impl From for BroCatli { } } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BroccoliCreateInstance() -> BroccoliState { BroCatli::new().into() } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BroccoliCreateInstanceWithWindowSize(window_size: u8) -> BroccoliState { match BroCatli::try_new_with_window_size(window_size) { Ok(bro_catli) => bro_catli.into(), Err(_) => BroCatli::new().into(), } } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BroccoliDestroyInstance(_state: BroccoliState) {} -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BroccoliNewBrotliFile(state: *mut BroccoliState) { - if let Err(panic_err) = catch_panic(|| { - let mut bro_catli: BroCatli = (*state).into(); - bro_catli.new_brotli_file(); - *state = BroccoliState::from(bro_catli); - BroCatliResult::Success - }) { - error_print(panic_err); + unsafe { + if let Err(panic_err) = catch_panic(|| { + let mut bro_catli: BroCatli = (*state).into(); + bro_catli.new_brotli_file(); + *state = BroccoliState::from(bro_catli); + BroCatliResult::Success + }) { + error_print(panic_err); + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BroccoliConcatStream( state: *mut BroccoliState, available_in: *mut usize, @@ -86,27 +88,30 @@ pub unsafe extern "C" fn BroccoliConcatStream( available_out: *mut usize, output_buf_ptr: *mut *mut u8, ) -> BroccoliResult { - catch_panic(|| { - let input_buf = slice_from_raw_parts_or_nil(*input_buf_ptr, *available_in); - let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); - let mut input_offset = 0usize; - let mut output_offset = 0usize; - let mut bro_catli: BroCatli = (*state).into(); - let ret = bro_catli.stream(input_buf, &mut input_offset, output_buf, &mut output_offset); - *input_buf_ptr = (*input_buf_ptr).add(input_offset); - *output_buf_ptr = (*output_buf_ptr).add(output_offset); - *available_in -= input_offset; - *available_out -= output_offset; - *state = BroccoliState::from(bro_catli); - ret - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - BroCatliResult::BrotliFileNotCraftedForConcatenation - }) -} - -#[no_mangle] + unsafe { + catch_panic(|| { + let input_buf = slice_from_raw_parts_or_nil(*input_buf_ptr, *available_in); + let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); + let mut input_offset = 0usize; + let mut output_offset = 0usize; + let mut bro_catli: BroCatli = (*state).into(); + let ret = + bro_catli.stream(input_buf, &mut input_offset, output_buf, &mut output_offset); + *input_buf_ptr = (*input_buf_ptr).add(input_offset); + *output_buf_ptr = (*output_buf_ptr).add(output_offset); + *available_in -= input_offset; + *available_out -= output_offset; + *state = BroccoliState::from(bro_catli); + ret + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); + BroCatliResult::BrotliFileNotCraftedForConcatenation + }) + } +} + +#[unsafe(no_mangle)] pub unsafe extern "C" fn BroccoliConcatStreaming( state: *mut BroccoliState, available_in: *mut usize, @@ -114,56 +119,62 @@ pub unsafe extern "C" fn BroccoliConcatStreaming( available_out: *mut usize, mut output_buf: *mut u8, ) -> BroccoliResult { - catch_panic(|| { - BroccoliConcatStream( - state, - available_in, - &mut input_buf, - available_out, - &mut output_buf, - ) - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - BroCatliResult::BrotliFileNotCraftedForConcatenation - }) + unsafe { + catch_panic(|| { + BroccoliConcatStream( + state, + available_in, + &mut input_buf, + available_out, + &mut output_buf, + ) + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); + BroCatliResult::BrotliFileNotCraftedForConcatenation + }) + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BroccoliConcatFinish( state: *mut BroccoliState, available_out: *mut usize, output_buf_ptr: *mut *mut u8, ) -> BroCatliResult { - catch_panic(|| { - let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); - let mut output_offset = 0usize; - let mut bro_catli: BroCatli = (*state).into(); - let ret = bro_catli.finish(output_buf, &mut output_offset); - *output_buf_ptr = (*output_buf_ptr).add(output_offset); - *available_out -= output_offset; - *state = BroccoliState::from(bro_catli); - ret - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - BroCatliResult::BrotliFileNotCraftedForConcatenation - }) + unsafe { + catch_panic(|| { + let output_buf = slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out); + let mut output_offset = 0usize; + let mut bro_catli: BroCatli = (*state).into(); + let ret = bro_catli.finish(output_buf, &mut output_offset); + *output_buf_ptr = (*output_buf_ptr).add(output_offset); + *available_out -= output_offset; + *state = BroccoliState::from(bro_catli); + ret + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); + BroCatliResult::BrotliFileNotCraftedForConcatenation + }) + } } // exactly the same as BrotliConcatFinish but without the indirect -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BroccoliConcatFinished( state: *mut BroccoliState, available_out: *mut usize, mut output_buf: *mut u8, ) -> BroCatliResult { - catch_panic(|| BroccoliConcatFinish(state, available_out, &mut output_buf)).unwrap_or_else( - |panic_err| { - error_print(panic_err); - BroCatliResult::BrotliFileNotCraftedForConcatenation - }, - ) + unsafe { + catch_panic(|| BroccoliConcatFinish(state, available_out, &mut output_buf)).unwrap_or_else( + |panic_err| { + error_print(panic_err); + BroCatliResult::BrotliFileNotCraftedForConcatenation + }, + ) + } } #[cfg(all(feature = "std", not(feature = "pass-through-ffi-panics")))] diff --git a/src/ffi/compressor.rs b/src/ffi/compressor.rs index fdd4245b..353f7bbe 100644 --- a/src/ffi/compressor.rs +++ b/src/ffi/compressor.rs @@ -6,7 +6,7 @@ use std::{io, panic, thread}; use brotli_decompressor::ffi::alloc_util::SubclassableAllocator; use brotli_decompressor::ffi::interface::{ - brotli_alloc_func, brotli_free_func, c_void, CAllocator, + CAllocator, brotli_alloc_func, brotli_free_func, c_void, }; use brotli_decompressor::ffi::{ alloc_util, slice_from_raw_parts_or_nil, slice_from_raw_parts_or_nil_mut, @@ -58,7 +58,9 @@ fn brotli_new_compressor_without_custom_alloc( } #[cfg(feature = "std")] unsafe fn free_compressor_no_custom_alloc(state_ptr: *mut BrotliEncoderState) { - let _state = alloc_util::Box::from_raw(state_ptr); + unsafe { + let _state = alloc_util::Box::from_raw(state_ptr); + } } #[cfg(not(feature = "std"))] @@ -66,129 +68,141 @@ unsafe fn free_compressor_no_custom_alloc(_state_ptr: *mut BrotliEncoderState) { unreachable!(); } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCreateInstance( alloc_func: brotli_alloc_func, free_func: brotli_free_func, opaque: *mut c_void, ) -> *mut BrotliEncoderState { - catch_panic_cstate(|| { - let allocators = CAllocator { - alloc_func, - free_func, - opaque, - }; - let to_box = BrotliEncoderState { - custom_allocator: allocators.clone(), - compressor: BrotliEncoderStateStruct::new(BrotliSubclassableAllocator::new( - SubclassableAllocator::new(allocators.clone()), - )), - }; - if let Some(alloc) = alloc_func { - if free_func.is_none() { - panic!("either both alloc and free must exist or neither"); - } - let ptr = alloc( - allocators.opaque, - core::mem::size_of::(), - ); - if ptr.is_null() { - return core::ptr::null_mut(); + unsafe { + catch_panic_cstate(|| { + let allocators = CAllocator { + alloc_func, + free_func, + opaque, + }; + let to_box = BrotliEncoderState { + custom_allocator: allocators.clone(), + compressor: BrotliEncoderStateStruct::new(BrotliSubclassableAllocator::new( + SubclassableAllocator::new(allocators.clone()), + )), + }; + if let Some(alloc) = alloc_func { + if free_func.is_none() { + panic!("either both alloc and free must exist or neither"); + } + let ptr = alloc( + allocators.opaque, + core::mem::size_of::(), + ); + if ptr.is_null() { + return core::ptr::null_mut(); + } + let brotli_decoder_state_ptr = + core::mem::transmute::<*mut c_void, *mut BrotliEncoderState>(ptr); + core::ptr::write(brotli_decoder_state_ptr, to_box); + brotli_decoder_state_ptr + } else { + brotli_new_compressor_without_custom_alloc(to_box) } - let brotli_decoder_state_ptr = - core::mem::transmute::<*mut c_void, *mut BrotliEncoderState>(ptr); - core::ptr::write(brotli_decoder_state_ptr, to_box); - brotli_decoder_state_ptr - } else { - brotli_new_compressor_without_custom_alloc(to_box) - } - }) - .unwrap_or_else(|err| { - error_print(err); - core::ptr::null_mut() - }) + }) + .unwrap_or_else(|err| { + error_print(err); + core::ptr::null_mut() + }) + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderSetParameter( state_ptr: *mut BrotliEncoderState, - param: ::enc::encode::BrotliEncoderParameter, + param: crate::enc::encode::BrotliEncoderParameter, value: u32, ) -> i32 { - if (*state_ptr).compressor.set_parameter(param, value) { - 1 - } else { - 0 + unsafe { + if (*state_ptr).compressor.set_parameter(param, value) { + 1 + } else { + 0 + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderDestroyInstance(state_ptr: *mut BrotliEncoderState) { - if state_ptr.is_null() { - return; - } - InternalBrotliEncoderDestroyInstance(&mut (*state_ptr).compressor); - if (*state_ptr).custom_allocator.alloc_func.is_some() { - if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { - let _to_free = core::ptr::read(state_ptr); - let ptr = core::mem::transmute::<*mut BrotliEncoderState, *mut c_void>(state_ptr); - free_fn((*state_ptr).custom_allocator.opaque, ptr); + unsafe { + if state_ptr.is_null() { + return; + } + InternalBrotliEncoderDestroyInstance(&mut (*state_ptr).compressor); + if (*state_ptr).custom_allocator.alloc_func.is_some() { + if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { + let _to_free = core::ptr::read(state_ptr); + let ptr = core::mem::transmute::<*mut BrotliEncoderState, *mut c_void>(state_ptr); + free_fn((*state_ptr).custom_allocator.opaque, ptr); + } + } else { + free_compressor_no_custom_alloc(state_ptr); } - } else { - free_compressor_no_custom_alloc(state_ptr); } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderIsFinished(state_ptr: *mut BrotliEncoderState) -> i32 { - if (*state_ptr).compressor.is_finished() { - 1 - } else { - 0 + unsafe { + if (*state_ptr).compressor.is_finished() { + 1 + } else { + 0 + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderHasMoreOutput(state_ptr: *mut BrotliEncoderState) -> i32 { - if (*state_ptr).compressor.has_more_output() { - 1 - } else { - 0 + unsafe { + if (*state_ptr).compressor.has_more_output() { + 1 + } else { + 0 + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderSetCustomDictionary( state_ptr: *mut BrotliEncoderState, size: usize, dict: *const u8, ) { - if let Err(panic_err) = catch_panic(|| { - let dict_slice = slice_from_raw_parts_or_nil(dict, size); - (*state_ptr) - .compressor - .set_custom_dictionary(size, dict_slice); - 0 - }) { - error_print(panic_err); + unsafe { + if let Err(panic_err) = catch_panic(|| { + let dict_slice = slice_from_raw_parts_or_nil(dict, size); + (*state_ptr) + .compressor + .set_custom_dictionary(size, dict_slice); + 0 + }) { + error_print(panic_err); + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderTakeOutput( state_ptr: *mut BrotliEncoderState, size: *mut usize, ) -> *const u8 { - (*state_ptr).compressor.take_output(&mut *size).as_ptr() + unsafe { (*state_ptr).compressor.take_output(&mut *size).as_ptr() } } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BrotliEncoderVersion() -> u32 { - ::enc::encode::BrotliEncoderVersion() + crate::enc::encode::BrotliEncoderVersion() } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BrotliEncoderMaxCompressedSize(input_size: usize) -> usize { - ::enc::encode::BrotliEncoderMaxCompressedSize(input_size) + crate::enc::encode::BrotliEncoderMaxCompressedSize(input_size) } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCompress( quality: i32, lgwin: i32, @@ -198,63 +212,65 @@ pub unsafe extern "C" fn BrotliEncoderCompress( encoded_size: *mut usize, encoded_buffer: *mut u8, ) -> i32 { - catch_panic(|| { - let input_buf = slice_from_raw_parts_or_nil(input_buffer, input_size); - let encoded_buf = slice_from_raw_parts_or_nil_mut(encoded_buffer, *encoded_size); - let allocators = CAllocator { - alloc_func: None, - free_func: None, - opaque: core::ptr::null_mut(), - }; - let translated_mode = match mode { - BrotliEncoderMode::BROTLI_MODE_GENERIC => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_GENERIC - } - BrotliEncoderMode::BROTLI_MODE_TEXT => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_TEXT - } - BrotliEncoderMode::BROTLI_MODE_FONT => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_FONT - } - BrotliEncoderMode::BROTLI_MODE_FORCE_LSB_PRIOR => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_LSB_PRIOR - } - BrotliEncoderMode::BROTLI_MODE_FORCE_MSB_PRIOR => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR - } - BrotliEncoderMode::BROTLI_MODE_FORCE_UTF8_PRIOR => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR - } - BrotliEncoderMode::BROTLI_MODE_FORCE_SIGNED_PRIOR => { - ::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR - } - }; - let mut m8 = - BrotliSubclassableAllocator::new(SubclassableAllocator::new(allocators.clone())); - let empty_m8 = - BrotliSubclassableAllocator::new(SubclassableAllocator::new(allocators.clone())); + unsafe { + catch_panic(|| { + let input_buf = slice_from_raw_parts_or_nil(input_buffer, input_size); + let encoded_buf = slice_from_raw_parts_or_nil_mut(encoded_buffer, *encoded_size); + let allocators = CAllocator { + alloc_func: None, + free_func: None, + opaque: core::ptr::null_mut(), + }; + let translated_mode = match mode { + BrotliEncoderMode::BROTLI_MODE_GENERIC => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_GENERIC + } + BrotliEncoderMode::BROTLI_MODE_TEXT => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_TEXT + } + BrotliEncoderMode::BROTLI_MODE_FONT => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_MODE_FONT + } + BrotliEncoderMode::BROTLI_MODE_FORCE_LSB_PRIOR => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_LSB_PRIOR + } + BrotliEncoderMode::BROTLI_MODE_FORCE_MSB_PRIOR => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR + } + BrotliEncoderMode::BROTLI_MODE_FORCE_UTF8_PRIOR => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR + } + BrotliEncoderMode::BROTLI_MODE_FORCE_SIGNED_PRIOR => { + crate::enc::backward_references::BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR + } + }; + let mut m8 = + BrotliSubclassableAllocator::new(SubclassableAllocator::new(allocators.clone())); + let empty_m8 = + BrotliSubclassableAllocator::new(SubclassableAllocator::new(allocators.clone())); - crate::enc::encode::encoder_compress( - empty_m8, - &mut m8, - quality, - lgwin, - translated_mode, - input_size, - input_buf, - &mut *encoded_size, - encoded_buf, - &mut |_a, _b, _c, _d| (), - ) - .into() - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - 0 - }) + crate::enc::encode::encoder_compress( + empty_m8, + &mut m8, + quality, + lgwin, + translated_mode, + input_size, + input_buf, + &mut *encoded_size, + encoded_buf, + &mut |_a, _b, _c, _d| (), + ) + .into() + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); + 0 + }) + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCompressStreaming( state_ptr: *mut BrotliEncoderState, op: BrotliEncoderOperation, @@ -263,18 +279,20 @@ pub unsafe extern "C" fn BrotliEncoderCompressStreaming( available_out: *mut usize, mut output_buf: *mut u8, ) -> i32 { - BrotliEncoderCompressStream( - state_ptr, - op, - available_in, - &mut input_buf, - available_out, - &mut output_buf, - core::ptr::null_mut(), - ) + unsafe { + BrotliEncoderCompressStream( + state_ptr, + op, + available_in, + &mut input_buf, + available_out, + &mut output_buf, + core::ptr::null_mut(), + ) + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCompressStream( state_ptr: *mut BrotliEncoderState, op: BrotliEncoderOperation, @@ -284,133 +302,139 @@ pub unsafe extern "C" fn BrotliEncoderCompressStream( output_buf_ptr: *mut *mut u8, total_out: *mut usize, ) -> i32 { - catch_panic(|| { - let mut input_offset = 0usize; - let mut output_offset = 0usize; - let result; - let translated_op = match op { - BrotliEncoderOperation::BROTLI_OPERATION_PROCESS => { - ::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_PROCESS - } - BrotliEncoderOperation::BROTLI_OPERATION_FLUSH => { - ::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_FLUSH - } - BrotliEncoderOperation::BROTLI_OPERATION_FINISH => { - ::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_FINISH - } - BrotliEncoderOperation::BROTLI_OPERATION_EMIT_METADATA => { - ::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_EMIT_METADATA - } - }; - { - let (input_buf, input_any): (&[u8], bool) = if *available_in != 0 { - ( - slice_from_raw_parts_or_nil(*input_buf_ptr, *available_in), - true, - ) - } else { - (&[], false) - }; - let (output_buf, output_any): (&mut [u8], bool) = if *available_out != 0 { - ( - slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out), - true, - ) - } else { - (&mut [], false) + unsafe { + catch_panic(|| { + let mut input_offset = 0usize; + let mut output_offset = 0usize; + let result; + let translated_op = match op { + BrotliEncoderOperation::BROTLI_OPERATION_PROCESS => { + crate::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_PROCESS + } + BrotliEncoderOperation::BROTLI_OPERATION_FLUSH => { + crate::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_FLUSH + } + BrotliEncoderOperation::BROTLI_OPERATION_FINISH => { + crate::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_FINISH + } + BrotliEncoderOperation::BROTLI_OPERATION_EMIT_METADATA => { + crate::enc::encode::BrotliEncoderOperation::BROTLI_OPERATION_EMIT_METADATA + } }; - let mut to = Some(0); - result = (*state_ptr).compressor.compress_stream( - translated_op, - &mut *available_in, - input_buf, - &mut input_offset, - &mut *available_out, - output_buf, - &mut output_offset, - &mut to, - &mut |_a, _b, _c, _d| (), - ); - if !total_out.is_null() { - *total_out = to.unwrap_or(0); - } - if input_any { - *input_buf_ptr = (*input_buf_ptr).add(input_offset); - } - if output_any { - *output_buf_ptr = (*output_buf_ptr).add(output_offset); + { + let (input_buf, input_any): (&[u8], bool) = if *available_in != 0 { + ( + slice_from_raw_parts_or_nil(*input_buf_ptr, *available_in), + true, + ) + } else { + (&[], false) + }; + let (output_buf, output_any): (&mut [u8], bool) = if *available_out != 0 { + ( + slice_from_raw_parts_or_nil_mut(*output_buf_ptr, *available_out), + true, + ) + } else { + (&mut [], false) + }; + let mut to = Some(0); + result = (*state_ptr).compressor.compress_stream( + translated_op, + &mut *available_in, + input_buf, + &mut input_offset, + &mut *available_out, + output_buf, + &mut output_offset, + &mut to, + &mut |_a, _b, _c, _d| (), + ); + if !total_out.is_null() { + *total_out = to.unwrap_or(0); + } + if input_any { + *input_buf_ptr = (*input_buf_ptr).add(input_offset); + } + if output_any { + *output_buf_ptr = (*output_buf_ptr).add(output_offset); + } } - } - if result { - 1 - } else { + if result { 1 } else { 0 } + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); 0 - } - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - 0 - }) + }) + } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderMallocU8( state_ptr: *mut BrotliEncoderState, size: usize, ) -> *mut u8 { - if let Some(alloc_fn) = (*state_ptr).custom_allocator.alloc_func { - core::mem::transmute::<*mut c_void, *mut u8>(alloc_fn( - (*state_ptr).custom_allocator.opaque, - size, - )) - } else { - alloc_util::alloc_stdlib(size) + unsafe { + if let Some(alloc_fn) = (*state_ptr).custom_allocator.alloc_func { + core::mem::transmute::<*mut c_void, *mut u8>(alloc_fn( + (*state_ptr).custom_allocator.opaque, + size, + )) + } else { + alloc_util::alloc_stdlib(size) + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderFreeU8( state_ptr: *mut BrotliEncoderState, data: *mut u8, size: usize, ) { - if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { - free_fn( - (*state_ptr).custom_allocator.opaque, - core::mem::transmute::<*mut u8, *mut c_void>(data), - ); - } else { - alloc_util::free_stdlib(data, size); + unsafe { + if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { + free_fn( + (*state_ptr).custom_allocator.opaque, + core::mem::transmute::<*mut u8, *mut c_void>(data), + ); + } else { + alloc_util::free_stdlib(data, size); + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderMallocUsize( state_ptr: *mut BrotliEncoderState, size: usize, ) -> *mut usize { - if let Some(alloc_fn) = (*state_ptr).custom_allocator.alloc_func { - core::mem::transmute::<*mut c_void, *mut usize>(alloc_fn( - (*state_ptr).custom_allocator.opaque, - size * core::mem::size_of::(), - )) - } else { - alloc_util::alloc_stdlib(size) + unsafe { + if let Some(alloc_fn) = (*state_ptr).custom_allocator.alloc_func { + core::mem::transmute::<*mut c_void, *mut usize>(alloc_fn( + (*state_ptr).custom_allocator.opaque, + size * core::mem::size_of::(), + )) + } else { + alloc_util::alloc_stdlib(size) + } } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderFreeUsize( state_ptr: *mut BrotliEncoderState, data: *mut usize, size: usize, ) { - if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { - free_fn( - (*state_ptr).custom_allocator.opaque, - core::mem::transmute::<*mut usize, *mut c_void>(data), - ); - } else { - alloc_util::free_stdlib(data, size); + unsafe { + if let Some(free_fn) = (*state_ptr).custom_allocator.free_func { + free_fn( + (*state_ptr).custom_allocator.opaque, + core::mem::transmute::<*mut usize, *mut c_void>(data), + ); + } else { + alloc_util::free_stdlib(data, size); + } } } diff --git a/src/ffi/decompressor.rs b/src/ffi/decompressor.rs index 089d700e..a8a4661b 100644 --- a/src/ffi/decompressor.rs +++ b/src/ffi/decompressor.rs @@ -1,12 +1,12 @@ pub use brotli_decompressor::ffi::interface::{brotli_alloc_func, brotli_free_func, c_void}; -pub use brotli_decompressor::{ffi, BrotliDecoderReturnInfo, HuffmanCode}; +pub use brotli_decompressor::{BrotliDecoderReturnInfo, HuffmanCode, ffi}; pub unsafe extern "C" fn CBrotliDecoderCreateInstance( alloc_func: brotli_alloc_func, free_func: brotli_free_func, opaque: *mut c_void, ) -> *mut ffi::BrotliDecoderState { - ffi::BrotliDecoderCreateInstance(alloc_func, free_func, opaque) + unsafe { ffi::BrotliDecoderCreateInstance(alloc_func, free_func, opaque) } } pub unsafe extern "C" fn CBrotliDecoderSetParameter( @@ -14,7 +14,7 @@ pub unsafe extern "C" fn CBrotliDecoderSetParameter( selector: ffi::interface::BrotliDecoderParameter, value: u32, ) { - ffi::BrotliDecoderSetParameter(state_ptr, selector, value) + unsafe { ffi::BrotliDecoderSetParameter(state_ptr, selector, value) } } #[cfg(feature = "std")] // this requires a default allocator @@ -24,7 +24,9 @@ pub unsafe extern "C" fn CBrotliDecoderDecompress( decoded_size: *mut usize, decoded_buffer: *mut u8, ) -> ffi::interface::BrotliDecoderResult { - ffi::BrotliDecoderDecompress(encoded_size, encoded_buffer, decoded_size, decoded_buffer) + unsafe { + ffi::BrotliDecoderDecompress(encoded_size, encoded_buffer, decoded_size, decoded_buffer) + } } pub unsafe extern "C" fn CBrotliDecoderDecompressStream( @@ -35,14 +37,16 @@ pub unsafe extern "C" fn CBrotliDecoderDecompressStream( output_buf_ptr: *mut *mut u8, total_out: *mut usize, ) -> ffi::interface::BrotliDecoderResult { - ffi::BrotliDecoderDecompressStream( - state_ptr, - available_in, - input_buf_ptr, - available_out, - output_buf_ptr, - total_out, - ) + unsafe { + ffi::BrotliDecoderDecompressStream( + state_ptr, + available_in, + input_buf_ptr, + available_out, + output_buf_ptr, + total_out, + ) + } } pub unsafe extern "C" fn CBrotliDecoderDecompressStreaming( @@ -52,13 +56,15 @@ pub unsafe extern "C" fn CBrotliDecoderDecompressStreaming( available_out: *mut usize, output_buf_ptr: *mut u8, ) -> ffi::interface::BrotliDecoderResult { - ffi::BrotliDecoderDecompressStreaming( - state_ptr, - available_in, - input_buf_ptr, - available_out, - output_buf_ptr, - ) + unsafe { + ffi::BrotliDecoderDecompressStreaming( + state_ptr, + available_in, + input_buf_ptr, + available_out, + output_buf_ptr, + ) + } } pub unsafe extern "C" fn CBrotliDecoderDecompressWithReturnInfo( @@ -67,12 +73,14 @@ pub unsafe extern "C" fn CBrotliDecoderDecompressWithReturnInfo( available_out_and_scratch: usize, output_buf_and_scratch: *mut u8, ) -> BrotliDecoderReturnInfo { - ffi::BrotliDecoderDecompressWithReturnInfo( - available_in, - input_buf_ptr, - available_out_and_scratch, - output_buf_and_scratch, - ) + unsafe { + ffi::BrotliDecoderDecompressWithReturnInfo( + available_in, + input_buf_ptr, + available_out_and_scratch, + output_buf_and_scratch, + ) + } } pub unsafe extern "C" fn CBrotliDecoderDecompressPrealloc( @@ -87,25 +95,27 @@ pub unsafe extern "C" fn CBrotliDecoderDecompressPrealloc( available_hc: usize, hc_ptr: *mut HuffmanCode, ) -> BrotliDecoderReturnInfo { - ffi::BrotliDecoderDecompressPrealloc( - available_in, - input_buf_ptr, - available_out, - output_buf_ptr, - available_u8, - u8_ptr, - available_u32, - u32_ptr, - available_hc, - hc_ptr, - ) + unsafe { + ffi::BrotliDecoderDecompressPrealloc( + available_in, + input_buf_ptr, + available_out, + output_buf_ptr, + available_u8, + u8_ptr, + available_u32, + u32_ptr, + available_hc, + hc_ptr, + ) + } } pub unsafe extern "C" fn CBrotliDecoderMallocU8( state_ptr: *mut ffi::BrotliDecoderState, size: usize, ) -> *mut u8 { - ffi::BrotliDecoderMallocU8(state_ptr, size) + unsafe { ffi::BrotliDecoderMallocU8(state_ptr, size) } } pub unsafe extern "C" fn CBrotliDecoderFreeU8( @@ -113,14 +123,14 @@ pub unsafe extern "C" fn CBrotliDecoderFreeU8( data: *mut u8, size: usize, ) { - ffi::BrotliDecoderFreeU8(state_ptr, data, size) + unsafe { ffi::BrotliDecoderFreeU8(state_ptr, data, size) } } pub unsafe extern "C" fn CBrotliDecoderMallocUsize( state_ptr: *mut ffi::BrotliDecoderState, size: usize, ) -> *mut usize { - ffi::BrotliDecoderMallocUsize(state_ptr, size) + unsafe { ffi::BrotliDecoderMallocUsize(state_ptr, size) } } pub unsafe extern "C" fn CBrotliDecoderFreeUsize( @@ -128,56 +138,56 @@ pub unsafe extern "C" fn CBrotliDecoderFreeUsize( data: *mut usize, size: usize, ) { - ffi::BrotliDecoderFreeUsize(state_ptr, data, size) + unsafe { ffi::BrotliDecoderFreeUsize(state_ptr, data, size) } } pub unsafe extern "C" fn CBrotliDecoderDestroyInstance(state_ptr: *mut ffi::BrotliDecoderState) { - ffi::BrotliDecoderDestroyInstance(state_ptr) + unsafe { ffi::BrotliDecoderDestroyInstance(state_ptr) } } pub extern "C" fn CBrotliDecoderVersion() -> u32 { ffi::BrotliDecoderVersion() } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn CBrotliDecoderErrorString(c: ffi::BrotliDecoderErrorCode) -> *const u8 { ffi::BrotliDecoderErrorString(c) } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderHasMoreOutput( state_ptr: *const ffi::BrotliDecoderState, ) -> i32 { - ffi::BrotliDecoderHasMoreOutput(state_ptr) + unsafe { ffi::BrotliDecoderHasMoreOutput(state_ptr) } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderTakeOutput( state_ptr: *mut ffi::BrotliDecoderState, size: *mut usize, ) -> *const u8 { - ffi::BrotliDecoderTakeOutput(state_ptr, size) + unsafe { ffi::BrotliDecoderTakeOutput(state_ptr, size) } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderIsUsed(state_ptr: *const ffi::BrotliDecoderState) -> i32 { - ffi::BrotliDecoderIsUsed(state_ptr) + unsafe { ffi::BrotliDecoderIsUsed(state_ptr) } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderIsFinished( state_ptr: *const ffi::BrotliDecoderState, ) -> i32 { - ffi::BrotliDecoderIsFinished(state_ptr) + unsafe { ffi::BrotliDecoderIsFinished(state_ptr) } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderGetErrorCode( state_ptr: *const ffi::BrotliDecoderState, ) -> ffi::BrotliDecoderErrorCode { - ffi::BrotliDecoderGetErrorCode(state_ptr) + unsafe { ffi::BrotliDecoderGetErrorCode(state_ptr) } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn CBrotliDecoderGetErrorString( state_ptr: *const ffi::BrotliDecoderState, ) -> *const u8 { - ffi::BrotliDecoderGetErrorString(state_ptr) + unsafe { ffi::BrotliDecoderGetErrorString(state_ptr) } } diff --git a/src/ffi/multicompress/mod.rs b/src/ffi/multicompress/mod.rs index df348010..4ef53fe5 100755 --- a/src/ffi/multicompress/mod.rs +++ b/src/ffi/multicompress/mod.rs @@ -9,16 +9,15 @@ use std::panic; use brotli_decompressor::ffi::alloc_util::SubclassableAllocator; use brotli_decompressor::ffi::interface::{ - brotli_alloc_func, brotli_free_func, c_void, CAllocator, + CAllocator, brotli_alloc_func, brotli_free_func, c_void, }; use brotli_decompressor::ffi::{slice_from_raw_parts_or_nil, slice_from_raw_parts_or_nil_mut}; -use {brotli_decompressor, core, enc}; use super::alloc_util::BrotliSubclassableAllocator; use super::compressor; use crate::enc::backward_references::{BrotliEncoderParams, UnionHasher}; use crate::enc::encode::{ - set_parameter, BrotliEncoderOperation, BrotliEncoderParameter, BrotliEncoderStateStruct, + BrotliEncoderOperation, BrotliEncoderParameter, BrotliEncoderStateStruct, set_parameter, }; use crate::enc::threading::{Owned, SendAlloc}; @@ -32,7 +31,7 @@ impl<'a> SliceWrapper for SliceRef<'a> { } macro_rules! make_send_alloc { - ($alloc_func: expr, $free_func: expr, $opaque: expr) => { + ($alloc_func: expr_2021, $free_func: expr_2021, $opaque: expr_2021) => { SendAlloc::new( BrotliSubclassableAllocator::new(SubclassableAllocator::new(CAllocator { alloc_func: $alloc_func, @@ -44,12 +43,12 @@ macro_rules! make_send_alloc { }; } -#[no_mangle] +#[unsafe(no_mangle)] pub extern "C" fn BrotliEncoderMaxCompressedSizeMulti( input_size: usize, num_threads: usize, ) -> usize { - ::enc::encode::BrotliEncoderMaxCompressedSizeMulti(input_size, num_threads) + crate::enc::encode::BrotliEncoderMaxCompressedSizeMulti(input_size, num_threads) } fn help_brotli_encoder_compress_single( @@ -88,7 +87,7 @@ fn help_brotli_encoder_compress_single( result } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCompressMulti( num_params: usize, param_keys: *const BrotliEncoderParameter, @@ -102,118 +101,121 @@ pub unsafe extern "C" fn BrotliEncoderCompressMulti( free_func: brotli_free_func, alloc_opaque_per_thread: *mut *mut c_void, ) -> i32 { - if desired_num_threads == 0 { - return 0; - } - let num_threads = min(desired_num_threads, MAX_THREADS); - compressor::catch_panic(|| { - let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); - let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); - let input_slice = slice_from_raw_parts_or_nil(input, input_size); - let output_slice = slice_from_raw_parts_or_nil_mut(encoded, *encoded_size); - if num_threads == 1 { - let allocators = CAllocator { - alloc_func, - free_func, - opaque: if alloc_opaque_per_thread.is_null() { - core::ptr::null_mut() - } else { - *alloc_opaque_per_thread - }, - }; - let m8 = - BrotliSubclassableAllocator::new(SubclassableAllocator::new(allocators.clone())); - return help_brotli_encoder_compress_single( - param_keys_slice, - param_values_slice, - input_slice, - output_slice, - &mut *encoded_size, - m8, - ) - .into(); + unsafe { + if desired_num_threads == 0 { + return 0; } - let null_opaques = [core::ptr::null_mut::(); MAX_THREADS]; - let alloc_opaque = if alloc_opaque_per_thread.is_null() { - &null_opaques[..] - } else { - slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) - }; - let mut params = BrotliEncoderParams::default(); - for (k, v) in param_keys_slice.iter().zip(param_values_slice.iter()) { - if !set_parameter(&mut params, *k, *v) { - return 0; + let num_threads = min(desired_num_threads, MAX_THREADS); + compressor::catch_panic(|| { + let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); + let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); + let input_slice = slice_from_raw_parts_or_nil(input, input_size); + let output_slice = slice_from_raw_parts_or_nil_mut(encoded, *encoded_size); + if num_threads == 1 { + let allocators = CAllocator { + alloc_func, + free_func, + opaque: if alloc_opaque_per_thread.is_null() { + core::ptr::null_mut() + } else { + *alloc_opaque_per_thread + }, + }; + let m8 = BrotliSubclassableAllocator::new(SubclassableAllocator::new( + allocators.clone(), + )); + return help_brotli_encoder_compress_single( + param_keys_slice, + param_values_slice, + input_slice, + output_slice, + &mut *encoded_size, + m8, + ) + .into(); } - } - let mut alloc_array: [_; MAX_THREADS] = [ - make_send_alloc!(alloc_func, free_func, alloc_opaque[0]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[1 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[2 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[3 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[4 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[5 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[6 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[7 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[8 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[9 % desired_num_threads]), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[10 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[11 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[12 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[13 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[14 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[15 % desired_num_threads] - ), - ]; + let null_opaques = [core::ptr::null_mut::(); MAX_THREADS]; + let alloc_opaque = if alloc_opaque_per_thread.is_null() { + &null_opaques[..] + } else { + slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) + }; + let mut params = BrotliEncoderParams::default(); + for (k, v) in param_keys_slice.iter().zip(param_values_slice.iter()) { + if !set_parameter(&mut params, *k, *v) { + return 0; + } + } + let mut alloc_array: [_; MAX_THREADS] = [ + make_send_alloc!(alloc_func, free_func, alloc_opaque[0]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[1 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[2 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[3 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[4 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[5 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[6 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[7 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[8 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[9 % desired_num_threads]), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[10 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[11 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[12 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[13 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[14 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[15 % desired_num_threads] + ), + ]; - let owned_input = &mut Owned::new(SliceRef(input_slice)); - let res = enc::compress_multi_no_threadpool( - ¶ms, - owned_input, - output_slice, - &mut alloc_array[..num_threads], - ); - match res { - Ok(size) => { - *encoded_size = size; - 1 + let owned_input = &mut Owned::new(SliceRef(input_slice)); + let res = crate::enc::compress_multi_no_threadpool( + ¶ms, + owned_input, + output_slice, + &mut alloc_array[..num_threads], + ); + match res { + Ok(size) => { + *encoded_size = size; + 1 + } + Err(_err) => 0, } - Err(_err) => 0, - } - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); - 0 - }) + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); + 0 + }) + } } #[repr(C)] pub struct BrotliEncoderWorkPool { custom_allocator: CAllocator, - work_pool: enc::WorkerPool< - enc::CompressionThreadResult, + work_pool: crate::enc::WorkerPool< + crate::enc::CompressionThreadResult, UnionHasher, BrotliSubclassableAllocator, (SliceRef<'static>, BrotliEncoderParams), @@ -235,50 +237,54 @@ fn brotli_new_work_pool_without_custom_alloc( brotli_decompressor::ffi::alloc_util::Box::::new(to_box), ) } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCreateWorkPool( num_threads: usize, alloc_func: brotli_alloc_func, free_func: brotli_free_func, opaque: *mut c_void, ) -> *mut BrotliEncoderWorkPool { - catch_panic_wstate(|| { - let allocators = CAllocator { - alloc_func, - free_func, - opaque, - }; - let to_box = BrotliEncoderWorkPool { - custom_allocator: allocators.clone(), - work_pool: enc::new_work_pool(min(num_threads, MAX_THREADS)), - }; - if let Some(alloc) = alloc_func { - if free_func.is_none() { - panic!("either both alloc and free must exist or neither"); - } - let ptr = alloc( - allocators.opaque, - core::mem::size_of::(), - ); - if ptr.is_null() { - return core::ptr::null_mut(); + unsafe { + catch_panic_wstate(|| { + let allocators = CAllocator { + alloc_func, + free_func, + opaque, + }; + let to_box = BrotliEncoderWorkPool { + custom_allocator: allocators.clone(), + work_pool: crate::enc::new_work_pool(min(num_threads, MAX_THREADS)), + }; + if let Some(alloc) = alloc_func { + if free_func.is_none() { + panic!("either both alloc and free must exist or neither"); + } + let ptr = alloc( + allocators.opaque, + core::mem::size_of::(), + ); + if ptr.is_null() { + return core::ptr::null_mut(); + } + let brotli_work_pool_ptr = + core::mem::transmute::<*mut c_void, *mut BrotliEncoderWorkPool>(ptr); + core::ptr::write(brotli_work_pool_ptr, to_box); + brotli_work_pool_ptr + } else { + brotli_new_work_pool_without_custom_alloc(to_box) } - let brotli_work_pool_ptr = - core::mem::transmute::<*mut c_void, *mut BrotliEncoderWorkPool>(ptr); - core::ptr::write(brotli_work_pool_ptr, to_box); - brotli_work_pool_ptr - } else { - brotli_new_work_pool_without_custom_alloc(to_box) - } - }) - .unwrap_or_else(|err| { - error_print(err); - core::ptr::null_mut() - }) + }) + .unwrap_or_else(|err| { + error_print(err); + core::ptr::null_mut() + }) + } } #[cfg(feature = "std")] unsafe fn free_work_pool_no_custom_alloc(_work_pool: *mut BrotliEncoderWorkPool) { - let _state = brotli_decompressor::ffi::alloc_util::Box::from_raw(_work_pool); + unsafe { + let _state = brotli_decompressor::ffi::alloc_util::Box::from_raw(_work_pool); + } } #[cfg(not(feature = "std"))] @@ -289,25 +295,32 @@ struct UnsafeUnwindBox(*mut BrotliEncoderWorkPool); #[cfg(all(feature = "std", not(feature = "pass-through-ffi-panics")))] impl panic::RefUnwindSafe for UnsafeUnwindBox {} -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderDestroyWorkPool(work_pool_ptr: *mut BrotliEncoderWorkPool) { - let wpp = UnsafeUnwindBox(work_pool_ptr); - if let Err(panic_err) = compressor::catch_panic(|| { - if (*wpp.0).custom_allocator.alloc_func.is_some() { - if let Some(free_fn) = (*wpp.0).custom_allocator.free_func { - let _to_free = core::ptr::read(wpp.0); - let ptr = core::mem::transmute::<*mut BrotliEncoderWorkPool, *mut c_void>(wpp.0); - free_fn((*wpp.0).custom_allocator.opaque, ptr); + unsafe { + let wpp = UnsafeUnwindBox(work_pool_ptr); + if let Err(panic_err) = compressor::catch_panic(|| { + // Capture the whole `UnsafeUnwindBox`, not just its pointer field. Since edition 2021 + // closures capture disjoint fields, which would capture the bare `*mut` and bypass the + // `RefUnwindSafe` impl the newtype exists to provide. + let wpp = &wpp; + if (*wpp.0).custom_allocator.alloc_func.is_some() { + if let Some(free_fn) = (*wpp.0).custom_allocator.free_func { + let _to_free = core::ptr::read(wpp.0); + let ptr = + core::mem::transmute::<*mut BrotliEncoderWorkPool, *mut c_void>(wpp.0); + free_fn((*wpp.0).custom_allocator.opaque, ptr); + } + } else { + free_work_pool_no_custom_alloc(wpp.0); } - } else { - free_work_pool_no_custom_alloc(wpp.0); + 0 + }) { + error_print(panic_err); } - 0 - }) { - error_print(panic_err); } } -#[no_mangle] +#[unsafe(no_mangle)] pub unsafe extern "C" fn BrotliEncoderCompressWorkPool( work_pool: *mut BrotliEncoderWorkPool, num_params: usize, @@ -322,108 +335,112 @@ pub unsafe extern "C" fn BrotliEncoderCompressWorkPool( free_func: brotli_free_func, alloc_opaque_per_thread: *mut *mut c_void, ) -> i32 { - if desired_num_threads == 0 { - return 0; - } - if work_pool.is_null() { - return compressor::catch_panic(|| { - BrotliEncoderCompressMulti( - num_params, - param_keys, - param_values, - input_size, - input, - encoded_size, - encoded, - desired_num_threads, - alloc_func, - free_func, - alloc_opaque_per_thread, - ) + unsafe { + if desired_num_threads == 0 { + return 0; + } + if work_pool.is_null() { + return compressor::catch_panic(|| { + BrotliEncoderCompressMulti( + num_params, + param_keys, + param_values, + input_size, + input, + encoded_size, + encoded, + desired_num_threads, + alloc_func, + free_func, + alloc_opaque_per_thread, + ) + }) + .unwrap_or_else(|panic_err| { + error_print(panic_err); // print panic + 0 // fail + }); + } + let work_pool_wrapper = UnsafeUnwindBox(work_pool); + compressor::catch_panic(|| { + // See `BrotliEncoderDestroyWorkPool`: capture the newtype whole, not its pointer field. + let work_pool_wrapper = &work_pool_wrapper; + let null_opaques = [core::ptr::null_mut::(); MAX_THREADS]; + let alloc_opaque = if alloc_opaque_per_thread.is_null() { + &null_opaques[..] + } else { + slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) + }; + let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); + let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); + let mut params = BrotliEncoderParams::default(); + for (k, v) in param_keys_slice.iter().zip(param_values_slice.iter()) { + if !set_parameter(&mut params, *k, *v) { + return 0; + } + } + let num_threads = min(desired_num_threads, MAX_THREADS); + let mut alloc_array: [_; MAX_THREADS] = [ + make_send_alloc!(alloc_func, free_func, alloc_opaque[0]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[1 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[2 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[3 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[4 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[5 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[6 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[7 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[8 % desired_num_threads]), + make_send_alloc!(alloc_func, free_func, alloc_opaque[9 % desired_num_threads]), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[10 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[11 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[12 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[13 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[14 % desired_num_threads] + ), + make_send_alloc!( + alloc_func, + free_func, + alloc_opaque[15 % desired_num_threads] + ), + ]; + let res = crate::enc::compress_worker_pool( + ¶ms, + &mut Owned::new(SliceRef(slice_from_raw_parts_or_nil(input, input_size))), + slice_from_raw_parts_or_nil_mut(encoded, *encoded_size), + &mut alloc_array[..num_threads], + &mut (*work_pool_wrapper.0).work_pool, + ); + match res { + Ok(size) => { + *encoded_size = size; + 1 + } + Err(_err) => 0, + } }) .unwrap_or_else(|panic_err| { error_print(panic_err); // print panic 0 // fail - }); + }) } - let work_pool_wrapper = UnsafeUnwindBox(work_pool); - compressor::catch_panic(|| { - let null_opaques = [core::ptr::null_mut::(); MAX_THREADS]; - let alloc_opaque = if alloc_opaque_per_thread.is_null() { - &null_opaques[..] - } else { - slice_from_raw_parts_or_nil(alloc_opaque_per_thread, desired_num_threads) - }; - let param_keys_slice = slice_from_raw_parts_or_nil(param_keys, num_params); - let param_values_slice = slice_from_raw_parts_or_nil(param_values, num_params); - let mut params = BrotliEncoderParams::default(); - for (k, v) in param_keys_slice.iter().zip(param_values_slice.iter()) { - if !set_parameter(&mut params, *k, *v) { - return 0; - } - } - let num_threads = min(desired_num_threads, MAX_THREADS); - let mut alloc_array: [_; MAX_THREADS] = [ - make_send_alloc!(alloc_func, free_func, alloc_opaque[0]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[1 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[2 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[3 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[4 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[5 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[6 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[7 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[8 % desired_num_threads]), - make_send_alloc!(alloc_func, free_func, alloc_opaque[9 % desired_num_threads]), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[10 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[11 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[12 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[13 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[14 % desired_num_threads] - ), - make_send_alloc!( - alloc_func, - free_func, - alloc_opaque[15 % desired_num_threads] - ), - ]; - let res = enc::compress_worker_pool( - ¶ms, - &mut Owned::new(SliceRef(slice_from_raw_parts_or_nil(input, input_size))), - slice_from_raw_parts_or_nil_mut(encoded, *encoded_size), - &mut alloc_array[..num_threads], - &mut (*work_pool_wrapper.0).work_pool, - ); - match res { - Ok(size) => { - *encoded_size = size; - 1 - } - Err(_err) => 0, - } - }) - .unwrap_or_else(|panic_err| { - error_print(panic_err); // print panic - 0 // fail - }) } #[cfg(all(feature = "std", not(feature = "pass-through-ffi-panics")))] diff --git a/src/lib.rs b/src/lib.rs index aa16a3a5..2bb6f90f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,11 +26,18 @@ pub mod enc; #[cfg(feature = "ffi-api")] pub mod ffi; -pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator}; +pub use crate::alloc::{ + AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator, +}; #[cfg(feature = "std")] pub use alloc_stdlib::HeapAlloc; #[cfg(feature = "std")] +pub use brotli_decompressor::BrotliDecompress; +#[cfg(feature = "std")] +pub use brotli_decompressor::BrotliDecompressCustomAlloc; +pub use brotli_decompressor::HuffmanCode; // so we can make custom allocator for decompression +#[cfg(feature = "std")] pub use brotli_decompressor::copy_from_to; pub use brotli_decompressor::io_wrappers::{CustomRead, CustomWrite}; #[cfg(feature = "std")] @@ -42,14 +49,9 @@ pub use brotli_decompressor::transform::TransformDictionaryWord; #[cfg(feature = "std")] pub use brotli_decompressor::writer::DecompressorWriter; pub use brotli_decompressor::writer::DecompressorWriterCustomIo; -#[cfg(feature = "std")] -pub use brotli_decompressor::BrotliDecompress; -#[cfg(feature = "std")] -pub use brotli_decompressor::BrotliDecompressCustomAlloc; -pub use brotli_decompressor::HuffmanCode; // so we can make custom allocator for decompression pub use brotli_decompressor::{ - dictionary, reader, transform, writer, BrotliDecompressCustomIo, - BrotliDecompressCustomIoCustomDict, BrotliDecompressStream, BrotliResult, BrotliState, + BrotliDecompressCustomIo, BrotliDecompressCustomIoCustomDict, BrotliDecompressStream, + BrotliResult, BrotliState, dictionary, reader, transform, writer, }; pub use self::enc::combined_alloc::CombiningAllocator; @@ -61,9 +63,9 @@ pub use crate::enc::reader::CompressorReaderCustomIo; #[cfg(feature = "std")] pub use crate::enc::writer::CompressorWriter; pub use crate::enc::writer::CompressorWriterCustomIo; -pub use crate::enc::{interface, BrotliCompressCustomIo, BrotliCompressCustomIoCustomDict}; #[cfg(feature = "std")] pub use crate::enc::{BrotliCompress, BrotliCompressCustomAlloc}; +pub use crate::enc::{BrotliCompressCustomIo, BrotliCompressCustomIoCustomDict, interface}; pub const VERSION: u8 = 1; From df1a70f0b3daed2ba5b9b5a4a18f64b56f7aa757 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 15:35:54 +0300 Subject: [PATCH 3/6] Vectorize modules after profiling --- .../hash_to_binary_tree.rs | 159 +++++++--- src/enc/backward_references/hq.rs | 288 +++++++++++++++--- src/enc/bit_cost.rs | 134 ++++++-- src/enc/block_splitter.rs | 51 ++-- src/enc/static_dict.rs | 121 +++++++- src/enc/vectorization.rs | 36 ++- 6 files changed, 645 insertions(+), 144 deletions(-) diff --git a/src/enc/backward_references/hash_to_binary_tree.rs b/src/enc/backward_references/hash_to_binary_tree.rs index 464a719f..6e76949f 100644 --- a/src/enc/backward_references/hash_to_binary_tree.rs +++ b/src/enc/backward_references/hash_to_binary_tree.rs @@ -2,15 +2,18 @@ use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core; use core::cmp::min; +use fearless_simd::Simd; + use super::{ AnyHasher, BrotliEncoderParams, CloneWithAlloc, H9Opts, HasherSearchResult, HowPrepared, Struct1, fix_unbroken_len, kHashMul32, }; use crate::enc::combined_alloc::allocate; use crate::enc::static_dict::{ - BROTLI_UNALIGNED_LOAD32, BrotliDictionary, FindMatchLengthWithLimit, + BROTLI_UNALIGNED_LOAD32, BrotliDictionary, FindMatchLengthWithLimitSimd, }; use crate::enc::util::floatX; +use crate::enc::vectorization::detect_level; pub const kInfinity: floatX = 1.7e38; @@ -201,6 +204,71 @@ where m32.free_cell(core::mem::take(&mut self.forest)); self.buckets_.free(m32); } + + /// `AnyHasher::Store` on an already-detected instruction set. + #[inline(always)] + fn store_simd(&mut self, simd: S, data: &[u8], mask: usize, ix: usize) { + let max_backward: usize = self.window_mask_.wrapping_sub(16).wrapping_add(1); + StoreAndFindMatchesH10Simd( + simd, + self, + data, + ix, + mask, + self.ringbuffer_break, + Params::max_tree_comp_length() as usize, + max_backward, + &mut 0, + &mut [], + ); + } + + /// `AnyHasher::StoreRange` on an already-detected instruction set, so the walk over + /// `ix_start..ix_end` pays for detection once rather than once per position. + #[inline(always)] + pub(crate) fn store_range_simd( + &mut self, + simd: S, + data: &[u8], + mask: usize, + ix_start: usize, + ix_end: usize, + ) { + let mut i: usize = ix_start; + let mut j: usize = ix_start; + if ix_start.wrapping_add(63) <= ix_end { + i = ix_end.wrapping_sub(63); + } + if ix_start.wrapping_add(512) <= i { + while j < i { + { + self.store_simd(simd, data, mask, j); + } + j = j.wrapping_add(8); + } + } + while i < ix_end { + { + self.store_simd(simd, data, mask, i); + } + i = i.wrapping_add(1); + } + } + + /// `AnyHasher::BulkStoreRange` on an already-detected instruction set. + #[inline(always)] + fn bulk_store_range_simd( + &mut self, + simd: S, + data: &[u8], + mask: usize, + ix_start: usize, + ix_end: usize, + ) { + for i in ix_start..ix_end { + self.store_simd(simd, data, mask, i); + } + } } impl< Alloc: Allocator + Allocator, @@ -282,44 +350,13 @@ where } #[inline(always)] fn Store(&mut self, data: &[u8], mask: usize, ix: usize) { - let max_backward: usize = self.window_mask_.wrapping_sub(16).wrapping_add(1); - StoreAndFindMatchesH10( - self, - data, - ix, - mask, - self.ringbuffer_break, - Params::max_tree_comp_length() as usize, - max_backward, - &mut 0, - &mut [], - ); + dispatch!(detect_level(), simd => self.store_simd(simd, data, mask, ix)) } fn StoreRange(&mut self, data: &[u8], mask: usize, ix_start: usize, ix_end: usize) { - let mut i: usize = ix_start; - let mut j: usize = ix_start; - if ix_start.wrapping_add(63) <= ix_end { - i = ix_end.wrapping_sub(63); - } - if ix_start.wrapping_add(512) <= i { - while j < i { - { - self.Store(data, mask, j); - } - j = j.wrapping_add(8); - } - } - while i < ix_end { - { - self.Store(data, mask, i); - } - i = i.wrapping_add(1); - } + dispatch!(detect_level(), simd => self.store_range_simd(simd, data, mask, ix_start, ix_end)) } fn BulkStoreRange(&mut self, data: &[u8], mask: usize, ix_start: usize, ix_end: usize) { - for i in ix_start..ix_end { - self.Store(data, mask, i); - } + dispatch!(detect_level(), simd => self.bulk_store_range_simd(simd, data, mask, ix_start, ix_end)) } fn Prepare(&mut self, _one_shot: bool, _input_size: usize, _data: &[u8]) -> HowPrepared { if self.common.is_prepared_ != 0 { @@ -434,9 +471,13 @@ fn RightChildIndexH10, } */ +/// Detects the instruction set per call, then runs [`StoreAndFindMatchesH10Simd`]. +/// +/// Callers walking a range of positions should detect once and call that directly. +#[allow(clippy::too_many_arguments)] pub fn StoreAndFindMatchesH10< AllocU32: Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Buckets: Allocable + SliceWrapperMut + SliceWrapper + PartialEq, Params: H10Params, >( xself: &mut H10, @@ -448,10 +489,45 @@ pub fn StoreAndFindMatchesH10< max_backward: usize, best_len: &mut usize, matches: &mut [u64], -) -> usize -where - Buckets: PartialEq, -{ +) -> usize { + dispatch!(detect_level(), simd => StoreAndFindMatchesH10Simd( + simd, + xself, + data, + cur_ix, + ring_buffer_mask, + ringbuffer_break, + max_length, + max_backward, + best_len, + matches, + )) +} + +/// Walks the binary tree rooted at `cur_ix`'s hash bucket, recording matches and +/// re-rooting the tree at the current position. +/// +/// The tree walk measures a match length per node, up to 64 of them, which is why this +/// takes an already-detected instruction set rather than probing per comparison. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub fn StoreAndFindMatchesH10Simd< + S: Simd, + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper + PartialEq, + Params: H10Params, +>( + simd: S, + xself: &mut H10, + data: &[u8], + cur_ix: usize, + ring_buffer_mask: usize, + ringbuffer_break: Option, + max_length: usize, + max_backward: usize, + best_len: &mut usize, + matches: &mut [u64], +) -> usize { let mut matches_offset = 0_usize; let cur_ix_masked = cur_ix & ring_buffer_mask; let max_comp_len = min(max_length, 128); @@ -483,7 +559,8 @@ where let cur_len = min(best_len_left, best_len_right); let len = fix_unbroken_len( - cur_len.wrapping_add(FindMatchLengthWithLimit( + cur_len.wrapping_add(FindMatchLengthWithLimitSimd( + simd, &data[cur_ix_masked.wrapping_add(cur_len)..], &data[prev_ix_masked.wrapping_add(cur_len)..], max_length.wrapping_sub(cur_len), diff --git a/src/enc/backward_references/hq.rs b/src/enc/backward_references/hq.rs index 3c967f2f..c50f7af6 100644 --- a/src/enc/backward_references/hq.rs +++ b/src/enc/backward_references/hq.rs @@ -2,8 +2,10 @@ use crate::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use core; use core::cmp::{max, min}; +use fearless_simd::Simd; + use super::hash_to_binary_tree::{ - Allocable, BackwardMatch, BackwardMatchMut, H10, H10Params, StoreAndFindMatchesH10, Union1, + Allocable, BackwardMatch, BackwardMatchMut, H10, H10Params, StoreAndFindMatchesH10Simd, Union1, ZopfliNode, kInfinity, }; use super::{ @@ -19,9 +21,10 @@ use crate::enc::constants::{kCopyExtra, kInsExtra}; use crate::enc::encode; use crate::enc::literal_cost::BrotliEstimateBitCostsForLiterals; use crate::enc::static_dict::{ - BrotliDictionary, BrotliFindAllStaticDictionaryMatches, FindMatchLengthWithLimit, + BrotliDictionary, BrotliFindAllStaticDictionaryMatches, FindMatchLengthWithLimitSimd, }; use crate::enc::util::{FastLog2, FastLog2f64, floatX}; +use crate::enc::vectorization::detect_level; const BROTLI_WINDOW_GAP: usize = 16; const BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN: usize = 37; @@ -267,6 +270,34 @@ pub fn StitchToPreviousBlockH10< ) where Buckets: PartialEq, { + dispatch!(detect_level(), simd => StitchToPreviousBlockH10Simd( + simd, + handle, + num_bytes, + position, + ringbuffer, + ringbuffer_mask, + ringbuffer_break, + )) +} + +/// [`StitchToPreviousBlockH10`] on an already-detected instruction set: the stitch walks +/// up to `max_tree_comp_length` positions, so detection is hoisted out of that loop. +#[inline(always)] +fn StitchToPreviousBlockH10Simd< + S: Simd, + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper + PartialEq, + Params: H10Params, +>( + simd: S, + handle: &mut H10, + num_bytes: usize, + position: usize, + ringbuffer: &[u8], + ringbuffer_mask: usize, + ringbuffer_break: Option, +) { if (num_bytes >= handle.HashTypeLength() - 1 && position >= Params::max_tree_comp_length() as usize) { @@ -285,7 +316,8 @@ pub fn StitchToPreviousBlockH10< /* We know that i + MAX_TREE_COMP_LENGTH <= position + num_bytes, i.e. the end of the current block and that we have at least MAX_TREE_COMP_LENGTH tail in the ring-buffer. */ - StoreAndFindMatchesH10( + StoreAndFindMatchesH10Simd( + simd, handle, ringbuffer, i, @@ -299,6 +331,12 @@ pub fn StitchToPreviousBlockH10< } } } +/// Every match this position can reach: recent short matches, the binary tree, then the +/// static dictionary. +/// +/// Detection is hoisted into this function rather than the match-length comparisons it +/// drives, which run tens of times per position. +#[inline(always)] fn FindAllMatchesH10< AllocU32: Allocator, Buckets: Allocable + SliceWrapperMut + SliceWrapper, @@ -319,6 +357,43 @@ fn FindAllMatchesH10< where Buckets: PartialEq, { + dispatch!(detect_level(), simd => FindAllMatchesH10Simd( + simd, + handle, + dictionary, + data, + ring_buffer_mask, + ring_buffer_break, + cur_ix, + max_length, + max_backward, + gap, + params, + matches, + )) +} + +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn FindAllMatchesH10Simd< + S: Simd, + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper + PartialEq, + Params: H10Params, +>( + simd: S, + handle: &mut H10, + dictionary: Option<&BrotliDictionary>, + data: &[u8], + ring_buffer_mask: usize, + ring_buffer_break: Option, + cur_ix: usize, + max_length: usize, + max_backward: usize, + gap: usize, + params: &BrotliEncoderParams, + matches: &mut [u64], +) -> usize { let mut matches_offset = 0usize; let cur_ix_masked: usize = cur_ix & ring_buffer_mask; let mut best_len: usize = 1usize; @@ -344,8 +419,12 @@ where if data[cur_ix_masked] == data[prev_ix] && data[cur_ix_masked.wrapping_add(1)] == data[prev_ix.wrapping_add(1)] { - let len = - FindMatchLengthWithLimit(&data[prev_ix..], &data[cur_ix_masked..], max_length); + let len = FindMatchLengthWithLimitSimd( + simd, + &data[prev_ix..], + &data[cur_ix_masked..], + max_length, + ); if len > best_len { best_len = len; BackwardMatchMut(&mut matches[matches_offset]).init(backward, len); @@ -355,7 +434,8 @@ where i = i.wrapping_sub(1); } if best_len < max_length { - let loc_offset = StoreAndFindMatchesH10( + let loc_offset = StoreAndFindMatchesH10Simd( + simd, handle, data, cur_ix, @@ -637,7 +717,16 @@ impl BackwardMatch { } } -fn UpdateNodes>( +/// Extends the shortest-path search from `pos`, both through the distance cache and +/// through this position's candidate matches. +/// +/// Takes an already-detected instruction set: the match-length comparisons below run once +/// per distance-cache slot and once per candidate match, far too often to probe the CPU +/// for each one. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn UpdateNodesSimd>( + simd: S, num_bytes: usize, block_start: usize, pos: usize, @@ -732,7 +821,8 @@ fn UpdateNodes>( continue; } len = fix_unbroken_len( - FindMatchLengthWithLimit( + FindMatchLengthWithLimitSimd( + simd, &ringbuffer[prev_ix..], &ringbuffer[cur_ix_masked..], max_len, @@ -850,13 +940,24 @@ fn ComputeShortestPathFromNodes(num_bytes: usize, nodes: &mut [ZopfliNode]) -> u } const MAX_NUM_MATCHES_H10: usize = 128; -pub fn BrotliZopfliComputeShortestPath< + +/// The per-position sweep of [`BrotliZopfliComputeShortestPath`], on an already-detected +/// instruction set. +/// +/// Split out so the whole block costs one detection: every position here reaches the match +/// finder, the node update and the hasher store, and each of those would otherwise probe +/// the CPU on its own. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn ShortestPathPositionsSimd< + S: Simd, AllocU32: Allocator, - Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Buckets: Allocable + SliceWrapperMut + SliceWrapper + PartialEq, Params: H10Params, AllocF: Allocator, >( - m: &mut AllocF, + simd: S, + handle: &mut H10, dictionary: Option<&BrotliDictionary>, num_bytes: usize, position: usize, @@ -866,42 +967,22 @@ pub fn BrotliZopfliComputeShortestPath< params: &BrotliEncoderParams, max_backward_limit: usize, dist_cache: &[i32], - handle: &mut H10, + model: &mut ZopfliCostModel, + queue: &mut StartPosQueue, nodes: &mut [ZopfliNode], -) -> usize -where - Buckets: PartialEq, -{ - let max_zopfli_len: usize = MaxZopfliLen(params); - let mut model: ZopfliCostModel; - let mut queue: StartPosQueue; - let mut matches = [0; MAX_NUM_MATCHES_H10]; - let store_end: usize = if num_bytes >= STORE_LOOKAHEAD_H_10 { - position - .wrapping_add(num_bytes) - .wrapping_sub(STORE_LOOKAHEAD_H_10) - .wrapping_add(1) - } else { - position - }; - let mut i: usize; - let gap: usize = 0usize; - let lz_matches_offset: usize = 0usize; - (nodes[0]).length = 0u32; - (nodes[0]).u = Union1::cost(0.0); - model = ZopfliCostModel::init(m, ¶ms.dist, num_bytes); - if !(0i32 == 0) { - return 0usize; - } - model.set_from_literal_costs(position, ringbuffer, ringbuffer_mask); - queue = StartPosQueue::default(); - i = 0usize; + matches: &mut [u64], + store_end: usize, + max_zopfli_len: usize, + gap: usize, +) { + let mut i = 0usize; while i.wrapping_add(handle.HashTypeLength()).wrapping_sub(1) < num_bytes { { let pos: usize = position.wrapping_add(i); let max_distance: usize = min(pos, max_backward_limit); let mut skip: usize; - let mut num_matches: usize = FindAllMatchesH10( + let mut num_matches: usize = FindAllMatchesH10Simd( + simd, handle, dictionary, ringbuffer, @@ -912,7 +993,7 @@ where max_distance, gap, params, - &mut matches[lz_matches_offset..], + matches, ); if num_matches > 0 && BackwardMatch(matches[num_matches.wrapping_sub(1)]).length() > max_zopfli_len @@ -920,7 +1001,8 @@ where matches[0] = matches[num_matches.wrapping_sub(1)]; num_matches = 1usize; } - skip = UpdateNodes( + skip = UpdateNodesSimd( + simd, num_bytes, position, i, @@ -931,9 +1013,9 @@ where max_backward_limit, dist_cache, num_matches, - &matches[..], - &mut model, - &mut queue, + matches, + model, + queue, nodes, ); if skip < 16384usize { @@ -943,7 +1025,8 @@ where skip = max(BackwardMatch(matches[0]).length(), skip); } if skip > 1usize { - handle.StoreRange( + handle.store_range_simd( + simd, ringbuffer, ringbuffer_mask, pos.wrapping_add(1), @@ -961,8 +1044,8 @@ where max_backward_limit, gap, dist_cache, - &mut model, - &mut queue, + model, + queue, nodes, ); skip = skip.wrapping_sub(1); @@ -971,6 +1054,72 @@ where } i = i.wrapping_add(1); } +} + +pub fn BrotliZopfliComputeShortestPath< + AllocU32: Allocator, + Buckets: Allocable + SliceWrapperMut + SliceWrapper, + Params: H10Params, + AllocF: Allocator, +>( + m: &mut AllocF, + dictionary: Option<&BrotliDictionary>, + num_bytes: usize, + position: usize, + ringbuffer: &[u8], + ringbuffer_mask: usize, + ringbuffer_break: Option, + params: &BrotliEncoderParams, + max_backward_limit: usize, + dist_cache: &[i32], + handle: &mut H10, + nodes: &mut [ZopfliNode], +) -> usize +where + Buckets: PartialEq, +{ + let max_zopfli_len: usize = MaxZopfliLen(params); + let mut model: ZopfliCostModel; + let mut queue: StartPosQueue; + let mut matches = [0; MAX_NUM_MATCHES_H10]; + let store_end: usize = if num_bytes >= STORE_LOOKAHEAD_H_10 { + position + .wrapping_add(num_bytes) + .wrapping_sub(STORE_LOOKAHEAD_H_10) + .wrapping_add(1) + } else { + position + }; + let gap: usize = 0usize; + let lz_matches_offset: usize = 0usize; + (nodes[0]).length = 0u32; + (nodes[0]).u = Union1::cost(0.0); + model = ZopfliCostModel::init(m, ¶ms.dist, num_bytes); + if !(0i32 == 0) { + return 0usize; + } + model.set_from_literal_costs(position, ringbuffer, ringbuffer_mask); + queue = StartPosQueue::default(); + dispatch!(detect_level(), simd => ShortestPathPositionsSimd( + simd, + handle, + dictionary, + num_bytes, + position, + ringbuffer, + ringbuffer_mask, + ringbuffer_break, + params, + max_backward_limit, + dist_cache, + &mut model, + &mut queue, + nodes, + &mut matches[lz_matches_offset..], + store_end, + max_zopfli_len, + gap, + )); model.cleanup(m); @@ -1150,6 +1299,8 @@ impl> ZopfliCostModel { } } +/// Detects once for the whole block; the per-position node update it drives measures +/// match lengths tens of times per position. fn ZopfliIterate>( num_bytes: usize, position: usize, @@ -1164,6 +1315,42 @@ fn ZopfliIterate>( num_matches: &[u32], matches: &[u64], nodes: &mut [ZopfliNode], +) -> usize { + dispatch!(detect_level(), simd => ZopfliIterateSimd( + simd, + num_bytes, + position, + ringbuffer, + ringbuffer_mask, + ringbuffer_break, + params, + max_backward_limit, + gap, + dist_cache, + model, + num_matches, + matches, + nodes, + )) +} + +#[inline(always)] +#[allow(clippy::too_many_arguments)] +fn ZopfliIterateSimd>( + simd: S, + num_bytes: usize, + position: usize, + ringbuffer: &[u8], + ringbuffer_mask: usize, + ringbuffer_break: Option, + params: &BrotliEncoderParams, + max_backward_limit: usize, + gap: usize, + dist_cache: &[i32], + model: &ZopfliCostModel, + num_matches: &[u32], + matches: &[u64], + nodes: &mut [ZopfliNode], ) -> usize { let max_zopfli_len: usize = MaxZopfliLen(params); let mut queue: StartPosQueue; @@ -1175,7 +1362,8 @@ fn ZopfliIterate>( i = 0usize; while i.wrapping_add(3) < num_bytes { { - let mut skip: usize = UpdateNodes( + let mut skip: usize = UpdateNodesSimd( + simd, num_bytes, position, i, diff --git a/src/enc/bit_cost.rs b/src/enc/bit_cost.rs index a79937c4..bc5dcee3 100644 --- a/src/enc/bit_cost.rs +++ b/src/enc/bit_cost.rs @@ -1,10 +1,12 @@ use crate::alloc::SliceWrapperMut; use core::cmp::{max, min}; +use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u32x8}; + use super::super::alloc::SliceWrapper; use super::histogram::CostAccessors; use super::util::{FastLog2, FastLog2u16}; -use super::vectorization::Mem256i; +use super::vectorization::{Mem256i, detect_level}; use crate::enc::floatX; const BROTLI_REPEAT_ZERO_CODE_LENGTH: usize = 17; @@ -175,37 +177,109 @@ pub fn BrotliPopulationCost + CostAccessors>( } bits += CostComputation(&mut depth_histo, nnz_data, nnz, total_count, log2total); } else { - let mut max_depth: usize = 1; - let mut depth_histo = [0u32; 18]; + let mut depth_histo = [0u32; BROTLI_CODE_LENGTH_CODES]; let log2total: floatX = FastLog2(histogram.total_count() as u64); // 64 bit here - let mut reps: u32 = 0; - for histo in histogram.slice()[..data_size].iter() { - if *histo != 0 { - if reps != 0 { - if reps < 3 { - depth_histo[0] += reps; - } else { - reps -= 2; - while reps > 0 { - depth_histo[17] += 1; - bits += 3.0; - reps >>= 3; - } - } - reps = 0; - } - let log2p = log2total - FastLog2u16(*histo as u16); - let mut depth = (log2p + 0.5) as usize; - bits += *histo as floatX * log2p; - depth = min(depth, 15); - max_depth = max(depth, max_depth); - depth_histo[depth] += 1; - } else { - reps += 1; - } - } + let max_depth = dispatch!(detect_level(), simd => accumulate_symbol_costs( + simd, + &histogram.slice()[..data_size], + log2total, + &mut bits, + &mut depth_histo, + )); bits += (18usize).wrapping_add((2usize).wrapping_mul(max_depth)) as floatX; - bits += BitsEntropy(&depth_histo[..], 18); + bits += BitsEntropy(&depth_histo[..], BROTLI_CODE_LENGTH_CODES); } bits } + +/// Charges one populated bucket, first flushing the run of empty buckets before it. +/// +/// Split out of [`accumulate_symbol_costs`] so the vectorized and remainder walks share it, +/// which keeps `bits` accumulating in bucket order and therefore bit-identical. +#[inline(always)] +fn accumulate_one_symbol( + histo: u32, + log2total: floatX, + bits: &mut floatX, + max_depth: &mut usize, + reps: &mut u32, + depth_histo: &mut [u32; BROTLI_CODE_LENGTH_CODES], +) { + if *reps != 0 { + if *reps < 3 { + depth_histo[0] += *reps; + } else { + let mut remaining = *reps - 2; + while remaining > 0 { + depth_histo[BROTLI_REPEAT_ZERO_CODE_LENGTH] += 1; + *bits += 3.0; + remaining >>= 3; + } + } + *reps = 0; + } + let log2p = log2total - FastLog2u16(histo as u16); + let depth = min((log2p + 0.5) as usize, 15); + *bits += histo as floatX * log2p; + *max_depth = max(depth, *max_depth); + depth_histo[depth] += 1; +} + +/// Charges every populated bucket of `histogram`, returning the deepest code length seen. +/// +/// Histograms are mostly empty — a distance alphabet has 544 buckets and a metablock +/// rarely touches a tenth of them — so the walk tests eight buckets per compare and only +/// falls back to per-bucket work for the ones that are actually populated. Empty runs +/// still land in `depth_histo` exactly as a bucket-at-a-time scan would leave them. +#[inline(always)] +fn accumulate_symbol_costs( + simd: S, + histogram: &[u32], + log2total: floatX, + bits: &mut floatX, + depth_histo: &mut [u32; BROTLI_CODE_LENGTH_CODES], +) -> usize { + let mut max_depth: usize = 1; + let mut reps: u32 = 0; + let empty = u32x8::splat(simd, 0); + + let mut buckets = histogram.chunks_exact(8); + for chunk in &mut buckets { + let mut populated = !u32x8::from_slice(simd, chunk).simd_eq(empty).to_bitmask() & 0xff; + if populated == 0 { + reps += 8; + continue; + } + let mut scanned = 0u32; + while populated != 0 { + let lane = populated.trailing_zeros(); + reps += lane - scanned; + scanned = lane + 1; + populated &= populated - 1; + accumulate_one_symbol( + chunk[lane as usize], + log2total, + bits, + &mut max_depth, + &mut reps, + depth_histo, + ); + } + reps += 8 - scanned; + } + for &histo in buckets.remainder() { + if histo != 0 { + accumulate_one_symbol( + histo, + log2total, + bits, + &mut max_depth, + &mut reps, + depth_histo, + ); + } else { + reps += 1; + } + } + max_depth +} diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index 9cadd7a8..2bfdd827 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -1,7 +1,7 @@ use core; use core::cmp::{max, min}; -use fearless_simd::{Simd, SimdBase, SimdFloat, SimdMask, f32x8}; +use fearless_simd::{Select, Simd, SimdBase, SimdFloat, SimdMask, f32x8, u32x8}; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; @@ -14,10 +14,13 @@ use super::histogram::{ HistogramClear, HistogramCommand, HistogramDistance, HistogramLiteral, }; use super::util::FastLog2; -use super::vectorization::{Mem256f, detect_level}; +use super::vectorization::{Mem256f, detect_level, min_lane_f32x8, min_lane_u32x8}; use crate::enc::combined_alloc::allocate; use crate::enc::floatX; +/// Lane offsets, added to a vector's base index to recover a histogram id. +static LANE_INDICES: [u32; 8] = [0, 1, 2, 3, 4, 5, 6, 7]; + static kMaxLiteralHistograms: usize = 100usize; static kMaxCommandHistograms: usize = 50usize; @@ -303,23 +306,35 @@ where let mut block_switch_cost: floatX = block_switch_bitcost; // main (vectorized) loop let insert_cost_slice = insert_cost.split_at(insert_cost_ix).1; - for (v_index, cost_iter) in cost - .split_at_mut(num_histograms >> 3) - .0 - .iter_mut() - .enumerate() - { + let num_vectors = num_histograms >> 3; + // Running per-lane winner, reduced across lanes once the row is done rather than + // once per vector. + let mut min_lanes = f32x8::splat(simd, min_cost); + let mut id_lanes = u32x8::splat(simd, u32::MAX); + for (v_index, cost_iter) in cost.split_at_mut(num_vectors).0.iter_mut().enumerate() { let base_index = v_index << 3; - let mut local_insert_cost = [0.0; 8]; - local_insert_cost - .clone_from_slice(insert_cost_slice.split_at(base_index).1.split_at(8).0); - for sub_index in 0usize..8usize { - cost_iter[sub_index] += local_insert_cost[sub_index]; - let final_cost = cost_iter[sub_index]; - if final_cost < min_cost { - min_cost = final_cost; - *block_id_ptr = (base_index + sub_index) as u8; - } + let updated = cost_iter.to_simd(simd) + + f32x8::from_slice(simd, insert_cost_slice.split_at(base_index).1.split_at(8).0); + *cost_iter = Mem256f::from_simd(updated); + // Strictly less, so a lane keeps the earliest histogram it tied with, exactly + // as the scalar scan this replaces did. + let improved = updated.simd_lt(min_lanes); + min_lanes = improved.select(updated, min_lanes); + id_lanes = improved.select( + u32x8::from_slice(simd, &LANE_INDICES) + base_index as u32, + id_lanes, + ); + } + if num_vectors != 0 { + let best = min_lane_f32x8(min_lanes); + if best < min_cost { + min_cost = best; + // Ties between lanes go to the lowest histogram id, again matching a scan. + *block_id_ptr = min_lane_u32x8( + min_lanes + .simd_eq(f32x8::splat(simd, best)) + .select(id_lanes, u32x8::splat(simd, u32::MAX)), + ) as u8; } } let vectorized_offset = ((num_histograms >> 3) << 3); diff --git a/src/enc/static_dict.rs b/src/enc/static_dict.rs index 58d31894..a3a6c062 100644 --- a/src/enc/static_dict.rs +++ b/src/enc/static_dict.rs @@ -1,4 +1,7 @@ use core::cmp::{max, min}; + +use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u8x32}; + pub const kNumDistanceCacheEntries: usize = 4; use super::super::dictionary::{ @@ -7,6 +10,7 @@ use super::super::dictionary::{ use super::static_dict_lut::{ DictWord, kDictHashMul32, kDictNumBits, kStaticDictionaryBuckets, kStaticDictionaryWords, }; +use super::vectorization::detect_level; #[allow(unused)] static kUppercaseFirst: u8 = 10u8; @@ -120,16 +124,127 @@ pub fn SlowerFindMatchLengthWithLimit(s1: &[u8], s2: &[u8], limit: usize) -> usi } limit } -// factor of 5 slower (example takes 90 seconds) +/// Length of the common prefix of `s1` and `s2`, capped at `limit`. +/// +/// Resolves short matches -- almost all of them -- without ever looking at the CPU's +/// feature set, and only detects an instruction set once a match has proven long enough +/// for wide compares to pay for that detection. Callers that already hold an instruction +/// set should use [`FindMatchLengthWithLimitSimd`]. #[allow(unused)] +#[inline] pub fn FindMatchLengthWithLimit(s1: &[u8], s2: &[u8], limit: usize) -> usize { - for (index, pair) in s1[..limit].iter().zip(s2[..limit].iter()).enumerate() { + let s1 = &s1[..limit]; + let s2 = &s2[..limit]; + match narrow_common_prefix(s1, s2, limit) { + Ok(len) => len, + Err(matched) => { + matched + detect_and_wide_common_prefix(&s1[matched..], &s2[matched..], limit - matched) + } + } +} + +/// Kept out of line so the caller only inlines the first stage, which is the one the +/// match finders run millions of times. Reaching here means the match has already run +/// [`WIDE_COMPARE_THRESHOLD`] bytes, which is long enough to absorb a CPU probe. +#[inline(never)] +fn detect_and_wide_common_prefix(s1: &[u8], s2: &[u8], limit: usize) -> usize { + dispatch!(detect_level(), simd => wide_common_prefix(simd, s1, s2, limit)) +} + +/// [`FindMatchLengthWithLimit`] on an already-detected instruction set. +#[inline(always)] +pub fn FindMatchLengthWithLimitSimd(simd: S, s1: &[u8], s2: &[u8], limit: usize) -> usize { + let s1 = &s1[..limit]; + let s2 = &s2[..limit]; + match narrow_common_prefix(s1, s2, limit) { + Ok(len) => len, + Err(matched) => { + matched + wide_common_prefix(simd, &s1[matched..], &s2[matched..], limit - matched) + } + } +} + +/// How far a match must already have run before wide compares are worth their setup. +const WIDE_COMPARE_THRESHOLD: usize = 32; + +/// The first stage of the match-length scan, in 64-bit steps. +/// +/// Most candidates differ within the first handful of bytes, and a 64-bit compare settles +/// that in a couple of cycles -- a wide compare would still be waiting on the horizontal +/// reduction that turns its result into a mask. Returns `Ok` with the answer whenever the +/// scan finishes here, and `Err(matched)` when the inputs are still equal after +/// [`WIDE_COMPARE_THRESHOLD`] bytes and at least 32 more remain to check. +#[inline(always)] +fn narrow_common_prefix(s1: &[u8], s2: &[u8], limit: usize) -> Result { + let mut matched = 0usize; + while matched < WIDE_COMPARE_THRESHOLD && limit - matched >= 8 { + if let Some(offset) = first_differing_byte(&s1[matched..], &s2[matched..]) { + return Ok(matched + offset); + } + matched += 8; + } + if limit - matched >= 32 { + return Err(matched); + } + Ok(matched + scalar_common_prefix(&s1[matched..], &s2[matched..], limit - matched)) +} + +/// The second stage, 32 bytes a step, for a match already known to be long. +/// +/// The branch is predictable by now and throughput is what matters, so a mismatch is +/// located as one `trailing_ones` over the compare mask. +#[inline(always)] +fn wide_common_prefix(simd: S, s1: &[u8], s2: &[u8], limit: usize) -> usize { + let mut matched = 0usize; + while limit - matched >= 32 { + let equal = u8x32::from_slice(simd, &s1[matched..matched + 32]) + .simd_eq(u8x32::from_slice(simd, &s2[matched..matched + 32])) + .to_bitmask() as u32; + if equal != u32::MAX { + return matched + equal.trailing_ones() as usize; + } + matched += 32; + } + matched + scalar_common_prefix(&s1[matched..], &s2[matched..], limit - matched) +} + +/// Common prefix length of fewer than 32 remaining bytes: 64-bit steps, then bytes. +#[inline(always)] +fn scalar_common_prefix(s1: &[u8], s2: &[u8], limit: usize) -> usize { + let mut matched = 0usize; + while limit - matched >= 8 { + if let Some(offset) = first_differing_byte(&s1[matched..], &s2[matched..]) { + return matched + offset; + } + matched += 8; + } + for (index, pair) in s1[matched..limit] + .iter() + .zip(s2[matched..limit].iter()) + .enumerate() + { if *pair.0 != *pair.1 { - return index; + return matched + index; } } limit } + +/// Offset of the first byte where the leading eight bytes of `s1` and `s2` differ, or +/// `None` if all eight match. +/// +/// [`BROTLI_UNALIGNED_LOAD64`] assembles its word little-endian on every target, so the +/// lowest differing bit always belongs to the earliest differing byte. +#[inline(always)] +fn first_differing_byte(s1: &[u8], s2: &[u8]) -> Option { + let diff = BROTLI_UNALIGNED_LOAD64(s1) ^ BROTLI_UNALIGNED_LOAD64(s2); + if diff == 0 { + None + } else { + Some((diff.trailing_zeros() >> 3) as usize) + } +} + #[allow(unused)] pub fn FindMatchLengthWithLimitMin4(s1: &[u8], s2: &[u8], limit: usize) -> usize { let (s1_start, s1_rest) = s1.split_at(5); diff --git a/src/enc/vectorization.rs b/src/enc/vectorization.rs index abfdc7b5..efb70b92 100644 --- a/src/enc/vectorization.rs +++ b/src/enc/vectorization.rs @@ -9,17 +9,49 @@ use core::ops::{Index, IndexMut}; use core::slice::SliceIndex; -use fearless_simd::{Level, Simd, SimdInto, f32x8, i16x16, i32x8}; +use fearless_simd::{ + Level, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x8, i16x16, i32x8, u32x8, +}; /// The instruction set the vectorized encoder paths run on. /// /// Detected at runtime where the platform allows it (`std` builds, wasm), otherwise the -/// best level this crate was compiled for. +/// best level this crate was compiled for. The `std` answer is cached: probing costs a +/// dozen feature tests, and callers such as [`crate::enc::bit_cost::BrotliPopulationCost`] +/// dispatch once per histogram, deep inside the clustering loops. +#[cfg(feature = "std")] +#[inline] +pub fn detect_level() -> Level { + static LEVEL: std::sync::OnceLock = std::sync::OnceLock::new(); + *LEVEL.get_or_init(|| Level::try_detect().unwrap_or_else(Level::baseline)) +} + +/// See the `std` variant above; without `std` there is nothing to cache, as +/// `try_detect` cannot probe the CPU and always resolves to the compiled-for level. +#[cfg(not(feature = "std"))] #[inline] pub fn detect_level() -> Level { Level::try_detect().unwrap_or_else(Level::baseline) } +/// The smallest lane of `v`, folded in `log2(8)` steps. +#[inline(always)] +pub fn min_lane_f32x8(v: f32x8) -> f32 { + let v = v.min(v.slide::<4>(v)); + let v = v.min(v.slide::<2>(v)); + let v = v.min(v.slide::<1>(v)); + v[0] +} + +/// The smallest lane of `v`, folded in `log2(8)` steps. +#[inline(always)] +pub fn min_lane_u32x8(v: u32x8) -> u32 { + let v = v.min(v.slide::<4>(v)); + let v = v.min(v.slide::<2>(v)); + let v = v.min(v.slide::<1>(v)); + v[0] +} + macro_rules! define_vector { ($(#[$attr:meta])* $name:ident, $elem:ty, $lanes:literal, $simd:ident) => { $(#[$attr])* From 064dc438d0b8fc154cf1e99761d6ff309463bfb3 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 17:11:58 +0300 Subject: [PATCH 4/6] Add profile and optimize hot pathes --- Cargo.toml | 11 ++ README.md | 31 +++++ src/bin/brotli.rs | 3 + src/enc/backward_references/hq.rs | 7 + src/enc/backward_references/mod.rs | 1 + src/enc/bit_cost.rs | 176 +++++++++++++++++++++----- src/enc/block_splitter.rs | 5 + src/enc/brotli_bit_stream.rs | 4 + src/enc/cluster.rs | 14 +- src/enc/compress_fragment.rs | 1 + src/enc/compress_fragment_two_pass.rs | 3 + src/enc/encode.rs | 7 + src/enc/literal_cost.rs | 1 + src/enc/metablock.rs | 3 + 14 files changed, 228 insertions(+), 39 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29dfe558..277aa149 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,10 @@ incremental = false "sha2" = { version = "~0.11", optional = true } +# Profiling instrumentation. Inert unless the `hotpath` feature is on: every call site is +# behind `cfg_attr`, so a default build never links it. +"hotpath" = { version = "~0.23", optional = true } + [dev-dependencies] # The test suite (src/enc/test.rs) builds calloc-backed memory pools, which on # alloc-no-stdlib 3.x live behind its "unsafe" feature (always present on 2.x). @@ -64,6 +68,13 @@ external-literal-probability = [] ffi-api = ["brotli-decompressor/ffi-api"] float64 = [] floating_point_context_mixing = [] +# Wall-clock profiling of the encoder pipeline. Requires `std`; run the `brotli` binary with +# `--features hotpath` and it prints a per-stage table on exit. +hotpath = ["dep:hotpath", "hotpath/hotpath", "std"] +# Same, but measuring allocation counts/bytes instead of time (installs a counting allocator). +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"] +# Same, but measuring CPU time instead of wall-clock. +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"] no-stdlib-ffi-binding = [] pass-through-ffi-panics = [] seccomp = ["brotli-decompressor/seccomp"] diff --git a/README.md b/README.md index 265849aa..47860eab 100644 --- a/README.md +++ b/README.md @@ -318,3 +318,34 @@ params.catable = true; // Sets catable=true, appendable=true, use_diction // All parameter dependencies are handled automatically by the library. // No manual fixups required - just set the primary flags you want. ``` + +## Profiling the encoder + +The encoder pipeline is instrumented with [hotpath](https://docs.rs/hotpath/). The +instrumentation is behind `cfg_attr`, so a default build neither links `hotpath` nor pays any +runtime cost; only `--features hotpath` turns it on. + +```bash +# wall-clock per pipeline stage +cargo run --release --features hotpath --bin brotli -- -c -q11 input.bin /dev/null + +# CPU time instead of wall-clock +cargo run --release --features hotpath-cpu --bin brotli -- -c -q11 input.bin /dev/null + +# allocation counts/bytes instead of time +cargo run --release --features hotpath-alloc --bin brotli -- -c -q11 input.bin /dev/null +``` + +The report prints on exit. `HOTPATH_OUTPUT_FORMAT=json-pretty` emits the full table as JSON +(the default table view truncates to fit the terminal). + +Measured stages: `encode_data`, `copy_input_to_ring_buffer`, `WriteMetaBlockInternal`, +`ChooseContextMap`, `DecideOverLiteralContextModeling`, `compress_stream_fast`, the three +`store_meta_block*` writers, `LogMetaBlock`, `BrotliCreateBackwardReferences` and the Zopfli +entry points, `BrotliBuildMetaBlock`/`Greedy`/`BrotliOptimizeHistograms`, `BrotliSplitBlock` +and its internals, the `cluster.rs` histogram-clustering functions, +`BrotliEstimateBitCostsForLiterals`, and the two `compress_fragment` fast paths. + +Instrumentation sits at metablock granularity, not per-byte, so overhead is under measurement +noise (q9/q10/q11 on a 3.9 MB corpus timed within 1% of an uninstrumented build). Leaf-level +attribution inside a stage needs a sampling profiler (`sample` on macOS, `perf` on Linux). diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs index 9bdb794b..216a5673 100644 --- a/src/bin/brotli.rs +++ b/src/bin/brotli.rs @@ -529,6 +529,9 @@ fn has_stdlib() -> bool { false } +// With `--features hotpath` this wraps `main` in a profiler guard that prints a per-stage +// table of the encoder pipeline on exit. Inert otherwise. +#[cfg_attr(feature = "hotpath", hotpath::main)] fn main() { let mut buffer_size = 65536; let mut do_compress = false; diff --git a/src/enc/backward_references/hq.rs b/src/enc/backward_references/hq.rs index c50f7af6..e5087588 100644 --- a/src/enc/backward_references/hq.rs +++ b/src/enc/backward_references/hq.rs @@ -97,6 +97,7 @@ impl ZopfliNode { } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliZopfliCreateCommands( num_bytes: usize, block_start: usize, @@ -1056,6 +1057,7 @@ fn ShortestPathPositionsSimd< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliZopfliComputeShortestPath< AllocU32: Allocator, Buckets: Allocable + SliceWrapperMut + SliceWrapper, @@ -1126,6 +1128,7 @@ where ComputeShortestPathFromNodes(num_bytes, nodes) } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliCreateZopfliBackwardReferences< Alloc: Allocator + Allocator + Allocator, Buckets: Allocable + SliceWrapperMut + SliceWrapper, @@ -1188,6 +1191,7 @@ pub fn BrotliCreateZopfliBackwardReferences< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn SetCost(histogram: &[u32], histogram_size: usize, literal_histogram: bool, cost: &mut [floatX]) { let mut sum: u64 = 0; for i in 0..histogram_size { @@ -1301,6 +1305,7 @@ impl> ZopfliCostModel { /// Detects once for the whole block; the per-position node update it drives measures /// match lengths tens of times per position. +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn ZopfliIterate>( num_bytes: usize, position: usize, @@ -1336,6 +1341,7 @@ fn ZopfliIterate>( #[inline(always)] #[allow(clippy::too_many_arguments)] +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn ZopfliIterateSimd>( simd: S, num_bytes: usize, @@ -1418,6 +1424,7 @@ fn ZopfliIterateSimd>( ComputeShortestPathFromNodes(num_bytes, nodes) } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliCreateHqZopfliBackwardReferences< Alloc: Allocator + Allocator + Allocator + Allocator, Buckets: Allocable + SliceWrapperMut + SliceWrapper, diff --git a/src/enc/backward_references/mod.rs b/src/enc/backward_references/mod.rs index 398ac94b..09876a5e 100644 --- a/src/enc/backward_references/mod.rs +++ b/src/enc/backward_references/mod.rs @@ -2550,6 +2550,7 @@ fn CreateBackwardReferences( *last_insert_len = insert_length; *num_commands = num_commands.wrapping_add(new_commands_count); } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliCreateBackwardReferences< Alloc: alloc::Allocator + alloc::Allocator diff --git a/src/enc/bit_cost.rs b/src/enc/bit_cost.rs index bc5dcee3..78394e24 100644 --- a/src/enc/bit_cost.rs +++ b/src/enc/bit_cost.rs @@ -1,7 +1,7 @@ use crate::alloc::SliceWrapperMut; use core::cmp::{max, min}; -use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u32x8}; +use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u32x16}; use super::super::alloc::SliceWrapper; use super::histogram::CostAccessors; @@ -75,25 +75,107 @@ fn CostComputation>( bits } +/// The bucket values a population cost is charged over. +/// +/// Clustering spends nearly all of its population-cost calls on the *sum* of two histograms — +/// [`BrotliHistogramBitCostDistance`](super::cluster::BrotliHistogramBitCostDistance) and the +/// pair queue in [`cluster`](super::cluster). Materializing that sum costs a full copy plus a +/// full add before the cost walk even starts, so [`Sum`] models it instead and lets the walk +/// the cost already makes do the adding. +trait Buckets { + fn len(&self) -> usize; + fn get(&self, i: usize) -> u32; + /// The sixteen buckets starting at `at`, which must leave sixteen in range. + fn chunk(&self, simd: S, at: usize) -> u32x16; +} + +/// One histogram's own buckets. +struct Own<'a>(&'a [u32]); + +/// Two histograms' buckets added lane-wise, never materialized. +/// +/// Wrapping addition and the left operand's length match +/// [`HistogramAddHistogram`](super::histogram::HistogramAddHistogram), which is what the +/// materializing form used to call. +struct Sum<'a>(&'a [u32], &'a [u32]); + +impl Buckets for Own<'_> { + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn get(&self, i: usize) -> u32 { + self.0[i] + } + #[inline(always)] + fn chunk(&self, simd: S, at: usize) -> u32x16 { + u32x16::from_slice(simd, &self.0[at..at + 16]) + } +} + +impl Buckets for Sum<'_> { + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn get(&self, i: usize) -> u32 { + self.0[i].wrapping_add(self.1[i]) + } + #[inline(always)] + fn chunk(&self, simd: S, at: usize) -> u32x16 { + u32x16::from_slice(simd, &self.0[at..at + 16]) + + u32x16::from_slice(simd, &self.1[at..at + 16]) + } +} + pub fn BrotliPopulationCost + CostAccessors>( histogram: &HistogramType, nnz_data: &mut HistogramType::i32vec, +) -> floatX { + population_cost(Own(histogram.slice()), histogram.total_count(), nnz_data) +} + +/// Cost of the histogram that adding `b` into `a` would produce, without building it. +/// +/// Bit-for-bit identical to cloning `a`, calling +/// [`HistogramAddHistogram`](super::histogram::HistogramAddHistogram) with `b` and costing the +/// result: the walk sees the same bucket values in the same order, so the float accumulation is +/// unchanged. It just skips the copy and the separate add pass. +pub fn BrotliPopulationCostOfSum + CostAccessors>( + a: &HistogramType, + b: &HistogramType, + nnz_data: &mut HistogramType::i32vec, +) -> floatX { + debug_assert_eq!(a.slice().len(), b.slice().len()); + population_cost( + Sum(a.slice(), b.slice()), + a.total_count() + b.total_count(), + nnz_data, + ) +} + +fn population_cost + SliceWrapperMut>( + buckets: B, + total_count: usize, + nnz_data: &mut Scratch, ) -> floatX { static kOneSymbolHistogramCost: floatX = 12.0; static kTwoSymbolHistogramCost: floatX = 20.0; static kThreeSymbolHistogramCost: floatX = 28.0; static kFourSymbolHistogramCost: floatX = 37.0; - let data_size: usize = histogram.slice().len(); + let data_size: usize = buckets.len(); let mut count = 0; let mut s: [usize; 5] = [0; 5]; let mut bits: floatX = 0.0; - if histogram.total_count() == 0 { + if total_count == 0 { return kOneSymbolHistogramCost; } for i in 0..data_size { - if histogram.slice()[i] > 0 { + if buckets.get(i) > 0 { s[count] = i; count += 1; if count > 4 { @@ -103,11 +185,11 @@ pub fn BrotliPopulationCost + CostAccessors>( } match count { 1 => return kOneSymbolHistogramCost, - 2 => return kTwoSymbolHistogramCost + histogram.total_count() as floatX, + 2 => return kTwoSymbolHistogramCost + total_count as floatX, 3 => { - let histo0: u32 = histogram.slice()[s[0]]; - let histo1: u32 = histogram.slice()[s[1]]; - let histo2: u32 = histogram.slice()[s[2]]; + let histo0: u32 = buckets.get(s[0]); + let histo1: u32 = buckets.get(s[1]); + let histo2: u32 = buckets.get(s[2]); let histomax: u32 = max(histo0, max(histo1, histo2)); return kThreeSymbolHistogramCost + (2u32).wrapping_mul(histo0.wrapping_add(histo1).wrapping_add(histo2)) as floatX @@ -117,7 +199,7 @@ pub fn BrotliPopulationCost + CostAccessors>( let mut histo: [u32; 4] = [0; 4]; for i in 0..4 { - histo[i] = histogram.slice()[s[i]]; + histo[i] = buckets.get(s[i]); } for i in 0..4 { for j in i + 1..4 { @@ -140,19 +222,20 @@ pub fn BrotliPopulationCost + CostAccessors>( // vectorization failed: it's faster to do things inline than split into two loops let mut nnz: usize = 0; let mut depth_histo = [0u32; 18]; - let total_count = histogram.total_count() as floatX; - let log2total = FastLog2(histogram.total_count() as u64); + let total_count_f = total_count as floatX; + let log2total = FastLog2(total_count as u64); let mut i: usize = 0; while i < data_size { - if histogram.slice()[i] > 0 { + if buckets.get(i) > 0 { + let histo = buckets.get(i); let nnz_val = &mut nnz_data.slice_mut()[nnz >> 3]; - nnz_val[nnz & 7] = histogram.slice()[i] as i32; + nnz_val[nnz & 7] = histo as i32; i += 1; nnz += 1; } else { let mut reps: u32 = 1; - for hd in histogram.slice()[i + 1..data_size].iter() { - if *hd != 0 { + for j in i + 1..data_size { + if buckets.get(j) != 0 { break; } reps += 1 @@ -175,13 +258,13 @@ pub fn BrotliPopulationCost + CostAccessors>( } } } - bits += CostComputation(&mut depth_histo, nnz_data, nnz, total_count, log2total); + bits += CostComputation(&mut depth_histo, nnz_data, nnz, total_count_f, log2total); } else { let mut depth_histo = [0u32; BROTLI_CODE_LENGTH_CODES]; - let log2total: floatX = FastLog2(histogram.total_count() as u64); // 64 bit here + let log2total: floatX = FastLog2(total_count as u64); // 64 bit here let max_depth = dispatch!(detect_level(), simd => accumulate_symbol_costs( simd, - &histogram.slice()[..data_size], + &buckets, log2total, &mut bits, &mut depth_histo, @@ -225,29 +308,54 @@ fn accumulate_one_symbol( depth_histo[depth] += 1; } -/// Charges every populated bucket of `histogram`, returning the deepest code length seen. +/// Charges every populated bucket of `buckets`, returning the deepest code length seen. /// -/// Histograms are mostly empty — a distance alphabet has 544 buckets and a metablock -/// rarely touches a tenth of them — so the walk tests eight buckets per compare and only -/// falls back to per-bucket work for the ones that are actually populated. Empty runs -/// still land in `depth_histo` exactly as a bucket-at-a-time scan would leave them. +/// Bucket occupancy is bimodal rather than uniformly sparse: measured over a mixed 3.9 MB +/// corpus at q10, 43% of buckets are populated overall, but they cluster — the 256-bucket +/// literal alphabet is dense over the ASCII range and empty above it, while the 704-bucket +/// command and 544-bucket distance alphabets are mostly empty. So the walk tests sixteen +/// buckets per compare and takes a straight-line branch for each extreme, falling back to the +/// bit-at-a-time scan only for mixed chunks. Both were measured to beat a plain scalar walk +/// and an eight-wide test. Empty runs still land in `depth_histo` exactly as a +/// bucket-at-a-time scan would leave them. #[inline(always)] -fn accumulate_symbol_costs( +fn accumulate_symbol_costs( simd: S, - histogram: &[u32], + buckets: &B, log2total: floatX, bits: &mut floatX, depth_histo: &mut [u32; BROTLI_CODE_LENGTH_CODES], ) -> usize { + const LANES: usize = 16; let mut max_depth: usize = 1; let mut reps: u32 = 0; - let empty = u32x8::splat(simd, 0); + let empty = u32x16::splat(simd, 0); - let mut buckets = histogram.chunks_exact(8); - for chunk in &mut buckets { - let mut populated = !u32x8::from_slice(simd, chunk).simd_eq(empty).to_bitmask() & 0xff; + let data_size = buckets.len(); + // Every histogram alphabet is a multiple of `LANES`, so the scalar tail below is normally + // dead; it is there to keep the walk correct for any bucket count. + let vectorized = data_size & !(LANES - 1); + let mut at = 0; + while at < vectorized { + let chunk = buckets.chunk(simd, at); + let mut populated = !chunk.simd_eq(empty).to_bitmask() & 0xffff; if populated == 0 { - reps += 8; + reps += LANES as u32; + at += LANES; + continue; + } + if populated == 0xffff { + for lane in 0..LANES { + accumulate_one_symbol( + chunk[lane], + log2total, + bits, + &mut max_depth, + &mut reps, + depth_histo, + ); + } + at += LANES; continue; } let mut scanned = 0u32; @@ -256,6 +364,8 @@ fn accumulate_symbol_costs( reps += lane - scanned; scanned = lane + 1; populated &= populated - 1; + // Read the lane back off `chunk` rather than through `buckets`: for `Sum` that is + // the difference between one register extract and re-loading both operands. accumulate_one_symbol( chunk[lane as usize], log2total, @@ -265,9 +375,11 @@ fn accumulate_symbol_costs( depth_histo, ); } - reps += 8 - scanned; + reps += LANES as u32 - scanned; + at += LANES; } - for &histo in buckets.remainder() { + for i in vectorized..data_size { + let histo = buckets.get(i); if histo != 0 { accumulate_one_symbol( histo, diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index 2bfdd827..e9a09aff 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -121,6 +121,7 @@ fn MyRand(seed: &mut u32) -> u32 { *seed } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn InitialEntropyCodes< HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors, IntegerType: Sized + Clone, @@ -170,6 +171,7 @@ fn RandomSample< HistogramAddVector(sample, &data[pos..], stride); } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn RefineEntropyCodes< HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors + core::default::Default, IntegerType: Sized + Clone, @@ -436,6 +438,7 @@ fn BuildBlockHistograms< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn ClusterBlocks< HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors + core::default::Default + Clone, Alloc: alloc::Allocator @@ -729,6 +732,7 @@ fn ClusterBlocks< >::free_cell(alloc, histogram_symbols); } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn SplitByteVector< HistogramType: SliceWrapper + SliceWrapperMut + CostAccessors + core::default::Default + Clone, Alloc: alloc::Allocator @@ -876,6 +880,7 @@ fn SplitByteVector< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliSplitBlock< Alloc: alloc::Allocator + alloc::Allocator diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs index 9b93c123..994b20a6 100755 --- a/src/enc/brotli_bit_stream.rs +++ b/src/enc/brotli_bit_stream.rs @@ -418,6 +418,7 @@ fn process_command_queue<'a, CmdProcessor: interface::CommandProcessor<'a>>( recoder_state } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn LogMetaBlock<'a, Alloc: BrotliAlloc, Cb>( alloc: &mut Alloc, commands: &[Command], @@ -2029,6 +2030,7 @@ pub fn JumpToByteBoundary(storage_ix: &mut usize, storage: &mut [u8]) { storage[(*storage_ix >> 3)] = 0u8; } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub(crate) fn store_meta_block( alloc: &mut Alloc, input: &[u8], @@ -2343,6 +2345,7 @@ fn StoreDataWithHuffmanCodes( } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub(crate) fn store_meta_block_trivial( alloc: &mut Alloc, input: &[u8], @@ -2572,6 +2575,7 @@ impl RecoderState { } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub(crate) fn store_meta_block_fast( m: &mut Alloc, input: &[u8], diff --git a/src/enc/cluster.rs b/src/enc/cluster.rs index 244786b2..b044e0ea 100644 --- a/src/enc/cluster.rs +++ b/src/enc/cluster.rs @@ -3,7 +3,7 @@ use core::cmp::min; use crate::alloc; -use super::bit_cost::BrotliPopulationCost; +use super::bit_cost::{BrotliPopulationCost, BrotliPopulationCostOfSum}; use super::histogram::{ CostAccessors, HistogramAddHistogram, HistogramClear, HistogramSelfAddHistogram, }; @@ -95,9 +95,8 @@ fn BrotliCompareAndPushToQueue< pairs[0].cost_diff.max(0.0) }; - let mut combo: HistogramType = out[idx1 as usize].clone(); - HistogramAddHistogram(&mut combo, &out[idx2 as usize]); - let cost_combo: super::util::floatX = BrotliPopulationCost(&combo, scratch_space); + let cost_combo: super::util::floatX = + BrotliPopulationCostOfSum(&out[idx1 as usize], &out[idx2 as usize], scratch_space); if cost_combo < threshold - p.cost_diff { p.cost_combo = cost_combo; is_good_pair = true; @@ -120,6 +119,7 @@ fn BrotliCompareAndPushToQueue< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliHistogramCombine< HistogramType: SliceWrapperMut + SliceWrapper + CostAccessors + Clone, >( @@ -252,9 +252,7 @@ pub fn BrotliHistogramBitCostDistance< if histogram.total_count() == 0usize { 0.0 } else { - let mut tmp: HistogramType = histogram.clone(); - HistogramAddHistogram(&mut tmp, candidate); - BrotliPopulationCost(&tmp, scratch_space) - candidate.bit_cost() + BrotliPopulationCostOfSum(histogram, candidate, scratch_space) - candidate.bit_cost() } } @@ -263,6 +261,7 @@ When called, clusters[0..num_clusters) contains the unique values from symbols[0..in_size), but this property is not preserved in this function. Note: we assume that out[]->bit_cost_ is already up-to-date. */ +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliHistogramRemap< HistogramType: SliceWrapperMut + SliceWrapper + CostAccessors + Clone, >( @@ -357,6 +356,7 @@ pub fn BrotliHistogramReindex< next_index as usize } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliClusterHistograms< HistogramType: SliceWrapperMut + SliceWrapper + CostAccessors + Clone, Alloc: alloc::Allocator + alloc::Allocator + alloc::Allocator, diff --git a/src/enc/compress_fragment.rs b/src/enc/compress_fragment.rs index ab6616e7..94b0a1c2 100644 --- a/src/enc/compress_fragment.rs +++ b/src/enc/compress_fragment.rs @@ -647,6 +647,7 @@ fn BuildAndStoreCommandPrefixCode( } #[allow(unused_assignments)] +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn compress_fragment_fast_impl>( m: &mut AllocHT, input_ptr: &[u8], diff --git a/src/enc/compress_fragment_two_pass.rs b/src/enc/compress_fragment_two_pass.rs index e0217175..77ec9d04 100644 --- a/src/enc/compress_fragment_two_pass.rs +++ b/src/enc/compress_fragment_two_pass.rs @@ -154,6 +154,7 @@ fn IsMatch(p1: &[u8], p2: &[u8], length: usize) -> bool { } #[allow(unused_assignments)] +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn CreateCommands( input_index: usize, block_size: usize, @@ -516,6 +517,7 @@ fn BuildAndStoreCommandPrefixCode( ); } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn StoreCommands>( mht: &mut AllocHT, mut literals: &[u8], @@ -643,6 +645,7 @@ fn EmitUncompressedMetaBlock( #[allow(unused_variables)] #[inline(always)] +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn compress_fragment_two_pass_impl>( m: &mut AllocHT, base_ip: &[u8], diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 7e1abf14..4840b52f 100644 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -809,6 +809,7 @@ fn RingBufferWrite>( } impl BrotliEncoderStateStruct { + #[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn copy_input_to_ring_buffer(&mut self, input_size: usize, input_buffer: &[u8]) { if !self.ensure_initialized() { return; @@ -1433,6 +1434,7 @@ fn MakeUncompressedStream(input: &[u8], input_size: usize, output: &mut [u8]) -> } #[cfg_attr(not(feature = "ffi-api"), cfg(test))] +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub(crate) fn encoder_compress< Alloc: BrotliAlloc, MetablockCallback: FnMut( @@ -1714,6 +1716,7 @@ fn MaxMetablockSize(params: &BrotliEncoderParams) -> usize { 1 << min(ComputeRbBits(params), 24) } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn ChooseContextMap( quality: i32, bigram_histo: &mut [u32], @@ -1870,6 +1873,7 @@ fn ShouldUseComplexStaticContextMap( } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn DecideOverLiteralContextModeling( input: &[u8], mut start_pos: usize, @@ -1938,6 +1942,7 @@ fn WriteEmptyLastBlocksInternal( BrotliWriteEmptyLastMetaBlock(storage_ix, storage) } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] fn WriteMetaBlockInternal( alloc: &mut Alloc, data: &[u8], @@ -2211,6 +2216,7 @@ fn ChooseDistanceParams(params: &mut BrotliEncoderParams) { } impl BrotliEncoderStateStruct { + #[cfg_attr(feature = "hotpath", hotpath::measure)] fn encode_data( &mut self, is_last: bool, @@ -2705,6 +2711,7 @@ impl BrotliEncoderStateStruct { ); } + #[cfg_attr(feature = "hotpath", hotpath::measure)] fn compress_stream_fast( &mut self, op: BrotliEncoderOperation, diff --git a/src/enc/literal_cost.rs b/src/enc/literal_cost.rs index 9634ae4a..374474a7 100644 --- a/src/enc/literal_cost.rs +++ b/src/enc/literal_cost.rs @@ -175,6 +175,7 @@ fn EstimateBitCostsForLiteralsUTF8( } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliEstimateBitCostsForLiterals( pos: usize, len: usize, diff --git a/src/enc/metablock.rs b/src/enc/metablock.rs index 2c7484a1..5d938d37 100644 --- a/src/enc/metablock.rs +++ b/src/enc/metablock.rs @@ -130,6 +130,7 @@ fn ComputeDistanceCost( true } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliBuildMetaBlock( alloc: &mut Alloc, ringbuffer: &[u8], @@ -1019,6 +1020,7 @@ pub fn BrotliBuildMetaBlockGreedyInternal< MapStaticContexts(alloc, num_contexts, static_context_map, mb); } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliBuildMetaBlockGreedy< Alloc: alloc::Allocator + alloc::Allocator @@ -1073,6 +1075,7 @@ pub fn BrotliBuildMetaBlockGreedy< } } +#[cfg_attr(feature = "hotpath", hotpath::measure)] pub fn BrotliOptimizeHistograms< Alloc: alloc::Allocator + alloc::Allocator From 8a48bc0d28dbfb17a019afa65d0f4c03b3c17276 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 21:49:58 +0300 Subject: [PATCH 5/6] Up fearless simd to 0.7 --- Cargo.toml | 2 +- src/enc/bit_cost.rs | 2 +- src/enc/block_splitter.rs | 2 +- src/enc/prior_eval.rs | 2 +- src/enc/static_dict.rs | 2 +- src/enc/vectorization.rs | 4 +--- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 277aa149..1b3c40ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ incremental = false "alloc-stdlib" = { version = "~0.2", optional = true } "brotli-decompressor" = { version = "~5.0", default-features = false } # `libm` is what makes the no-stdlib build possible; the `std` feature below overrides it. -"fearless_simd" = { version = "~0.6", default-features = false, features = ["libm"] } +"fearless_simd" = { version = "~0.7", default-features = false, features = ["libm"] } "sha2" = { version = "~0.11", optional = true } diff --git a/src/enc/bit_cost.rs b/src/enc/bit_cost.rs index 78394e24..3cad11ba 100644 --- a/src/enc/bit_cost.rs +++ b/src/enc/bit_cost.rs @@ -1,7 +1,7 @@ use crate::alloc::SliceWrapperMut; use core::cmp::{max, min}; -use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u32x16}; +use fearless_simd::{Simd, SimdBase, SimdMask, u32x16}; use super::super::alloc::SliceWrapper; use super::histogram::CostAccessors; diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index e9a09aff..a79009e4 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -1,7 +1,7 @@ use core; use core::cmp::{max, min}; -use fearless_simd::{Select, Simd, SimdBase, SimdFloat, SimdMask, f32x8, u32x8}; +use fearless_simd::{Select, Simd, SimdBase, SimdMask, f32x8, u32x8}; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; use super::backward_references::BrotliEncoderParams; diff --git a/src/enc/prior_eval.rs b/src/enc/prior_eval.rs index 0a86d071..40fbc7a0 100644 --- a/src/enc/prior_eval.rs +++ b/src/enc/prior_eval.rs @@ -1,7 +1,7 @@ use core; use core::cmp::min; -use fearless_simd::{Level, Select, Simd, SimdBase, SimdInt, f32x8, i16x16}; +use fearless_simd::{Level, Select, Simd, SimdBase, f32x8, i16x16}; use super::super::alloc; use super::super::alloc::{Allocator, SliceWrapper, SliceWrapperMut}; diff --git a/src/enc/static_dict.rs b/src/enc/static_dict.rs index a3a6c062..22f0356c 100644 --- a/src/enc/static_dict.rs +++ b/src/enc/static_dict.rs @@ -1,6 +1,6 @@ use core::cmp::{max, min}; -use fearless_simd::{Simd, SimdBase, SimdInt, SimdMask, u8x32}; +use fearless_simd::{Simd, SimdBase, SimdMask, u8x32}; pub const kNumDistanceCacheEntries: usize = 4; diff --git a/src/enc/vectorization.rs b/src/enc/vectorization.rs index efb70b92..bd9050aa 100644 --- a/src/enc/vectorization.rs +++ b/src/enc/vectorization.rs @@ -9,9 +9,7 @@ use core::ops::{Index, IndexMut}; use core::slice::SliceIndex; -use fearless_simd::{ - Level, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x8, i16x16, i32x8, u32x8, -}; +use fearless_simd::{Level, Simd, SimdBase, SimdInto, f32x8, i16x16, i32x8, u32x8}; /// The instruction set the vectorized encoder paths run on. /// From 1522ec9cafc48558da0025e0355bc554b2ec89f8 Mon Sep 17 00:00:00 2001 From: Mnwa Date: Tue, 11 Aug 2026 22:19:55 +0300 Subject: [PATCH 6/6] Replace clone from to copy from --- src/bin/integration_tests.rs | 12 +++---- src/bin/tests.rs | 2 +- src/concat/mod.rs | 20 +++++------ .../hash_to_binary_tree.rs | 4 +-- src/enc/backward_references/mod.rs | 22 ++++++------ src/enc/block_splitter.rs | 14 ++++---- src/enc/brotli_bit_stream.rs | 12 +++---- src/enc/cluster.rs | 2 +- src/enc/compress_fragment.rs | 2 +- src/enc/compress_fragment_two_pass.rs | 4 +-- src/enc/context_map_entropy.rs | 12 +++---- src/enc/encode.rs | 35 +++++++++---------- src/enc/find_stride.rs | 16 ++++----- src/enc/interface.rs | 2 +- src/enc/metablock.rs | 8 ++--- src/enc/reader.rs | 2 +- src/enc/static_dict.rs | 12 +++---- src/enc/test.rs | 2 +- src/enc/threading/mod.rs | 2 +- src/ffi/broccoli.rs | 2 +- 20 files changed, 93 insertions(+), 94 deletions(-) diff --git a/src/bin/integration_tests.rs b/src/bin/integration_tests.rs index d4fda543..b8fdf1af 100644 --- a/src/bin/integration_tests.rs +++ b/src/bin/integration_tests.rs @@ -233,7 +233,7 @@ impl io::Read for Buffer { let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); if bytes_to_read > 0 { buf[0..bytes_to_read] - .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); + .copy_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); } self.read_offset += bytes_to_read; Ok(bytes_to_read) @@ -256,7 +256,7 @@ impl io::Read for UnlimitedBuffer { let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); if bytes_to_read > 0 { buf[0..bytes_to_read] - .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); + .copy_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); } self.read_offset += bytes_to_read; Ok(bytes_to_read) @@ -869,7 +869,7 @@ impl<'a> LimitedBuffer<'a> { fn reset(&mut self) { self.write_offset = 0; self.read_offset = 0; - self.data.split_at_mut(32).0.clone_from_slice(&[0u8; 32]); // clear the first 256 bits + self.data.split_at_mut(32).0.copy_from_slice(&[0u8; 32]); // clear the first 256 bits } fn reset_read(&mut self) { self.read_offset = 0; @@ -883,7 +883,7 @@ impl<'a> io::Read for LimitedBuffer<'a> { let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); if bytes_to_read > 0 { buf[0..bytes_to_read] - .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); + .copy_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); } self.read_offset += bytes_to_read; Ok(bytes_to_read) @@ -895,7 +895,7 @@ impl<'a> io::Write for LimitedBuffer<'a> { let bytes_to_write = min(buf.len(), self.data.len() - self.write_offset); if bytes_to_write > 0 { self.data[self.write_offset..self.write_offset + bytes_to_write] - .clone_from_slice(&buf[..bytes_to_write]); + .copy_from_slice(&buf[..bytes_to_write]); } else { return Err(io::Error::new(io::ErrorKind::WriteZero, "OutOfBufferSpace")); } @@ -1406,7 +1406,7 @@ impl io::Read for SoonErrorReader { self.1 = false; if first { let len = min(self.0.len(), data.len()); - data[..len].clone_from_slice(&self.0[..len]); + data[..len].copy_from_slice(&self.0[..len]); return Ok(len); } Err(io::Error::new(io::ErrorKind::PermissionDenied, "err")) diff --git a/src/bin/tests.rs b/src/bin/tests.rs index 512693b8..8d071b43 100644 --- a/src/bin/tests.rs +++ b/src/bin/tests.rs @@ -25,7 +25,7 @@ impl io::Read for Buffer { let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); if bytes_to_read > 0 { buf[0..bytes_to_read] - .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); + .copy_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); } self.read_offset += bytes_to_read; Ok(bytes_to_read) diff --git a/src/concat/mod.rs b/src/concat/mod.rs index bdc0025c..92b8dbe2 100644 --- a/src/concat/mod.rs +++ b/src/concat/mod.rs @@ -173,7 +173,7 @@ impl BroCatli { let xlen = possible_new_stream_pending.bytes_so_far.len(); possible_new_stream_pending .bytes_so_far - .clone_from_slice(&buffer[16..16 + xlen]); + .copy_from_slice(&buffer[16..16 + xlen]); let new_stream_pending: Option = if has_new_stream_pending { Some(possible_new_stream_pending) } else { @@ -192,7 +192,7 @@ impl BroCatli { return Err(()); } let xlen = ret.last_bytes.len(); - ret.last_bytes.clone_from_slice(&buffer[..xlen]); + ret.last_bytes.copy_from_slice(&buffer[..xlen]); Ok(ret) } #[inline(always)] @@ -200,7 +200,7 @@ impl BroCatli { if 16 + NUM_STREAM_HEADER_BYTES > buffer.len() { return Err(()); } - buffer[..self.last_bytes.len()].clone_from_slice(&self.last_bytes[..]); + buffer[..self.last_bytes.len()].copy_from_slice(&self.last_bytes[..]); buffer[8] = self.last_bytes_len; buffer[9] = (self.last_byte_sanitized as u8) | ((self.new_stream_pending.is_some() as u8) << 6) @@ -215,7 +215,7 @@ impl BroCatli { buffer[13] = new_stream_pending.num_bytes_written.unwrap_or(0); // 14, 15 reserved buffer[16..16 + new_stream_pending.bytes_so_far.len()] - .clone_from_slice(&new_stream_pending.bytes_so_far[..]); + .copy_from_slice(&new_stream_pending.bytes_so_far[..]); } Ok(()) } @@ -402,7 +402,7 @@ impl BroCatli { new_stream_pending.num_bytes_written = Some(0); new_stream_pending .bytes_so_far - .clone_from_slice(&realigned_header[1..]); + .copy_from_slice(&realigned_header[1..]); } } else { assert_ne!(self.window_size, 0); @@ -418,7 +418,7 @@ impl BroCatli { .1 .split_at_mut(to_copy) .0 - .clone_from_slice( + .copy_from_slice( new_stream_pending .bytes_so_far .split_at(usize::from(new_stream_pending.num_bytes_written.unwrap())) @@ -465,7 +465,7 @@ impl BroCatli { [usize::from(new_stream_pending.num_bytes_read)..]; let to_copy = min(dst.len(), in_bytes.len() - *in_offset); dst[..to_copy] - .clone_from_slice(in_bytes.split_at(*in_offset).1.split_at(to_copy).0); + .copy_from_slice(in_bytes.split_at(*in_offset).1.split_at(to_copy).0); *in_offset += to_copy; new_stream_pending.num_bytes_read += to_copy as u8; } @@ -540,7 +540,7 @@ impl BroCatli { .1 .split_at_mut(2) .0 - .clone_from_slice(&self.last_bytes[..]); + .copy_from_slice(&self.last_bytes[..]); *out_offset += 2; let (new_in_offset, last_two) = in_bytes .split_at(*in_offset) @@ -548,7 +548,7 @@ impl BroCatli { .split_at(to_copy) .0 .split_at(to_copy - 2); - self.last_bytes.clone_from_slice(last_two); + self.last_bytes.copy_from_slice(last_two); *in_offset += 2; // add this after the clone since we grab the last 2 bytes, not the first to_copy -= 2; out_bytes @@ -556,7 +556,7 @@ impl BroCatli { .1 .split_at_mut(to_copy) .0 - .clone_from_slice(new_in_offset); + .copy_from_slice(new_in_offset); *out_offset += to_copy; *in_offset += to_copy; if *out_offset == out_bytes.len() { diff --git a/src/enc/backward_references/hash_to_binary_tree.rs b/src/enc/backward_references/hash_to_binary_tree.rs index 6e76949f..11dc36ea 100644 --- a/src/enc/backward_references/hash_to_binary_tree.rs +++ b/src/enc/backward_references/hash_to_binary_tree.rs @@ -290,8 +290,8 @@ where }; ret.buckets_ .slice_mut() - .clone_from_slice(self.buckets_.slice()); - ret.forest.slice_mut().clone_from_slice(self.forest.slice()); + .copy_from_slice(self.buckets_.slice()); + ret.forest.slice_mut().copy_from_slice(self.forest.slice()); ret } } diff --git a/src/enc/backward_references/mod.rs b/src/enc/backward_references/mod.rs index 09876a5e..7e7609b4 100644 --- a/src/enc/backward_references/mod.rs +++ b/src/enc/backward_references/mod.rs @@ -1256,7 +1256,7 @@ impl< let shift = self.specialization.hash_shift(); for chunk_id in 0..del { let ix_offset = ix_start + chunk_id * REG_SIZE; - data64[..REG_SIZE + lookahead4 - 1].clone_from_slice( + data64[..REG_SIZE + lookahead4 - 1].copy_from_slice( data.split_at(ix_offset) .1 .split_at(REG_SIZE + lookahead4 - 1) @@ -1345,7 +1345,7 @@ impl< for chunk_id in 0..del { let ix_offset = ix_start + chunk_id * REG_SIZE; data64[..REG_SIZE + lookahead4] - .clone_from_slice(data.split_at(ix_offset).1.split_at(REG_SIZE + lookahead4).0); + .copy_from_slice(data.split_at(ix_offset).1.split_at(REG_SIZE + lookahead4).0); for quad_index in 0..(REG_SIZE >> 2) { let i = quad_index << 2; let ffffffff = 0xffff_ffff; @@ -1429,7 +1429,7 @@ impl< for chunk_id in 0..del { let ix_offset = ix_start + chunk_id * REG_SIZE; data64[..REG_SIZE + lookahead4] - .clone_from_slice(data.split_at(ix_offset).1.split_at(REG_SIZE + lookahead4).0); + .copy_from_slice(data.split_at(ix_offset).1.split_at(REG_SIZE + lookahead4).0); for i in 0..REG_SIZE { let mixed_word = ((u32::from(data64[i]) | (u32::from(data64[i + 1]) << 8) @@ -2001,7 +2001,7 @@ impl + alloc::Allocator> CloneWithAlloc ret.buckets_ .buckets_ .slice_mut() - .clone_from_slice(self.buckets_.buckets_.slice()); + .copy_from_slice(self.buckets_.buckets_.slice()); ret } } @@ -2019,7 +2019,7 @@ impl + alloc::Allocator> CloneWithAlloc ret.buckets_ .buckets_ .slice_mut() - .clone_from_slice(self.buckets_.buckets_.slice()); + .copy_from_slice(self.buckets_.buckets_.slice()); ret } } @@ -2037,7 +2037,7 @@ impl + alloc::Allocator> CloneWithAlloc ret.buckets_ .buckets_ .slice_mut() - .clone_from_slice(self.buckets_.buckets_.slice()); + .copy_from_slice(self.buckets_.buckets_.slice()); ret } } @@ -2055,16 +2055,16 @@ impl + alloc::Allocator> CloneWithAlloc ret.buckets_ .buckets_ .slice_mut() - .clone_from_slice(self.buckets_.buckets_.slice()); + .copy_from_slice(self.buckets_.buckets_.slice()); ret } } impl + alloc::Allocator> CloneWithAlloc for H9 { fn clone_with_alloc(&self, m: &mut Alloc) -> Self { let mut num = allocate::(m, self.num_.len()); - num.slice_mut().clone_from_slice(self.num_.slice()); + num.slice_mut().copy_from_slice(self.num_.slice()); let mut buckets = allocate::(m, self.buckets_.len()); - buckets.slice_mut().clone_from_slice(self.buckets_.slice()); + buckets.slice_mut().copy_from_slice(self.buckets_.slice()); H9:: { num_: num, buckets_: buckets, @@ -2080,9 +2080,9 @@ impl< { fn clone_with_alloc(&self, m: &mut Alloc) -> Self { let mut num = allocate::(m, self.num.len()); - num.slice_mut().clone_from_slice(self.num.slice()); + num.slice_mut().copy_from_slice(self.num.slice()); let mut buckets = allocate::(m, self.buckets.len()); - buckets.slice_mut().clone_from_slice(self.buckets.slice()); + buckets.slice_mut().copy_from_slice(self.buckets.slice()); AdvHasher:: { GetHasherCommon: self.GetHasherCommon.clone(), specialization: self.specialization.clone(), diff --git a/src/enc/block_splitter.rs b/src/enc/block_splitter.rs index a79009e4..fce28014 100644 --- a/src/enc/block_splitter.rs +++ b/src/enc/block_splitter.rs @@ -96,14 +96,14 @@ fn CopyLiteralsToByteArray( if from_pos.wrapping_add(insert_len) > mask { let head_size: usize = mask.wrapping_add(1).wrapping_sub(from_pos); literals[pos..(pos + head_size)] - .clone_from_slice(&data[from_pos..(from_pos + head_size)]); + .copy_from_slice(&data[from_pos..(from_pos + head_size)]); from_pos = 0usize; pos = pos.wrapping_add(head_size); insert_len = insert_len.wrapping_sub(head_size); } if insert_len > 0usize { literals[pos..(pos + insert_len)] - .clone_from_slice(&data[from_pos..(from_pos + insert_len)]); + .copy_from_slice(&data[from_pos..(from_pos + insert_len)]); pos = pos.wrapping_add(insert_len); } from_pos = from_pos @@ -566,7 +566,7 @@ fn ClusterBlocks< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..cluster_size_capacity] - .clone_from_slice(&cluster_size.slice()[..cluster_size_capacity]); + .copy_from_slice(&cluster_size.slice()[..cluster_size_capacity]); >::free_cell( alloc, core::mem::replace(&mut cluster_size, new_array), @@ -681,7 +681,7 @@ fn ClusterBlocks< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..split.types_alloc_size()] - .clone_from_slice(&split.types.slice()[..split.types_alloc_size()]); + .copy_from_slice(&split.types.slice()[..split.types_alloc_size()]); >::free_cell( alloc, core::mem::replace(&mut split.types, new_array), @@ -700,7 +700,7 @@ fn ClusterBlocks< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..split.lengths_alloc_size()] - .clone_from_slice(split.lengths.slice()); + .copy_from_slice(split.lengths.slice()); >::free_cell( alloc, core::mem::replace(&mut split.lengths, new_array), @@ -779,7 +779,7 @@ fn SplitByteVector< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..split.types_alloc_size()] - .clone_from_slice(&split.types.slice()[..split.types_alloc_size()]); + .copy_from_slice(&split.types.slice()[..split.types_alloc_size()]); >::free_cell( alloc, core::mem::replace(&mut split.types, new_array), @@ -798,7 +798,7 @@ fn SplitByteVector< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..split.lengths_alloc_size()] - .clone_from_slice(&split.lengths.slice()[..split.lengths_alloc_size()]); + .copy_from_slice(&split.lengths.slice()[..split.lengths_alloc_size()]); >::free_cell( alloc, core::mem::replace(&mut split.lengths, new_array), diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs index 994b20a6..918c0bc1 100755 --- a/src/enc/brotli_bit_stream.rs +++ b/src/enc/brotli_bit_stream.rs @@ -156,7 +156,7 @@ impl<'a, Alloc: BrotliAlloc> interface::CommandProcessor<'a> for CommandQueue<'a tmp.slice_mut() .split_at_mut(self.queue.slice().len()) .0 - .clone_from_slice(self.queue.slice()); + .copy_from_slice(self.queue.slice()); >::free_cell( self.mc, core::mem::replace(&mut self.queue, tmp), @@ -230,7 +230,7 @@ fn process_command_queue<'a, CmdProcessor: interface::CommandProcessor<'a>>( ) -> RecoderState { let mut input_iter = input; let mut local_dist_cache = [0i32; kNumDistanceCacheEntries]; - local_dist_cache.clone_from_slice(&dist_cache[..]); + local_dist_cache.copy_from_slice(&dist_cache[..]); let mut btypel_counter = 0usize; let mut btypec_counter = 0usize; let mut btyped_counter = 0usize; @@ -377,8 +377,8 @@ fn process_command_queue<'a, CmdProcessor: interface::CommandProcessor<'a>>( if prev_dist_index != 1 || dist_offset != 0 { // update distance cache unless it's the "0 distance symbol" let mut tmp_dist_cache = [0i32; kNumDistanceCacheEntries - 1]; - tmp_dist_cache.clone_from_slice(&local_dist_cache[..kNumDistanceCacheEntries - 1]); - local_dist_cache[1..].clone_from_slice(&tmp_dist_cache[..]); + tmp_dist_cache.copy_from_slice(&local_dist_cache[..kNumDistanceCacheEntries - 1]); + local_dist_cache[1..].copy_from_slice(&tmp_dist_cache[..]); local_dist_cache[0] = final_distance as i32; } } @@ -2798,10 +2798,10 @@ pub(crate) fn store_uncompressed_meta_block( BrotliStoreUncompressedMetaBlockHeader(len, storage_ix, storage); JumpToByteBoundary(storage_ix, storage); let dst_start0 = (*storage_ix >> 3); - storage[dst_start0..(dst_start0 + input0.len())].clone_from_slice(input0); + storage[dst_start0..(dst_start0 + input0.len())].copy_from_slice(input0); *storage_ix = storage_ix.wrapping_add(input0.len() << 3); let dst_start1 = (*storage_ix >> 3); - storage[dst_start1..(dst_start1 + input1.len())].clone_from_slice(input1); + storage[dst_start1..(dst_start1 + input1.len())].copy_from_slice(input1); *storage_ix = storage_ix.wrapping_add(input1.len() << 3); BrotliWriteBitsPrepareStorage(*storage_ix, storage); if params.log_meta_block && !suppress_meta_block_logging { diff --git a/src/enc/cluster.rs b/src/enc/cluster.rs index b044e0ea..b8350c39 100644 --- a/src/enc/cluster.rs +++ b/src/enc/cluster.rs @@ -429,7 +429,7 @@ pub fn BrotliClusterHistograms< } new_array = alloc_or_default::(alloc, _new_size); new_array.slice_mut()[..pairs_capacity] - .clone_from_slice(&pairs.slice()[..pairs_capacity]); + .copy_from_slice(&pairs.slice()[..pairs_capacity]); >::free_cell( alloc, core::mem::replace(&mut pairs, new_array), diff --git a/src/enc/compress_fragment.rs b/src/enc/compress_fragment.rs index 94b0a1c2..4ae2e5f3 100644 --- a/src/enc/compress_fragment.rs +++ b/src/enc/compress_fragment.rs @@ -708,7 +708,7 @@ fn compress_fragment_fast_impl>( 'continue_to_next_block: loop { let mut ip_index: usize; if code_block_selection == CodeBlockState::EMIT_COMMANDS { - cmd_histo[..128].clone_from_slice(&kCmdHistoSeed[..]); + cmd_histo[..128].copy_from_slice(&kCmdHistoSeed[..]); ip_index = input_index; last_distance = -1i32; ip_end = input_index.wrapping_add(block_size); diff --git a/src/enc/compress_fragment_two_pass.rs b/src/enc/compress_fragment_two_pass.rs index 77ec9d04..57148a28 100644 --- a/src/enc/compress_fragment_two_pass.rs +++ b/src/enc/compress_fragment_two_pass.rs @@ -250,7 +250,7 @@ fn CreateCommands( ip_index = ip_index.wrapping_add(matched); *num_commands += EmitInsertLen(insert as u32, commands); (*literals)[..(insert as usize)] - .clone_from_slice(&base_ip[next_emit..(next_emit + insert as usize)]); + .copy_from_slice(&base_ip[next_emit..(next_emit + insert as usize)]); *num_literals += insert as usize; let new_literals = core::mem::take(literals); let _ = core::mem::replace(literals, &mut new_literals[(insert as usize)..]); @@ -378,7 +378,7 @@ fn CreateCommands( let insert: u32 = ip_end.wrapping_sub(next_emit) as u32; *num_commands += EmitInsertLen(insert, commands); literals[..insert as usize] - .clone_from_slice(&base_ip[next_emit..(next_emit + insert as usize)]); + .copy_from_slice(&base_ip[next_emit..(next_emit + insert as usize)]); let mut xliterals = core::mem::take(literals); *literals = &mut core::mem::take(&mut xliterals)[(insert as usize)..]; *num_literals += insert as usize; diff --git a/src/enc/context_map_entropy.rs b/src/enc/context_map_entropy.rs index 5161a4e5..dd2e08f5 100644 --- a/src/enc/context_map_entropy.rs +++ b/src/enc/context_map_entropy.rs @@ -96,7 +96,7 @@ fn compute_combined_cost( assert_eq!(cdfs.len(), 16 * NUM_SPEEDS_TO_TRY); let nibble = nibble_u8 as usize & 0xf; let mut stride_pdf = [0u16; NUM_SPEEDS_TO_TRY]; - stride_pdf.clone_from_slice( + stride_pdf.copy_from_slice( cdfs.split_at(NUM_SPEEDS_TO_TRY * nibble) .1 .split_at(NUM_SPEEDS_TO_TRY) @@ -105,7 +105,7 @@ fn compute_combined_cost( let mut cm_pdf: u16 = mixing_cdf[nibble]; if nibble_u8 != 0 { let mut tmp = [0u16; NUM_SPEEDS_TO_TRY]; - tmp.clone_from_slice( + tmp.copy_from_slice( cdfs.split_at(NUM_SPEEDS_TO_TRY * (nibble - 1)) .1 .split_at(NUM_SPEEDS_TO_TRY) @@ -117,7 +117,7 @@ fn compute_combined_cost( cm_pdf -= mixing_cdf[nibble - 1] } let mut stride_max = [0u16; NUM_SPEEDS_TO_TRY]; - stride_max.clone_from_slice(cdfs.split_at(NUM_SPEEDS_TO_TRY * 15).1); + stride_max.copy_from_slice(cdfs.split_at(NUM_SPEEDS_TO_TRY * 15).1); let cm_max = mixing_cdf[15]; for i in 0..NUM_SPEEDS_TO_TRY { if stride_pdf[i] == 0 { @@ -141,7 +141,7 @@ fn compute_cost(singleton_cost: &mut [floatX; NUM_SPEEDS_TO_TRY], cdfs: &[u16], assert_eq!(cdfs.len(), 16 * NUM_SPEEDS_TO_TRY); let nibble = nibble_u8 as usize & 0xf; let mut pdf = [0u16; NUM_SPEEDS_TO_TRY]; - pdf.clone_from_slice( + pdf.copy_from_slice( cdfs.split_at(NUM_SPEEDS_TO_TRY * nibble) .1 .split_at(NUM_SPEEDS_TO_TRY) @@ -149,7 +149,7 @@ fn compute_cost(singleton_cost: &mut [floatX; NUM_SPEEDS_TO_TRY], cdfs: &[u16], ); if nibble_u8 != 0 { let mut tmp = [0u16; NUM_SPEEDS_TO_TRY]; - tmp.clone_from_slice( + tmp.copy_from_slice( cdfs.split_at(NUM_SPEEDS_TO_TRY * (nibble - 1)) .1 .split_at(NUM_SPEEDS_TO_TRY) @@ -160,7 +160,7 @@ fn compute_cost(singleton_cost: &mut [floatX; NUM_SPEEDS_TO_TRY], cdfs: &[u16], } } let mut max = [0u16; NUM_SPEEDS_TO_TRY]; - max.clone_from_slice(cdfs.split_at(NUM_SPEEDS_TO_TRY * 15).1); + max.copy_from_slice(cdfs.split_at(NUM_SPEEDS_TO_TRY * 15).1); for i in 0..NUM_SPEEDS_TO_TRY { if pdf[i] == 0 { assert_ne!(pdf[i], 0); diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 4840b52f..ffb12ef2 100644 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -652,9 +652,9 @@ fn InitCommandPrefixCodes( 0x88, 0x54, 0x94, 0x46, 0xe1, 0xb0, 0xd0, 0x4e, 0xb2, 0xf7, 0x4, 0x0, ]; static kDefaultCommandCodeNumBits: usize = 448usize; - cmd_depths[..].clone_from_slice(&kDefaultCommandDepths[..]); - cmd_bits[..].clone_from_slice(&kDefaultCommandBits[..]); - cmd_code[..kDefaultCommandCode.len()].clone_from_slice(&kDefaultCommandCode[..]); + cmd_depths[..].copy_from_slice(&kDefaultCommandDepths[..]); + cmd_bits[..].copy_from_slice(&kDefaultCommandBits[..]); + cmd_code[..kDefaultCommandCode.len()].copy_from_slice(&kDefaultCommandCode[..]); *cmd_code_numbits = kDefaultCommandCodeNumBits; } @@ -718,7 +718,7 @@ fn RingBufferInitBuffer>( if !rb.data_mo.slice().is_empty() { let lim: usize = ((2u32).wrapping_add(rb.cur_size_) as usize) .wrapping_add(kSlackForEightByteHashingEverywhere); - new_data.slice_mut()[..lim].clone_from_slice(&rb.data_mo.slice()[..lim]); + new_data.slice_mut()[..lim].copy_from_slice(&rb.data_mo.slice()[..lim]); m.free_cell(core::mem::take(&mut rb.data_mo)); } let _ = core::mem::replace(&mut rb.data_mo, new_data); @@ -744,7 +744,7 @@ fn RingBufferWriteTail>( let p: usize = (rb.size_ as usize).wrapping_add(masked_pos); let begin = rb.buffer_index.wrapping_add(p); let lim = min(n, (rb.tail_size_ as usize).wrapping_sub(masked_pos)); - rb.data_mo.slice_mut()[begin..(begin + lim)].clone_from_slice(&bytes[..lim]); + rb.data_mo.slice_mut()[begin..(begin + lim)].copy_from_slice(&bytes[..lim]); } } @@ -757,8 +757,7 @@ fn RingBufferWrite>( if rb.pos_ == 0u32 && (n < rb.tail_size_ as usize) { rb.pos_ = n as u32; RingBufferInitBuffer(m, rb.pos_, rb); - rb.data_mo.slice_mut()[rb.buffer_index..(rb.buffer_index + n)] - .clone_from_slice(&bytes[..n]); + rb.data_mo.slice_mut()[rb.buffer_index..(rb.buffer_index + n)].copy_from_slice(&bytes[..n]); return; } if rb.cur_size_ < rb.total_size_ { @@ -778,18 +777,18 @@ fn RingBufferWrite>( if masked_pos.wrapping_add(n) <= rb.size_ as usize { // a single write fits let start = rb.buffer_index.wrapping_add(masked_pos); - rb.data_mo.slice_mut()[start..(start + n)].clone_from_slice(&bytes[..n]); + rb.data_mo.slice_mut()[start..(start + n)].copy_from_slice(&bytes[..n]); } else { { let start = rb.buffer_index.wrapping_add(masked_pos); let mid = min(n, (rb.total_size_ as usize).wrapping_sub(masked_pos)); - rb.data_mo.slice_mut()[start..(start + mid)].clone_from_slice(&bytes[..mid]); + rb.data_mo.slice_mut()[start..(start + mid)].copy_from_slice(&bytes[..mid]); } let xstart = rb.buffer_index.wrapping_add(0); let size = n.wrapping_sub((rb.size_ as usize).wrapping_sub(masked_pos)); let bytes_start = (rb.size_ as usize).wrapping_sub(masked_pos); rb.data_mo.slice_mut()[xstart..(xstart + size)] - .clone_from_slice(&bytes[bytes_start..(bytes_start + size)]); + .copy_from_slice(&bytes[bytes_start..(bytes_start + size)]); } } let data_2 = rb.data_mo.slice()[rb @@ -1423,7 +1422,7 @@ fn MakeUncompressedStream(input: &[u8], input_size: usize, output: &mut [u8]) -> result = result.wrapping_add(1); } output[result..(result + chunk_size as usize)] - .clone_from_slice(&input[offset..(offset + chunk_size as usize)]); + .copy_from_slice(&input[offset..(offset + chunk_size as usize)]); result = result.wrapping_add(chunk_size as usize); offset = offset.wrapping_add(chunk_size as usize); size = size.wrapping_sub(chunk_size as usize); @@ -1584,7 +1583,7 @@ impl BrotliEncoderStateStruct { if self.available_out_ != 0usize && (*available_out != 0usize) { let copy_output_size: usize = min(self.available_out_, *available_out); (*next_out_array)[(*next_out_offset)..(*next_out_offset + copy_output_size)] - .clone_from_slice(&GetNextOut!(self)[..copy_output_size]); + .copy_from_slice(&GetNextOut!(self)[..copy_output_size]); //memcpy(*next_out, s.next_out_, copy_output_size); *next_out_offset = next_out_offset.wrapping_add(copy_output_size); *available_out = available_out.wrapping_sub(copy_output_size); @@ -1996,7 +1995,7 @@ fn WriteMetaBlockInternal( num_literals, num_commands, ) { - dist_cache[..4].clone_from_slice(&saved_dist_cache[..4]); + dist_cache[..4].copy_from_slice(&saved_dist_cache[..4]); store_uncompressed_meta_block( alloc, is_last, @@ -2144,7 +2143,7 @@ fn WriteMetaBlockInternal( mb.destroy(alloc); } if bytes + 4 + saved_byte_location < (*storage_ix >> 3) { - dist_cache[..4].clone_from_slice(&saved_dist_cache[..4]); + dist_cache[..4].copy_from_slice(&saved_dist_cache[..4]); //memcpy(dist_cache, // saved_dist_cache, // (4usize).wrapping_mul(::core::mem::size_of::())); @@ -2411,7 +2410,7 @@ impl BrotliEncoderStateStruct { let mut new_commands = allocate::(&mut self.m8, newsize); if !self.commands_.slice().is_empty() { new_commands.slice_mut()[..self.num_commands_] - .clone_from_slice(&self.commands_.slice()[..self.num_commands_]); + .copy_from_slice(&self.commands_.slice()[..self.num_commands_]); >::free_cell( &mut self.m8, core::mem::take(&mut self.commands_), @@ -2541,7 +2540,7 @@ impl BrotliEncoderStateStruct { self.num_commands_ = 0usize; self.num_literals_ = 0usize; self.saved_dist_cache_ - .clone_from_slice(self.dist_cache_.split_at(4).0); + .copy_from_slice(self.dist_cache_.split_at(4).0); self.next_out_ = NextOut::DynamicStorage(0); // this always returns that *out_size = storage_ix >> 3; true @@ -2655,7 +2654,7 @@ impl BrotliEncoderStateStruct { let copy: u32 = min(self.remaining_metadata_bytes_ as usize, *available_out) as u32; next_out_array[*next_out_offset..(*next_out_offset + copy as usize)] - .clone_from_slice( + .copy_from_slice( &next_in_array[*next_in_offset..(*next_in_offset + copy as usize)], ); //memcpy(*next_out, *next_in, copy as usize); @@ -2670,7 +2669,7 @@ impl BrotliEncoderStateStruct { } else { let copy: u32 = min(self.remaining_metadata_bytes_, 16u32); self.next_out_ = NextOut::TinyBuf(0); - GetNextOut!(self)[..(copy as usize)].clone_from_slice( + GetNextOut!(self)[..(copy as usize)].copy_from_slice( &next_in_array[*next_in_offset..(*next_in_offset + copy as usize)], ); //memcpy(s.next_out_, *next_in, copy as usize); diff --git a/src/enc/find_stride.rs b/src/enc/find_stride.rs index fd816f25..5a80e017 100644 --- a/src/enc/find_stride.rs +++ b/src/enc/find_stride.rs @@ -51,7 +51,7 @@ impl> EntropyBucketPopulation { fn clone_from(&mut self, other: &EntropyBucketPopulation) { self.bucket_populations .slice_mut() - .clone_from_slice(other.bucket_populations.slice()); + .copy_from_slice(other.bucket_populations.slice()); } fn add_assign(&mut self, other: &EntropyBucketPopulation) { assert_eq!( @@ -93,7 +93,7 @@ impl> EntropyBucketPopulation { if do_clear && !found_any { self.bucket_populations .slice_mut() - .clone_from_slice(item.bucket_populations.slice()); + .copy_from_slice(item.bucket_populations.slice()); found_any = true; } else { for (dst, src) in self @@ -125,7 +125,7 @@ impl> EntropyBucketPopulation { scratch .bucket_populations .slice_mut() - .clone_from_slice(self.bucket_populations.slice()); + .copy_from_slice(self.bucket_populations.slice()); scratch.bucket_populations.slice_mut()[65535] += 1; // to demonstrate that we have scratch.bucket_populations.slice_mut()[65535] -= 1; // to demonstrate that we have write capability let mut stray_count = 0.0 as floatY; @@ -227,7 +227,7 @@ impl> EntropyPyramid { } pub fn stride_last_level_range(&self) -> [u8; NUM_LEAF_NODES] { let mut ret = [0u8; NUM_LEAF_NODES]; - ret.clone_from_slice(self.stride.split_at(self.stride.len() - NUM_LEAF_NODES).1); + ret.copy_from_slice(self.stride.split_at(self.stride.len() - NUM_LEAF_NODES).1); ret } pub fn free(&mut self, m32: &mut AllocU32) { @@ -723,8 +723,8 @@ impl> EntropyTally { } { let mut tmp = [0u8; NUM_STRIDES - 1]; - tmp.clone_from_slice(&priors[..(NUM_STRIDES - 1)]); - priors[1..].clone_from_slice(&tmp[..]); + tmp.copy_from_slice(&priors[..(NUM_STRIDES - 1)]); + priors[1..].copy_from_slice(&tmp[..]); priors[0] = *val; } } @@ -823,8 +823,8 @@ impl> EntropyTally { { //reset prior values for the next item let mut tmp = [0u8; 7]; - tmp.clone_from_slice(&priors[..7]); - priors[1..].clone_from_slice(&tmp[..]); + tmp.copy_from_slice(&priors[..7]); + priors[1..].copy_from_slice(&tmp[..]); priors[0] = *val; } } diff --git a/src/enc/interface.rs b/src/enc/interface.rs index e6e21113..2045f2ba 100644 --- a/src/enc/interface.rs +++ b/src/enc/interface.rs @@ -149,7 +149,7 @@ impl + SliceWrapperMut> PredictionModeContextMap pub fn set_mixing_values(&mut self, mixing_mask: &[u8; NUM_MIXING_VALUES]) { let cm_slice = self.predmode_speed_and_distance_context_map.slice_mut(); cm_slice[MIXING_OFFSET..(MIXING_OFFSET + NUM_MIXING_VALUES)] - .clone_from_slice(&mixing_mask[..]); + .copy_from_slice(&mixing_mask[..]); } #[inline] pub fn get_mixing_values_mut(&mut self) -> &mut [u8] { diff --git a/src/enc/metablock.rs b/src/enc/metablock.rs index 5d938d37..8ca94ab3 100644 --- a/src/enc/metablock.rs +++ b/src/enc/metablock.rs @@ -426,7 +426,7 @@ fn InitBlockSplitter< new_array = allocate::(alloc, _new_size); if (!split.types.slice().is_empty()) { new_array.slice_mut()[..split.types.slice().len()] - .clone_from_slice(split.types.slice()); + .copy_from_slice(split.types.slice()); } >::free_cell( alloc, @@ -446,7 +446,7 @@ fn InitBlockSplitter< } let mut new_array = allocate::(alloc, _new_size); new_array.slice_mut()[..split.lengths.slice().len()] - .clone_from_slice(split.lengths.slice()); + .copy_from_slice(split.lengths.slice()); >::free_cell( alloc, core::mem::replace(&mut split.lengths, new_array), @@ -510,7 +510,7 @@ fn InitContextBlockSplitter< let mut new_array = allocate::(alloc, _new_size); if (!split.types.slice().is_empty()) { new_array.slice_mut()[..split.types.slice().len()] - .clone_from_slice(split.types.slice()); + .copy_from_slice(split.types.slice()); } >::free_cell( alloc, @@ -531,7 +531,7 @@ fn InitContextBlockSplitter< let mut new_array = allocate::(alloc, _new_size); if (!split.lengths.slice().is_empty()) { new_array.slice_mut()[..split.lengths.slice().len()] - .clone_from_slice(split.lengths.slice()); + .copy_from_slice(split.lengths.slice()); } >::free_cell( alloc, diff --git a/src/enc/reader.rs b/src/enc/reader.rs index 38c538c9..1149ba4e 100644 --- a/src/enc/reader.rs +++ b/src/enc/reader.rs @@ -167,7 +167,7 @@ impl, BufferType: SliceWrapperMut, Alloc: Br .input_buffer .slice_mut() .split_at_mut(self.input_offset); - first[0..avail_in].clone_from_slice(&second[0..avail_in]); + first[0..avail_in].copy_from_slice(&second[0..avail_in]); self.input_len -= self.input_offset; self.input_offset = 0; } diff --git a/src/enc/static_dict.rs b/src/enc/static_dict.rs index 22f0356c..0c8cc15e 100644 --- a/src/enc/static_dict.rs +++ b/src/enc/static_dict.rs @@ -36,7 +36,7 @@ pub fn BrotliGetDictionary() -> &'static BrotliDictionary { #[inline(always)] pub fn BROTLI_UNALIGNED_LOAD32(sl: &[u8]) -> u32 { let mut p = [0u8; 4]; - p[..].clone_from_slice(sl.split_at(4).0); + p[..].copy_from_slice(sl.split_at(4).0); (p[0] as u32) | ((p[1] as u32) << 8) | ((p[2] as u32) << 16) | ((p[3] as u32) << 24) } #[inline(always)] @@ -47,7 +47,7 @@ pub fn Hash(data: &[u8]) -> u32 { #[inline(always)] pub fn BROTLI_UNALIGNED_LOAD64(sl: &[u8]) -> u64 { let mut p = [0u8; 8]; - p[..].clone_from_slice(sl.split_at(8).0); + p[..].copy_from_slice(sl.split_at(8).0); (p[0] as u64) | ((p[1] as u64) << 8) | ((p[2] as u64) << 16) @@ -69,16 +69,16 @@ pub fn BROTLI_UNALIGNED_STORE64(outp: &mut [u8], v: u64) { ((v >> 48) & 0xff) as u8, ((v >> 56) & 0xff) as u8, ]; - outp.split_at_mut(8).0.clone_from_slice(&p[..]); + outp.split_at_mut(8).0.copy_from_slice(&p[..]); } macro_rules! sub_match { ($s1 : expr_2021, $s2 : expr_2021, $limit : expr_2021, $matched : expr_2021, $split_pair1 : expr_2021, $split_pair2 : expr_2021, $s1_lo : expr_2021, $s2_lo : expr_2021, $s1_as_64 : expr_2021, $s2_as_64 : expr_2021, $vec_len: expr_2021) => { $split_pair1 = $s1.split_at($vec_len); - $s1_lo[..$vec_len].clone_from_slice($split_pair1.0); + $s1_lo[..$vec_len].copy_from_slice($split_pair1.0); $s1 = $split_pair1.1; $split_pair2 = $s2.split_at($vec_len); - $s2_lo[..$vec_len].clone_from_slice($split_pair2.0); + $s2_lo[..$vec_len].copy_from_slice($split_pair2.0); $s2 = $split_pair2.1; $limit -= $vec_len; for index in 0..($vec_len >> 3) { @@ -1422,7 +1422,7 @@ pub fn BrotliFindAllStaticDictionaryMatches( mod test { #[allow(unused)] fn construct_situation(seed: &[u8], mut output: &mut [u8], limit: usize, matchfor: usize) { - output[..].clone_from_slice(seed); + output[..].copy_from_slice(seed); if matchfor >= limit { return; } diff --git a/src/enc/test.rs b/src/enc/test.rs index 187888bd..11ab6ec8 100644 --- a/src/enc/test.rs +++ b/src/enc/test.rs @@ -628,7 +628,7 @@ impl io::Read for Buffer { let bytes_to_read = min(buf.len(), self.data.len() - self.read_offset); if bytes_to_read > 0 { buf[0..bytes_to_read] - .clone_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); + .copy_from_slice(&self.data[self.read_offset..self.read_offset + bytes_to_read]); } self.read_offset += bytes_to_read; return Ok(bytes_to_read); diff --git a/src/enc/threading/mod.rs b/src/enc/threading/mod.rs index c6eebaaa..63696f11 100644 --- a/src/enc/threading/mod.rs +++ b/src/enc/threading/mod.rs @@ -311,7 +311,7 @@ where { let input = if let InternalSendAlloc::A(ref mut alloc, ref _extra) = alloc_per_thread[0].0 { let mut input = allocate::(alloc, input_slice.len()); - input.slice_mut().clone_from_slice(input_slice); + input.slice_mut().copy_from_slice(input_slice); input } else { alloc_default::() diff --git a/src/ffi/broccoli.rs b/src/ffi/broccoli.rs index c8b89dad..0ee86bd5 100644 --- a/src/ffi/broccoli.rs +++ b/src/ffi/broccoli.rs @@ -21,7 +21,7 @@ pub struct BroccoliState { impl Clone for BroccoliState { fn clone(&self) -> BroccoliState { let mut cd = [0u8; 120]; - cd.clone_from_slice(&self.current_data[..]); + cd.copy_from_slice(&self.current_data[..]); BroccoliState { more_data: self.more_data, current_data: cd,