diff --git a/crates/fff-core/benches/memmem_bench.rs b/crates/fff-core/benches/memmem_bench.rs index 284c5d784..cacc78629 100644 --- a/crates/fff-core/benches/memmem_bench.rs +++ b/crates/fff-core/benches/memmem_bench.rs @@ -1,5 +1,5 @@ use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; -use fff_search::case_insensitive_memmem; +use fff_search::simd_string_utils::memmem; use std::path::Path; /// Load real source files from the repository as benchmark haystacks. @@ -41,7 +41,7 @@ fn load_real_files() -> Vec<(&'static str, Vec)> { } fn bench_memmem(c: &mut Criterion) { - let mut group = c.benchmark_group("case_insensitive_memmem"); + let mut group = c.benchmark_group("simd_string_utils_memmem"); let files = load_real_files(); assert!(!files.is_empty(), "No source files found for benchmarking"); @@ -69,18 +69,10 @@ fn bench_memmem(c: &mut Criterion) { let id = format!("{file_label}/{needle_label}"); group.bench_with_input( - BenchmarkId::new("packed_pair", &id), + BenchmarkId::new("find", &id), &(haystack, &needle_lower), |b, &(h, n)| { - b.iter(|| black_box(case_insensitive_memmem::search_packed_pair(h, n))); - }, - ); - - group.bench_with_input( - BenchmarkId::new("memchr2_search", &id), - &(haystack, &needle_lower), - |b, &(h, n)| { - b.iter(|| black_box(case_insensitive_memmem::search(h, n))); + b.iter(|| black_box(memmem::find(h, n))); }, ); } diff --git a/crates/fff-core/src/bigram_query.rs b/crates/fff-core/src/bigram_query.rs index f39afecb2..cea488a41 100644 --- a/crates/fff-core/src/bigram_query.rs +++ b/crates/fff-core/src/bigram_query.rs @@ -1,26 +1,48 @@ -//! Regex → bigram decomposition for the inverted bigram index. -//! -//! Parses a regex pattern with `regex-syntax`, walks the HIR to extract -//! guaranteed bigram keys (u16), and evaluates them as an AND/OR query tree -//! against [`BigramFilter`]'s inverted posting lists. -//! -//! Two bigram types are extracted: -//! - **Consecutive** (gap=0): adjacent byte pairs `(pattern[i], pattern[i+1])` -//! - **Sparse-1** (gap=1): pairs across a single-byte wildcard, e.g. `a.b → (a,b)` -//! -//! The sparse-1 extraction is the key feature: regex patterns like `foo.bar` -//! yield the cross-boundary sparse-1 bigram `(o,b)` that provides strong -//! filtering even when the `.` prevents any consecutive cross-boundary bigram. - use crate::bigram_filter::BigramFilter; use regex_syntax::hir::{Class, Hir, HirKind}; use smallvec::SmallVec; use std::borrow::Cow; -/// Maximum byte values to enumerate from a character class. -/// Larger classes are treated as unknown (no bigram extractable). const MAX_CLASS_EXPAND: usize = 16; +// stack inlined array padded with 0 and tracked length +#[derive(Clone, Copy, PartialEq, Eq)] +struct InlineArray { + bytes: [u8; MAX_CLASS_EXPAND], + len: usize, +} + +impl InlineArray { + const fn new() -> Self { + Self { + bytes: [0; MAX_CLASS_EXPAND], + len: 0, + } + } + + fn from_byte(b: u8) -> Self { + let mut set = Self::new(); + set.push(b); + set + } + + /// Append a byte; no-op if already full (callers guard against this). + fn push(&mut self, b: u8) { + if self.len < MAX_CLASS_EXPAND { + self.bytes[self.len] = b; + self.len += 1; + } + } +} + +impl std::ops::Deref for InlineArray { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + #[inline] fn consec_key(a: u8, b: u8) -> Option { let al = a.to_ascii_lowercase(); @@ -121,19 +143,15 @@ impl BigramQuery { } let mut result: Option> = None; for child in children { - match child.evaluate_cow(index) { - // Any branch can't be filtered → whole OR can't be filtered - None => return None, - Some(child_bits) => { - result = Some(match result { - None => child_bits.into_owned(), - Some(mut r) => { - bitset_or(&mut r, &child_bits); - r - } - }); + // Any branch can't be filtered -> whole OR can't be filtered + let child_bits = child.evaluate_cow(index)?; + result = Some(match result { + None => child_bits.into_owned(), + Some(mut r) => { + bitset_or(&mut r, &child_bits); + r } - } + }); } result.map(Cow::Owned) } @@ -141,14 +159,10 @@ impl BigramQuery { } } -/// Intermediate state tracked during HIR traversal for bigram extraction. struct HirInfo { query: BigramQuery, - /// Possible first bytes (lowercased, printable ASCII) when this node matches. - first: Option>, - /// Possible last bytes. - last: Option>, - /// Whether this node can match the empty string. + first: Option, + last: Option, can_be_empty: bool, } @@ -185,7 +199,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue return BigramQuery::Any; } - // For very short queries (0 typos), AND all bigrams — exact subsequence. + // the simpliest case, just check that every bigram is present either consec or not if max_typos == 0 { return simplify_and( bigram_keys @@ -225,7 +239,7 @@ pub(crate) fn fuzzy_to_bigram_query(query: &str, num_probes: usize) -> BigramQue return simplify_and(probes.iter().map(|&k| BigramQuery::Consec(k)).collect()); } - // Generate all C(n, required) subsets → OR(AND(subset), ...) + // Generate all C(n, required) subsets as OR(AND(subset), ...) let mut branches = Vec::new(); let mut combo = vec![0u16; required]; combine(&probes, required, 0, 0, &mut combo, &mut branches); @@ -282,7 +296,7 @@ fn decompose(hir: &Hir) -> HirInfo { match bytes { Some(b) if !b.is_empty() => HirInfo { query: BigramQuery::Any, - first: Some(b.clone()), + first: Some(b), last: Some(b), can_be_empty, }, @@ -344,13 +358,13 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo { if lower.len() == 1 { let b = lower[0]; let first = if (32..=126).contains(&b) { - Some(SmallVec::from_slice(&[b])) + Some(InlineArray::from_byte(b)) } else { None }; return HirInfo { query: BigramQuery::Any, - first: first.clone(), + first, last: first, can_be_empty: false, }; @@ -380,12 +394,12 @@ fn decompose_literal(bytes: &[u8]) -> HirInfo { HirInfo { query: simplify_and(qs), first: if (32..=126).contains(&first_byte) { - Some(SmallVec::from_slice(&[first_byte])) + Some(InlineArray::from_byte(first_byte)) } else { None }, last: if (32..=126).contains(&last_byte) { - Some(SmallVec::from_slice(&[last_byte])) + Some(InlineArray::from_byte(last_byte)) } else { None }, @@ -401,22 +415,20 @@ fn decompose_concat(parts: &[Hir]) -> HirInfo { let infos: Vec = parts.iter().map(decompose).collect(); let mut qs: Vec = Vec::new(); - // 1. Collect child bigrams for info in &infos { if !info.query.is_any() { qs.push(info.query.clone()); } } - // 2. Dense cross-boundary between adjacent mandatory parts + // Dense cross-boundary between adjacent mandatory parts for pair in infos.windows(2) { if !pair[0].can_be_empty && !pair[1].can_be_empty { push_cross_consec(&mut qs, pair[0].last.as_deref(), pair[1].first.as_deref()); } } - // 3. Sparse-1 cross-boundary: across a single 1-byte-wide middle part. - // Catches `foo.bar` → sparse-1 `(o,b)` across the dot. + // Sparse-1 cross-boundary: across a single 1 byte wide middle part if parts.len() >= 3 { for i in 0..parts.len() - 2 { let left = &infos[i]; @@ -464,8 +476,8 @@ fn decompose_alternation(alts: &[Hir]) -> HirInfo { } } -fn expand_class(class: &Class) -> Option> { - let mut bytes: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new(); +fn expand_class(class: &Class) -> Option { + let mut bytes = InlineArray::new(); match class { Class::Bytes(bc) => { for range in bc.ranges() { @@ -473,6 +485,7 @@ fn expand_class(class: &Class) -> Option> { if bytes.len() + count > MAX_CLASS_EXPAND { return None; } + for b in range.start()..=range.end() { if (32..=126).contains(&b) { let lower = b.to_ascii_lowercase(); @@ -554,11 +567,11 @@ fn cross_product(last: Option<&[u8]>, first: Option<&[u8]>, skip: bool) -> Optio } } -fn collect_first(infos: &[HirInfo]) -> Option> { - let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new(); +fn collect_first(infos: &[HirInfo]) -> Option { + let mut result = InlineArray::new(); for info in infos { if let Some(ref bytes) = info.first { - for &b in bytes { + for &b in bytes.iter() { if !result.contains(&b) { if result.len() >= MAX_CLASS_EXPAND { return None; @@ -580,11 +593,11 @@ fn collect_first(infos: &[HirInfo]) -> Option> } } -fn collect_last(infos: &[HirInfo]) -> Option> { - let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new(); +fn collect_last(infos: &[HirInfo]) -> Option { + let mut result = InlineArray::new(); for info in infos.iter().rev() { if let Some(ref bytes) = info.last { - for &b in bytes { + for &b in bytes.iter() { if !result.contains(&b) { if result.len() >= MAX_CLASS_EXPAND { return None; @@ -606,22 +619,17 @@ fn collect_last(infos: &[HirInfo]) -> Option> { } } -fn merge_byte_sets<'a>( - iter: impl Iterator>>, -) -> Option> { - let mut result: SmallVec<[u8; MAX_CLASS_EXPAND]> = SmallVec::new(); +fn merge_byte_sets<'a>(iter: impl Iterator>) -> Option { + let mut result = InlineArray::new(); for opt in iter { - match opt { - None => return None, - Some(bytes) => { - for &b in bytes { - if !result.contains(&b) { - if result.len() >= MAX_CLASS_EXPAND { - return None; - } - result.push(b); - } + let bytes = opt.as_ref()?; + + for &b in bytes.iter() { + if !result.contains(&b) { + if result.len() >= MAX_CLASS_EXPAND { + return None; } + result.push(b); } } } @@ -757,7 +765,7 @@ mod tests { #[test] fn sparse1_across_digit() { - // "foo\dbar" → sparse-1 (o,b) across \d + // "foo\dbar" -> sparse-1 (o,b) across \d let idx = build_test_index(&[ b"foo3bar baz", // 0: has all bigrams b"foobar baz", // 1: has consecutive (o,b) but pattern needs sparse-1 @@ -795,7 +803,6 @@ mod tests { #[test] fn optional_group_excluded() { - // (bar)? is optional — its bigrams are not required let q = regex_to_bigram_query("foo(bar)?baz"); assert!(!q.is_any()); @@ -813,7 +820,7 @@ mod tests { #[test] fn repetition_min2_cross_boundary() { - // (ab){2,} → bigram "ab" + cross-boundary "b","a" + // (ab){2,} -> bigram "ab" + cross-boundary "b","a" let q = regex_to_bigram_query("(ab){2,}"); assert!(!q.is_any()); @@ -830,16 +837,14 @@ mod tests { #[test] fn two_dots_no_sparse1() { - // "a..b" — two 1-byte parts between a and b, not a single 1-byte part - // No sparse-1 (a,b) should be extracted let q = regex_to_bigram_query("a..b"); - // Single-char literals with 2 unknown bytes between → Any + // Single-char literals with 2 unknown bytes between -> Any assert!(q.is_any()); } #[test] fn character_class_cross_boundary() { - // [abc]de → cross-boundary OR(ad,bd,cd) + bigram de + // [abc]de -> cross-boundary OR(ad,bd,cd) + bigram de // All three class variants must appear in the corpus so the OR // branches are tracked in the index (untracked bigrams make the // OR conservatively return None, which is correct but untestable). @@ -861,8 +866,6 @@ mod tests { assert!(!BigramFilter::is_candidate(&candidates, 3)); } - // ── Helpers for inspecting query trees ────────────────────────── - fn has_consec(q: &BigramQuery, a: u8, b: u8) -> bool { let Some(key) = consec_key(a, b) else { return false; @@ -896,13 +899,12 @@ mod tests { /// plus typical grep patterns used by agentic tools. /// /// Each entry: `(regex, Option<&[Bg]>)`. - /// - `None` → pure classes / unsupported syntax, Any is acceptable. - /// - `Some(&[..])` → must be non-Any, and every listed bigram must appear. + /// - `None` -> pure classes / unsupported syntax, Any is acceptable. + /// - `Some(&[..])` -> must be non-Any, and every listed bigram must appear. #[test] fn common_regex_patterns() { #[rustfmt::skip] let cases: &[(&str, Option<&[Bg]>)] = &[ - // ── Pure-class / anchor / unsupported → Any is fine ────── (r"^\d+$", None), // 1. whole numbers (r"^\d*\.\d+$", None), // 2. decimals (r"^\d*(\.\d+)?$", None), // 3. whole + decimal @@ -931,11 +933,9 @@ mod tests { (r"^[\w,\s-]+\.[A-Za-z]{3}$", None), // 27. filename (r"^[A-PR-WY][1-9]\d\s?\d{4}[1-9]$", None), // 28. HK ID - // ── Patterns with extractable literal bigrams ──────────── - // 13. URL with required protocol (r"https?://(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)", Some(&[ - ("ht", C), ("tt", C), ("tp", C), // from "http" + ("ht", C), ("tt", C), ("tp", C), // from "http" ("ht", S), ("tp", S), // from "http" skip-1 (":/", C), ("//", C), // from "://" ])), @@ -943,31 +943,31 @@ mod tests { // 29. fn\s+\w+ (r"fn\s+\w+", Some(&[ ("fn", C), // from "fn" - ("n ", C), // cross-boundary: 'n' → \s starts ' ' + ("n ", C), // cross-boundary: 'n' -> \s starts ' ' ])), // 30. use\s+crate:: (r"use\s+crate::", Some(&[ - ("us", C), ("se", C), ("ue", S), // from "use" - ("cr", C), ("ra", C), ("at", C), // from "crate" + ("us", C), ("se", C), ("ue", S), // from "use" + ("cr", C), ("ra", C), ("at", C), // from "crate" ("te", C), ("::", C), - ("ca", S), ("rt", S), ("ae", S), // "crate" skip-1 + ("ca", S), ("rt", S), ("ae", S), // "crate" skip-1 ])), // 31. unwrap\(\)|expect\( (r"unwrap\(\)|expect\(", Some(&[ - ("nw", C), ("wr", C), ("ra", C), // "unwrap(" + ("nw", C), ("wr", C), ("ra", C), // "unwrap(" ("ap", C), ("p(", C), - ("xp", C), ("pe", C), ("ec", C), // "expect(" + ("xp", C), ("pe", C), ("ec", C), // "expect(" ("ct", C), ("t(", C), ])), // 32. TODO|FIXME|HACK (r"TODO|FIXME|HACK", Some(&[ - ("to", C), ("od", C), ("do", C), // "TODO" - ("fi", C), ("ix", C), ("xm", C), // "FIXME" + ("to", C), ("od", C), ("do", C), // "TODO" + ("fi", C), ("ix", C), ("xm", C), // "FIXME" ("me", C), - ("ha", C), ("ac", C), ("ck", C), // "HACK" + ("ha", C), ("ac", C), ("ck", C), // "HACK" ("hc", S), ("ak", S), // "HACK" skip-1 ])), ]; diff --git a/crates/fff-core/src/case_insensitive_memmem.rs b/crates/fff-core/src/case_insensitive_memmem.rs deleted file mode 100644 index bdbb8b843..000000000 --- a/crates/fff-core/src/case_insensitive_memmem.rs +++ /dev/null @@ -1,666 +0,0 @@ -//! SIMD-accelerated case-insensitive substring search. -//! -//! Implementations (fastest → simplest): -//! - `search_packed_pair`: AVX2 packed-pair scan (two rare bytes at known offsets) -//! - `search`: memchr2 first-byte scan + verify -//! -//! The packed-pair approach mirrors what `memchr::memmem` does internally for -//! case-sensitive search — pick two rare bytes from the needle, SIMD-scan for -//! both simultaneously, verify candidates. This gives quadratic selectivity -//! over the single-byte memchr2 approach. - -// this is stolen from the memchr2 crate -const BYTE_FREQUENCIES: [u8; 256] = [ - 55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00 - 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10 - 255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20 - 208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30 - 120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185, - 167, // 0x40 A-O - 186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114, - 223, // 0x50 P-_ - 151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246, - 244, // 0x60 a-o - 231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127, - 27, // 0x70 p-DEL - 212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80 - 207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90 - 118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0 - 166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0 - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0 - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0 - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0 - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0 -]; - -#[inline] -fn ascii_fold_byte(b: u8) -> u8 { - if b.is_ascii_uppercase() { b | 0x20 } else { b } -} - -/// Toggle ASCII letter case by flipping bit 5. -/// `'n' → 'N'`, `'N' → 'n'`. -#[inline] -fn ascii_swap_case(b: u8) -> u8 { - b ^ 0x20 -} - -/// Effective frequency rank for a case-insensitive byte position. -/// Takes the max of lower/upper ranks because we must scan for both. -#[inline] -fn case_insensitive_rank(lower: u8) -> u8 { - if lower.is_ascii_lowercase() { - let upper = ascii_swap_case(lower); - BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[upper as usize]) - } else { - BYTE_FREQUENCIES[lower as usize] - } -} - -/// Pick two needle positions with the rarest bytes (case-insensitive). -/// Returns (index1, index2) where index1 <= index2. -fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) { - debug_assert!(needle_lower.len() >= 2); - - let mut best1 = (u8::MAX, 0usize); // (rank, position) - let mut best2 = (u8::MAX, 1usize); - - for (i, &b) in needle_lower.iter().enumerate() { - let r = case_insensitive_rank(b); - if r < best1.0 { - best2 = best1; - best1 = (r, i); - } else if r < best2.0 && i != best1.1 { - best2 = (r, i); - } - } - - let i1 = best1.1.min(best2.1); - let i2 = best1.1.max(best2.1); - (i1, i2) -} - -#[inline] -fn verify_scalar(h: *const u8, needle_lower: &[u8]) -> bool { - for (i, _) in needle_lower.iter().enumerate() { - if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] { - return false; - } - } - true -} - -/// AVX2 case-insensitive verify: checks whether `needle_lower` matches -/// the haystack bytes starting at `h`, treating ASCII uppercase as lowercase. -/// -/// Processes 32 bytes at a time using a SIMD trick: AVX2 only has a -/// **signed** byte compare (`cmpgt`), but we need an **unsigned** range -/// check (`'A' <= byte <= 'Z'`). The trick is to XOR every byte with -/// `0x80`, which maps the unsigned range `[0, 255]` into the signed range -/// `[-128, 127]` while preserving order. After the flip, signed `cmpgt` -/// gives correct unsigned comparisons. -/// -/// Once we know which bytes are uppercase, we set bit 5 (`0x20`) on them -/// — this converts `'A'..'Z'` to `'a'..'z'` — then compare against the -/// pre-lowered needle. -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn verify_avx2(h: *const u8, needle_lower: &[u8]) -> bool { - use core::arch::x86_64::*; - - let len = needle_lower.len(); - let mut i = 0usize; - - // Broadcast constants used every iteration: - // - // flip = 0x80 in every lane — XOR converts unsigned→signed domain - // a_minus_1 = ('A' - 1) ^ 0x80 — lower bound for the range check (signed) - // z_plus_1 = ('Z' + 1) ^ 0x80 — upper bound for the range check (signed) - // bit20 = 0x20 in every lane — OR this onto uppercase bytes to lowercase them - let flip = _mm256_set1_epi8(0x80u8 as i8); - let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8); - let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8); - let bit20 = _mm256_set1_epi8(0x20u8 as i8); - - while i + 32 <= len { - // Load 32 bytes from the haystack candidate position. - let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) }; - // Load 32 bytes from the pre-lowercased needle. - let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) }; - - // Flip into signed domain: x = hv ^ 0x80. - // After this, unsigned ordering is preserved under signed compare. - let x = _mm256_xor_si256(hv, flip); - - // ge_a[lane] = 0xFF if x[lane] > a_minus_1, i.e. hv[lane] >= 'A' (unsigned). - let ge_a = _mm256_cmpgt_epi8(x, a_minus_1); - // le_z[lane] = 0xFF if z_plus_1 > x[lane], i.e. hv[lane] <= 'Z' (unsigned). - let le_z = _mm256_cmpgt_epi8(z_plus_1, x); - // upper[lane] = 0xFF only for bytes in the range 'A'..='Z'. - let upper = _mm256_and_si256(ge_a, le_z); - - // Case-fold: set bit 5 on uppercase bytes → converts 'A'..'Z' to 'a'..'z'. - // Non-letter bytes are untouched because their `upper` lane is 0x00. - let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20)); - - // Compare the folded haystack against the lowercase needle. - let eq = _mm256_cmpeq_epi8(folded, nv); - // movemask extracts the high bit of each lane into a 32-bit mask. - // All-equal → all high bits set → mask == 0xFFFFFFFF == -1i32. - if _mm256_movemask_epi8(eq) != -1i32 { - return false; - } - - i += 32; - } - - // Scalar tail: handle remaining bytes that don't fill a full 32-byte vector. - while i < len { - if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] { - return false; - } - i += 1; - } - true -} - -// ======== NEON + dotprod (aarch64) =========================================== - -/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF). -/// Bit *i* of the result corresponds to byte *i* of the input vector. -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "neon")] -#[inline] -unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 { - use core::arch::aarch64::*; - - // AND each byte with its bit-position mask, then horizontally sum each half. - // Max possible sum per half = 1+2+4+8+16+32+64+128 = 255, fits in u8. - static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128]; - let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) }; - let masked = vandq_u8(v, bit_mask); - let lo = vaddv_u8(vget_low_u8(masked)); - let hi = vaddv_u8(vget_high_u8(masked)); - (lo as u16) | ((hi as u16) << 8) -} - -/// NEON + dotprod case-insensitive verify. -/// -/// Uses unsigned range checks (NEON has `vcge`/`vcle` for unsigned bytes -/// no XOR-0x80 trick needed unlike AVX2) to detect uppercase ASCII, folds -/// to lowercase, then checks equality via UDOT: XOR the folded haystack -/// with the pre-lowered needle and dot-product the difference with itself. -/// Any non-zero byte produces a non-zero u32 lane. -/// -/// The UDOT instruction is emitted via inline asm because the `vdotq_u32` -/// intrinsic is still behind an unstable feature gate on stable Rust. -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "neon,dotprod")] -unsafe fn verify_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool { - use core::arch::aarch64::*; - - let len = needle_lower.len(); - let mut i = 0usize; - - let a_val = vdupq_n_u8(b'A'); - let z_val = vdupq_n_u8(b'Z'); - let bit20 = vdupq_n_u8(0x20); - - while i + 16 <= len { - let hv = unsafe { vld1q_u8(h.add(i)) }; - let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) }; - - // Unsigned range check: 'A' <= byte <= 'Z' - let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val)); - // Case-fold: set bit 5 on uppercase bytes → 'A'..'Z' → 'a'..'z' - let folded = vorrq_u8(hv, vandq_u8(upper, bit20)); - - // XOR with needle — all-zero iff every byte matches. - let xored = veorq_u8(folded, nv); - - // UDOT: dot(xored, xored) sums squares of 4 consecutive byte - // differences into each of the 4 u32 lanes (accumulates into zero). - // Any non-zero byte produces a positive u32 contribution. - let dots: uint32x4_t; - let zero = vdupq_n_u32(0); - unsafe { - core::arch::asm!( - "udot {d:v}.4s, {a:v}.16b, {b:v}.16b", - d = inlateout(vreg) zero => dots, - a = in(vreg) xored, - b = in(vreg) xored, - ); - } - - if vmaxvq_u32(dots) != 0 { - return false; - } - - i += 16; - } - - // Scalar tail - while i < len { - if ascii_fold_byte(unsafe { *h.add(i) }) != needle_lower[i] { - return false; - } - i += 1; - } - true -} - -/// NEON packed-pair kernel: scan 16 haystack positions per iteration, -/// checking two rare bytes (case-insensitive) simultaneously. -/// Same algorithm as the AVX2 version but with 128-bit vectors. -#[cfg(target_arch = "aarch64")] -#[target_feature(enable = "neon")] -unsafe fn search_packed_pair_neon( - haystack: &[u8], - needle_lower: &[u8], - i1: usize, - i2: usize, -) -> bool { - use core::arch::aarch64::*; - - let n = needle_lower.len(); - let hlen = haystack.len(); - let ptr = haystack.as_ptr(); - let last_start = hlen - n; - - let b1 = needle_lower[i1]; - let b1_alt = if b1.is_ascii_lowercase() { - ascii_swap_case(b1) - } else { - b1 - }; - let b2 = needle_lower[i2]; - let b2_alt = if b2.is_ascii_lowercase() { - ascii_swap_case(b2) - } else { - b2 - }; - - let v1_lo = vdupq_n_u8(b1); - let v1_hi = vdupq_n_u8(b1_alt); - let v2_lo = vdupq_n_u8(b2); - let v2_hi = vdupq_n_u8(b2_alt); - - let max_idx = i1.max(i2); - let max_offset = hlen.saturating_sub(max_idx + 16); - let mut offset = 0usize; - - while offset <= max_offset { - let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) }; - let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) }; - - // Case-insensitive match: OR both case variants, then AND the two positions. - let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi)); - let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi)); - - let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) }; - - while mask != 0 { - let bit = mask.trailing_zeros() as usize; - let candidate = offset + bit; - if candidate > last_start { - return false; - } - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - mask &= mask - 1; - } - - offset += 16; - } - - // Tail: remaining positions that couldn't fill a full vector. - if offset <= last_start { - let rare_pos = - if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) { - i1 - } else { - i2 - }; - let rare_byte = needle_lower[rare_pos]; - let tail_start = offset + rare_pos; - let tail_end = last_start + rare_pos + 1; - if tail_start < tail_end { - let tail_space = &haystack[tail_start..tail_end]; - if rare_byte.is_ascii_lowercase() { - for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { - let candidate = offset + pos; - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - } - } else { - for pos in memchr::memchr_iter(rare_byte, tail_space) { - let candidate = offset + pos; - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - } - } - } - } - - false -} - -#[inline] -unsafe fn verify_dispatch(h: *const u8, needle_lower: &[u8]) -> bool { - #[cfg(target_arch = "x86_64")] - { - if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") { - return unsafe { verify_avx2(h, needle_lower) }; - } - } - #[cfg(target_arch = "aarch64")] - { - if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") { - return unsafe { verify_neon_dotprod(h, needle_lower) }; - } - } - - verify_scalar(h, needle_lower) -} - -// ── Packed-pair search (AVX2) ─────────────────────────────────────────── - -/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration, -/// checking two rare bytes (case-insensitive) simultaneously. -/// 4 cmpeq + 2 or + 1 and + 1 movemask per 32 bytes — same memory -/// bandwidth as memchr2 but quadratic selectivity. -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn search_packed_pair_avx2( - haystack: &[u8], - needle_lower: &[u8], - i1: usize, - i2: usize, -) -> bool { - use core::arch::x86_64::*; - - let n = needle_lower.len(); - let hlen = haystack.len(); - let ptr = haystack.as_ptr(); - let last_start = hlen - n; // last valid match-start position - - let b1 = needle_lower[i1]; - let b1_alt = if b1.is_ascii_lowercase() { - ascii_swap_case(b1) - } else { - b1 - }; - let b2 = needle_lower[i2]; - let b2_alt = if b2.is_ascii_lowercase() { - ascii_swap_case(b2) - } else { - b2 - }; - - let v1_lo = _mm256_set1_epi8(b1 as i8); - let v1_hi = _mm256_set1_epi8(b1_alt as i8); - let v2_lo = _mm256_set1_epi8(b2 as i8); - let v2_hi = _mm256_set1_epi8(b2_alt as i8); - - // Main loop: process 32 candidate positions per iteration. - // We load from ptr+offset+i1 and ptr+offset+i2, so we need - // offset + max(i1,i2) + 31 < hlen. - let max_idx = i1.max(i2); - let max_offset = hlen.saturating_sub(max_idx + 32); - let mut offset = 0usize; - - while offset <= max_offset { - let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) }; - let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) }; - - // Case-insensitive match: OR both case variants, then AND the two positions. - let eq1 = _mm256_or_si256( - _mm256_cmpeq_epi8(chunk1, v1_lo), - _mm256_cmpeq_epi8(chunk1, v1_hi), - ); - let eq2 = _mm256_or_si256( - _mm256_cmpeq_epi8(chunk2, v2_lo), - _mm256_cmpeq_epi8(chunk2, v2_hi), - ); - - let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32; - - while mask != 0 { - let bit = mask.trailing_zeros() as usize; - let candidate = offset + bit; - if candidate > last_start { - // Past the end — no more valid positions in this or future chunks. - return false; - } - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - mask &= mask - 1; - } - - offset += 32; - } - - // Tail: remaining positions that couldn't fill a full vector. - // Use memchr2 on the rarest byte for these last few positions. - if offset <= last_start { - let rare_pos = - if case_insensitive_rank(needle_lower[i1]) <= case_insensitive_rank(needle_lower[i2]) { - i1 - } else { - i2 - }; - let rare_byte = needle_lower[rare_pos]; - let tail_start = offset + rare_pos; - let tail_end = last_start + rare_pos + 1; - if tail_start < tail_end { - let tail_space = &haystack[tail_start..tail_end]; - if rare_byte.is_ascii_lowercase() { - for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { - let candidate = offset + pos; - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - } - } else { - for pos in memchr::memchr_iter(rare_byte, tail_space) { - let candidate = offset + pos; - if unsafe { verify_dispatch(ptr.add(candidate), needle_lower) } { - return true; - } - } - } - } - } - - false -} - -/// Packed-pair case-insensitive substring search. -/// -/// Selects the two rarest bytes from the needle (using the memchr byte -/// frequency heuristic), then SIMD-scans for both at their known offsets -/// simultaneously. Falls back to `search` for needles shorter than 2 bytes. -pub fn search_packed_pair(haystack: &[u8], needle_lower: &[u8]) -> bool { - let n = needle_lower.len(); - if n == 0 { - return true; - } - if n < 2 { - return search(haystack, needle_lower); - } - if n > haystack.len() { - return false; - } - - #[cfg_attr( - not(any(target_arch = "x86_64", target_arch = "aarch64")), - allow(unused_variables) - )] - let (i1, i2) = select_rare_pair(needle_lower); - - #[cfg(target_arch = "x86_64")] - { - if std::is_x86_feature_detected!("avx2") { - // Need enough haystack for at least one vector load. - let max_idx = i1.max(i2); - if haystack.len() >= max_idx + 32 { - return unsafe { search_packed_pair_avx2(haystack, needle_lower, i1, i2) }; - } - } - } - - #[cfg(target_arch = "aarch64")] - { - // The NEON packed-pair scan checks 16 bytes/iteration with ~7 ops, - // while memchr's optimized loop processes more bytes with fewer ops. - // Packed-pair wins when the first byte is common (lots of false - // positives for memchr2 that we avoid). But when the first byte is - // rare (z, q, x, ...) memchr2 has no false positives and its raw - // throughput dominates. Threshold 200 on the frequency table splits - // common letters (s=243, e=253, f=227) from rare ones (z=152, q=139). - let first_byte_rank = case_insensitive_rank(needle_lower[0]); - let max_idx = i1.max(i2); - if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 { - return unsafe { search_packed_pair_neon(haystack, needle_lower, i1, i2) }; - } - } - - // Fallback for short haystacks or non-SIMD platforms. - search(haystack, needle_lower) -} - -// ── Original memchr2 first-byte search ────────────────────────────────── - -/// Case-insensitive search using memchr2 on the first byte. -pub fn search(haystack: &[u8], needle_lower: &[u8]) -> bool { - let n = needle_lower.len(); - if n == 0 { - return true; - } - if n > haystack.len() { - return false; - } - - let search_space = &haystack[..=haystack.len() - n]; - let first = needle_lower[0]; - - if first.is_ascii_lowercase() { - let alt = ascii_swap_case(first); - for pos in memchr::memchr2_iter(first, alt, search_space) { - if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } { - return true; - } - } - } else { - for pos in memchr::memchr_iter(first, search_space) { - if unsafe { verify_dispatch(haystack.as_ptr().add(pos), needle_lower) } { - return true; - } - } - } - false -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic_case_insensitive() { - assert!(search_packed_pair(b"Hello World", b"hello")); - assert!(search_packed_pair(b"Hello World", b"world")); - assert!(search_packed_pair(b"NOMORE bugs", b"nomore")); - assert!(!search_packed_pair(b"Hello World", b"xyz")); - } - - #[test] - fn edge_cases() { - assert!(search_packed_pair(b"ab", b"ab")); - assert!(search_packed_pair(b"AB", b"ab")); - assert!(!search_packed_pair(b"a", b"ab")); - assert!(search_packed_pair(b"anything", b"")); - assert!(!search_packed_pair(b"", b"x")); - } - - #[test] - fn packed_pair_matches_search() { - let haystacks: &[&[u8]] = &[ - b"The quick brown fox jumps over the lazy dog", - b"int mutex_lock(struct mutex *lock) { return 0; }", - b"#define NOMORE_RETRIES 5\nif (nomore) return;", - b"abcdefghijklmnopqrstuvwxyz", - b"short", - ]; - let needles: &[&[u8]] = &[b"fox", b"mutex", b"nomore", b"xyz", b"the", b"short", b"qr"]; - for h in haystacks { - for n in needles { - let lower: Vec = n.iter().map(|b| b.to_ascii_lowercase()).collect(); - assert_eq!( - search_packed_pair(h, &lower), - search(h, &lower), - "mismatch for haystack={:?} needle={:?}", - std::str::from_utf8(h), - std::str::from_utf8(n), - ); - } - } - } - - #[test] - fn long_haystack_neon_path() { - // Haystack > 16 bytes exercises NEON packed-pair search loop - let haystack = - b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb"; - assert!(search_packed_pair(haystack, b"this_is_a_long_needle_test")); - assert!(!search_packed_pair( - haystack, - b"this_is_a_long_needle_testz" - )); - - // Needle >= 16 bytes exercises NEON dotprod verify - let long_needle = b"struct mutex *lock"; - let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }"; - assert!(search_packed_pair(haystack2, long_needle)); - - // All uppercase haystack, lowercase needle - let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - assert!(search_packed_pair(upper_hay, b"qrstuvwxyz0123456789a")); - assert!(!search_packed_pair(upper_hay, b"qrstuvwxyz01234567899")); - - // Needle at very end - let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me"; - assert!(search_packed_pair(end_hay, b"find_me")); - - // Needle at very start - assert!(search_packed_pair(end_hay, b"xx")); - - // 1KB haystack with needle near the end - let mut big = vec![b'z'; 1024]; - big[1000..1010].copy_from_slice(b"hElLo_WoRl"); - assert!(search_packed_pair(&big, b"hello_wo")); - assert!(!search_packed_pair(&big, b"hello_world")); - } - - #[test] - fn rare_pair_selection() { - // For "nomore": n=246, o=244, m=233, o=244, r=245, e=253 - // Rarest positions should include 'm' (pos 2, rank 233) - let (i1, i2) = select_rare_pair(b"nomore"); - let ranks: Vec = b"nomore" - .iter() - .map(|&b| case_insensitive_rank(b)) - .collect(); - let r1 = ranks[i1]; - let r2 = ranks[i2]; - // Both selected ranks should be <= all other ranks - for (i, &r) in ranks.iter().enumerate() { - if i != i1 && i != i2 { - assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal"); - } - } - } -} diff --git a/crates/fff-core/src/constraints.rs b/crates/fff-core/src/constraints.rs index 6a11abe42..c70e9bbb7 100644 --- a/crates/fff-core/src/constraints.rs +++ b/crates/fff-core/src/constraints.rs @@ -5,31 +5,7 @@ use smallvec::SmallVec; use crate::git::is_modified_status; use crate::simd_path::ArenaPtr; - -/// `needle` must already be lowercase. -#[inline] -fn contains_ascii_ci(haystack: &str, needle: &str) -> bool { - let h = haystack.as_bytes(); - let n = needle.as_bytes(); - if n.len() > h.len() { - return false; - } - if n.is_empty() { - return true; - } - let first = n[0]; - for i in 0..=(h.len() - n.len()) { - if h[i].to_ascii_lowercase() == first - && h[i..i + n.len()] - .iter() - .zip(n) - .all(|(a, b)| a.to_ascii_lowercase() == *b) - { - return true; - } - } - false -} +use crate::simd_string_utils::memmem::find_case_insensitive_short; const PAR_THRESHOLD: usize = 10_000; @@ -216,7 +192,7 @@ impl<'q, 'c> ConstraintPlan<'q, 'c> { overflow_arena: ArenaPtr, ) -> Self { let mut extensions = SmallVec::new(); - let mut rest = SmallVec::new(); + let mut rest: SmallVec<[&'c Constraint<'q>; 8]> = SmallVec::new(); for c in constraints { match c { Constraint::Extension(ext) => extensions.push(*ext), @@ -284,56 +260,16 @@ impl<'q, 'c> ConstraintPlan<'q, 'c> { let mut glob_idx = 0; self.rest.iter().all(|c| { - let glob: &GlobStrategy = &self.glob; - let glob_idx: &mut usize = &mut glob_idx; - let negate = false; - let raw = match c { - Constraint::Glob(_) => { - let m = match glob { - GlobStrategy::None => true, - GlobStrategy::Prepass(masks) => masks - .get(*glob_idx) - .and_then(|mask| mask.get(index).copied()) - .unwrap_or(false), - GlobStrategy::Inline(patterns) => { - item.write_relative_path(arena, &mut scratch.path); - patterns - .get(*glob_idx) - .and_then(|p| p.as_ref()) - .map(|p| compiled_matches(p, &scratch.path)) - .unwrap_or(false) - } - }; - *glob_idx += 1; - m - } - // Reachable only via `Not(Extension(_))` — bare extensions are split out - // up front and handled in `passes_extensions`. - Constraint::Extension(ext) => { - item.write_file_name(arena, &mut scratch.fname); - file_has_extension(&scratch.fname, ext) - } - Constraint::PathSegment(segment) => { - item.write_relative_path(arena, &mut scratch.path); - path_contains_segment(&scratch.path, segment) - } - Constraint::FilePath(suffix) => { - item.write_relative_path(arena, &mut scratch.path); - path_ends_with_suffix(&scratch.path, suffix) - } - Constraint::Text(text) => { - // Only meaningful under negation (used as exclude filter). - item.write_relative_path(arena, &mut scratch.path); - contains_ascii_ci(&scratch.path, text) - } - Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter), - Constraint::Not(inner) => { - return evaluate(item, index, inner, glob, glob_idx, !negate, arena, scratch); - } - // Pass-throughs — handled at higher levels. - Constraint::Parts(_) | Constraint::Exclude(_) | Constraint::FileType(_) => true, - }; - if negate { !raw } else { raw } + evaluate( + item, + index, + c, + &self.glob, + &mut glob_idx, + false, + arena, + scratch, + ) }) } @@ -403,7 +339,7 @@ fn evaluate( Constraint::Text(text) => { // Only meaningful under negation (used as exclude filter). item.write_relative_path(arena, &mut scratch.path); - contains_ascii_ci(&scratch.path, text) + find_case_insensitive_short(scratch.path.as_bytes(), text.as_bytes()).is_some() } Constraint::GitStatus(filter) => matches_git_status(item.git_status(), filter), Constraint::Not(inner) => { diff --git a/crates/fff-core/src/dbs/mod.rs b/crates/fff-core/src/dbs/mod.rs index 59f5f1a98..63073cc82 100644 --- a/crates/fff-core/src/dbs/mod.rs +++ b/crates/fff-core/src/dbs/mod.rs @@ -1,4 +1,10 @@ +pub(crate) mod lmdb; + pub mod db_healthcheck; +pub use db_healthcheck::{DbHealth, DbHealthChecker}; + pub mod frecency; -pub(crate) mod lmdb; +pub use frecency::*; + pub mod query_tracker; +pub use query_tracker::*; diff --git a/crates/fff-core/src/grep.rs b/crates/fff-core/src/grep.rs index ac7535b62..3eecc434e 100644 --- a/crates/fff-core/src/grep.rs +++ b/crates/fff-core/src/grep.rs @@ -1,9 +1,8 @@ use crate::{ - BigramFilter, BigramOverlay, + bigram_filter::{BigramFilter, BigramOverlay, extract_bigrams}, bigram_query::{fuzzy_to_bigram_query, regex_to_bigram_query}, - case_insensitive_memmem, constraints::{ConstraintPlan, ConstraintsBuffers}, - extract_bigrams, + simd_string_utils::memmem, sort_buffer::sort_with_buffer, types::{ContentCacheBudget, FileItem, FileSliceExt, MmapSlot}, }; @@ -21,14 +20,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use tracing::Level; -/// Detect if a line looks like a code definition (struct, fn, class, etc.). -/// -/// Used at match time to tag `GrepMatch::is_definition` so that output -/// formatters can sort/annotate definitions without re-scanning lines. -/// -/// Hand-rolled keyword scanner — avoids regex overhead entirely. -/// Strips optional visibility/modifier keywords, then checks if the next -/// token is a definition keyword followed by a word boundary. +/// Detect if a line looks like a code definition (struct, fn, class, etc.) pub fn is_definition_line(line: &str) -> bool { let s = line.trim_start().as_bytes(); let s = skip_modifiers(s); @@ -231,9 +223,7 @@ fn replace_unescaped_newline_escapes(text: &str) -> String { /// Controls how the grep pattern is interpreted. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum GrepMode { - /// Default mode: the query is treated as literal text. - /// The pattern is searched using SIMD-accelerated `memchr::memmem`. - /// Special regex characters in the query have no special meaning. + /// Literal plain text match: default path that doesn't require any regex machinery #[default] PlainText, /// Regex mode: the query is treated as a regular expression. @@ -469,9 +459,7 @@ impl Matcher for PlainTextMatcher<'_> { let hay = &haystack[at..]; let found = if self.case_insensitive { - // ASCII case-insensitive: lowercase the haystack slice on the fly. - // We scan with a rolling window to avoid allocating a full copy. - ascii_case_insensitive_find(hay, self.needle) + memmem::find(hay, self.needle) } else { memchr::memmem::find(hay, self.needle) }; @@ -485,93 +473,6 @@ impl Matcher for PlainTextMatcher<'_> { } } -/// ASCII case-insensitive substring search. -/// -/// Uses a SIMD-accelerated two-byte scan (first + last byte of needle) via -/// `memchr2_iter`, then verifies candidates with a fast byte comparison that -/// leverages the fact that ASCII case differs only in bit 0x20. -#[inline] -fn ascii_case_insensitive_find(haystack: &[u8], needle_lower: &[u8]) -> Option { - let nlen = needle_lower.len(); - if nlen == 0 { - return Some(0); - } - - if haystack.len() < nlen { - return None; - } - - let first_lo = needle_lower[0]; - let first_hi = first_lo.to_ascii_uppercase(); - - // Single-byte needle: just find either case variant. - if nlen == 1 { - return memchr::memchr2(first_lo, first_hi, haystack); - } - - let tail = &needle_lower[1..]; - let end = haystack.len() - nlen; - - // Scan for candidates where the first byte matches (either case). - for pos in memchr::memchr2_iter(first_lo, first_hi, &haystack[..=end]) { - // Verify the remaining bytes with bitwise ASCII case-insensitive compare. - // For ASCII letters, (a ^ b) & ~0x20 == 0 when they match ignoring case. - // For non-letters, exact equality is required; OR-ing with 0x20 maps both - // cases to lowercase and is correct for non-alpha bytes that are already equal. - let candidate = unsafe { haystack.get_unchecked(pos + 1..pos + nlen) }; - if ascii_case_eq(candidate, tail) { - return Some(pos); - } - } - None -} - -/// Fast ASCII case-insensitive byte slice comparison. -/// -/// Returns true if `a` and `b` are equal when compared case-insensitively -/// for ASCII bytes. Both slices must have the same length. -#[inline] -fn ascii_case_eq(a: &[u8], b: &[u8]) -> bool { - debug_assert_eq!(a.len(), b.len()); - // Process 8 bytes at a time using u64 bitwise operations. - // For each byte: (x | 0x20) maps uppercase ASCII to lowercase. - // This is correct for letters. For non-letter bytes where the original - // values are equal, OR-ing with 0x20 preserves equality. For non-letter - // bytes where values differ, this can produce false positives only when - // they differ exactly by 0x20 — we do a fast exact-match check first - // to catch those rare cases. - let len = a.len(); - let mut i = 0; - - // Fast path: compare 8 bytes at a time - while i + 8 <= len { - let va = u64::from_ne_bytes(unsafe { *(a.as_ptr().add(i) as *const [u8; 8]) }); - let vb = u64::from_ne_bytes(unsafe { *(b.as_ptr().add(i) as *const [u8; 8]) }); - - // Quick exact-match shortcut (common for non-alpha content) - if va != vb { - // Case-insensitive: OR each byte with 0x20 to fold case - const MASK: u64 = 0x2020_2020_2020_2020; - if (va | MASK) != (vb | MASK) { - return false; - } - } - i += 8; - } - - // Handle remaining bytes - while i < len { - let ha = unsafe { *a.get_unchecked(i) }; - let hb = unsafe { *b.get_unchecked(i) }; - if ha != hb && (ha | 0x20) != (hb | 0x20) { - return false; - } - i += 1; - } - - true -} - /// Maximum bytes of a matched line to keep for display. Prevents minified /// JS or huge single-line files from blowing up memory. const MAX_LINE_DISPLAY_LEN: usize = 512; @@ -720,10 +621,9 @@ fn truncate_display_bytes(bytes: &[u8]) -> &[u8] { /// Sink for `PlainText` mode. /// -/// Highlights are extracted with SIMD-accelerated `memchr::memmem::Finder`. -/// Case-insensitive matching lowercases the line into a stack buffer before -/// searching, keeping positions 1:1 for ASCII. -/// No regex engine is involved at any point. +/// Highlights are extracted with `memchr::memmem::Finder` (case-sensitive) +/// or the SIMD `simd_string_utils::memmem` search (case-insensitive). No regex engine is +/// involved at any point. struct PlainTextSink<'r> { state: SinkState, finder: &'r memchr::memmem::Finder<'r>, @@ -749,16 +649,11 @@ impl Sink for PlainTextSink<'_> { let mut first = true; if self.case_insensitive { - // Lowercase the display bytes into a stack buffer; positions are 1:1 - // for ASCII so no mapping is needed. - let mut lowered = [0u8; MAX_LINE_DISPLAY_LEN]; - let len = display_bytes.len().min(MAX_LINE_DISPLAY_LEN); - for (dst, &src) in lowered[..len].iter_mut().zip(display_bytes) { - *dst = src.to_ascii_lowercase(); - } - + // The finder was built over the lowered pattern, so its needle is + // exactly the `needle_lower` expected by `memmem::find`. + let needle_lower = self.finder.needle(); let mut start_pos = 0usize; - while let Some(pos) = self.finder.find(&lowered[start_pos..len]) { + while let Some(pos) = memmem::find(&display_bytes[start_pos..], needle_lower) { let abs_start = (start_pos + pos) as u32; let abs_end = (abs_start + self.pattern_len).min(display_len); if first { @@ -1278,7 +1173,7 @@ where // setup, and line-splitting for files that can't match. if let Some(pf) = ctx.prefilter { let found = if ctx.prefilter_case_insensitive { - case_insensitive_memmem::search_packed_pair(content, pf.needle()) + memmem::find(content, pf.needle()).is_some() } else { pf.find(content).is_some() }; diff --git a/crates/fff-core/src/lib.rs b/crates/fff-core/src/lib.rs index e9a78dbad..434fcd833 100644 --- a/crates/fff-core/src/lib.rs +++ b/crates/fff-core/src/lib.rs @@ -100,69 +100,62 @@ compile_error!( Enable one, e.g. `--features ripgrep` or `--features zlob`." ); -mod background_watcher; -mod git_status_worker; -pub(crate) mod parallelism; -mod scan; -// public only for benchmarks — the inverted index is still re-exported via -// `pub use bigram_filter::*` below for external consumers. -#[doc(hidden)] -pub mod bigram_filter; -pub mod bigram_query; -pub mod constants; -mod constraints; -mod error; -mod score; -mod sort_buffer; -pub(crate) mod stable_vec; -// this is pub only for benchmarks -pub mod case_insensitive_memmem; - -pub(crate) mod simd_path; +/// Primary entry points with thread-safe [`SharedFilePicker`](shared::FilePicker) instance +pub mod shared; +pub use shared::*; -/// Core file picker: filesystem indexing, background watching, and fuzzy search. -/// +/// Core file picker single thread: filesystem indexing, background watching, and fuzzy search. /// See [`FilePicker`](file_picker::FilePicker) for the main entry point. pub mod file_picker; +pub use file_picker::*; /// Database-backed persistence: frecency, query history, LMDB plumbing. pub mod dbs; -pub use dbs::frecency; +pub use dbs::*; /// Git status caching and repository detection utilities. pub mod git; /// Live grep search with regex, plain-text, and fuzzy matching modes. -/// -/// Supports constraint filtering (file extensions, path segments, globs) -/// and parallel execution via rayon. pub mod grep; +pub use grep::*; /// Tracing/logging initialization and panic hook setup. pub mod log; -/// Path manipulation utilities: cross platform canonicalization, tilde expansion, and -/// directory distance penalties for search scoring. +/// Various path utils might be handy for you to work with fff paths pub mod path_utils; -pub use dbs::query_tracker; - /// Core data types shared across the crate. pub mod types; +pub use types::*; + +pub mod constants; +// ================================== +// these are public only for benchmarks, no backward compatibility guaranteed +#[doc(hidden)] +pub mod bigram_filter; +#[doc(hidden)] +pub mod simd_string_utils; +// ================================== + +mod background_watcher; +mod constraints; +mod error; +mod git_status_worker; mod ignore; -/// Thread-safe shared handles for [`FilePicker`], [`FrecencyTracker`], -/// and [`QueryTracker`]. -pub mod shared; -pub mod walk; +mod scan; +mod score; +mod sort_buffer; + +pub(crate) mod bigram_query; +pub(crate) mod parallelism; +pub(crate) mod simd_path; +pub(crate) mod stable_vec; +pub(crate) mod walk; -pub use bigram_filter::*; -pub use dbs::db_healthcheck::{DbHealth, DbHealthChecker}; +// fff error pub use error::{Error, Result}; + pub use fff_query_parser::*; -pub use file_picker::*; -pub use frecency::*; -pub use grep::*; -pub use query_tracker::*; -pub use shared::*; -pub use types::*; diff --git a/crates/fff-core/src/simd_string_utils/case.rs b/crates/fff-core/src/simd_string_utils/case.rs new file mode 100644 index 000000000..fc806f4d1 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/case.rs @@ -0,0 +1,168 @@ +#[inline] +pub fn ascii_swap_case(b: u8) -> u8 { + b ^ 0x20 +} + +#[inline] +fn eq_lowered_scalar(h: *const u8, needle_lower: &[u8]) -> bool { + for (i, &n) in needle_lower.iter().enumerate() { + if unsafe { *h.add(i) }.to_ascii_lowercase() != n { + return false; + } + } + true +} + +/// AVX2 only has a **signed** byte compare (`cmpgt`), but we need an +/// **unsigned** range check (`'A' <= byte <= 'Z'`). XOR-ing every byte with +/// `0x80` maps the unsigned range `[0, 255]` into the signed range +/// `[-128, 127]` preserving order, so signed `cmpgt` becomes correct +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn eq_lowered_avx2(h: *const u8, needle_lower: &[u8]) -> bool { + use core::arch::x86_64::*; + + let len = needle_lower.len(); + let mut i = 0usize; + + let flip = _mm256_set1_epi8(0x80u8 as i8); + let a_minus_1 = _mm256_set1_epi8((b'A' - 1) as i8 ^ 0x80u8 as i8); + let z_plus_1 = _mm256_set1_epi8((b'Z' + 1) as i8 ^ 0x80u8 as i8); + let bit20 = _mm256_set1_epi8(0x20u8 as i8); + + while i + 32 <= len { + let hv = unsafe { _mm256_loadu_si256(h.add(i) as *const __m256i) }; + let nv = unsafe { _mm256_loadu_si256(needle_lower.as_ptr().add(i) as *const __m256i) }; + + // Signed-domain range check selects uppercase lanes, OR bit 5 folds them. + let x = _mm256_xor_si256(hv, flip); + let ge_a = _mm256_cmpgt_epi8(x, a_minus_1); + let le_z = _mm256_cmpgt_epi8(z_plus_1, x); + let upper = _mm256_and_si256(ge_a, le_z); + let folded = _mm256_or_si256(hv, _mm256_and_si256(upper, bit20)); + + let eq = _mm256_cmpeq_epi8(folded, nv); + if _mm256_movemask_epi8(eq) != -1i32 { + return false; + } + + i += 32; + } + + while i < len { + if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] { + return false; + } + i += 1; + } + true +} + +/// Unsigned range checks (`vcge`/`vcle`) detect uppercase ASCII, bit 5 folds +/// to lowercase, then equality is checked via udot: xors the folded haystack +/// with the pre-lowered needle and dot-product the difference with itself +/// any non-zero byte produces a non-zero u32 lane. udot is emitted via inline +/// asm because `vdotq_u32` is still behind an unstable feature gate. +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon,dotprod")] +unsafe fn eq_lowered_neon_dotprod(h: *const u8, needle_lower: &[u8]) -> bool { + use core::arch::aarch64::*; + + let len = needle_lower.len(); + let mut i = 0usize; + + let a_val = vdupq_n_u8(b'A'); + let z_val = vdupq_n_u8(b'Z'); + let bit20 = vdupq_n_u8(0x20); + + while i + 16 <= len { + let hv = unsafe { vld1q_u8(h.add(i)) }; + let nv = unsafe { vld1q_u8(needle_lower.as_ptr().add(i)) }; + + let upper = vandq_u8(vcgeq_u8(hv, a_val), vcleq_u8(hv, z_val)); + let folded = vorrq_u8(hv, vandq_u8(upper, bit20)); + let xored = veorq_u8(folded, nv); + + let dots: uint32x4_t; + let zero = vdupq_n_u32(0); + unsafe { + core::arch::asm!( + "udot {d:v}.4s, {a:v}.16b, {b:v}.16b", + d = inlateout(vreg) zero => dots, + a = in(vreg) xored, + b = in(vreg) xored, + ); + } + + if vmaxvq_u32(dots) != 0 { + return false; + } + + i += 16; + } + + while i < len { + if unsafe { *h.add(i) }.to_ascii_lowercase() != needle_lower[i] { + return false; + } + i += 1; + } + true +} + +/// Case-insensitive equality of `needle_lower` against the haystack bytes +/// starting at `h`. `needle_lower` must be pre-lowercased (ASCII). +/// +/// # Safety +/// `h` must be valid for reads of `needle_lower.len()` bytes. +#[inline] +pub(crate) unsafe fn eq_lowered_case(haystack: *const u8, needle_lower: &[u8]) -> bool { + #[cfg(target_arch = "x86_64")] + { + if needle_lower.len() >= 32 && std::is_x86_feature_detected!("avx2") { + return unsafe { eq_lowered_avx2(haystack, needle_lower) }; + } + } + #[cfg(target_arch = "aarch64")] + { + if needle_lower.len() >= 16 && std::arch::is_aarch64_feature_detected!("dotprod") { + return unsafe { eq_lowered_neon_dotprod(haystack, needle_lower) }; + } + } + + eq_lowered_scalar(haystack, needle_lower) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn eq_lowered(haystack: &[u8], needle_lower: &[u8]) -> bool { + assert!(haystack.len() >= needle_lower.len()); + unsafe { eq_lowered_case(haystack.as_ptr(), needle_lower) } + } + + #[test] + fn swap_case_toggles_letters() { + assert_eq!(ascii_swap_case(b'n'), b'N'); + assert_eq!(ascii_swap_case(b'N'), b'n'); + assert_eq!(ascii_swap_case(b'z'), b'Z'); + } + + #[test] + fn eq_matches_std_semantics() { + assert!(eq_lowered(b"Hello", b"hello")); + assert!(eq_lowered(b"HELLO WORLD", b"hello")); + assert!(!eq_lowered(b"Hellp", b"hello")); + // Non-letters must not fold: '[' (0x5B) vs '{' (0x7B) differ only in bit 5. + assert!(!eq_lowered(b"A[", b"a{")); + assert!(eq_lowered(b"A{", b"a{")); + // Long inputs exercise the SIMD kernels. + let hay = b"INT STRUCT MUTEX *LOCK(STRUCT MUTEX *LOCK) { RETURN 0; }"; + let needle: Vec = hay.iter().map(|b| b.to_ascii_lowercase()).collect(); + assert!(eq_lowered(hay, &needle)); + let mut bad = needle.clone(); + *bad.last_mut().unwrap() = b'!'; + assert!(!eq_lowered(hay, &bad)); + } +} diff --git a/crates/fff-core/src/simd_string_utils/memmem.rs b/crates/fff-core/src/simd_string_utils/memmem.rs new file mode 100644 index 000000000..a5bda6011 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/memmem.rs @@ -0,0 +1,494 @@ +use super::case::{ascii_swap_case, eq_lowered_case}; +use smallvec::SmallVec; + +// Byte frequency table stolen from memchr +const BYTE_FREQUENCIES: [u8; 256] = [ + 55, 52, 51, 50, 49, 48, 47, 46, 45, 103, 242, 66, 67, 229, 44, 43, // 0x00 + 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 56, 32, 31, 30, 29, 28, // 0x10 + 255, 148, 164, 149, 136, 160, 155, 173, 221, 222, 134, 122, 232, 202, 215, 224, // 0x20 + 208, 220, 204, 187, 183, 179, 177, 168, 178, 200, 226, 195, 154, 184, 174, 126, // 0x30 + 120, 191, 157, 194, 170, 189, 162, 161, 150, 193, 142, 137, 171, 176, 185, + 167, // 0x40 A-O + 186, 112, 175, 192, 188, 156, 140, 143, 123, 133, 128, 147, 138, 146, 114, + 223, // 0x50 P-_ + 151, 249, 216, 238, 236, 253, 227, 218, 230, 247, 135, 180, 241, 233, 246, + 244, // 0x60 a-o + 231, 139, 245, 243, 251, 235, 201, 196, 240, 214, 152, 182, 205, 181, 127, + 27, // 0x70 p-DEL + 212, 211, 210, 213, 228, 197, 169, 159, 131, 172, 105, 80, 98, 96, 97, 81, // 0x80 + 207, 145, 116, 115, 144, 130, 153, 121, 107, 132, 109, 110, 124, 111, 82, 108, // 0x90 + 118, 141, 113, 129, 119, 125, 165, 117, 92, 106, 83, 72, 99, 93, 65, 79, // 0xa0 + 166, 237, 163, 199, 190, 225, 209, 203, 198, 217, 219, 206, 234, 248, 158, 239, // 0xb0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xc0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xd0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xe0 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, // 0xf0 +]; + +#[inline] +fn rank(lower: u8) -> u8 { + if lower.is_ascii_lowercase() { + BYTE_FREQUENCIES[lower as usize].max(BYTE_FREQUENCIES[ascii_swap_case(lower) as usize]) + } else { + BYTE_FREQUENCIES[lower as usize] + } +} + +/// Pick two needle positions with the rarest bytes (case-insensitive) +fn select_rare_pair(needle_lower: &[u8]) -> (usize, usize) { + debug_assert!(needle_lower.len() >= 2); + + let mut best1 = (u8::MAX, 0usize); // (rank, position) + let mut best2 = (u8::MAX, 1usize); + + for (i, &b) in needle_lower.iter().enumerate() { + let r = rank(b); + if r < best1.0 { + best2 = best1; + best1 = (r, i); + } else if r < best2.0 && i != best1.1 { + best2 = (r, i); + } + } + + let i1 = best1.1.min(best2.1); + let i2 = best1.1.max(best2.1); + (i1, i2) +} + +/// Extract a 16-bit bitmask from a NEON comparison result (each byte 0x00 or 0xFF) +/// Bit *i* of the result corresponds to byte *i* of the input vector +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +#[inline] +unsafe fn neon_movemask(v: core::arch::aarch64::uint8x16_t) -> u16 { + use core::arch::aarch64::*; + + // AND each byte with its bit-position mask, then horizontally sum each half. + static BITS: [u8; 16] = [1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128]; + let bit_mask = unsafe { vld1q_u8(BITS.as_ptr()) }; + let masked = vandq_u8(v, bit_mask); + let lo = vaddv_u8(vget_low_u8(masked)); + let hi = vaddv_u8(vget_high_u8(masked)); + (lo as u16) | ((hi as u16) << 8) +} + +/// AVX2 packed-pair kernel: scan 32 haystack positions per iteration, +/// checking two rare bytes (case-insensitive) simultaneously +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn find_packed_pair_avx2( + haystack: &[u8], + needle_lower: &[u8], + i1: usize, + i2: usize, +) -> Option { + use core::arch::x86_64::*; + + let n = needle_lower.len(); + let hlen = haystack.len(); + let ptr = haystack.as_ptr(); + let last_start = hlen - n; // last valid match-start position + + let b1 = needle_lower[i1]; + let b1_alt = if b1.is_ascii_lowercase() { + ascii_swap_case(b1) + } else { + b1 + }; + let b2 = needle_lower[i2]; + let b2_alt = if b2.is_ascii_lowercase() { + ascii_swap_case(b2) + } else { + b2 + }; + + let v1_lo = _mm256_set1_epi8(b1 as i8); + let v1_hi = _mm256_set1_epi8(b1_alt as i8); + let v2_lo = _mm256_set1_epi8(b2 as i8); + let v2_hi = _mm256_set1_epi8(b2_alt as i8); + + // Loads come from ptr+offset+i1 and ptr+offset+i2, so we need offset + max(i1,i2) + 31 < hlen. + let max_idx = i1.max(i2); + let max_offset = hlen.saturating_sub(max_idx + 32); + let mut offset = 0usize; + + while offset <= max_offset { + let chunk1 = unsafe { _mm256_loadu_si256(ptr.add(offset + i1) as *const __m256i) }; + let chunk2 = unsafe { _mm256_loadu_si256(ptr.add(offset + i2) as *const __m256i) }; + + // Case-insensitive match: OR both case variants, then AND the two positions. + let eq1 = _mm256_or_si256( + _mm256_cmpeq_epi8(chunk1, v1_lo), + _mm256_cmpeq_epi8(chunk1, v1_hi), + ); + let eq2 = _mm256_or_si256( + _mm256_cmpeq_epi8(chunk2, v2_lo), + _mm256_cmpeq_epi8(chunk2, v2_hi), + ); + + let mut mask = _mm256_movemask_epi8(_mm256_and_si256(eq1, eq2)) as u32; + + // Candidates are visited in increasing position order, so the first + // verified candidate is the leftmost match + while mask != 0 { + let bit = mask.trailing_zeros() as usize; + let candidate = offset + bit; + if candidate > last_start { + return None; + } + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + mask &= mask - 1; + } + + offset += 32; + } + + // handle remaining characters + if offset <= last_start { + let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) { + i1 + } else { + i2 + }; + let rare_byte = needle_lower[rare_pos]; + let tail_start = offset + rare_pos; + let tail_end = last_start + rare_pos + 1; + if tail_start < tail_end { + let tail_space = &haystack[tail_start..tail_end]; + if rare_byte.is_ascii_lowercase() { + for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } else { + for pos in memchr::memchr_iter(rare_byte, tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } + } + } + + None +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn find_packed_pair_neon( + haystack: &[u8], + needle_lower: &[u8], + i1: usize, + i2: usize, +) -> Option { + use core::arch::aarch64::*; + + let n = needle_lower.len(); + let hlen = haystack.len(); + let ptr = haystack.as_ptr(); + let last_start = hlen - n; + + let b1 = needle_lower[i1]; + let b1_alt = if b1.is_ascii_lowercase() { + ascii_swap_case(b1) + } else { + b1 + }; + let b2 = needle_lower[i2]; + let b2_alt = if b2.is_ascii_lowercase() { + ascii_swap_case(b2) + } else { + b2 + }; + + let v1_lo = vdupq_n_u8(b1); + let v1_hi = vdupq_n_u8(b1_alt); + let v2_lo = vdupq_n_u8(b2); + let v2_hi = vdupq_n_u8(b2_alt); + + let max_idx = i1.max(i2); + let max_offset = hlen.saturating_sub(max_idx + 16); + let mut offset = 0usize; + + while offset <= max_offset { + let chunk1 = unsafe { vld1q_u8(ptr.add(offset + i1)) }; + let chunk2 = unsafe { vld1q_u8(ptr.add(offset + i2)) }; + + // Case-insensitive match: OR both case variants, then AND the two positions. + let eq1 = vorrq_u8(vceqq_u8(chunk1, v1_lo), vceqq_u8(chunk1, v1_hi)); + let eq2 = vorrq_u8(vceqq_u8(chunk2, v2_lo), vceqq_u8(chunk2, v2_hi)); + + let mut mask = unsafe { neon_movemask(vandq_u8(eq1, eq2)) }; + + while mask != 0 { + let bit = mask.trailing_zeros() as usize; + let candidate = offset + bit; + if candidate > last_start { + return None; + } + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + mask &= mask - 1; + } + + offset += 16; + } + + // Tail: remaining positions that couldn't fill a full vector. + if offset <= last_start { + let rare_pos = if rank(needle_lower[i1]) <= rank(needle_lower[i2]) { + i1 + } else { + i2 + }; + let rare_byte = needle_lower[rare_pos]; + let tail_start = offset + rare_pos; + let tail_end = last_start + rare_pos + 1; + if tail_start < tail_end { + let tail_space = &haystack[tail_start..tail_end]; + if rare_byte.is_ascii_lowercase() { + for pos in memchr::memchr2_iter(rare_byte, ascii_swap_case(rare_byte), tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } else { + for pos in memchr::memchr_iter(rare_byte, tail_space) { + let candidate = offset + pos; + if unsafe { eq_lowered_case(ptr.add(candidate), needle_lower) } { + return Some(candidate); + } + } + } + } + } + + None +} + +fn find_first_byte_with_memchr(haystack: &[u8], needle_lower: &[u8]) -> Option { + let n = needle_lower.len(); + debug_assert!(n >= 1 && n <= haystack.len()); + + let search_space = &haystack[..=haystack.len() - n]; + let first = needle_lower[0]; + + if first.is_ascii_lowercase() { + let alt = ascii_swap_case(first); + for pos in memchr::memchr2_iter(first, alt, search_space) { + if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } { + return Some(pos); + } + } + } else { + for pos in memchr::memchr_iter(first, search_space) { + if unsafe { eq_lowered_case(haystack.as_ptr().add(pos), needle_lower) } { + return Some(pos); + } + } + } + None +} + +/// ASCII case-insensitive substring search returning the leftmost match +/// position. `needle_lower` must be pre-lowercased (ASCII). +// pub because it is used in out of the crate benchmarks +#[doc(hidden)] // it's pub only for benches +pub fn find(haystack: &[u8], needle_lower: &[u8]) -> Option { + let n = needle_lower.len(); + if n == 0 { + return Some(0); + } + if n > haystack.len() { + return None; + } + + if n == 1 { + let first = needle_lower[0]; + return if first.is_ascii_lowercase() { + memchr::memchr2(first, ascii_swap_case(first), haystack) + } else { + memchr::memchr(first, haystack) + }; + } + + #[cfg_attr( + not(any(target_arch = "x86_64", target_arch = "aarch64")), + allow(unused_variables) + )] + let (i1, i2) = select_rare_pair(needle_lower); + + #[cfg(target_arch = "x86_64")] + { + if std::is_x86_feature_detected!("avx2") { + // Need enough haystack for at least one vector load. + let max_idx = i1.max(i2); + if haystack.len() >= max_idx + 32 { + return unsafe { find_packed_pair_avx2(haystack, needle_lower, i1, i2) }; + } + } + } + + #[cfg(target_arch = "aarch64")] + { + // Packed-pair wins when the first byte is common (memchr2 drowns in + // false positives), but a rare first byte (z, q, x, ...) makes + // memchr2's raw throughput dominate. Threshold 200 on the frequency + // table splits common letters (s=243, e=253) from rare ones (z=152). + let first_byte_rank = rank(needle_lower[0]); + let max_idx = i1.max(i2); + if first_byte_rank >= 200 && haystack.len() >= max_idx + 16 { + return unsafe { find_packed_pair_neon(haystack, needle_lower, i1, i2) }; + } + } + + // fallbacks to memchr based implementation cause we still have it and it supports more SIMD backends + // TODO convert all the supported backend by memchr and get rid of the fallback + find_first_byte_with_memchr(haystack, needle_lower) +} + +/// A case insensitive find that works better with smaller strings, doesn't unwrap a complicated +/// AVX backend we use for grep because only cpu flags check takes usually more time than find itself +pub fn find_case_insensitive_short(haystack: &[u8], needle: &[u8]) -> Option { + debug_assert!(haystack.len() < 1024); + let mut needle_lower: SmallVec<[u8; 64]> = SmallVec::from_slice(needle); + needle_lower.make_ascii_lowercase(); + + find(haystack, &needle_lower) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reference_find(haystack: &[u8], needle_lower: &[u8]) -> Option { + if needle_lower.is_empty() { + return Some(0); + } + if needle_lower.len() > haystack.len() { + return None; + } + haystack + .windows(needle_lower.len()) + .position(|w| w.eq_ignore_ascii_case(needle_lower)) + } + + #[test] + fn basic_case_insensitive() { + assert_eq!(find(b"Hello World", b"hello"), Some(0)); + assert_eq!(find(b"Hello World", b"world"), Some(6)); + assert_eq!(find(b"NOMORE bugs", b"nomore"), Some(0)); + assert_eq!(find(b"Hello World", b"xyz"), None); + assert!(find(b"Hello World", b"o w").is_some()); + } + + #[test] + fn edge_cases() { + assert_eq!(find(b"ab", b"ab"), Some(0)); + assert_eq!(find(b"AB", b"ab"), Some(0)); + assert_eq!(find(b"a", b"ab"), None); + assert_eq!(find(b"anything", b""), Some(0)); + assert_eq!(find(b"", b"x"), None); + assert_eq!(find(b"xxA", b"a"), Some(2)); + assert_eq!(find(b"xx:", b":"), Some(2)); + } + + #[test] + fn returns_leftmost_match() { + assert_eq!(find(b"foo FOO foo", b"foo"), Some(0)); + let mut big = vec![b'.'; 300]; + big[100..103].copy_from_slice(b"FoO"); + big[200..203].copy_from_slice(b"foo"); + assert_eq!(find(&big, b"foo"), Some(100)); + } + + #[test] + fn non_letter_bytes_do_not_case_fold() { + // '[' (0x5B) and '{' (0x7B) differ only in bit 5 but are not letters. + // A fold implemented as a bare `| 0x20` would falsely match these. + assert_eq!(find(b"A[", b"a{"), None); + assert_eq!(find(b"x@y", b"x`y"), None); + assert_eq!(find(b"a]b", b"a}b"), None); + assert_eq!(find(b"A{", b"a{"), Some(0)); + } + + #[test] + fn matches_reference_on_random_inputs() { + // Deterministic xorshift PRNG — no external deps. + let mut state = 0x9E3779B97F4A7C15u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + + // Alphabet with letters, both-case pairs, and 0x20-differing symbols. + let alphabet = b"aAbBzZ [{@`]}^~_0.\n"; + for _ in 0..2000 { + let hlen = (next() % 200) as usize; + let nlen = (next() % 8) as usize; + let haystack: Vec = (0..hlen) + .map(|_| alphabet[(next() % alphabet.len() as u64) as usize]) + .collect(); + let needle: Vec = (0..nlen) + .map(|_| alphabet[(next() % alphabet.len() as u64) as usize].to_ascii_lowercase()) + .collect(); + + assert_eq!( + find(&haystack, &needle), + reference_find(&haystack, &needle), + "mismatch for haystack={:?} needle={:?}", + haystack, + needle, + ); + } + } + + #[test] + fn long_haystack_simd_paths() { + let haystack = + b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaTHIS_IS_A_LONG_NEEDLE_TESTbbbbbbbbbbbbbbbbbb"; + assert_eq!(find(haystack, b"this_is_a_long_needle_test"), Some(32)); + assert_eq!(find(haystack, b"this_is_a_long_needle_testz"), None); + + // Needle >= 16 bytes exercises SIMD verify. + let haystack2 = b"int STRUCT MUTEX *LOCK(struct mutex *lock) { return 0; }"; + assert_eq!(find(haystack2, b"struct mutex *lock"), Some(4)); + + let upper_hay = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + assert_eq!(find(upper_hay, b"qrstuvwxyz0123456789a"), Some(16)); + assert_eq!(find(upper_hay, b"qrstuvwxyz01234567899"), None); + + // Needle at very end / very start. + let end_hay = b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxfind_me"; + assert_eq!(find(end_hay, b"find_me"), Some(end_hay.len() - 7)); + assert_eq!(find(end_hay, b"xx"), Some(0)); + + // 1KB haystack with needle near the end. + let mut big = vec![b'z'; 1024]; + big[1000..1010].copy_from_slice(b"hElLo_WoRl"); + assert_eq!(find(&big, b"hello_wo"), Some(1000)); + assert_eq!(find(&big, b"hello_world"), None); + } + + #[test] + fn rare_pair_selection() { + let (i1, i2) = select_rare_pair(b"nomore"); + let ranks: Vec = b"nomore".iter().map(|&b| rank(b)).collect(); + let (r1, r2) = (ranks[i1], ranks[i2]); + for (i, &r) in ranks.iter().enumerate() { + if i != i1 && i != i2 { + assert!(r1 <= r || r2 <= r, "pair ({i1},{i2}) not optimal"); + } + } + } +} diff --git a/crates/fff-core/src/simd_string_utils/mod.rs b/crates/fff-core/src/simd_string_utils/mod.rs new file mode 100644 index 000000000..bb66b4388 --- /dev/null +++ b/crates/fff-core/src/simd_string_utils/mod.rs @@ -0,0 +1,5 @@ +//! SIMD-accelerated string utilities: case flipping/folding and +//! case-insensitive substring search. + +pub mod case; +pub mod memmem; diff --git a/crates/fff-nvim/src/bin/bench_ci_memmem.rs b/crates/fff-nvim/src/bin/bench_ci_memmem.rs deleted file mode 100644 index 0de50230d..000000000 --- a/crates/fff-nvim/src/bin/bench_ci_memmem.rs +++ /dev/null @@ -1,222 +0,0 @@ -/// Benchmark: AVX2 vs scalar case-insensitive memmem prefilter. -/// -/// Loads all non-binary file contents from a repo, then times both -/// implementations scanning every file for the query. -/// -/// Usage: -/// cargo build --release --bin bench_ci_memmem -/// ./target/release/bench_ci_memmem --path ./big-repo --query "nomore" --iters 5 -use fff::case_insensitive_memmem; -use std::io::Read; -use std::path::Path; -use std::time::Instant; - -fn fmt_dur(us: u128) -> String { - if us > 1_000_000 { - format!("{:.2}s", us as f64 / 1_000_000.0) - } else if us > 1000 { - format!("{:.2}ms", us as f64 / 1000.0) - } else { - format!("{}µs", us) - } -} - -fn stats(times_us: &mut [u128]) -> (u128, u128, u128, u128) { - times_us.sort(); - let sum: u128 = times_us.iter().sum(); - let mean = sum / times_us.len() as u128; - let median = times_us[times_us.len() / 2]; - (mean, median, times_us[0], times_us[times_us.len() - 1]) -} - -fn detect_binary(path: &Path, size: u64) -> bool { - if size == 0 { - return false; - } - let Ok(file) = std::fs::File::open(path) else { - return false; - }; - let mut reader = std::io::BufReader::with_capacity(1024, file); - let mut buf = [0u8; 512]; - let n = reader.read(&mut buf).unwrap_or(0); - buf[..n].contains(&0) -} - -#[cfg(not(feature = "zlob"))] -fn load_file_contents(base_path: &Path) -> Vec> { - use ignore::WalkBuilder; - - let mut contents = Vec::new(); - let max_size = 10 * 1024 * 1024u64; - - WalkBuilder::new(base_path) - .hidden(false) - .git_ignore(true) - .git_exclude(true) - .git_global(true) - .ignore(true) - .follow_links(false) - .build() - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().is_some_and(|ft| ft.is_file())) - .for_each(|entry| { - let path = entry.path(); - let size = entry.metadata().ok().map_or(0, |m| m.len()); - if size == 0 || size > max_size || detect_binary(path, size) { - return; - } - if let Ok(data) = std::fs::read(path) { - contents.push(data); - } - }); - - contents -} - -#[cfg(feature = "zlob")] -fn load_file_contents(base_path: &Path) -> Vec> { - use std::cell::RefCell; - use zlob::walk::{WalkBuilder, WalkFlags, WalkMetadata, WalkState}; - - let contents = RefCell::new(Vec::new()); - let max_size = 10 * 1024 * 1024u64; - - let _ = WalkBuilder::new(base_path) - .expect("WalkBuilder::new") - .options(WalkFlags::GITIGNORE) - .metadata(WalkMetadata::SIZE) - .run_serial(|entry| { - if !entry.is_file() { - return WalkState::Continue; - } - let path = entry.path(); - let size = entry.size().unwrap_or(0); - if size == 0 || size > max_size || detect_binary(path, size) { - return WalkState::Continue; - } - if let Ok(data) = std::fs::read(path) { - contents.borrow_mut().push(data); - } - WalkState::Continue - }); - - contents.into_inner() -} - -fn bench_impl( - label: &str, - contents: &[Vec], - needle_lower: &[u8], - total_bytes: u64, - iters: usize, - search_fn: fn(&[u8], &[u8]) -> bool, -) { - eprintln!("\n [{}]", label); - let mut times = Vec::with_capacity(iters); - let mut hit_count = 0u32; - - for i in 0..iters { - let t = Instant::now(); - let mut hits = 0u32; - for content in contents { - if search_fn(content, needle_lower) { - hits += 1; - } - } - let us = t.elapsed().as_micros(); - times.push(us); - hit_count = hits; - let tp = total_bytes as f64 / (us as f64 / 1_000_000.0) / (1024.0 * 1024.0 * 1024.0); - eprintln!( - " iter {}: {} ({} hits, {:.2} GB/s)", - i + 1, - fmt_dur(us), - hits, - tp - ); - } - - let (mean, median, min, max) = stats(&mut times); - let med_tp = total_bytes as f64 / (median as f64 / 1_000_000.0) / (1024.0 * 1024.0 * 1024.0); - eprintln!( - " mean: {} median: {} ({:.2} GB/s) min: {} max: {} hits: {}", - fmt_dur(mean), - fmt_dur(median), - med_tp, - fmt_dur(min), - fmt_dur(max), - hit_count - ); -} - -fn main() { - let args: Vec = std::env::args().collect(); - - let path = args - .iter() - .position(|a| a == "--path") - .and_then(|i| args.get(i + 1)) - .map(|s| s.as_str()) - .unwrap_or("."); - - let query = args - .iter() - .position(|a| a == "--query") - .and_then(|i| args.get(i + 1)) - .map(|s| s.as_str()) - .unwrap_or("TODO"); - - let iters: usize = args - .iter() - .position(|a| a == "--iters") - .and_then(|i| args.get(i + 1)) - .and_then(|s| s.parse().ok()) - .unwrap_or(5); - - let repo = std::path::PathBuf::from(path); - if !repo.exists() { - eprintln!("Path not found: {}", path); - eprintln!("Usage: bench_ci_memmem --path --query [--iters N]"); - std::process::exit(1); - } - - let canonical = fff::path_utils::canonicalize(&repo).expect("Failed to canonicalize path"); - let needle_lower: Vec = query.bytes().map(|b| b.to_ascii_lowercase()).collect(); - - eprintln!("=== bench_ci_memmem: AVX2 vs Scalar ==="); - eprintln!("Path: {}", canonical.display()); - eprintln!("Query: \"{}\"", query); - eprintln!("Needle: {:?}", std::str::from_utf8(&needle_lower).unwrap()); - eprintln!("Iters: {}", iters); - - eprint!("\n[1/2] Loading files into memory... "); - let t = Instant::now(); - let contents = load_file_contents(&canonical); - let total_bytes: u64 = contents.iter().map(|c| c.len() as u64).sum(); - eprintln!( - "{} files, {:.1} MB in {:.2}s", - contents.len(), - total_bytes as f64 / (1024.0 * 1024.0), - t.elapsed().as_secs_f64() - ); - - eprintln!("\n[2/2] Benchmarking memmem prefilter (scanning ALL files)"); - - bench_impl( - "Packed pair: (AVX2 two-byte scan)", - &contents, - &needle_lower, - total_bytes, - iters, - case_insensitive_memmem::search_packed_pair, - ); - - bench_impl( - "scalar: memchr2 first-byte + AVX2 verify", - &contents, - &needle_lower, - total_bytes, - iters, - case_insensitive_memmem::search, - ); -} diff --git a/crates/fff-query-parser/src/parser.rs b/crates/fff-query-parser/src/parser.rs index c3834e1e7..88a32e91f 100644 --- a/crates/fff-query-parser/src/parser.rs +++ b/crates/fff-query-parser/src/parser.rs @@ -331,7 +331,8 @@ fn parse_token<'a, C: ParserConfig>(token: &'a str, config: &C) -> Option Option { haystack.iter().position(|&b| b == needle)