From 80ee097e462195e2da5b4f0bc64f9858be42c4ea Mon Sep 17 00:00:00 2001 From: Benjamin Demaille Date: Tue, 11 Aug 2026 22:04:57 +0200 Subject: [PATCH] perf: prefetch the next candidates' text in the merge loop Phase 4 had become the largest ext-mem cost, and its merge CPU (19.2 s on 80 MB of DNA) exceeded the entire CPU of the in-memory path. The merge is latency-bound: the tied branch dereferences the text at two random addresses, and the address for step i+1 is not known until step i retires, so there is no memory-level parallelism and the hardware prefetcher cannot see the pattern. The candidate positions themselves live in the two index arrays, which are sequential and already in cache, so the addresses several steps ahead are known even though the dependent loads are not. Issue them as prefetches, offset by the current boundary LCP `m`, which estimates where the next scans start. This is not the prefetch recorded as a negative result in `lcp.rs`. That one sat inside the strided scan loop, which the hardware prefetcher already covers. This one targets the random access, which it cannot. Apple M4 Max, 12 threads, ext-mem, output verified identical to the in-memory suffix array on both inputs: chr21.0123, 80 MB phase4 merge CPU 19.20 s -> 12.42 s phase4 wall 2.17 s -> 1.46 s total 3.55 s -> 2.90 s CPU 33.8 -> 27.0 s chr21.fa, 47.5 MB phase4 merge CPU 7.99 s -> 6.76 s total 2.56 s -> 1.99 s CPU 23.3 -> 18.0 s Co-Authored-By: Claude Opus 5 (1M context) --- src/sample_sort.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/sample_sort.rs b/src/sample_sort.rs index daa9264..0d6f3dd 100644 --- a/src/sample_sort.rs +++ b/src/sample_sort.rs @@ -35,6 +35,36 @@ use crate::lcp::{LcpDispatch, Symbol}; use crate::limits::{LimitProvider, PlainText}; use rayon::join; +/// How many merge steps ahead the text prefetch runs. Large enough to cover a +/// DRAM round trip at the merge's step rate, small enough that the prefetched +/// line is still resident when the step that needs it arrives. +const PREFETCH_DISTANCE: usize = 8; + +/// Hint the CPU to start pulling `text[at]` into cache. +/// +/// A no-op on targets without a stable prefetch intrinsic, and harmless when +/// `at` is out of bounds: the address is never dereferenced, only used as a +/// prefetch operand, and prefetch instructions on both supported targets +/// ignore faulting addresses. +#[inline(always)] +fn prefetch_symbol(text: &[S], at: usize) { + let _ = (text, at); + #[cfg(target_arch = "x86_64")] + unsafe { + std::arch::x86_64::_mm_prefetch( + text.as_ptr().add(at.min(text.len())) as *const i8, + std::arch::x86_64::_MM_HINT_T0, + ); + } + #[cfg(target_arch = "aarch64")] + unsafe { + // `core::arch::aarch64::_prefetch` is still unstable, so emit the + // instruction directly. `prfm` never faults. + let p = text.as_ptr().add(at.min(text.len())); + std::arch::asm!("prfm pldl1keep, [{p}]", p = in(reg) p, options(nostack, readonly, preserves_flags)); + } +} + /// Tunable options for SA construction. #[derive(Clone, Debug)] pub struct Opts { @@ -295,6 +325,26 @@ pub(crate) fn merge( let mut lim_b_cache: Option<(usize, usize)> = None; while i_a < len_a && i_b < len_b { + // The tied branch below dereferences the text at two *random* + // addresses, and the address for step `i + 1` is not known until step + // `i` retires, so there is no memory-level parallelism to exploit and + // the hardware prefetcher cannot see the pattern either. But the + // candidate positions themselves live in `arr_a` / `arr_b`, which are + // sequential and already in cache, so the addresses a few steps ahead + // *are* known. Issue them now. + // + // This is not the prefetch that was tried and reverted in `lcp.rs`: + // that one sat inside the strided scan loop, which the hardware + // prefetcher already covers. Here the access is random, which is the + // case hardware cannot predict. `m` is the current boundary LCP and a + // good estimate of where the next scans will start. + if i_a + PREFETCH_DISTANCE < len_a { + prefetch_symbol(text, arr_a[i_a + PREFETCH_DISTANCE].to_usize() + m); + } + if i_b + PREFETCH_DISTANCE < len_b { + prefetch_symbol(text, arr_b[i_b + PREFETCH_DISTANCE].to_usize() + m); + } + let l_a = lcp_a[i_a].to_usize(); // (output_a, lcp_for_output, new_m)