From ba4800d8ece769a8420ab91fe4a4b2df341375f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Sun, 15 Mar 2026 12:52:56 +0100 Subject: [PATCH] Optimize insert probing with early EMPTY check In find_or_find_insert_index_inner, when the first empty-or-deleted slot found is EMPTY (not a tombstone), return immediately without the expensive match_empty() SIMD group scan. This is the common case when there are no tombstones in the table. An EMPTY byte at the computed index also guarantees the index is valid (not a small-table wrapping false positive), so the fix_insert_index check is skipped too. Benchmarks (aarch64, 10 runs, median): insert_foldhash_serial: 11,734 -> 11,410 ns (-2.8%) rehash_in_place: 220,745 -> 189,163 ns (-14.3%) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/raw.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/raw.rs b/src/raw.rs index 778ccbb60..adc42939a 100644 --- a/src/raw.rs +++ b/src/raw.rs @@ -1832,6 +1832,24 @@ impl RawTableInner { // insertion slot from the group if we don't have one yet. if likely(insert_index.is_none()) { insert_index = self.find_insert_index_in_group(&group, &probe_seq); + + // When the found slot is EMPTY (not DELETED), we can stop probing + // immediately without computing the more expensive match_empty() on + // the whole group. This is the common case when there are no + // tombstones in the table. + // + // An EMPTY slot in this group guarantees the key we're looking for + // can't be in a subsequent group. Additionally, a true EMPTY byte + // at the computed index means the index is valid (not a small-table + // false positive from wrapping), so fix_insert_index is unnecessary. + if let Some(idx) = insert_index { + // SAFETY: The index was found by `find_insert_index_in_group` which + // guarantees it is in the range `0..=self.bucket_mask`, so calling + // `self.ctrl(idx)` is safe. + if likely(unsafe { *self.ctrl(idx) } == Tag::EMPTY) { + return Err(idx); + } + } } if let Some(insert_index) = insert_index {