Fix EliasFanoVec::rank returning out-of-range index for last element of large bucket

src/elias_fano/mod.rs

@@ -381,18 +381,18 @@ impl EliasFanoVec {
                             }
                         }
 
-                        // update the cursor because we use it for the final index calculation
+                        // `final_bound` is absolute; the final return adds `start_index_lower`,
+                        // so we have to subtract it here to match the relative cursor coming out of the linear search.
                         if INDEX {
-                            cursor = final_bound as isize + direction;
+                            cursor = final_bound as isize + direction - start_index_lower as isize;
                         }
                         break;
                     }
                 }
 
                 return if INDEX {
                     // the loop ended because the element at cursor has a larger upper index,
-                    // so we return the previous element count
-                    // (element at curser - 1, +1 because count is not 0 based)
+                    // so we return the previous element count (element at curser - 1 + 1 because count is not 0 based)
                     start_index_lower as u64 + cursor as u64
                 } else {
                     (query_masked_upper | lower_candidate) + self.universe_zero

src/elias_fano/tests.rs

@@ -524,3 +524,15 @@ fn test_empty_ef_vec() {
     assert_eq!(ef.rank(3), 0);
     assert_eq!(ef.delta(0), None);
 }
+
+/// Regression for a bin-search-fallback off-by-`start_index_lower` in `rank` when
+/// the query hits the last element of a bucket (in the upper array) and has already fallen back to binary search.
+/// Indices 5..8 share upper-bucket 4, querying 1004 should return 8, but was off by the `start_index_lower` of the search.
+#[test]
+fn test_rank_last_element_of_large_bucket() {
+    let term_ids: Vec<u64> = vec![0, 500, 600, 800, 900, 1_001, 1_002, 1_003, 1_004, 3_000];
+    let ef = EliasFanoVec::from_slice(&term_ids);
+    for (i, &t) in term_ids.iter().enumerate() {
+        assert_eq!(ef.rank(t) as usize, i, "rank({t}) expected {i}");
+    }
+}