From 5251fc48180c0d4ba25a396c00971aea4ea48a63 Mon Sep 17 00:00:00 2001 From: kevin9327 Date: Fri, 21 Aug 2026 20:23:34 +0900 Subject: [PATCH] fix(fff-search): keep filename_offset on a char boundary for non-UTF-8 names new_from_walk_bytes stored the caller's basename_offset, which indexes the raw path bytes, as the filename_offset into a String produced by from_utf8_lossy. For a name with invalid UTF-8 the lossy conversion replaces each ill-formed byte with a 3-byte U+FFFD, shifting every later index, so the offset could land inside a replacement char. Slicing the relative path there then panicked on the Rayon scan thread and aborted the process: end byte index N is not a char boundary; it is inside '\u{fffd}' Re-derive the split from the lossy string when a byte was replaced (Cow::Owned), matching what the pure-Rust walker path already does; valid UTF-8 keeps the fast raw offset with no rfind. Adds a unit test. Fixes #799. Co-Authored-By: Claude Opus 4.8 --- crates/fff-core/src/file_picker.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/fff-core/src/file_picker.rs b/crates/fff-core/src/file_picker.rs index bbb62096b..54f5b8857 100644 --- a/crates/fff-core/src/file_picker.rs +++ b/crates/fff-core/src/file_picker.rs @@ -504,11 +504,17 @@ impl FileItem { modified: u64, ) -> (Self, String) { let is_binary = is_known_binary_extension(path); - // SAFETY-ish: paths on macOS/Linux are bytes; lossy conversion mirrors - // the existing `to_string_lossy()` behavior on non-UTF8 names. - let rel_str = String::from_utf8_lossy(relative_path).into_owned(); - let item = Self::new_raw(basename_offset, size, modified, git_status, is_binary); - (item, rel_str) + // basename_offset indexes the raw bytes; from_utf8_lossy keeps those + // indices only for valid UTF-8, so re-derive when a byte was replaced (#799). + let rel = String::from_utf8_lossy(relative_path); + let fname_offset = match &rel { + std::borrow::Cow::Borrowed(_) => basename_offset, + std::borrow::Cow::Owned(s) => { + s.rfind(std::path::is_separator).map_or(0, |i| i + 1) as u16 + } + }; + let item = Self::new_raw(fname_offset, size, modified, git_status, is_binary); + (item, rel.into_owned()) } pub(crate) fn update_frecency_scores( @@ -2395,6 +2401,17 @@ pub(crate) fn hint_allocator_collect() { mod tests { use super::*; + #[test] + fn walk_bytes_offset_stays_on_a_char_boundary_for_non_utf8_names() { + // 0xFF becomes a 3-byte U+FFFD, so the raw basename_offset (2, past + // "\xff/") would slice inside it; the split must stay on a boundary (#799). + let (item, rel) = + FileItem::new_from_walk_bytes(Path::new("x.txt"), b"\xff/.txt", 2, None, 0, 0); + let (dir, file) = rel.split_at(item.path.filename_offset as usize); + assert_eq!(dir, "\u{fffd}/"); + assert_eq!(file, ".txt"); + } + /// The watcher must watch every ancestor directory up to `base_path`, /// not just the immediate parents of indexed files. The dir table is /// built from the walker's visited dirs, so pure ancestors (dirs that