From 795647067bfeccba4a7e4dfddafb2a1a10fee837 Mon Sep 17 00:00:00 2001 From: flupkede Date: Tue, 22 Sep 2026 23:20:44 +0200 Subject: [PATCH 01/77] fix(embed): reuse serve's embedding pool in incremental refresh (todo #135) The incremental-refresh and watcher paths constructed a fresh EmbeddingService per call; in serve the process-global LMDB registry refused the second open of the persistent cache, silently degrading every refresh to cache-less re-embedding. Thread the serve-wide pool through and reuse pool.get(model) so the process-shared cache is hit; CLI/standalone-MCP processes keep direct construction (no pool there). --- src/index/manager.rs | 142 ++++++++++++++++++++++++++++++++++++++----- src/mcp/runtime.rs | 3 +- src/serve/mod.rs | 23 ++++++- src/serve/tui.rs | 2 + 4 files changed, 151 insertions(+), 19 deletions(-) diff --git a/src/index/manager.rs b/src/index/manager.rs index 5076374f..1a0ddaa2 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -20,7 +20,7 @@ use crate::constants::{ DB_DIR_NAME, DEFAULT_FSW_DEBOUNCE_MS, FILE_META_DB_NAME, LANG_CSHARP, LANG_TYPESCRIPT, SCIP_CSHARP_DEBOUNCE_MS, SCIP_TYPESCRIPT_DEBOUNCE_MS, WRITER_LOCK_FILE, }; -use crate::embed::ModelType; +use crate::embed::{EmbeddingServicePool, ModelType}; use crate::fts::FtsStore; use crate::symbols::{RebuildScope, SymbolIndexer, SymbolIndexerRegistry}; use crate::vectordb::VectorStore; @@ -297,6 +297,9 @@ pub struct IndexManager { git_head_watcher: Option, /// Shared stores for concurrent access stores: Arc, + /// Serve-wide embedding-service pool (`None` outside serve — CLI/standalone + /// MCP run in their own process where direct construction is correct). + embedding_pool: Option>, /// Per-language symbol indexer registry (C# etc.) symbol_registry: Arc, } @@ -311,6 +314,24 @@ fn is_ts_extension(path: &Path) -> bool { ) } +/// Where an embed pass gets its `EmbeddingService`: the serve-wide pool +/// (which already holds the persistent cache's LMDB env open) or a fresh +/// service (CLI / standalone MCP, own process). +enum EmbeddingSource { + Pool(Arc), + Fresh, +} + +/// Decide the embed source from the optional pool. Pure seam so the +/// `Some`/`None` arm choice shared by both embed call sites is unit-testable +/// without loading a model. +fn embed_service_choice(pool: Option<&Arc>) -> EmbeddingSource { + match pool { + Some(p) => EmbeddingSource::Pool(Arc::clone(p)), + None => EmbeddingSource::Fresh, + } +} + impl IndexManager { /// Cancellation guard shared by every cancellable indexing function. /// @@ -394,6 +415,7 @@ impl IndexManager { watcher, git_head_watcher: Some(git_head_watcher), stores, + embedding_pool: None, symbol_registry: Arc::new(SymbolIndexerRegistry::new()), }) } @@ -445,9 +467,13 @@ impl IndexManager { /// # Arguments /// * `codebase_path` - Path to the codebase to index /// * `stores` - Shared stores for concurrent access (created by caller) + /// * `embedding_pool` - Serve-wide embedding-service pool; `Some` in serve + /// (reuses the process-shared persistent cache), `None` in standalone + /// processes (direct construction — no pool to conflict with). pub async fn new_without_refresh>( codebase_path: P, stores: Arc, + embedding_pool: Option>, ) -> Result { let path_buf = codebase_path.as_ref().to_path_buf(); let db_path = path_buf.join(DB_DIR_NAME); @@ -490,6 +516,7 @@ impl IndexManager { watcher, git_head_watcher: Some(git_head_watcher), stores, + embedding_pool, symbol_registry: Arc::new(SymbolIndexerRegistry::new()), }) } @@ -548,11 +575,18 @@ impl IndexManager { /// /// This checks for changed/deleted files since last index and updates /// the index accordingly. Uses the shared stores to avoid lock conflicts. + /// + /// `embedding_pool` — when `Some`, embeddings are produced by the serve + /// pool's service so the process-shared persistent cache is reused; a + /// fresh `EmbeddingService` in the serve process would be refused by the + /// process-global LMDB registry (the pool already holds the cache open) + /// and silently degrade to cache-less re-embedding. pub async fn perform_incremental_refresh_with_stores( codebase_path: &Path, db_path: &Path, stores: &SharedStores, cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::chunker::SemanticChunker; @@ -767,6 +801,7 @@ impl IndexManager { // DURING the (long, core-saturating) embed pass is observed // per-file, not only once the whole batch returns. let batch_cancel = cancel_token.clone(); + let embed_source = embed_service_choice(embedding_pool); let embedded_chunks = tokio::task::spawn_blocking( move || -> Result> { let mut chunker = SemanticChunker::new(100, 2000, 10); @@ -792,17 +827,39 @@ impl IndexManager { return Ok(Vec::new()); } - let mut embedding_service = EmbeddingService::with_cache_dir( - embed_model, - Some(cache_dir_for_batch.as_path()), - )?; // NOTE: embed_chunks runs a single ONNX inference over // the whole batch atomically, so it is not interruptible // mid-call. Worst-case cancel latency is bounded to one // batch's embed (INCREMENTAL_REFRESH_BATCH_SIZE=200 // files); the per-file check above bounds the read/chunk // phase that precedes it. - embedding_service.embed_chunks(all_chunks) + match embed_source { + // Serve process: the pool already holds the + // persistent cache's LMDB env open — a fresh + // EmbeddingService here is refused by the + // process-global registry and would re-embed + // everything without cache. + EmbeddingSource::Pool(pool) => { + let service = pool.get(embed_model)?; + // Recover from poisoning: a panicked embed + // service still yields a valid guard (the + // batch's embed_chunks result is what carries + // correctness); a poison here must not + // hard-fail every later batch for the + // process lifetime. + let mut guard = service + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.embed_chunks(all_chunks) + } + EmbeddingSource::Fresh => { + let mut embedding_service = EmbeddingService::with_cache_dir( + embed_model, + Some(cache_dir_for_batch.as_path()), + )?; + embedding_service.embed_chunks(all_chunks) + } + } }, ) .await @@ -968,6 +1025,7 @@ impl IndexManager { stores: &SharedStores, model_override: Option, cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, ) -> Result<()> { use crate::cache::FileMetaStore; use anyhow::Context; @@ -1085,8 +1143,14 @@ impl IndexManager { info!("✅ Stores cleared, metadata preserved. Starting full reindex..."); // ── Step 5: Reindex — all files treated as "changed" since metadata is empty ── - Self::perform_incremental_refresh_with_stores(codebase_path, db_path, stores, cancel_token) - .await + Self::perform_incremental_refresh_with_stores( + codebase_path, + db_path, + stores, + cancel_token, + embedding_pool, + ) + .await } /// Start the file system watcher (begin collecting events) without starting the processing loop. @@ -1285,6 +1349,7 @@ impl IndexManager { let stores = self.stores.clone(); let git_head_watcher = self.git_head_watcher.clone(); let symbol_registry = self.symbol_registry.clone(); + let embedding_pool = self.embedding_pool.clone(); let indexing_cb = indexing_status_cb.clone(); info!("🚀 Starting background file watcher..."); @@ -1373,6 +1438,7 @@ impl IndexManager { &db_path, &stores, &cancel_token, + embedding_pool.as_ref(), ) .await { @@ -1545,6 +1611,7 @@ impl IndexManager { to_index, to_remove, &cancel_token, + embedding_pool.as_ref(), ) .await { @@ -1849,6 +1916,7 @@ impl IndexManager { files_to_index: Vec, files_to_remove: Vec, cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, ) -> Result<()> { use crate::output::set_quiet; @@ -1952,7 +2020,9 @@ impl IndexManager { Self::ensure_indexing_active(cancel_token)?; for file_path in &files_to_index { debug!("📄 Indexing: {}", file_path.display()); - if let Err(e) = Self::index_single_file(codebase_path, file_path, stores).await { + if let Err(e) = + Self::index_single_file(codebase_path, file_path, stores, embedding_pool).await + { warn!("⚠️ Failed to index {}: {}", file_path.display(), e); } } @@ -2004,6 +2074,7 @@ impl IndexManager { db_path: &Path, stores: &SharedStores, cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::file::FileWalker; @@ -2180,7 +2251,9 @@ impl IndexManager { Self::ensure_indexing_active(cancel_token)?; let reindex_count = files_to_reindex.len(); for file_path in &files_to_reindex { - if let Err(e) = Self::index_single_file(codebase_path, file_path, stores).await { + if let Err(e) = + Self::index_single_file(codebase_path, file_path, stores, embedding_pool).await + { warn!("⚠️ Failed to re-index {}: {}", file_path.display(), e); } } @@ -2276,6 +2349,7 @@ impl IndexManager { codebase_path: &Path, file_path: &Path, stores: &SharedStores, + embedding_pool: Option<&Arc>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::chunker::{Chunker, SemanticChunker}; @@ -2327,11 +2401,25 @@ impl IndexManager { let (embed_model, dimensions) = Self::resolve_embed_model(&db_path)?; let model_name = embed_model.short_name(); - // Generate embeddings - let cache_dir = crate::constants::get_global_models_cache_dir()?; - let mut embedding_service = - EmbeddingService::with_cache_dir(embed_model, Some(cache_dir.as_path()))?; - let embedded_chunks = embedding_service.embed_chunks(chunks)?; + // Generate embeddings. Reuse the serve pool's service when available: + // the pool already holds the persistent cache's LMDB env open, so a + // fresh EmbeddingService here is refused by the process-global + // registry and would degrade this path to cache-less re-embedding. + let embedded_chunks = match embed_service_choice(embedding_pool) { + EmbeddingSource::Pool(pool) => { + let service = pool.get(embed_model)?; + let mut guard = service + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.embed_chunks(chunks)? + } + EmbeddingSource::Fresh => { + let cache_dir = crate::constants::get_global_models_cache_dir()?; + let mut embedding_service = + EmbeddingService::with_cache_dir(embed_model, Some(cache_dir.as_path()))?; + embedding_service.embed_chunks(chunks)? + } + }; // ── Delete stale chunks for this file BEFORE inserting new ones ── // When a file is re-indexed (e.g. after a content change), the old @@ -2523,6 +2611,20 @@ mod tests { .expect("db dir must be deletable after SharedStores drops (no leaked LMDB handles)"); } + /// The pool arm must be taken exactly when a pool is present, carrying + /// that same pool; `None` must select a fresh service. Pins the arm + /// selection both embed call sites share — no model is loaded here + /// (`EmbeddingServicePool::new` is a plain constructor). + #[test] + fn embed_service_choice_selects_pool_when_present_and_fresh_when_none() { + let pool = Arc::new(EmbeddingServicePool::new(None)); + match embed_service_choice(Some(&pool)) { + EmbeddingSource::Pool(p) => assert!(Arc::ptr_eq(&p, &pool)), + EmbeddingSource::Fresh => panic!("Some(pool) must select Pool, got Fresh"), + } + assert!(matches!(embed_service_choice(None), EmbeddingSource::Fresh)); + } + /// Helper: create metadata.json in db_path with given dimensions fn create_metadata_json(db_path: &Path, dimensions: usize) { let metadata = serde_json::json!({ @@ -2605,6 +2707,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -2649,6 +2752,7 @@ mod tests { &db_path, &stores, &token, + None, ) .await; @@ -2699,6 +2803,7 @@ mod tests { &stores, None, &task_token, + None, ) .await }); @@ -2775,6 +2880,7 @@ mod tests { &stores, None, &CancellationToken::new(), + None, ) .await .expect("force reindex on empty codebase should succeed"); @@ -2839,6 +2945,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -2900,6 +3007,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -2942,6 +3050,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -2989,6 +3098,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -3044,6 +3154,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; @@ -3088,6 +3199,7 @@ mod tests { &db_path, &stores, &CancellationToken::new(), + None, ) .await; diff --git a/src/mcp/runtime.rs b/src/mcp/runtime.rs index f44999c1..415ed9bb 100644 --- a/src/mcp/runtime.rs +++ b/src/mcp/runtime.rs @@ -805,7 +805,7 @@ pub async fn run_mcp_server_with_options( // Create IndexManager with shared stores (skip initial refresh - do in background) tracing::info!("🔍 Initializing index manager..."); let index_manager = - IndexManager::new_without_refresh(&project_path, shared_stores.clone()).await?; + IndexManager::new_without_refresh(&project_path, shared_stores.clone(), None).await?; // Background: refresh FIRST, then file watcher (sequential, not concurrent) // Both write to SharedStores, so they must not run concurrently @@ -828,6 +828,7 @@ pub async fn run_mcp_server_with_options( &db_path_clone, &shared_stores_clone, &bg_cancel_token, + None, ) .await { diff --git a/src/serve/mod.rs b/src/serve/mod.rs index e17690df..60ca5e03 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -2244,8 +2244,9 @@ impl ServeState { } }; let db_path = path.join(DB_DIR_NAME); + let pool = self.embedding_pool(); - match IndexManager::new_without_refresh(&path, stores.clone()).await { + match IndexManager::new_without_refresh(&path, stores.clone(), Some(pool.clone())).await { Ok(im) => { let im_arc = Arc::new(im); let token = CancellationToken::new(); @@ -2269,6 +2270,7 @@ impl ServeState { &db_path_bg, &stores_bg, &token_for_task, + Some(&pool), ) .await { @@ -2457,11 +2459,13 @@ impl ServeState { // so it is given a fresh token that is never cancelled — the refresh runs // to completion. A real user-initiated cancel routes through the // RepoState::Write token owned by the live task instead. + let pool = self.embedding_pool(); if let Err(e) = IndexManager::perform_incremental_refresh_with_stores( &path, &db_path, &stores_arc, &CancellationToken::new(), + Some(&pool), ) .await { @@ -2642,7 +2646,10 @@ impl ServeState { // On failure, still store as Write — searches keep working, live updates disabled. let (index_manager_opt, cancel_token) = { let alias_clone = alias.to_string(); - match IndexManager::new_without_refresh(&path, stores_arc.clone()).await { + let pool = self.embedding_pool(); + match IndexManager::new_without_refresh(&path, stores_arc.clone(), Some(pool.clone())) + .await + { Ok(im) => { let im_arc = Arc::new(im); let token = CancellationToken::new(); @@ -2666,6 +2673,7 @@ impl ServeState { &db_path_clone, &stores_for_task, &token_for_task, + Some(&pool), ) .await { @@ -2750,6 +2758,7 @@ impl ServeState { let cancel_token = CancellationToken::new(); let token_for_task = cancel_token.clone(); + let pool_bg = self.embedding_pool(); // Fire-and-forget: create IndexManager + start FSW in background. // We don't block the first query — the repo is already searchable from the Warm state. @@ -2758,7 +2767,9 @@ impl ServeState { return; } - match IndexManager::new_without_refresh(&path_bg, stores_bg.clone()).await { + match IndexManager::new_without_refresh(&path_bg, stores_bg.clone(), Some(pool_bg)) + .await + { Ok(im) => { let im_arc = Arc::new(im); let im_for_task = im_arc.clone(); @@ -4282,12 +4293,14 @@ async fn reindex_handler( ); // 2. Clear data and reindex + let pool = g_state.embedding_pool(); match IndexManager::force_reindex_with_stores( &project_path, &db_path, &stores, None, &reindex_token_task, + Some(&pool), ) .await { @@ -4370,11 +4383,13 @@ async fn reindex_handler( "🔄 Incremental reindex triggered for '{}' via HTTP API", alias_bg ); + let pool = g_state.embedding_pool(); match IndexManager::perform_incremental_refresh_with_stores( &project_path, &db_path, &stores, &reindex_token_task, + Some(&pool), ) .await { @@ -4688,12 +4703,14 @@ async fn add_repo_handler( project_path.display() ); + let pool = state_bg.embedding_pool(); match IndexManager::force_reindex_with_stores( &project_path, &db_path, &stores, model_override, &token_for_task, + Some(&pool), ) .await { diff --git a/src/serve/tui.rs b/src/serve/tui.rs index b26393b6..6176bd32 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -1152,6 +1152,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei let alias_bg = alias.clone(); let state_bg = state.clone(); + let pool = state.embedding_pool(); // Fresh cancellation token for this reindex task, registered in // `index_tasks` so `remove_repo` can cancel + await it (BUG1). let reindex_token = CancellationToken::new(); @@ -1168,6 +1169,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei &stores, None, &reindex_token_task, + Some(&pool), ) .await { From 2c1382626ceef116db44b4e5a1385c20aed5974d Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 09:24:56 +0200 Subject: [PATCH 02/77] feat(index): relative chunk paths, index add --local, clean build logs - Chunk paths stored relative to the indexed project root so built DBs are portable across machines/containers (snapshot federation previously worked only because build path == serve path). One-time consequence: existing DBs refreshed by this binary see all files as changed and re-embed once. - index add --local: skip serve delegation, always build in-process. - index/index add: write a clean, unmangled build log to /.codesearch.db/build.log when stdout is not a TTY (or --log-file). - current_git_head: strip GIT_DIR/GIT_WORK_TREE/GIT_INDEX_FILE/GIT_OBJECT_DIRECTORY from the child env so repo discovery keys off the given root (pre-existing test failure under push hooks). --- src/cache/file_meta.rs | 52 +++++++---- src/cache/file_meta_tests.rs | 30 ++++++- src/cache/mod.rs | 2 +- src/cli/doctor.rs | 11 ++- src/cli/mod.rs | 30 ++++++- src/index/build_log.rs | 91 +++++++++++++++++++ src/index/manager.rs | 168 +++++++++++++++++++++++++++-------- src/index/mod.rs | 119 +++++++++++++++++++------ src/mcp/helpers.rs | 29 ++++-- src/mcp/tests.rs | 35 ++++++++ src/search/mod.rs | 14 +-- src/symbols/mod.rs | 7 ++ 12 files changed, 480 insertions(+), 108 deletions(-) create mode 100644 src/index/build_log.rs diff --git a/src/cache/file_meta.rs b/src/cache/file_meta.rs index dab5f38e..21bfdb78 100644 --- a/src/cache/file_meta.rs +++ b/src/cache/file_meta.rs @@ -201,6 +201,18 @@ pub fn normalize_path_relative(path: &str, project_root_normalized: &str) -> Str } } +/// Canonical storage key for a file: its path **relative to the project root**, +/// normalized (UNC stripped, forward slashes on Windows). Already-relative +/// inputs and paths outside `project_root` fall back to the normalized input. +/// +/// Chunk metadata and `FileMetaStore` entries are keyed by this relative form +/// so a built database directory is machine-portable (snapshot tarballs move +/// the DB between hosts where the absolute build path does not exist). +pub fn storage_key(path: &Path, project_root: &Path) -> String { + let root_normalized = normalize_path(project_root); + normalize_path_relative(&normalize_path(path), &root_normalized) +} + /// Check whether a path matches a normalized filter prefix. /// /// `project_root_normalized` should be pre-normalized with `normalize_path_str`. @@ -249,7 +261,7 @@ pub struct FileMeta { /// 3. Stores chunk count for statistics #[derive(Debug, Serialize, Deserialize)] pub struct FileMetaStore { - /// Map of absolute file path -> metadata + /// Map of project-relative file path (storage key) -> metadata files: HashMap, /// Model used for indexing (invalidate if model changes) pub model_name: String, @@ -358,17 +370,21 @@ impl FileMetaStore { Ok(mtime.duration_since(SystemTime::UNIX_EPOCH)?.as_secs()) } - /// Check if a file needs re-indexing /// Check whether a path is already tracked (regardless of chunk count). /// Used by doctor to distinguish "never indexed" from "indexed but unchunkable". - pub fn is_tracked(&self, path: &Path) -> bool { - let path_str = normalize_path(path); + /// + /// `key` is the project-relative storage key (see [`storage_key`]). + pub fn is_tracked(&self, key: &str) -> bool { + let path_str = normalize_path_str(key); self.files.contains_key(&path_str) } /// Returns: (needs_reindex, existing_chunk_ids_to_delete) - pub fn check_file(&self, path: &Path) -> Result<(bool, Vec)> { - let path_str = normalize_path(path); + /// + /// `path` is the absolute path used for filesystem stats (mtime/size/hash); + /// `key` is the project-relative storage key the metadata is filed under. + pub fn check_file(&self, path: &Path, key: &str) -> Result<(bool, Vec)> { + let path_str = normalize_path_str(key); // Get current file stats let current_mtime = Self::get_mtime(path)?; @@ -395,9 +411,12 @@ impl FileMetaStore { } } - /// Update metadata for a file after indexing - pub fn update_file(&mut self, path: &Path, chunk_ids: Vec) -> Result<()> { - let path_str = normalize_path(path); + /// Update metadata for a file after indexing. + /// + /// `path` is the absolute path used for filesystem stats; `key` is the + /// project-relative storage key the metadata is filed under. + pub fn update_file(&mut self, path: &Path, key: &str, chunk_ids: Vec) -> Result<()> { + let path_str = normalize_path_str(key); let hash = Self::compute_hash(path)?; let mtime = Self::get_mtime(path)?; let size = fs::metadata(path)?.len(); @@ -416,9 +435,9 @@ impl FileMetaStore { Ok(()) } - /// Mark a file as deleted - pub fn remove_file(&mut self, path: &Path) -> Option { - let path_str = normalize_path(path); + /// Mark a file as deleted. `key` is the project-relative storage key. + pub fn remove_file(&mut self, key: &str) -> Option { + let path_str = normalize_path_str(key); self.files.remove(&path_str) } @@ -433,11 +452,14 @@ impl FileMetaStore { self.files.is_empty() } - /// Find files that were deleted (exist in store but not on disk) - pub fn find_deleted_files(&self) -> Vec<(String, Vec)> { + /// Find files that were deleted (exist in store but not on disk). + /// + /// `project_root` resolves relative keys back onto disk for the + /// existence check — keys are project-relative so the store is portable. + pub fn find_deleted_files(&self, project_root: &Path) -> Vec<(String, Vec)> { self.files .iter() - .filter(|(path, _)| !Path::new(path).exists()) + .filter(|(path, _)| !project_root.join(path).exists()) .map(|(path, meta)| (path.clone(), meta.chunk_ids.clone())) .collect() } diff --git a/src/cache/file_meta_tests.rs b/src/cache/file_meta_tests.rs index 011d4743..fa2a51ee 100644 --- a/src/cache/file_meta_tests.rs +++ b/src/cache/file_meta_tests.rs @@ -354,22 +354,24 @@ fn test_file_meta_store() { fs::write(&test_file, "hello world").unwrap(); // Check new file - let (needs_reindex, old_chunks) = store.check_file(&test_file).unwrap(); + let (needs_reindex, old_chunks) = store.check_file(&test_file, "test.txt").unwrap(); assert!(needs_reindex); assert!(old_chunks.is_empty()); // Update metadata - store.update_file(&test_file, vec![1, 2, 3]).unwrap(); + store + .update_file(&test_file, "test.txt", vec![1, 2, 3]) + .unwrap(); // Check again - should not need reindex - let (needs_reindex, _) = store.check_file(&test_file).unwrap(); + let (needs_reindex, _) = store.check_file(&test_file, "test.txt").unwrap(); assert!(!needs_reindex); // Modify file fs::write(&test_file, "hello world modified").unwrap(); // Now should need reindex - let (needs_reindex, old_chunks) = store.check_file(&test_file).unwrap(); + let (needs_reindex, old_chunks) = store.check_file(&test_file, "test.txt").unwrap(); assert!(needs_reindex); assert_eq!(old_chunks, vec![1, 2, 3]); @@ -377,6 +379,26 @@ fn test_file_meta_store() { store.save(db_path).unwrap(); let loaded = FileMetaStore::load_or_create(db_path, "test-model", 384).unwrap(); assert_eq!(loaded.files.len(), 1); + assert!(loaded.is_tracked("test.txt")); +} + +#[test] +fn test_storage_key_relative_and_portable() { + let dir = tempdir().unwrap(); + let root = dir.path(); + let file = root.join("src").join("main.rs"); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write(&file, "fn main() {}").unwrap(); + + let key = storage_key(&file, root); + assert_eq!(key, "src/main.rs"); + // Already-relative input round-trips unchanged + assert_eq!(storage_key(Path::new("src/main.rs"), root), "src/main.rs"); + // Path outside the root falls back to the normalized absolute form + let outside = storage_key(Path::new("C:/elsewhere/x.rs"), root); + assert!(outside.ends_with("elsewhere/x.rs")); + // Root itself yields an empty-ish key without panicking + let _ = storage_key(root, root); } // ========================================================================= diff --git a/src/cache/mod.rs b/src/cache/mod.rs index a70183e8..1e44b3de 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -2,7 +2,7 @@ mod file_meta; pub use file_meta::{ normalize_filter_path, normalize_path, normalize_path_str, normalize_user_path, - path_matches_filter, safe_canonicalize, strip_unc_prefix, FileMetaStore, + path_matches_filter, safe_canonicalize, storage_key, strip_unc_prefix, FileMetaStore, }; use moka::sync::Cache; diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index 2cb3f413..84713ace 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -306,7 +306,7 @@ fn check_file_integrity(db_path: &Path, project_path: &Path) -> CheckResult { }; // Stale files: in index but deleted from disk - let stale_files = store.find_deleted_files(); + let stale_files = store.find_deleted_files(project_path); let stale_count = stale_files.len(); // Walk disk to find all indexable files (uses the real FileWalker) @@ -327,14 +327,15 @@ fn check_file_integrity(db_path: &Path, project_path: &Path) -> CheckResult { let mut unindexed = 0; for file in &files { - match store.check_file(&file.path) { + let key = crate::cache::storage_key(&file.path, project_path); + match store.check_file(&file.path, &key) { Ok((needs_reindex, old_ids)) => { if needs_reindex && old_ids.is_empty() { // check_file returns (true, []) for two cases: // 1. File has NO entry in the store → genuinely unindexed // 2. File IS tracked but produced 0 chunks (minified JS, empty file, etc.) // Distinguish them with is_tracked() — case 2 is not an error. - if store.is_tracked(&file.path) { + if store.is_tracked(&key) { // Unchunkable file — tracked with 0 chunks, not a problem up_to_date += 1; } else { @@ -1085,7 +1086,9 @@ mod tests { fs::write(&test_file, "fn stale() {}").unwrap(); let mut store = FileMetaStore::new("minilm-l6-q".to_string(), 384); - store.update_file(&test_file, vec![1, 2, 3]).unwrap(); + store + .update_file(&test_file, "will_be_deleted.rs", vec![1, 2, 3]) + .unwrap(); store.save(&db_dir).unwrap(); // Now delete the file — it becomes stale diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 0b64ced8..64a052e0 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -26,6 +26,14 @@ pub enum IndexCommands { #[arg(long)] model: Option, + /// Build locally even if a serve instance is running + #[arg(long)] + local: bool, + + /// Write a clean build log to this path (default: /.codesearch.db/build.log when stdout is not a TTY) + #[arg(long)] + log_file: Option, + /// Register the repo on a remote peer (name from `codesearch remote list`). /// When set, is a path on the remote's filesystem. #[arg(long)] @@ -371,6 +379,10 @@ pub enum Commands { #[arg(long)] symbols: bool, + /// Write a clean build log to this path (default: /.codesearch.db/build.log when stdout is not a TTY) + #[arg(long)] + log_file: Option, + // Backward-compat flags (predate subcommands) /// Add a repository to the index (creates local or global index) #[arg(long)] @@ -1059,6 +1071,7 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { remove, keep_config, list, + log_file, } => { // Subcommand path (preferred) if let Some(cmd) = command { @@ -1067,6 +1080,8 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { path: add_path, global, model, + local, + log_file: add_log_file, remote, } => { if let Some(peer_name) = &remote { @@ -1087,8 +1102,15 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Some(mt) = mt { warn_if_heavier_model(mt); } - crate::index::add_to_index(add_path, global, mt, cancel_token.clone()) - .await + crate::index::add_to_index( + add_path, + global, + local, + add_log_file, + mt, + cancel_token.clone(), + ) + .await } } IndexCommands::Remove { @@ -1152,6 +1174,8 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { crate::index::add_to_index( effective_path, global, + false, + log_file, model_type, cancel_token.clone(), ) @@ -1190,6 +1214,8 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { force, false, model_type, + false, + log_file, cancel_token.clone(), ) .await diff --git a/src/index/build_log.rs b/src/index/build_log.rs new file mode 100644 index 00000000..f623a797 --- /dev/null +++ b/src/index/build_log.rs @@ -0,0 +1,91 @@ +//! Clean build log for `index` / `index add` builds. +//! +//! When stdout is not a TTY (CI, redirected output, agents), the console +//! progress bar mangles output with `\r` redraws. In that mode a plain, +//! line-oriented `build.log` is written next to the DB instead; a +//! `--log-file ` override forces it on regardless of TTY state. + +use anyhow::Result; +use std::fs::{File, OpenOptions}; +use std::io::{IsTerminal, Write}; +use std::path::{Path, PathBuf}; + +/// True when stdout is attached to an interactive terminal. +pub fn stdout_is_interactive() -> bool { + std::io::stdout().is_terminal() +} + +/// Append-only structured build log (no progress-bar redraw spam). +pub struct BuildLog { + file: Option, + path: PathBuf, +} + +impl BuildLog { + /// Open the build log for a project. + /// + /// - `override_path` (from `--log-file`): always log to this file. + /// - Otherwise: log to `/.codesearch.db/build.log` only when + /// stdout is not a TTY; on an interactive console no log is written and + /// console output behaves exactly as before. + pub fn open(project_root: &Path, override_path: Option<&Path>) -> Result { + let path = match override_path { + Some(p) => p.to_path_buf(), + None => { + if stdout_is_interactive() { + return Ok(Self { + file: None, + path: PathBuf::new(), + }); + } + project_root + .join(crate::constants::DB_DIR_NAME) + .join("build.log") + } + }; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let file = OpenOptions::new().create(true).append(true).open(&path)?; + Ok(Self { + file: Some(file), + path, + }) + } + + /// The log file path when logging is active, for reporting to the user. + pub fn path(&self) -> Option<&Path> { + self.file.as_ref().map(|_| self.path.as_path()) + } + + /// Write one structured line. Best-effort: logging must never fail a build. + /// ANSI color codes are stripped so the file stays plain text — callers + /// pass pre-styled (`colored`) strings meant for the console. + pub fn line(&self, msg: impl AsRef) { + if let Some(file) = &self.file { + let mut file = file; + let _ = writeln!(file, "{}", strip_ansi(msg.as_ref())); + let _ = file.flush(); + } + } +} + +/// Remove ANSI escape sequences (CSI: ESC `[` params final-byte). +fn strip_ansi(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + if c == '\x1b' && chars.peek() == Some(&'[') { + chars.next(); + // Consume up to and including the final byte (@-~) + for f in chars.by_ref() { + if ('\u{40}'..='\u{7e}').contains(&f) { + break; + } + } + } else { + out.push(c); + } + } + out +} diff --git a/src/index/manager.rs b/src/index/manager.rs index 5076374f..bd0fa548 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -631,7 +631,8 @@ impl IndexManager { let mut unchanged_count = 0; for file in &files { - let (needs_reindex, _old_chunk_ids) = file_meta_store.check_file(&file.path)?; + let key = crate::cache::storage_key(&file.path, codebase_path); + let (needs_reindex, _old_chunk_ids) = file_meta_store.check_file(&file.path, &key)?; if needs_reindex { changed_files.push(file.clone()); debug!("📝 File changed: {}", file.path.display()); @@ -641,7 +642,7 @@ impl IndexManager { } // Find deleted files - let deleted_files = file_meta_store.find_deleted_files(); + let deleted_files = file_meta_store.find_deleted_files(codebase_path); info!( " Unchanged: {}, Changed: {}, Deleted: {}", @@ -688,12 +689,13 @@ impl IndexManager { } } } - file_meta_store.remove_file(Path::new(file_path)); + file_meta_store.remove_file(file_path); } // Delete old chunks for changed files for file in &changed_files { - let (_, old_chunk_ids) = file_meta_store.check_file(&file.path)?; + let key = crate::cache::storage_key(&file.path, codebase_path); + let (_, old_chunk_ids) = file_meta_store.check_file(&file.path, &key)?; if !old_chunk_ids.is_empty() { debug!( "🔄 Deleting {} old chunks for: {}", @@ -763,6 +765,7 @@ impl IndexManager { // are not needed on the async side and may not be `Send`. let files_for_embed = file_batch.to_vec(); let cache_dir_for_batch = cache_dir.clone(); + let root_for_batch = codebase_path.to_path_buf(); // Clone the token into the blocking closure so a cancel arriving // DURING the (long, core-saturating) embed pass is observed // per-file, not only once the whole batch returns. @@ -783,8 +786,13 @@ impl IndexManager { Ok(c) => c, Err(_) => continue, }; - let chunks = - chunker.chunk_semantic(file.language, &file.path, &content)?; + // Chunk under the project-relative storage key so + // chunk metadata stays machine-portable. + let rel = std::path::PathBuf::from(crate::cache::storage_key( + &file.path, + &root_for_batch, + )); + let chunks = chunker.chunk_semantic(file.language, &rel, &content)?; all_chunks.extend(chunks); } @@ -869,14 +877,14 @@ impl IndexManager { } for file in file_batch { - let path_str = normalize_path(&file.path); + let path_str = crate::cache::storage_key(&file.path, codebase_path); if let Some(ids) = chunks_by_file.get(&path_str) { - file_meta_store.update_file(&file.path, ids.clone())?; + file_meta_store.update_file(&file.path, &path_str, ids.clone())?; } else { // File was processed but produced 0 chunks (e.g. minified JS, // empty file). Track it with empty chunk list so it is not // re-processed on every run and doctor doesn't flag it. - file_meta_store.update_file(&file.path, vec![])?; + file_meta_store.update_file(&file.path, &path_str, vec![])?; } } @@ -886,7 +894,8 @@ impl IndexManager { // them so they are not flagged as unindexed on every // subsequent run. for file in file_batch { - file_meta_store.update_file(&file.path, vec![])?; + let key = crate::cache::storage_key(&file.path, codebase_path); + file_meta_store.update_file(&file.path, &key, vec![])?; } } } @@ -2050,7 +2059,9 @@ impl IndexManager { let mut chunks_to_delete: Vec = Vec::new(); for file_info in &files { - let (needs_reindex, old_chunk_ids) = file_meta_store.check_file(&file_info.path)?; + let key = crate::cache::storage_key(&file_info.path, codebase_path); + let (needs_reindex, old_chunk_ids) = + file_meta_store.check_file(&file_info.path, &key)?; if needs_reindex { chunks_to_delete.extend(old_chunk_ids); files_to_reindex.push(file_info.path.clone()); @@ -2058,7 +2069,7 @@ impl IndexManager { } // Find files that were deleted (tracked in metadata but not on disk) - let deleted_files = file_meta_store.find_deleted_files(); + let deleted_files = file_meta_store.find_deleted_files(codebase_path); if files_to_reindex.is_empty() && deleted_files.is_empty() { info!("✅ Branch refresh: index is up to date, no changes needed"); @@ -2095,7 +2106,7 @@ impl IndexManager { // Remove deleted files from FileMetaStore let mut deleted_count = deleted_files.len(); for (file_path, _chunk_ids) in &deleted_files { - file_meta_store.remove_file(std::path::Path::new(file_path)); + file_meta_store.remove_file(file_path); } // Save metadata after deletions (before re-indexing, since @@ -2126,7 +2137,9 @@ impl IndexManager { let mut orphan_file_count = 0usize; for (vs_path, chunk_ids) in &vs_file_chunks { - if !std::path::Path::new(vs_path).exists() { + // Stored chunk paths are project-relative; resolve against + // the codebase root for the on-disk existence check. + if !codebase_path.join(vs_path).exists() { orphan_chunk_ids.extend(chunk_ids); orphan_file_count += 1; } @@ -2233,6 +2246,8 @@ impl IndexManager { false, false, None, + false, + None, CancellationToken::new(), ) .await?; @@ -2305,9 +2320,11 @@ impl IndexManager { } }; - // Chunk the file + // Chunk the file under its project-relative storage key so chunk + // metadata stays machine-portable. let chunker = SemanticChunker::new(100, 4000, 2); - let chunks = chunker.chunk_file(file_path, &content)?; + let rel_key = crate::cache::storage_key(file_path, codebase_path); + let chunks = chunker.chunk_file(std::path::Path::new(&rel_key), &content)?; if chunks.is_empty() { debug!("No chunks created for file: {}", file_path.display()); @@ -2342,7 +2359,7 @@ impl IndexManager { { let mut file_meta_store = FileMetaStore::load_or_create(&db_path, model_name, dimensions)?; - if let Some(old_meta) = file_meta_store.remove_file(file_path) { + if let Some(old_meta) = file_meta_store.remove_file(&rel_key) { let old_ids = old_meta.chunk_ids; if !old_ids.is_empty() { debug!( @@ -2398,7 +2415,7 @@ impl IndexManager { // Update file metadata (separate store, not shared) let mut file_meta_store = FileMetaStore::load_or_create(&db_path, model_name, dimensions)?; - file_meta_store.update_file(file_path, chunk_ids)?; + file_meta_store.update_file(file_path, &rel_key, chunk_ids)?; file_meta_store.save(&db_path)?; info!( @@ -2413,7 +2430,7 @@ impl IndexManager { /// Remove a file from the index using shared stores (for FSW delete events). /// This version uses the shared stores to avoid LMDB conflicts. async fn remove_file_from_index_with_stores( - _codebase_path: &Path, + codebase_path: &Path, db_path: &Path, stores: &SharedStores, file_path: &Path, @@ -2438,7 +2455,8 @@ impl IndexManager { // Get chunk IDs from file metadata directly (not check_file which reads from disk) // The file is already deleted, so we can't read mtime/size/hash - let meta = file_meta_store.remove_file(file_path); + let key = crate::cache::storage_key(file_path, codebase_path); + let meta = file_meta_store.remove_file(&key); let chunk_ids = match meta { Some(m) if !m.chunk_ids.is_empty() => m.chunk_ids, Some(_) => { @@ -2810,14 +2828,16 @@ mod tests { // Track the ghost file in FileMetaStore let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); - file_meta.update_file(&ghost_file, vec![100, 101]).unwrap(); + file_meta + .update_file(&ghost_file, "ghost.rs", vec![100, 101]) + .unwrap(); file_meta.save(&db_path).unwrap(); // Now delete the ghost file from disk — simulates branch switch std::fs::remove_file(&ghost_file).unwrap(); // Verify precondition: ghost file IS tracked but NOT on disk - let deleted_before = file_meta.find_deleted_files(); + let deleted_before = file_meta.find_deleted_files(&codebase_path); assert_eq!( deleted_before.len(), 1, @@ -2846,7 +2866,7 @@ mod tests { // Verify: reload FileMetaStore and confirm ghost entry is gone let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); - let deleted_after = reloaded.find_deleted_files(); + let deleted_after = reloaded.find_deleted_files(&codebase_path); assert!( deleted_after.is_empty(), "Ghost file should have been removed from FileMetaStore after refresh, found: {:?}", @@ -2875,9 +2895,15 @@ mod tests { // Track all ghost files let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); - file_meta.update_file(&ghost1, vec![10, 11]).unwrap(); - file_meta.update_file(&ghost2, vec![20, 21, 22]).unwrap(); - file_meta.update_file(&ghost3, vec![30]).unwrap(); + file_meta + .update_file(&ghost1, "ghost1.rs", vec![10, 11]) + .unwrap(); + file_meta + .update_file(&ghost2, "ghost2.rs", vec![20, 21, 22]) + .unwrap(); + file_meta + .update_file(&ghost3, "ghost3.rs", vec![30]) + .unwrap(); file_meta.save(&db_path).unwrap(); // Delete all ghost files @@ -2886,7 +2912,7 @@ mod tests { std::fs::remove_file(&ghost3).unwrap(); // Verify precondition - let deleted_before = file_meta.find_deleted_files(); + let deleted_before = file_meta.find_deleted_files(&codebase_path); assert_eq!( deleted_before.len(), 3, @@ -2907,7 +2933,7 @@ mod tests { // All ghost entries should be removed let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); - let deleted_after = reloaded.find_deleted_files(); + let deleted_after = reloaded.find_deleted_files(&codebase_path); assert!( deleted_after.is_empty(), "All 3 ghost files should be removed, found: {:?}", @@ -2932,7 +2958,9 @@ mod tests { // Track it in FileMetaStore (update_file reads mtime/size/hash) let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); - file_meta.update_file(&real_file, vec![1, 2]).unwrap(); + file_meta + .update_file(&real_file, "main.rs", vec![1, 2]) + .unwrap(); file_meta.save(&db_path).unwrap(); let stores = create_test_stores(&db_path, 4).await; @@ -2949,7 +2977,7 @@ mod tests { // Verify: real file entry should still be in FileMetaStore let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); - let deleted = reloaded.find_deleted_files(); + let deleted = reloaded.find_deleted_files(&codebase_path); assert!( deleted.is_empty(), "Real file should NOT be removed from FileMetaStore" @@ -2975,8 +3003,12 @@ mod tests { // Track both files let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); - file_meta.update_file(&real_file, vec![1, 2]).unwrap(); - file_meta.update_file(&ghost_file, vec![3, 4, 5]).unwrap(); + file_meta + .update_file(&real_file, "real.rs", vec![1, 2]) + .unwrap(); + file_meta + .update_file(&ghost_file, "ghost.rs", vec![3, 4, 5]) + .unwrap(); file_meta.save(&db_path).unwrap(); // Delete ghost file — simulates branch switch removing it @@ -2996,7 +3028,7 @@ mod tests { // Verify: ghost is removed, real is preserved let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); - let deleted = reloaded.find_deleted_files(); + let deleted = reloaded.find_deleted_files(&codebase_path); assert!( deleted.is_empty(), "Ghost entry should be removed, real should remain. Found deleted: {:?}", @@ -3004,7 +3036,7 @@ mod tests { ); // Verify the real file is still tracked by checking it doesn't need reindex - let (needs_reindex, _chunk_ids) = reloaded.check_file(&real_file).unwrap(); + let (needs_reindex, _chunk_ids) = reloaded.check_file(&real_file, "real.rs").unwrap(); assert!( !needs_reindex, "Real file should still be tracked and up-to-date in FileMetaStore" @@ -3029,8 +3061,12 @@ mod tests { std::fs::write(&file2, "pub fn util_fn() {}").unwrap(); let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); - file_meta.update_file(&file1, vec![1, 2, 3]).unwrap(); - file_meta.update_file(&file2, vec![4, 5]).unwrap(); + file_meta + .update_file(&file1, "lib.rs", vec![1, 2, 3]) + .unwrap(); + file_meta + .update_file(&file2, "util.rs", vec![4, 5]) + .unwrap(); file_meta.save(&db_path).unwrap(); // Delete ALL files — simulates switching to a branch with no source @@ -3051,7 +3087,7 @@ mod tests { // All entries should be cleaned let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); - let deleted = reloaded.find_deleted_files(); + let deleted = reloaded.find_deleted_files(&codebase_path); assert!(deleted.is_empty(), "All stale entries should be removed"); } @@ -3078,7 +3114,7 @@ mod tests { let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); // update_file hashes current on-disk content, so a subsequent check_file // on the unmodified file returns needs_reindex = false. - file_meta.update_file(&file, vec![1]).unwrap(); + file_meta.update_file(&file, "lib.rs", vec![1]).unwrap(); file_meta.save(&db_path).unwrap(); let stores = create_test_stores(&db_path, 4).await; @@ -3100,8 +3136,62 @@ mod tests { // The tracked file must still be tracked and not flagged as deleted. let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); assert!( - reloaded.find_deleted_files().is_empty(), + reloaded.find_deleted_files(&codebase_path).is_empty(), "No files should be flagged deleted on an up-to-date refresh" ); } + + #[tokio::test] + async fn test_incremental_refresh_round_trips_relative_keys() { + // Chunk/file-meta paths are stored RELATIVE to the project root so the + // built DB is portable across machines. A file tracked under its + // relative storage key must be seen as UNCHANGED (not re-indexed, not + // deleted) by a refresh pass that re-walks the same codebase. + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + + create_metadata_json(&db_path, 4); + + let file = codebase_path.join("lib.rs"); + std::fs::write(&file, "pub fn lib_fn() {}").unwrap(); + + let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); + let key = crate::cache::storage_key(&file, &codebase_path); + assert_eq!(key, "lib.rs", "storage key must be project-relative"); + file_meta.update_file(&file, &key, vec![1]).unwrap(); + file_meta.save(&db_path).unwrap(); + + let stores = create_test_stores(&db_path, 4).await; + + let result = IndexManager::perform_incremental_refresh_with_stores( + &codebase_path, + &db_path, + &stores, + &CancellationToken::new(), + ) + .await; + + assert!( + result.is_ok(), + "Second pass over unchanged files must succeed: {:?}", + result + ); + + let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); + assert!( + reloaded.is_tracked("lib.rs"), + "Relative key must survive the refresh round-trip, keys: {:?}", + reloaded.tracked_files().collect::>() + ); + assert!( + !reloaded + .tracked_files() + .any(|k| std::path::Path::new(k).is_absolute()), + "No stored key may be absolute, keys: {:?}", + reloaded.tracked_files().collect::>() + ); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 273c460a..07e0fe1c 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -7,7 +7,7 @@ use std::time::Instant; use tokio_util::sync::CancellationToken; use tracing::{debug, info}; -use crate::cache::{normalize_path, safe_canonicalize, FileMetaStore}; +use crate::cache::{normalize_path, safe_canonicalize, storage_key, FileMetaStore}; use crate::chunker::SemanticChunker; use crate::db_discovery::{find_best_database, is_registered_repository, register_repository}; use crate::embed::{EmbeddingService, ModelType}; @@ -22,6 +22,9 @@ pub use manager::{ SymbolRebuildSignal, }; +mod build_log; +pub use build_log::BuildLog; + /// Ensure the HNSW vector index is built if it was never built in a previous /// (possibly cancelled) run. /// @@ -505,17 +508,20 @@ fn get_global_db_path(path: Option) -> Result<(PathBuf, PathBuf)> { /// * `global` - Create global index instead of local /// * `model` - Override embedding model /// * `quiet` - Suppress verbose output (for server/MCP mode) +#[allow(clippy::too_many_arguments)] pub async fn index( path: Option, dry_run: bool, force: bool, global: bool, model: Option, + local: bool, + log_file: Option, cancel_token: CancellationToken, ) -> Result<()> { // Always try to delegate to a running serve instance via HTTP. // This avoids file-lock conflicts between CLI and serve holding the same LMDB. - if !dry_run { + if !dry_run && !local { // Try to delegate; if serve is unresponsive (warming up), wait and retry. let delegate_result = serve_delegate_with_warmup_wait(|| { let path = path.clone(); @@ -566,7 +572,17 @@ pub async fn index( } } } - index_with_options(path, dry_run, force, global, model, false, cancel_token).await + index_with_options( + path, + dry_run, + force, + global, + model, + false, + log_file, + cancel_token, + ) + .await } /// Index a repository with quiet mode option (for server/MCP use) @@ -587,10 +603,11 @@ pub async fn index_quiet_with_model( model: Option, cancel_token: CancellationToken, ) -> Result<()> { - index_with_options(path, false, force, global, model, true, cancel_token).await + index_with_options(path, false, force, global, model, true, None, cancel_token).await } /// Internal index function with all options +#[allow(clippy::too_many_arguments)] async fn index_with_options( path: Option, dry_run: bool, @@ -598,6 +615,7 @@ async fn index_with_options( global: bool, model: Option, quiet: bool, + log_file: Option, cancel_token: CancellationToken, ) -> Result<()> { let (db_path, project_path) = get_db_path_smart(path, global, force)?; @@ -607,13 +625,33 @@ async fn index_with_options( // embed 384-dim MiniLM vectors into a 768-dim index. See `resolve_index_model`. let model_type = resolve_index_model(&db_path, force, model)?; + // Clean build log: written next to the DB when stdout is redirected + // (progress-bar redraws mangle piped output), or to --log-file always. + let build_log = match BuildLog::open(&project_path, log_file.as_deref()) { + Ok(l) => Some(l), + Err(e) => { + tracing::warn!("could not open build log: {}", e); + None + } + }; + // Macro to conditionally print macro_rules! log_print { - ($($arg:tt)*) => { + ($($arg:tt)*) => {{ + let msg = format!($($arg)*); if !quiet { - println!($($arg)*); + println!("{}", msg); } - }; + if let Some(log) = &build_log { + log.line(&msg); + } + }}; + } + + if let (Some(l), false) = (&build_log, quiet) { + if let Some(p) = l.path() { + log_print!("📝 Build log: {}", p.display()); + } } log_print!("{}", "🚀 Codesearch Indexer".bright_cyan().bold()); @@ -709,7 +747,8 @@ async fn index_with_options( let mut unchanged_files = 0; for file in &files { - let (needs_reindex, _old_chunk_ids) = file_meta_store.check_file(&file.path)?; + let key = storage_key(&file.path, &project_path); + let (needs_reindex, _old_chunk_ids) = file_meta_store.check_file(&file.path, &key)?; if needs_reindex { changed_files.push(file.clone()); @@ -721,7 +760,7 @@ async fn index_with_options( } // Find deleted files (in metadata but not on disk) - let deleted_files = file_meta_store.find_deleted_files(); + let deleted_files = file_meta_store.find_deleted_files(&project_path); for (file_path, _chunk_ids) in &deleted_files { debug!("🗑️ File deleted from disk: {}", file_path); @@ -763,7 +802,8 @@ async fn index_with_options( total_chunks_to_delete += chunk_ids.len() as u32; } for file in &changed_files { - let (_, chunk_ids) = file_meta_store.check_file(&file.path)?; + let key = storage_key(&file.path, &project_path); + let (_, chunk_ids) = file_meta_store.check_file(&file.path, &key)?; total_chunks_to_delete += chunk_ids.len() as u32; } @@ -787,12 +827,13 @@ async fn index_with_options( fts_store.delete_chunk(*chunk_id)?; } } - file_meta_store.remove_file(Path::new(&file_path)); + file_meta_store.remove_file(file_path.as_str()); } // Delete changed files' old chunks for file in &changed_files { - let (_, old_chunk_ids) = file_meta_store.check_file(&file.path)?; + let key = storage_key(&file.path, &project_path); + let (_, old_chunk_ids) = file_meta_store.check_file(&file.path, &key)?; if !old_chunk_ids.is_empty() { let file_path_str = file.path.to_string_lossy().to_string(); info!( @@ -928,7 +969,10 @@ async fn index_with_options( }; // Phase 2a: Chunk this file only (memory efficient!) - let chunks = chunker.chunk_semantic(file.language, &file.path, &source_code)?; + // The path stored in chunk metadata is project-relative so the built + // database stays machine-portable (snapshot tarballs move the DB dir). + let rel_path = PathBuf::from(storage_key(&file.path, &project_path)); + let chunks = chunker.chunk_semantic(file.language, &rel_path, &source_code)?; let chunk_count = chunks.len(); debug!( " Created {} chunks for {}", @@ -941,7 +985,7 @@ async fn index_with_options( // A file with 0 chunks (e.g. minified JS, empty file) is "processed // but unchunkable" — record it with an empty chunk list so check_file() // returns (unchanged) on future runs and doctor doesn't flag it. - let path_str = file.path.to_string_lossy().to_string(); + let path_str = storage_key(&file.path, &project_path); file_chunks.insert(path_str, vec![]); pb.inc(1); continue; @@ -999,7 +1043,7 @@ async fn index_with_options( } // Track chunk IDs per file for metadata (only paths and IDs, not chunk content) - let file_path = file.path.to_string_lossy().to_string(); + let file_path = storage_key(&file.path, &project_path); file_chunks.insert(file_path, chunk_ids.clone()); total_chunks += chunk_count; @@ -1075,7 +1119,9 @@ async fn index_with_options( let save_result = if is_incremental { let mut meta = file_meta_store.take().unwrap(); for (file_path, chunk_ids) in file_chunks { - if let Err(e) = meta.update_file(Path::new(&file_path), chunk_ids) { + if let Err(e) = + meta.update_file(&project_path.join(&file_path), &file_path, chunk_ids) + { log_print!( "{} file-meta update warning for '{}': {}", "⚠️ ".yellow(), @@ -1091,7 +1137,9 @@ async fn index_with_options( model_type.dimensions(), ); for (file_path, chunk_ids) in file_chunks { - if let Err(e) = meta.update_file(Path::new(&file_path), chunk_ids) { + if let Err(e) = + meta.update_file(&project_path.join(&file_path), &file_path, chunk_ids) + { log_print!( "{} file-meta update warning for '{}': {}", "⚠️ ".yellow(), @@ -1171,7 +1219,7 @@ async fn index_with_options( let mut store = file_meta_store.take().unwrap(); let file_count = file_chunks.len(); for (file_path, chunk_ids) in file_chunks { - store.update_file(Path::new(&file_path), chunk_ids)?; + store.update_file(&project_path.join(&file_path), &file_path, chunk_ids)?; } store.save(&db_path)?; log_print!( @@ -1184,7 +1232,7 @@ async fn index_with_options( model_type.dimensions(), ); for (file_path, chunk_ids) in file_chunks { - store.update_file(Path::new(&file_path), chunk_ids)?; + store.update_file(&project_path.join(&file_path), &file_path, chunk_ids)?; } store.save(&db_path)?; } @@ -1230,7 +1278,7 @@ async fn index_with_options( // Update FileMetaStore with new/changed files (unchanged files are already preserved) for (file_path, chunk_ids) in file_chunks { - file_meta_store.update_file(Path::new(&file_path), chunk_ids)?; + file_meta_store.update_file(&project_path.join(&file_path), &file_path, chunk_ids)?; } // Save FileMetaStore (includes both unchanged + updated files) @@ -1247,7 +1295,7 @@ async fn index_with_options( // Update FileMetaStore for (file_path, chunk_ids) in file_chunks { - file_meta_store.update_file(Path::new(&file_path), chunk_ids)?; + file_meta_store.update_file(&project_path.join(&file_path), &file_path, chunk_ids)?; } // Save FileMetaStore @@ -1481,6 +1529,8 @@ pub async fn prune_index() -> Result<()> { pub async fn add_to_index( path: Option, global: bool, + local: bool, + log_file: Option, model: Option, cancel_token: CancellationToken, ) -> Result<()> { @@ -1493,13 +1543,22 @@ pub async fn add_to_index( // Try delegating to a running serve instance first. // Serve handles: register in repos.json + create index + warmup. - let add_delegate = serve_delegate_with_warmup_wait(|| { - let path = path.clone(); - // Alias is always derived from the directory name; the CLI no longer - // lets the user set it. Pass None so serve derives it consistently. - async move { try_delegate_add_to_serve(&path, &None, global, &model).await } - }) - .await; + // `--local` short-circuits the delegation attempt entirely. + let add_delegate = if local { + println!( + "{}", + "🏠 --local: building in-process (serve delegation skipped).".cyan() + ); + Err(DelegateError::ServeDown) + } else { + serve_delegate_with_warmup_wait(|| { + let path = path.clone(); + // Alias is always derived from the directory name; the CLI no longer + // lets the user set it. Pass None so serve derives it consistently. + async move { try_delegate_add_to_serve(&path, &None, global, &model).await } + }) + .await + }; match add_delegate { Ok((assigned_alias, _)) => { @@ -1618,6 +1677,8 @@ pub async fn add_to_index( false, true, model, + local, + log_file, cancel_token.clone(), ) .await?; @@ -1631,6 +1692,8 @@ pub async fn add_to_index( false, false, model, + local, + log_file, cancel_token, ) .await?; diff --git a/src/mcp/helpers.rs b/src/mcp/helpers.rs index 49001c98..243e2aea 100644 --- a/src/mcp/helpers.rs +++ b/src/mcp/helpers.rs @@ -38,6 +38,11 @@ impl HasScore for crate::fts::FtsResult { // === Simple Glob Matcher === +/// Resolve a tool path for store lookups to its canonical form: normalized and +/// relative to the project root. Chunk paths are stored relative to the root, +/// so `chunks_for_file` needles must be relative too. Absolute inputs under +/// the root are relativized; absolute inputs outside the root and unresolvable +/// relative inputs fall back to the normalized input. pub(crate) fn normalize_tool_path(path: &str, project_root: &Path) -> String { let p = Path::new(path); let resolved = if p.is_absolute() { @@ -45,7 +50,7 @@ pub(crate) fn normalize_tool_path(path: &str, project_root: &Path) -> String { } else { project_root.join(p) }; - crate::cache::normalize_path_str(resolved.to_string_lossy().as_ref()) + crate::cache::storage_key(&resolved, project_root) } /// Strip a project-alias prefix from a tool path. @@ -68,6 +73,10 @@ pub(crate) fn strip_alias_prefix(path: &str, alias: Option<&String>) -> String { /// Prefix a result path with its repo alias for group queries, normalizing /// Windows backslashes to forward slashes in the process. When `alias` is /// None or empty, the path is still normalized (useful for stdio mode). +/// +/// Chunk paths are stored project-relative, so a path that is neither under +/// `project_root` nor absolute is treated as already-relative: it just gets +/// the alias prepended (or is returned unchanged when there is no alias). pub(crate) fn prefix_path_with_alias( path: &str, alias: Option<&str>, @@ -77,14 +86,16 @@ pub(crate) fn prefix_path_with_alias( let normalized_root = crate::cache::normalize_path_str(project_root) .trim_end_matches('/') .to_string(); - match normalized.strip_prefix(&normalized_root) { - Some(rest) => { - let relative = rest.trim_start_matches('/'); - match alias { - Some(a) if !a.is_empty() => format!("{}/{}", a, relative), - _ => relative.to_string(), - } - } + let relative = match normalized.strip_prefix(&normalized_root) { + Some(rest) => Some(rest.trim_start_matches('/')), + None if !Path::new(&normalized).has_root() => Some(normalized.as_str()), + None => None, + }; + match relative { + Some(relative) => match alias { + Some(a) if !a.is_empty() => format!("{}/{}", a, relative), + _ => relative.to_string(), + }, None => normalized, } } diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index 3495b472..e3541b40 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -544,6 +544,41 @@ fn test_stdio_mode_paths_not_prefixed() { assert_eq!(result, "src/main.rs"); } +// === relative stored paths (portable snapshots) === + +#[test] +fn test_relative_path_with_alias() { + // Chunk paths are stored project-relative; serve must yield "/" + let result = super::prefix_path_with_alias("src/main.rs", Some("myrepo"), "C:/repo"); + assert_eq!(result, "myrepo/src/main.rs"); +} + +#[test] +fn test_relative_path_without_alias() { + let result = super::prefix_path_with_alias("src/main.rs", None, "C:/repo"); + assert_eq!(result, "src/main.rs"); +} + +#[test] +fn test_relative_path_empty_alias() { + let result = super::prefix_path_with_alias("docs/guide.md", Some(""), "/data/vendor"); + assert_eq!(result, "docs/guide.md"); +} + +#[test] +fn test_normalize_tool_path_relativizes_absolute() { + let root = std::path::Path::new("/tmp/proj"); + let result = super::normalize_tool_path("/tmp/proj/src/main.rs", root); + assert_eq!(result, "src/main.rs"); +} + +#[test] +fn test_normalize_tool_path_relative_stays_relative() { + let root = std::path::Path::new("/tmp/proj"); + let result = super::normalize_tool_path("src/main.rs", root); + assert_eq!(result, "src/main.rs"); +} + #[test] fn test_dedup_key_includes_alias() { // Two stores each returning chunk_id=1, different content. diff --git a/src/search/mod.rs b/src/search/mod.rs index 05beec06..314d7ed8 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -1121,7 +1121,8 @@ fn sync_database(db_path: &Path, model_type: ModelType) -> Result<()> { // Check for changed files for file in &files { - let (needs_reindex, old_chunk_ids) = file_meta.check_file(&file.path)?; + let key = crate::cache::storage_key(&file.path, project_path); + let (needs_reindex, old_chunk_ids) = file_meta.check_file(&file.path, &key)?; if !needs_reindex { continue; @@ -1144,28 +1145,29 @@ fn sync_database(db_path: &Path, model_type: ModelType) -> Result<()> { Err(_) => continue, }; - let chunks = chunker.chunk_semantic(file.language, &file.path, &source_code)?; + let rel = std::path::PathBuf::from(key.clone()); + let chunks = chunker.chunk_semantic(file.language, &rel, &source_code)?; if chunks.is_empty() { - file_meta.update_file(&file.path, vec![])?; + file_meta.update_file(&file.path, &key, vec![])?; continue; } // Embed and insert let embedded_chunks = embedding_service.embed_chunks(chunks)?; let chunk_ids = store.insert_chunks_with_ids(embedded_chunks)?; - file_meta.update_file(&file.path, chunk_ids)?; + file_meta.update_file(&file.path, &key, chunk_ids)?; } // Check for deleted files - let deleted_files = file_meta.find_deleted_files(); + let deleted_files = file_meta.find_deleted_files(project_path); for (path, chunk_ids) in &deleted_files { changes += 1; println!(" 🗑️ {} (deleted)", sanitize_for_terminal(path)); if !chunk_ids.is_empty() { store.delete_chunks(chunk_ids)?; } - file_meta.remove_file(std::path::Path::new(path)); + file_meta.remove_file(path); } // Rebuild index if changes were made diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 9295618d..c3542077 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -153,6 +153,13 @@ pub(crate) fn current_git_head(repo_root: &Path) -> Option { let output = std::process::Command::new("git") .args(["rev-parse", "HEAD"]) .current_dir(repo_root) + // Repo discovery must key off repo_root alone: a caller running under + // git (e.g. a push hook) exports GIT_DIR/GIT_WORK_TREE, which would + // otherwise override discovery and report the WRONG repo's HEAD. + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .env_remove("GIT_OBJECT_DIRECTORY") .output() .ok()?; if !output.status.success() { From 96f1eb03218aa1d759f0e24ac63cf5dec52a8f48 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 14:20:33 +0200 Subject: [PATCH 03/77] docs: changelog for 1.4.10 pending work (relative-path migration, --local, pool reuse) --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 016fdec1..f80b1874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,22 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.4.10] + +### Changed + +- **⚠️ MIGRATION — chunk paths are stored project-relative; every existing index re-embeds ONCE on first refresh with this binary.** Chunk metadata, tantivy documents and the file-meta store now key files by their path relative to the codebase root instead of the absolute path, making a built database portable across machines and mount points (the cloud snapshot only worked before because build path and serve path happened to be identical). Consequence, by design: an existing database's absolute-path entries no longer match the new relative keys, so the first incremental refresh sees every file as changed and re-chunks + re-embeds the whole corpus one time (expect a full-reindex-length CPU/embedding burst per repo; orphaned absolute-path entries are cleaned as deleted). After that one pass, incremental behaviour is identical to before — refreshes are again change-only. **Tooling impact:** paths RETURNED by search results, `stats` and the TUI change from absolute to `/`; paths SUPPLIED to tools still accept both forms — absolute needles are relativised internally (`normalize_tool_path`), so feeding a returned relative path or an old absolute path back into `chunks_for_file`/`get_chunk` keeps working. The SCIP/symbol index uses its own separate key space and is untouched. + +### Added + +- **`codesearch index add --local` — build in-process, never delegating to a running serve.** Deliberate local builds (snapshot assembly, CI, offline tooling) previously either raced the serve delegation or had to fake a dead port; `--local` skips delegation explicitly. Pair it with `CODESEARCH_REPOS_CONFIG` (existing override, honours load AND save) to keep build registrations out of a developer's live `~/.codesearch/repos.json` entirely — the docs-harvester's `index-local` pipeline does exactly that for strict hub separation. +- **Non-TTY index runs write a clean `build.log`.** When stdout is not a terminal (CI, piped runs), phase lines are also written ANSI-free to `/.codesearch.db/build.log`; `--log-file` forces a location on any run. + +### Fixed + +- **Incremental refresh reuses the serve hub's embedding pool (todo #135).** The refresh path constructed its own fastembed pool alongside the serve pool, doubling model memory per repo and re-initialising ONNX on every refresh; it now shares the serve pool. Single-serve runs are unchanged. +- **`current_git_head` no longer leaks caller `GIT_DIR`/`GIT_WORK_TREE`/`GIT_INDEX_FILE`/`GIT_OBJECT_DIRECTORY` into the queried repo.** A `codesearch index` invocation from inside a git hook (QC gate) inherits the hook's git environment and answered with the HOOK's repo state instead of the target project's, producing wrong head SHAs and spurious rebuild decisions; the four variables are stripped before `git rev-parse`. + ## [1.4.9] - 2026-09-23 ### Changed From a0338e37c2a835c8019454176d941a8a94ddd241 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 21:01:40 +0200 Subject: [PATCH 04/77] Fix legacy-path-migration fallout: key migration, FTS writer, warmup locks, status labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2c13826 made chunk paths project-relative, which re-embedded every legacy index and duplicated chunks; it also exposed three latent failure modes. One commit carries four fixes for that fallout: - Key migration (index/key_migration.rs, cache/file_meta.rs): re-key legacy absolute FileMetaStore keys to project-relative form, rewrite the chunk paths in both stores, and sweep chunks no tracked file references — unchanged files are recognised as up to date instead of re-embedded, and the duplicates an earlier interrupted re-embed left are removed. - FTS writer (fts/tantivy_store.rs): prepare_commit joins and respawns the indexing workers and can abort half-way, leaving zero workers behind a live 10k-doc pipeline; that writer is now discarded and an error returned instead of silently dropping every later batch and blocking add_document. - Warmup (serve/mod.rs): register Warm and release the per-alias open_lock before the refresh so queries are answered from existing data; track the refresh with begin/end_indexing so the reaper leaves the repo alone and FSW does not start mid-warmup; queries acquire open_lock with a deadline instead of queueing behind a cold open forever. - Status (mcp/status.rs): report indexing/busy/held-in-process instead of mislabelling our own activity as locked-externally, and fetch live stats via try_read so status answers while a batch holds the store lock. --- src/cache/file_meta.rs | 60 ++++++++++++ src/constants.rs | 5 + src/fts/tantivy_store.rs | 143 ++++++++++++++++----------- src/index/key_migration.rs | 196 +++++++++++++++++++++++++++++++++++++ src/index/manager.rs | 91 +++++++++++++++++ src/index/mod.rs | 16 +++ src/mcp/status.rs | 43 ++++++-- src/search/mod.rs | 7 ++ src/serve/mod.rs | 79 +++++++++++---- src/serve/tests.rs | 25 +++++ src/vectordb/mod.rs | 2 +- src/vectordb/store.rs | 67 ++++++++++++- 12 files changed, 648 insertions(+), 86 deletions(-) create mode 100644 src/index/key_migration.rs diff --git a/src/cache/file_meta.rs b/src/cache/file_meta.rs index 21bfdb78..f3f78533 100644 --- a/src/cache/file_meta.rs +++ b/src/cache/file_meta.rs @@ -441,6 +441,50 @@ impl FileMetaStore { self.files.remove(&path_str) } + /// Re-key entries still filed under an absolute path inside `project_root` + /// (the pre-relative-key format) to their project-relative storage key. + /// + /// Without this every legacy entry misses its lookup: the file is seen as + /// new and re-embedded, while the absolute entry — which + /// `find_deleted_files` resolves to an existing file — survives forever + /// together with its now-duplicate chunks. An absolute entry whose relative + /// key is already tracked is dropped (its chunks are orphans). + pub fn relativize_legacy_keys(&mut self, project_root: &Path) -> LegacyKeyMigration { + let legacy: Vec<(String, String)> = self + .files + .keys() + .filter(|key| Path::new(key.as_str()).has_root()) + .filter_map(|key| { + let rel = storage_key(Path::new(key), project_root); + (rel != *key).then(|| (key.clone(), rel)) + }) + .collect(); + + let mut migration = LegacyKeyMigration::default(); + for (key, rel) in legacy { + let Some(meta) = self.files.remove(&key) else { + continue; + }; + if self.files.contains_key(&rel) { + migration.superseded += 1; + continue; + } + migration + .rekeyed + .push((rel.clone(), meta.chunk_ids.clone())); + self.files.insert(rel, meta); + } + migration + } + + /// Every chunk id referenced by a tracked file. + pub fn tracked_chunk_ids(&self) -> std::collections::HashSet { + self.files + .values() + .flat_map(|m| m.chunk_ids.iter().copied()) + .collect() + } + /// Get all tracked files #[allow(dead_code)] // Reserved for file listing feature pub fn tracked_files(&self) -> impl Iterator { @@ -496,6 +540,22 @@ impl FileMetaStore { } } +/// Outcome of [`FileMetaStore::relativize_legacy_keys`]. +#[derive(Debug, Default)] +pub struct LegacyKeyMigration { + /// `(relative key, chunk ids)` of every re-keyed entry — their chunks + /// still carry the absolute path. + pub rekeyed: Vec<(String, Vec)>, + /// Absolute entries dropped because the relative key was already tracked. + pub superseded: usize, +} + +impl LegacyKeyMigration { + pub fn is_empty(&self) -> bool { + self.rekeyed.is_empty() && self.superseded == 0 + } +} + #[derive(Debug)] #[allow(dead_code)] // Used with stats() method pub struct FileMetaStats { diff --git a/src/constants.rs b/src/constants.rs index f0bd457e..fe6b1e9b 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -404,6 +404,11 @@ pub const REPO_IDLE_TIMEOUT_SECS: u64 = 30 * 60; // 30 minutes /// How often the idle-reaper background task checks for repos to evict. pub const REAPER_INTERVAL_SECS: u64 = 5 * 60; // 5 minutes +/// Longest a query waits for another task's cold open of the same repo. +/// Kept well under the MCP client's request timeout so the caller gets a +/// "retry shortly" error instead of a cancelled request whose handler lingers. +pub const REPO_OPEN_WAIT_SECS: u64 = 20; + /// Environment variable to override the repo idle timeout. pub const REPO_IDLE_TIMEOUT_ENV: &str = "CODESEARCH_REPO_IDLE_TIMEOUT_SECS"; diff --git a/src/fts/tantivy_store.rs b/src/fts/tantivy_store.rs index 45e7eba4..4b21fef6 100644 --- a/src/fts/tantivy_store.rs +++ b/src/fts/tantivy_store.rs @@ -29,6 +29,12 @@ pub struct FtsResult { pub score: f32, } +enum CommitAttempt { + Committed, + PrepareFailed(String), + PublishFailed(String), +} + /// Full-text search store using Tantivy /// /// Single connection type that supports both read and write operations. @@ -351,9 +357,7 @@ impl FtsStore { Ok(_) => Ok(()), Err(e) => { let error_str = e.to_string(); - if error_str.contains("writer was killed") - || error_str.contains("index writer was killed") - { + if Self::is_writer_killed(&error_str) { tracing::debug!( "FTS writer was killed, recreating and retrying add_chunk for chunk {}", chunk_id @@ -406,88 +410,117 @@ impl FtsStore { Ok(()) } + fn is_writer_killed(error: &str) -> bool { + error.contains("writer was killed") || error.contains("index writer was killed") + } + + fn is_transient_commit_error(error: &str) -> bool { + error.contains("Access is denied") + || error.contains("PermissionDenied") + || error.contains("IoError") + } + + /// One commit attempt, split so a failure in `prepare_commit` is told + /// apart from a failure while publishing an already-prepared commit. + /// + /// `prepare_commit` joins every indexing worker and respawns it one by + /// one; a worker error aborts that loop half-way and leaves the writer + /// with fewer (often zero) workers behind a live 10k-doc pipeline. + /// Committing that writer again "succeeds" while every later document is + /// queued and silently dropped, and once the pipeline fills, + /// `add_document` blocks forever — so it must never be reused. + fn commit_once(&mut self) -> CommitAttempt { + let Some(writer) = self.writer.as_mut() else { + return CommitAttempt::Committed; + }; + match writer.prepare_commit() { + Ok(prepared) => match prepared.commit() { + Ok(_) => CommitAttempt::Committed, + Err(e) => CommitAttempt::PublishFailed(e.to_string()), + }, + Err(e) => CommitAttempt::PrepareFailed(e.to_string()), + } + } + /// Commit pending changes with retry logic for Windows file locking. /// /// If the writer was killed (background merge panic), it is recreated. /// Data since the last successful commit will be lost in that case, but /// indexing can continue rather than aborting entirely. + /// + /// Any other failure while flushing the workers discards the writer and + /// returns an error: its pending documents are gone, and the caller must + /// learn that instead of seeing a commit that silently indexed nothing. pub fn commit(&mut self) -> Result<()> { if self.writer.is_none() { return Ok(()); } let max_retries = 5; - let mut last_error: Option = None; + let mut last_error = String::new(); for attempt in 0..max_retries { if attempt > 0 { - // Wait before retry (exponential backoff: 100ms, 200ms, 400ms, 800ms) + // Wait before retry (exponential backoff: 200ms, 400ms, 800ms, 1600ms) std::thread::sleep(std::time::Duration::from_millis(100 * (1 << attempt))); } - let writer = self.writer.as_mut().unwrap(); - match writer.commit() { - Ok(_) => { - // Reload reader to see changes + let error_str = match self.commit_once() { + CommitAttempt::Committed => { if let Err(e) = self.reader.reload() { // Non-fatal: reader will eventually catch up tracing::debug!("Reader reload warning: {}", e); } return Ok(()); } - Err(e) => { - let error_str = e.to_string(); - last_error = Some(error_str.clone()); - - // Writer was killed by background thread panic — recreate it - if error_str.contains("writer was killed") - || error_str.contains("index writer was killed") - { - tracing::debug!( - "FTS writer was killed during commit (attempt {}/{}). \ - Recreating writer. Data since last commit may be lost.", - attempt + 1, - max_retries - ); - self.writer = None; - self.ensure_writer()?; - // After recreating, the pending data is gone, so commit - // the new (empty) writer to ensure a clean state - if let Some(ref mut w) = self.writer { - w.commit() - .map_err(|e| anyhow!("FTS commit after recovery failed: {}", e))?; - } - if let Err(e) = self.reader.reload() { - tracing::debug!("Reader reload warning: {}", e); - } - return Ok(()); - } - - // File locking error — retry with backoff - if error_str.contains("Access is denied") - || error_str.contains("PermissionDenied") - || error_str.contains("IoError") - { - tracing::debug!( - "FTS commit retry {}/{}: {}", - attempt + 1, - max_retries, - error_str - ); - // Continue to retry - } else { - // Non-recoverable error, fail immediately - return Err(anyhow!("FTS commit failed: {}", error_str)); - } + CommitAttempt::PrepareFailed(e) if !Self::is_writer_killed(&e) => { + self.writer = None; + return Err(anyhow!( + "FTS commit failed while flushing indexing workers: {}. The writer was \ + discarded; uncommitted FTS changes are lost and must be re-indexed", + e + )); + } + CommitAttempt::PrepareFailed(e) | CommitAttempt::PublishFailed(e) => e, + }; + last_error = error_str.clone(); + + if Self::is_writer_killed(&error_str) { + tracing::debug!( + "FTS writer was killed during commit (attempt {}/{}). \ + Recreating writer. Data since last commit may be lost.", + attempt + 1, + max_retries + ); + self.writer = None; + self.ensure_writer()?; + // After recreating, the pending data is gone, so commit + // the new (empty) writer to ensure a clean state + if let Some(ref mut w) = self.writer { + w.commit() + .map_err(|e| anyhow!("FTS commit after recovery failed: {}", e))?; + } + if let Err(e) = self.reader.reload() { + tracing::debug!("Reader reload warning: {}", e); } + return Ok(()); } + + if !Self::is_transient_commit_error(&error_str) { + return Err(anyhow!("FTS commit failed: {}", error_str)); + } + tracing::debug!( + "FTS commit retry {}/{}: {}", + attempt + 1, + max_retries, + error_str + ); } - // All retries exhausted Err(anyhow!( "FTS commit failed after {} retries: {}", max_retries, - last_error.unwrap_or_default() + last_error )) } diff --git a/src/index/key_migration.rs b/src/index/key_migration.rs new file mode 100644 index 00000000..ab224418 --- /dev/null +++ b/src/index/key_migration.rs @@ -0,0 +1,196 @@ +//! Repair of indexes written before chunk paths became project-relative. +//! +//! 1. absolute `FileMetaStore` keys are re-keyed to their relative form, so +//! unchanged files are recognised as up to date instead of re-embedded; +//! 2. the chunks of those files get the relative path in both stores; +//! 3. chunks no tracked file references are deleted — the duplicates a +//! previous (possibly interrupted) re-embed under relative paths left. + +use anyhow::{anyhow, Result}; +use std::path::Path; +use std::sync::Arc; +use tracing::info; + +use super::SharedStores; +use crate::cache::FileMetaStore; +use crate::fts::FtsStore; +use crate::vectordb::{ChunkMetadata, VectorStore}; + +/// In-process variant for callers that own both stores exclusively (CLI). +/// Returns `true` when `file_meta` changed and must be saved by the caller. +pub(crate) fn repair_legacy_index( + project_root: &Path, + file_meta: &mut FileMetaStore, + vector_store: &mut VectorStore, + fts_store: &mut FtsStore, +) -> Result { + let migration = file_meta.relativize_legacy_keys(project_root); + let rewritten = rewrite_vector_paths(vector_store, &migration.rekeyed)?; + mirror_paths_into_fts(fts_store, &rewritten)?; + log_migration( + migration.rekeyed.len(), + rewritten.len(), + migration.superseded, + ); + + let untracked = untracked_chunk_ids(file_meta, vector_store)?; + if !untracked.is_empty() { + delete_from_vector_store(vector_store, &untracked)?; + delete_from_fts(fts_store, &untracked)?; + log_sweep(untracked.len()); + } + Ok(!migration.is_empty()) +} + +/// Shared-stores variant. Each phase takes ONE store lock on the blocking +/// pool, never both at once, so it cannot deadlock against a reader that +/// holds the other lock. +pub(crate) async fn repair_legacy_index_shared( + project_root: &Path, + file_meta: &mut FileMetaStore, + stores: &SharedStores, +) -> Result { + let migration = file_meta.relativize_legacy_keys(project_root); + + let rekeyed = migration.rekeyed.clone(); + let rewritten = on_store(&stores.vector_store, move |vs| { + rewrite_vector_paths(vs, &rekeyed) + }) + .await?; + let rewritten_count = rewritten.len(); + on_store(&stores.fts_store, move |fts| { + mirror_paths_into_fts(fts, &rewritten) + }) + .await?; + log_migration( + migration.rekeyed.len(), + rewritten_count, + migration.superseded, + ); + + let tracked = file_meta.tracked_chunk_ids(); + let meta_empty = file_meta.is_empty(); + let untracked = on_store(&stores.vector_store, move |vs| { + if meta_empty { + return Ok(Vec::new()); + } + let untracked = untracked_among(vs, &tracked)?; + if !untracked.is_empty() { + delete_from_vector_store(vs, &untracked)?; + } + Ok(untracked) + }) + .await?; + if !untracked.is_empty() { + let count = untracked.len(); + on_store(&stores.fts_store, move |fts| { + delete_from_fts(fts, &untracked) + }) + .await?; + log_sweep(count); + } + Ok(!migration.is_empty()) +} + +async fn on_store(store: &Arc>, f: F) -> Result +where + S: Send + Sync + 'static, + T: Send + 'static, + F: FnOnce(&mut S) -> Result + Send + 'static, +{ + let store = Arc::clone(store); + tokio::task::spawn_blocking(move || f(&mut store.blocking_write())) + .await + .map_err(|e| anyhow!("legacy index repair task panicked: {}", e))? +} + +fn rewrite_vector_paths( + vector_store: &mut VectorStore, + rekeyed: &[(String, Vec)], +) -> Result> { + if rekeyed.is_empty() { + return Ok(Vec::new()); + } + let updates: Vec<(u32, String)> = rekeyed + .iter() + .flat_map(|(key, ids)| ids.iter().map(move |id| (*id, key.clone()))) + .collect(); + vector_store.rewrite_chunk_paths(&updates) +} + +fn mirror_paths_into_fts( + fts_store: &mut FtsStore, + rewritten: &[(u32, ChunkMetadata)], +) -> Result<()> { + if rewritten.is_empty() { + return Ok(()); + } + for (id, meta) in rewritten { + fts_store.delete_chunk(*id)?; + fts_store.add_chunk( + *id, + &meta.content, + &meta.path, + meta.signature.as_deref(), + &meta.kind, + )?; + } + fts_store.commit() +} + +fn untracked_chunk_ids(file_meta: &FileMetaStore, vector_store: &VectorStore) -> Result> { + if file_meta.is_empty() { + return Ok(Vec::new()); + } + untracked_among(vector_store, &file_meta.tracked_chunk_ids()) +} + +/// Chunk ids in the store that no tracked file references. The full scan +/// only runs when the store holds more chunks than the metadata accounts for. +fn untracked_among( + vector_store: &VectorStore, + tracked: &std::collections::HashSet, +) -> Result> { + let (total_chunks, _) = vector_store.index_health()?; + if total_chunks <= tracked.len() { + return Ok(Vec::new()); + } + Ok(vector_store + .get_chunks_by_file()? + .into_values() + .flatten() + .filter(|id| !tracked.contains(id)) + .collect()) +} + +fn delete_from_vector_store(vector_store: &mut VectorStore, ids: &[u32]) -> Result<()> { + vector_store.delete_chunks(ids)?; + // Deletes invalidate the vector graph; a refresh with nothing else to + // embed would otherwise leave search failing with "Index not built". + vector_store.build_index() +} + +fn delete_from_fts(fts_store: &mut FtsStore, ids: &[u32]) -> Result<()> { + for id in ids { + fts_store.delete_chunk(*id)?; + } + fts_store.commit() +} + +fn log_migration(rekeyed: usize, rewritten: usize, superseded: usize) { + if rekeyed + superseded > 0 { + info!( + "🔄 Migrated {} legacy absolute file key(s) to project-relative ({} chunk path(s) \ + rewritten, {} superseded entr(ies) dropped)", + rekeyed, rewritten, superseded + ); + } +} + +fn log_sweep(count: usize) { + info!( + "🧹 Deleted {} chunk(s) not referenced by any tracked file (duplicates from an \ + earlier re-index)", + count + ); +} diff --git a/src/index/manager.rs b/src/index/manager.rs index da063fcb..8a84345c 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -650,6 +650,16 @@ impl IndexManager { } } + if super::key_migration::repair_legacy_index_shared( + codebase_path, + &mut file_meta_store, + stores, + ) + .await? + { + file_meta_store.save(db_path)?; + } + // Walk files. // // `FileWalker::walk()` is synchronous and I/O-heavy (recursive directory @@ -2124,6 +2134,15 @@ impl IndexManager { let mut file_meta_store = FileMetaStore::load_or_create(db_path, model_name, dimensions)?; + if super::key_migration::repair_legacy_index_shared( + codebase_path, + &mut file_meta_store, + stores, + ) + .await? + { + file_meta_store.save(db_path)?; + } // Find files that need re-indexing (new or content changed) let mut files_to_reindex: Vec = Vec::new(); @@ -3307,4 +3326,76 @@ mod tests { reloaded.tracked_files().collect::>() ); } + + #[tokio::test] + async fn test_incremental_refresh_migrates_legacy_absolute_keys() { + use crate::chunker::{Chunk, ChunkKind}; + use crate::embed::EmbeddedChunk; + + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + + let file = codebase_path.join("lib.rs"); + std::fs::write(&file, "pub fn lib_fn() {}").unwrap(); + let absolute = file.to_string_lossy().to_string(); + + let stores = create_test_stores(&db_path, 4).await; + let chunk = |path: &str| { + EmbeddedChunk::new( + Chunk::new( + "pub fn lib_fn() {}".to_string(), + 0, + 1, + ChunkKind::Function, + path.to_string(), + ), + vec![1.0, 0.0, 0.0, 0.0], + ) + }; + let (legacy_id, duplicate_id) = { + let mut vs = stores.vector_store.write().await; + let ids = vs + .insert_chunks_with_ids(vec![chunk(&absolute), chunk("lib.rs")]) + .unwrap(); + vs.build_index().unwrap(); + (ids[0], ids[1]) + }; + + // Pre-2c13826 layout: the file is tracked under its absolute path. + let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); + file_meta + .update_file(&file, &absolute, vec![legacy_id]) + .unwrap(); + file_meta.save(&db_path).unwrap(); + + IndexManager::perform_incremental_refresh_with_stores( + &codebase_path, + &db_path, + &stores, + &CancellationToken::new(), + None, + ) + .await + .expect("refresh over a legacy index must succeed"); + + let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); + assert!(reloaded.is_tracked("lib.rs")); + assert!(!reloaded.is_tracked(&absolute)); + assert_eq!( + reloaded.tracked_chunk_ids(), + std::collections::HashSet::from([legacy_id]), + "an unchanged file must keep its chunks, not be re-embedded" + ); + + let vs = stores.vector_store.read().await; + assert_eq!(vs.get_chunk(legacy_id).unwrap().unwrap().path, "lib.rs"); + assert!( + vs.get_chunk(duplicate_id).unwrap().is_none(), + "the untracked duplicate chunk must be swept" + ); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 07e0fe1c..ab368e09 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -25,6 +25,9 @@ pub use manager::{ mod build_log; pub use build_log::BuildLog; +mod key_migration; +pub(crate) use key_migration::repair_legacy_index; + /// Ensure the HNSW vector index is built if it was never built in a previous /// (possibly cancelled) run. /// @@ -742,6 +745,19 @@ async fn index_with_options( } } + if !file_meta_store.is_empty() { + let mut vs = VectorStore::new(&db_path, model_type.dimensions())?; + let mut fts = FtsStore::new_with_writer(&db_path)?; + if key_migration::repair_legacy_index( + &project_path, + file_meta_store, + &mut vs, + &mut fts, + )? { + file_meta_store.save(&db_path)?; + } + } + // Find changed and deleted files let mut changed_files = Vec::new(); let mut unchanged_files = 0; diff --git a/src/mcp/status.rs b/src/mcp/status.rs index 969f3d4c..6fbfba3c 100644 --- a/src/mcp/status.rs +++ b/src/mcp/status.rs @@ -10,6 +10,12 @@ use rmcp::{ tool, tool_router, ErrorData as McpError, }; +const LOCK_STATUS_INDEXING: &str = "indexing"; +const LOCK_STATUS_BUSY: &str = "busy"; +const LOCK_STATUS_HELD_IN_PROCESS: &str = "held-in-process"; +const LOCK_STATUS_LOCKED_EXTERNALLY: &str = "locked-externally"; +const LOCK_STATUS_AVAILABLE: &str = "available"; + #[tool_router(router = status_router, vis = "pub(crate)")] impl CodesearchService { /// Unified status tool — dispatches based on `kind`. @@ -323,11 +329,13 @@ impl CodesearchService { // For repos already opened in DashMap, use the live SharedStores for stats // WITHOUT opening a new VectorStore connection. // For unopened repos, just report metadata — do NOT open the DB. - if let Some(stores) = serve_state.get_opened_stores(alias) { - let stats_result = { - let vs = stores.vector_store.read().await; - vs.stats() - }; + let opened = serve_state.get_opened_stores(alias); + // Never wait on the store lock: an indexing run holds it for + // whole batches, and status must answer immediately. + let stats_result = opened.as_ref().and_then(|stores| { + stores.vector_store.try_read().ok().map(|vs| vs.stats()) + }); + if let Some(stats_result) = stats_result { // `0 chunks` alone reads exactly like "not indexed yet" — the // repo may in fact be full and simply failing to answer (the // read-only-incident shape this branch exists for). Attribute @@ -352,14 +360,29 @@ impl CodesearchService { error, ) } else { - // Repo NOT opened — read persisted stats from metadata.json + // Not opened, or its store is busy — persisted stats from + // metadata.json; do NOT open the DB. Our own holders are + // checked before the file-lock probe, which cannot tell + // this process apart from another one. let (md_chunks, md_files) = read_metadata_stats(&db_path); - let lock_status = if crate::index::is_database_locked(&db_path) { - "locked-externally".to_string() + let lock_status = if serve_state.is_indexing(alias) { + LOCK_STATUS_INDEXING + } else if opened.is_some() { + LOCK_STATUS_BUSY + } else if serve_state.is_held_in_process(alias, &db_path) { + LOCK_STATUS_HELD_IN_PROCESS + } else if crate::index::is_database_locked(&db_path) { + LOCK_STATUS_LOCKED_EXTERNALLY } else { - "available".to_string() + LOCK_STATUS_AVAILABLE }; - (md_chunks, md_files, model_name, lock_status, None) + ( + md_chunks, + md_files, + model_name, + lock_status.to_string(), + None, + ) } } else { (0, 0, "not indexed".to_string(), "unknown".to_string(), None) diff --git a/src/search/mod.rs b/src/search/mod.rs index 314d7ed8..3011bb49 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -1106,6 +1106,13 @@ fn sync_database(db_path: &Path, model_type: ModelType) -> Result<()> { // Load file metadata store let mut file_meta = FileMetaStore::load_or_create(db_path, model_type.short_name(), model_type.dimensions())?; + if !file_meta.is_empty() { + let mut vs = VectorStore::new(db_path, model_type.dimensions())?; + let mut fts = FtsStore::new_with_writer(db_path)?; + if crate::index::repair_legacy_index(project_path, &mut file_meta, &mut vs, &mut fts)? { + file_meta.save(db_path)?; + } + } // Walk the file system let walker = FileWalker::new(project_path.to_path_buf()); diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 60ca5e03..2c09cb52 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -422,6 +422,25 @@ impl ServeState { .clone() } + /// [`Self::open_lock`] acquired with a deadline, for query paths: a cold + /// open or warmup that holds it for long must surface as an error, never as + /// a request that hangs until the client gives up. + async fn acquire_open_lock_bounded( + &self, + alias: &str, + ) -> std::result::Result, String> { + let wait = Duration::from_secs(crate::constants::REPO_OPEN_WAIT_SECS); + tokio::time::timeout(wait, self.open_lock(alias).lock_owned()) + .await + .map_err(|_| { + format!( + "Repository '{}' is still being opened (waited {}s) — retry shortly", + alias, + wait.as_secs() + ) + }) + } + /// Fast-path lookup: `Some(result)` when `alias` already has opened stores /// in the cache, `None` when a cold open is needed. Shared by the pre-lock /// fast path and the re-check after the single-flight acquire — identical @@ -861,7 +880,7 @@ impl ServeState { /// The eviction uses the atomic `remove_if` primitive so that a concurrent /// `begin_indexing` that inserts a fresh timestamp between the staleness /// check and the removal cannot be wrongly evicted. - fn is_indexing(&self, alias: &str) -> bool { + pub(crate) fn is_indexing(&self, alias: &str) -> bool { let max = self.indexing_timeout(); // Atomically evict a stale entry. `remove_if` holds the shard's write // lock for the predicate check + removal, so a racing `begin_indexing` @@ -2348,7 +2367,7 @@ impl ServeState { // twice, or the loser trips the LMDB double-open guard and the repo // wedges as an incurable Conflicted (todo #131). let open_lock = self.open_lock(alias); - let _open_guard = open_lock.lock().await; + let open_guard = open_lock.lock().await; if let Some(entry) = self.repos.get(alias) { match entry.value() { RepoState::Write { .. } | RepoState::Warm { .. } | RepoState::Readonly { .. } => { @@ -2453,34 +2472,49 @@ impl ServeState { } } - let stores_arc = stores; + // Register as Warm BEFORE the refresh and release the single-flight + // lock: a refresh of a large repo takes minutes, and queries must be + // answered from the existing data meanwhile instead of queueing on + // `open_lock`. FSW is started lazily on first query. + self.repos.insert( + alias.to_string(), + RepoState::Warm { + stores: Arc::clone(&stores), + }, + ); + // Start the idle timer at warmup. A real query will reset it via + // touch_access; without this, repos that are warmed but never queried + // would never appear in `last_access` and therefore never be evicted + // by `evict_idle_repos`, holding LMDB envs and embedder state forever. + self.touch_access(alias); + drop(open_guard); + // Tracked like any reindex: status reports "indexing", the reaper + // leaves the repo alone, and a concurrent reindex is not doubled. + if !self.begin_indexing(alias) { + info!( + "Warmup '{}': another indexing run is active, skipping refresh", + alias + ); + return Ok(()); + } // Warmup runs at startup (pre-warm), never in response to a user action, // so it is given a fresh token that is never cancelled — the refresh runs // to completion. A real user-initiated cancel routes through the // RepoState::Write token owned by the live task instead. let pool = self.embedding_pool(); - if let Err(e) = IndexManager::perform_incremental_refresh_with_stores( + let refreshed = IndexManager::perform_incremental_refresh_with_stores( &path, &db_path, - &stores_arc, + &stores, &CancellationToken::new(), Some(&pool), ) - .await - { + .await; + self.end_indexing(alias); + if let Err(e) = refreshed { tracing::warn!("Warmup '{}': incremental refresh failed: {}", alias, e); } - - // Store as Warm — FSW will be started lazily on first query. - self.repos - .insert(alias.to_string(), RepoState::Warm { stores: stores_arc }); - - // Start the idle timer at warmup. A real query will reset it via - // touch_access; without this, repos that are warmed but never queried - // would never appear in `last_access` and therefore never be evicted - // by `evict_idle_repos`, holding LMDB envs and embedder state forever. - self.touch_access(alias); Ok(()) } @@ -2555,8 +2589,7 @@ impl ServeState { // opens both reach try_open_stores; the second trips the LMDB // double-open guard and caches Conflicted — incurable while the first // opener holds its env (todo #131). - let open_lock = self.open_lock(alias); - let _open_guard = open_lock.lock().await; + let _open_guard = self.acquire_open_lock_bounded(alias).await?; if let Some(result) = self.try_cached_stores(alias, touch) { return result; } @@ -2952,6 +2985,14 @@ impl ServeState { } } + /// True when this process itself keeps `alias`'s DB busy: an indexing run, + /// or an LMDB env still held by an evicted-but-referenced `SharedStores` + /// (e.g. an in-flight handler). A file-lock probe cannot tell that apart + /// from another process, so status must ask this first. + pub(crate) fn is_held_in_process(&self, alias: &str, db_path: &Path) -> bool { + self.is_indexing(alias) || !crate::lmdb_registry::open_holders_under(db_path).is_empty() + } + /// Get the SharedStores for an already-opened repo (no DB open). /// Returns None if the repo is not opened or is in Conflicted state. pub(crate) fn get_opened_stores(&self, alias: &str) -> Option> { diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 1316de05..9d211f73 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3103,3 +3103,28 @@ async fn self_clean_keeps_the_db_dir_of_a_still_registered_repo() { "an unregistered alias's orphaned DB dir must still be cleaned up" ); } + +#[tokio::test] +#[serial] +async fn own_lmdb_holder_and_indexing_count_as_held_in_process() { + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "600")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let tmp = tempfile::tempdir().unwrap(); + let db_path = crate::cache::safe_canonicalize(tmp.path()) + .unwrap() + .join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + + assert!(!state.is_held_in_process("own", &db_path)); + let store = crate::vectordb::VectorStore::new(&db_path, 4).unwrap(); + assert!( + state.is_held_in_process("own", &db_path), + "an env this process holds must never read as an external lock" + ); + drop(store); + assert!(!state.is_held_in_process("own", &db_path)); + + state.begin_indexing("own"); + assert!(state.is_held_in_process("own", &db_path)); + state.end_indexing("own"); +} diff --git a/src/vectordb/mod.rs b/src/vectordb/mod.rs index 8a987483..170f721c 100644 --- a/src/vectordb/mod.rs +++ b/src/vectordb/mod.rs @@ -1,3 +1,3 @@ mod store; -pub use store::{merge_metadata_atomic, SearchResult, StoreStats, VectorStore}; +pub use store::{merge_metadata_atomic, ChunkMetadata, SearchResult, StoreStats, VectorStore}; diff --git a/src/vectordb/store.rs b/src/vectordb/store.rs index 2ac6654b..7fa9999f 100644 --- a/src/vectordb/store.rs +++ b/src/vectordb/store.rs @@ -403,6 +403,9 @@ pub struct VectorStore { /// Key in the "meta" database holding the highest chunk id ever assigned. const META_KEY_ID_HWM: &str = "id_hwm"; +/// Chunks per write transaction in [`VectorStore::rewrite_chunk_paths`]. +const PATH_REWRITE_BATCH: usize = 1000; + /// Derive `next_id` so ids are NEVER reused across reopens. /// /// Takes the max of (highest live key + 1) and (persisted high-water mark + 1): @@ -965,6 +968,57 @@ impl VectorStore { Ok(file_chunks) } + /// Rewrite the stored `path` of chunks in place — metadata only, vectors + /// untouched. Returns the rewritten records so the caller can mirror the + /// new path into the FTS index. Missing ids are skipped. + /// + /// Batched: one transaction over every chunk of a large index would need + /// copy-on-write space for the whole chunks table at once. + pub fn rewrite_chunk_paths( + &mut self, + updates: &[(u32, String)], + ) -> Result> { + let mut rewritten = Vec::with_capacity(updates.len()); + for batch in updates.chunks(PATH_REWRITE_BATCH) { + let done = match self.rewrite_chunk_paths_batch(batch) { + Err(e) if self.is_map_full_error(e.as_ref()) => { + let new_size = self.map_size_mb * 2; + warn!( + "MDB_MAP_FULL rewriting {} chunk path(s), resizing to {}MB", + batch.len(), + new_size + ); + self.resize_environment(new_size)?; + self.rewrite_chunk_paths_batch(batch)? + } + other => other?, + }; + rewritten.extend(done); + } + Ok(rewritten) + } + + fn rewrite_chunk_paths_batch( + &mut self, + batch: &[(u32, String)], + ) -> Result> { + let mut wtxn = self.env.write_txn()?; + let mut rewritten = Vec::with_capacity(batch.len()); + for (id, path) in batch { + let Some(mut meta) = self.chunks.get(&wtxn, id)? else { + continue; + }; + if meta.path == *path { + continue; + } + meta.path = path.clone(); + self.chunks.put(&mut wtxn, id, &meta)?; + rewritten.push((*id, meta)); + } + wtxn.commit()?; + Ok(rewritten) + } + /// Delete chunks by their IDs /// /// Returns the number of chunks deleted @@ -1073,7 +1127,18 @@ impl VectorStore { let result = self.insert_chunks_with_ids_impl(&chunks); match &result { - Ok(_) => return result, + Ok(_) => { + if attempts > 1 { + tracing::info!( + "✅ Insert of {} chunk(s) succeeded on attempt {}/{} at {}MB", + chunks.len(), + attempts, + max_attempts, + self.map_size_mb + ); + } + return result; + } Err(e) => { self.next_id = id_before; if !self.is_map_full_error(e.as_ref()) { From 146518e02b86de896222184f6cdcfeb1b10fbdd7 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 21:45:10 +0200 Subject: [PATCH 05/77] Make legacy repair safe against concurrent refreshes The orphan sweep ran on every refresh and could delete chunks a concurrent refresh had inserted but not yet recorded; it now runs only after a real migration, and the FSW no longer starts while warmup indexes. Path rewrites lock per batch so queries interleave, every re-keyed chunk is mirrored into FTS so a failed mirror is retried, and a killed FTS writer now returns an error instead of silently dropping documents. --- src/fts/tantivy_store.rs | 25 ++++++------- src/index/key_migration.rs | 74 +++++++++++++++++++++++--------------- src/index/manager.rs | 60 +++++++++++++++++++++++++++++++ src/serve/mod.rs | 5 ++- src/serve/tests.rs | 36 +++++++++++++++++++ src/vectordb/mod.rs | 4 ++- src/vectordb/store.rs | 14 ++++---- 7 files changed, 166 insertions(+), 52 deletions(-) diff --git a/src/fts/tantivy_store.rs b/src/fts/tantivy_store.rs index 4b21fef6..2f05ade2 100644 --- a/src/fts/tantivy_store.rs +++ b/src/fts/tantivy_store.rs @@ -444,13 +444,10 @@ impl FtsStore { /// Commit pending changes with retry logic for Windows file locking. /// - /// If the writer was killed (background merge panic), it is recreated. - /// Data since the last successful commit will be lost in that case, but - /// indexing can continue rather than aborting entirely. - /// - /// Any other failure while flushing the workers discards the writer and - /// returns an error: its pending documents are gone, and the caller must - /// learn that instead of seeing a commit that silently indexed nothing. + /// A killed writer (background merge panic) is recreated so the store stays + /// usable; any other flush failure discards it. Both return an error: the + /// pending documents are gone, and the caller must learn that instead of + /// seeing a commit that silently indexed nothing. pub fn commit(&mut self) -> Result<()> { if self.writer.is_none() { return Ok(()); @@ -486,12 +483,6 @@ impl FtsStore { last_error = error_str.clone(); if Self::is_writer_killed(&error_str) { - tracing::debug!( - "FTS writer was killed during commit (attempt {}/{}). \ - Recreating writer. Data since last commit may be lost.", - attempt + 1, - max_retries - ); self.writer = None; self.ensure_writer()?; // After recreating, the pending data is gone, so commit @@ -503,7 +494,13 @@ impl FtsStore { if let Err(e) = self.reader.reload() { tracing::debug!("Reader reload warning: {}", e); } - return Ok(()); + // The store stays usable, but the caller must not record the + // dropped documents as indexed. + return Err(anyhow!( + "FTS writer was killed during commit: {}. A fresh writer was created; \ + uncommitted FTS changes are lost and must be re-indexed", + error_str + )); } if !Self::is_transient_commit_error(&error_str) { diff --git a/src/index/key_migration.rs b/src/index/key_migration.rs index ab224418..2a8adbab 100644 --- a/src/index/key_migration.rs +++ b/src/index/key_migration.rs @@ -14,7 +14,7 @@ use tracing::info; use super::SharedStores; use crate::cache::FileMetaStore; use crate::fts::FtsStore; -use crate::vectordb::{ChunkMetadata, VectorStore}; +use crate::vectordb::{ChunkMetadata, VectorStore, PATH_REWRITE_BATCH}; /// In-process variant for callers that own both stores exclusively (CLI). /// Returns `true` when `file_meta` changed and must be saved by the caller. @@ -33,35 +33,49 @@ pub(crate) fn repair_legacy_index( migration.superseded, ); + if migration.is_empty() { + return Ok(false); + } let untracked = untracked_chunk_ids(file_meta, vector_store)?; if !untracked.is_empty() { delete_from_vector_store(vector_store, &untracked)?; delete_from_fts(fts_store, &untracked)?; log_sweep(untracked.len()); } - Ok(!migration.is_empty()) + Ok(true) } -/// Shared-stores variant. Each phase takes ONE store lock on the blocking +/// Shared-stores variant. Each step takes ONE store lock on the blocking /// pool, never both at once, so it cannot deadlock against a reader that -/// holds the other lock. +/// holds the other lock; path rewrites lock per batch so queries interleave. +/// +/// The orphan sweep runs only after an actual migration: without one, a +/// chunk missing from `file_meta` may belong to a concurrent refresh that +/// inserted it but has not saved its metadata yet. pub(crate) async fn repair_legacy_index_shared( project_root: &Path, file_meta: &mut FileMetaStore, stores: &SharedStores, ) -> Result { let migration = file_meta.relativize_legacy_keys(project_root); + if migration.is_empty() { + return Ok(false); + } - let rekeyed = migration.rekeyed.clone(); - let rewritten = on_store(&stores.vector_store, move |vs| { - rewrite_vector_paths(vs, &rekeyed) - }) - .await?; - let rewritten_count = rewritten.len(); - on_store(&stores.fts_store, move |fts| { - mirror_paths_into_fts(fts, &rewritten) - }) - .await?; + let updates = path_updates(&migration.rekeyed); + let mut rewritten_count = 0; + for batch in updates.chunks(PATH_REWRITE_BATCH) { + let batch = batch.to_vec(); + let rewritten = on_store(&stores.vector_store, move |vs| { + vs.rewrite_chunk_paths(&batch) + }) + .await?; + rewritten_count += rewritten.len(); + on_store(&stores.fts_store, move |fts| { + mirror_paths_into_fts(fts, &rewritten) + }) + .await?; + } log_migration( migration.rekeyed.len(), rewritten_count, @@ -69,27 +83,26 @@ pub(crate) async fn repair_legacy_index_shared( ); let tracked = file_meta.tracked_chunk_ids(); - let meta_empty = file_meta.is_empty(); - let untracked = on_store(&stores.vector_store, move |vs| { - if meta_empty { - return Ok(Vec::new()); - } - let untracked = untracked_among(vs, &tracked)?; - if !untracked.is_empty() { - delete_from_vector_store(vs, &untracked)?; - } - Ok(untracked) + let vector_store = Arc::clone(&stores.vector_store); + let untracked = tokio::task::spawn_blocking(move || { + untracked_among(&vector_store.blocking_read(), &tracked) }) - .await?; + .await + .map_err(|e| anyhow!("legacy index repair task panicked: {}", e))??; if !untracked.is_empty() { let count = untracked.len(); + let ids = untracked.clone(); + on_store(&stores.vector_store, move |vs| { + delete_from_vector_store(vs, &ids) + }) + .await?; on_store(&stores.fts_store, move |fts| { delete_from_fts(fts, &untracked) }) .await?; log_sweep(count); } - Ok(!migration.is_empty()) + Ok(true) } async fn on_store(store: &Arc>, f: F) -> Result @@ -111,11 +124,14 @@ fn rewrite_vector_paths( if rekeyed.is_empty() { return Ok(Vec::new()); } - let updates: Vec<(u32, String)> = rekeyed + vector_store.rewrite_chunk_paths(&path_updates(rekeyed)) +} + +fn path_updates(rekeyed: &[(String, Vec)]) -> Vec<(u32, String)> { + rekeyed .iter() .flat_map(|(key, ids)| ids.iter().map(move |id| (*id, key.clone()))) - .collect(); - vector_store.rewrite_chunk_paths(&updates) + .collect() } fn mirror_paths_into_fts( diff --git a/src/index/manager.rs b/src/index/manager.rs index 8a84345c..06846962 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -3327,6 +3327,66 @@ mod tests { ); } + #[tokio::test] + async fn legacy_repair_never_sweeps_without_a_migration() { + use crate::chunker::{Chunk, ChunkKind}; + use crate::embed::EmbeddedChunk; + + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + let file = codebase_path.join("lib.rs"); + std::fs::write(&file, "pub fn lib_fn() {}").unwrap(); + + let stores = create_test_stores(&db_path, 4).await; + let ids = { + let mut vs = stores.vector_store.write().await; + let chunk = |path: &str| { + EmbeddedChunk::new( + Chunk::new( + "fn f() {}".to_string(), + 0, + 1, + ChunkKind::Function, + path.to_string(), + ), + vec![1.0, 0.0, 0.0, 0.0], + ) + }; + vs.insert_chunks_with_ids(vec![chunk("lib.rs"), chunk("new.rs")]) + .unwrap() + }; + // ids[1] stands for a chunk a concurrent refresh inserted but whose + // file meta it has not saved yet. + let mut file_meta = FileMetaStore::new("test-model".to_string(), 4); + file_meta + .update_file(&file, "lib.rs", vec![ids[0]]) + .unwrap(); + + let changed = super::super::key_migration::repair_legacy_index_shared( + &codebase_path, + &mut file_meta, + &stores, + ) + .await + .unwrap(); + + assert!(!changed); + assert!( + stores + .vector_store + .read() + .await + .get_chunk(ids[1]) + .unwrap() + .is_some(), + "an untracked chunk must survive when no legacy key was migrated" + ); + } + #[tokio::test] async fn test_incremental_refresh_migrates_legacy_absolute_keys() { use crate::chunker::{Chunk, ChunkKind}; diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 2c09cb52..cfa8eea8 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -461,7 +461,10 @@ impl ServeState { // Fan-out/candidate-detection callers pass touch=false and must not // trigger Warm → Write or start FSW. let stores = stores.clone(); - if !touch { + // While warmup's refresh runs, the FSW must not start: its own + // refresh would race warmup's unsaved file meta, and its + // indexing callback would clear warmup's marker. + if !touch || self.is_indexing(alias) { return Some(Ok(stores)); } drop(entry); // release DashMap read guard before mutation diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 9d211f73..dac2b092 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3128,3 +3128,39 @@ async fn own_lmdb_holder_and_indexing_count_as_held_in_process() { assert!(state.is_held_in_process("own", &db_path)); state.end_indexing("own"); } + +#[tokio::test] +#[serial] +async fn warm_query_does_not_start_fsw_while_warmup_indexes() { + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "600")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let tmp = tempfile::tempdir().unwrap(); + let db_path = crate::cache::safe_canonicalize(tmp.path()) + .unwrap() + .join(DB_DIR_NAME); + let OpenedStores::Write(stores) = state + .try_open_stores("warming", &db_path, true, false, None) + .expect("open must succeed") + else { + panic!("brand-new repo must open Write"); + }; + state + .repos + .insert("warming".to_string(), RepoState::Warm { stores }); + state.begin_indexing("warming"); + + let result = state.try_cached_stores("warming", true); + assert!( + matches!(result, Some(Ok(_))), + "queries must be answered from Warm" + ); + assert!( + matches!( + state.repos.get("warming").unwrap().value(), + RepoState::Warm { .. } + ), + "a query during warmup indexing must not transition Warm to Write" + ); + state.end_indexing("warming"); + state.repos.remove("warming"); +} diff --git a/src/vectordb/mod.rs b/src/vectordb/mod.rs index 170f721c..330021bb 100644 --- a/src/vectordb/mod.rs +++ b/src/vectordb/mod.rs @@ -1,3 +1,5 @@ mod store; -pub use store::{merge_metadata_atomic, ChunkMetadata, SearchResult, StoreStats, VectorStore}; +pub use store::{ + merge_metadata_atomic, ChunkMetadata, SearchResult, StoreStats, VectorStore, PATH_REWRITE_BATCH, +}; diff --git a/src/vectordb/store.rs b/src/vectordb/store.rs index 7fa9999f..58f51370 100644 --- a/src/vectordb/store.rs +++ b/src/vectordb/store.rs @@ -404,7 +404,7 @@ pub struct VectorStore { const META_KEY_ID_HWM: &str = "id_hwm"; /// Chunks per write transaction in [`VectorStore::rewrite_chunk_paths`]. -const PATH_REWRITE_BATCH: usize = 1000; +pub const PATH_REWRITE_BATCH: usize = 1000; /// Derive `next_id` so ids are NEVER reused across reopens. /// @@ -969,8 +969,9 @@ impl VectorStore { } /// Rewrite the stored `path` of chunks in place — metadata only, vectors - /// untouched. Returns the rewritten records so the caller can mirror the - /// new path into the FTS index. Missing ids are skipped. + /// untouched. Returns every found record, already-rewritten ones included, + /// so a caller retrying after a failed FTS mirror still mirrors them all. + /// Missing ids are skipped. /// /// Batched: one transaction over every chunk of a large index would need /// copy-on-write space for the whole chunks table at once. @@ -1008,11 +1009,10 @@ impl VectorStore { let Some(mut meta) = self.chunks.get(&wtxn, id)? else { continue; }; - if meta.path == *path { - continue; + if meta.path != *path { + meta.path = path.clone(); + self.chunks.put(&mut wtxn, id, &meta)?; } - meta.path = path.clone(); - self.chunks.put(&mut wtxn, id, &meta)?; rewritten.push((*id, meta)); } wtxn.commit()?; From 1b12a1e05e4bc16217e4c199954b26924d5d29ef Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 22:12:36 +0200 Subject: [PATCH 06/77] Renew the warmup indexing marker with a per-batch heartbeat BOIN-scale warmups run past MAX_INDEXING_SECS (30 min), so the lazy stale-marker eviction dropped the marker mid-refresh and the idle reaper could evict the repo while the refresh still held its stores (LMDB double-open / todo #131 shape). perform_incremental_refresh_with_stores now fires an IndexingHeartbeat at start, before each slow phase, and per batch; warmup passes a closure that renews its marker via ServeState::renew_indexing (upsert, so a raced eviction cannot strip protection for the rest of the run). Other callers pass None for now. --- src/index/manager.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++ src/index/mod.rs | 4 ++-- src/mcp/runtime.rs | 1 + src/serve/mod.rs | 25 +++++++++++++++++++++++- src/serve/tests.rs | 35 ++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 3 deletions(-) diff --git a/src/index/manager.rs b/src/index/manager.rs index 06846962..301d2b2d 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -72,6 +72,13 @@ pub type CSharpRebuildNotifier = Arc; /// during file-watcher-triggered refreshes (branch changes, batch flushes). pub type IndexingStatusCallback = Arc; +/// Heartbeat fired by long indexing loops (once per batch, plus before each +/// slow phase) so the caller can renew its indexing marker before +/// `MAX_INDEXING_SECS` expires. BOIN-scale warmups run well past 30 minutes; +/// without renewal the stale-marker eviction drops the marker mid-refresh and +/// the reaper/FSW can then race the live refresh (todo #131 shape). +pub type IndexingHeartbeat = Arc; + /// Batch flush timeout in milliseconds. /// Events are batched and flushed when: /// 1. No new events for this duration, OR @@ -581,12 +588,18 @@ impl IndexManager { /// fresh `EmbeddingService` in the serve process would be refused by the /// process-global LMDB registry (the pool already holds the cache open) /// and silently degrade to cache-less re-embedding. + /// + /// `heartbeat` — when `Some`, fired once at start, before each slow phase, + /// and once per batch. Callers tracking an `active_reindexes` marker pass + /// a closure that renews it; without renewal a refresh running past + /// `MAX_INDEXING_SECS` loses the marker and races the reaper/FSW. pub async fn perform_incremental_refresh_with_stores( codebase_path: &Path, db_path: &Path, stores: &SharedStores, cancel_token: &CancellationToken, embedding_pool: Option<&Arc>, + heartbeat: Option<&IndexingHeartbeat>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::chunker::SemanticChunker; @@ -599,6 +612,9 @@ impl IndexManager { // Bail out before reading/deriving anything if a cancellation already // arrived (e.g. remove_repo ran while this task was scheduled). Self::ensure_indexing_active(cancel_token)?; + if let Some(hb) = heartbeat { + hb(); + } // Read model name + dims (lenient) for the FileMetaStore. The strict, // fail-fast embedding-model resolution happens lazily below, only when @@ -705,6 +721,9 @@ impl IndexManager { // destructive store mutation below (stale-chunk deletion), so a // half-cleaned index is never left behind by a removed repo. Self::ensure_indexing_active(cancel_token)?; + if let Some(hb) = heartbeat { + hb(); + } // There is work to do. Resolve the embedding model NOW — before any // destructive store mutation below — so that a corrupt index (unknown @@ -800,6 +819,12 @@ impl IndexManager { for (batch_idx, file_batch) in changed_files.chunks(batch_size).enumerate() { // Abort between batches if the repo was removed mid-index. Self::ensure_indexing_active(cancel_token)?; + // A batch of a large delta can run for minutes — renew the + // caller's indexing marker so it survives past the stale + // threshold while real progress is being made. + if let Some(hb) = heartbeat { + hb(); + } // Read + chunk + embed is synchronous, CPU/I/O-heavy work // (file reads, tree-sitter parsing, fastembed/ONNX inference that @@ -1168,6 +1193,7 @@ impl IndexManager { stores, cancel_token, embedding_pool, + None, ) .await } @@ -2790,6 +2816,7 @@ mod tests { &stores, &token, None, + None, ) .await; @@ -3249,12 +3276,24 @@ mod tests { let stores = create_test_stores(&db_path, 4).await; + // The heartbeat must be threaded through and invoked. Only the + // start-of-refresh beat is reachable here: a batch with changed files + // needs a real embedder, which no unit test spins up. + let beats = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let heartbeat: IndexingHeartbeat = { + let beats = Arc::clone(&beats); + Arc::new(move || { + beats.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }) + }; + let result = IndexManager::perform_incremental_refresh_with_stores( &codebase_path, &db_path, &stores, &CancellationToken::new(), None, + Some(&heartbeat), ) .await; @@ -3263,6 +3302,10 @@ mod tests { "Up-to-date incremental refresh should succeed: {:?}", result ); + assert!( + beats.load(std::sync::atomic::Ordering::SeqCst) >= 1, + "heartbeat must fire at least once per refresh" + ); // The tracked file must still be tracked and not flagged as deleted. let reloaded = FileMetaStore::load_or_create(&db_path, "test-model", 4).unwrap(); @@ -3303,6 +3346,7 @@ mod tests { &stores, &CancellationToken::new(), None, + None, ) .await; @@ -3438,6 +3482,7 @@ mod tests { &stores, &CancellationToken::new(), None, + None, ) .await .expect("refresh over a legacy index must succeed"); diff --git a/src/index/mod.rs b/src/index/mod.rs index ab368e09..4b0877c7 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -18,8 +18,8 @@ use crate::vectordb::{merge_metadata_atomic, VectorStore}; // Index manager module mod manager; pub use manager::{ - is_database_locked, CSharpRebuildNotifier, IndexManager, IndexingStatusCallback, SharedStores, - SymbolRebuildSignal, + is_database_locked, CSharpRebuildNotifier, IndexManager, IndexingHeartbeat, + IndexingStatusCallback, SharedStores, SymbolRebuildSignal, }; mod build_log; diff --git a/src/mcp/runtime.rs b/src/mcp/runtime.rs index 415ed9bb..221e94e7 100644 --- a/src/mcp/runtime.rs +++ b/src/mcp/runtime.rs @@ -829,6 +829,7 @@ pub async fn run_mcp_server_with_options( &shared_stores_clone, &bg_cancel_token, None, + None, ) .await { diff --git a/src/serve/mod.rs b/src/serve/mod.rs index cfa8eea8..2944bb5b 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -45,7 +45,8 @@ use crate::constants::{ }; use crate::db_discovery::repos::{config_dir, ReposConfig}; use crate::index::{ - CSharpRebuildNotifier, IndexManager, IndexingStatusCallback, SharedStores, SymbolRebuildSignal, + CSharpRebuildNotifier, IndexManager, IndexingHeartbeat, IndexingStatusCallback, SharedStores, + SymbolRebuildSignal, }; use crate::mcp::types::HealthResponse; use crate::symbols::{csharp, RebuildScope, SymbolIndexerRegistry}; @@ -657,6 +658,17 @@ impl ServeState { self.active_reindexes.remove(alias); } + /// Renew the indexing marker for `alias` — the per-batch heartbeat a long + /// refresh uses to stay alive past the stale-marker threshold + /// (`is_indexing` would otherwise evict it mid-run and let the reaper and + /// FSW race the live refresh). Upsert on purpose: if the marker was + /// already evicted by a race, re-inserting still protects the remainder + /// of the refresh. + fn renew_indexing(&self, alias: &str) { + self.active_reindexes + .insert(alias.to_string(), Instant::now()); + } + /// True iff an error chain indicates LMDB storage-format corruption — /// data written by an older storage-major (arroy/heed) that the current /// one refuses to read — rather than a transient or unrelated failure. @@ -2293,6 +2305,7 @@ impl ServeState { &stores_bg, &token_for_task, Some(&pool), + None, ) .await { @@ -2506,12 +2519,20 @@ impl ServeState { // to completion. A real user-initiated cancel routes through the // RepoState::Write token owned by the live task instead. let pool = self.embedding_pool(); + // BOIN-scale warmups run past MAX_INDEXING_SECS; the per-batch heartbeat + // keeps the marker alive so the reaper and FSW keep honouring it. + let heartbeat: IndexingHeartbeat = { + let state = Arc::clone(self); + let alias_hb = alias.to_string(); + Arc::new(move || state.renew_indexing(&alias_hb)) + }; let refreshed = IndexManager::perform_incremental_refresh_with_stores( &path, &db_path, &stores, &CancellationToken::new(), Some(&pool), + Some(&heartbeat), ) .await; self.end_indexing(alias); @@ -2710,6 +2731,7 @@ impl ServeState { &stores_for_task, &token_for_task, Some(&pool), + None, ) .await { @@ -4434,6 +4456,7 @@ async fn reindex_handler( &stores, &reindex_token_task, Some(&pool), + None, ) .await { diff --git a/src/serve/tests.rs b/src/serve/tests.rs index dac2b092..4be041ea 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3164,3 +3164,38 @@ async fn warm_query_does_not_start_fsw_while_warmup_indexes() { state.end_indexing("warming"); state.repos.remove("warming"); } + +#[tokio::test] +#[serial] +async fn indexing_marker_heartbeat_survives_past_max() { + // Regression: a BOIN-scale warmup runs longer than MAX_INDEXING_SECS, so + // the lazy stale-marker eviction dropped its marker mid-refresh and the + // reaper could evict the repo while the refresh still held its stores. + // The per-batch heartbeat must keep the marker alive for as long as the + // refresh is making progress. + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "2")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + + // Control: without renewal the marker expires at the threshold. + state.begin_indexing("ctrl"); + tokio::time::sleep(Duration::from_millis(2100)).await; + assert!( + !state.is_indexing("ctrl"), + "an unrenewed marker must expire at the threshold" + ); + + // Heartbeat: renewal inside the window keeps the marker alive past max. + state.begin_indexing("hb"); + tokio::time::sleep(Duration::from_millis(1000)).await; + state.renew_indexing("hb"); + tokio::time::sleep(Duration::from_millis(1000)).await; + assert!( + state.is_indexing("hb"), + "a heartbeat-renewed marker must stay alive past the threshold" + ); + state.end_indexing("hb"); + assert!( + !state.is_indexing("hb"), + "end_indexing must clear the marker" + ); +} From ee2e1df14f34ece96e14c9bcb3fc569c2a0db4a7 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 22:29:39 +0200 Subject: [PATCH 07/77] Scope indexing markers per owner so completions cannot erase each other MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FSW's completion callback cleared the whole alias entry with a plain bool, so a watcher refresh finishing mid-run erased a concurrent force reindex's marker — the reaper could then evict a repo the reindex still held (LMDB double-open shape). active_reindexes now maps alias -> owner -> last heartbeat (IndexingOwner: Warmup/Watcher/Reindex/Symbol); begin/end/ renew are owner-scoped, is_indexing evicts only stale owners and still cancels a leaked index task, and begin_indexing keeps rejecting while any owner's marker is fresh. Adds the race, exclusivity and per-owner eviction tests. Review-fixes: - [Critical] Empty-map cleanup dropped the shard guard before remove(alias), so a concurrent begin_indexing insert landing in the gap was erased (re-opened the plain-bool marker race) → all three sites (is_indexing, end_indexing, FSW callback) now use atomic remove_if(alias, |_, m| m.is_empty()) which holds the shard lock across predicate + removal. --- src/serve/mod.rs | 217 +++++++++++++++++++++++++++------------------ src/serve/tests.rs | 116 +++++++++++++++++++++--- src/serve/tui.rs | 20 ++--- 3 files changed, 241 insertions(+), 112 deletions(-) diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 2944bb5b..ed168d5b 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -181,6 +181,23 @@ pub(crate) enum OpenedStores { Readonly(Arc), } +/// Which flow holds an indexing marker in [`ServeState::active_reindexes`]. +/// +/// Owners make marker completion scoped: the file-watcher callback ends the +/// [`IndexingOwner::Watcher`] marker only, so it can never erase a concurrent +/// reindex's or warmup's marker (the plain-bool callback bug). +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub(crate) enum IndexingOwner { + /// Background warmup refresh (`warmup_repo`). + Warmup, + /// File-watcher-triggered refresh batches (`make_indexing_status_callback`). + Watcher, + /// User-initiated reindex (HTTP handler, TUI, force reindex). + Reindex, + /// Watcher-triggered SCIP symbol rebuild (`trigger_symbol_rebuild`). + Symbol, +} + /// Shared state for the serve mode. pub(crate) struct ServeState { /// Repo alias → opened stores (or conflicted marker). @@ -256,9 +273,11 @@ pub(crate) struct ServeState { config_mtime: std::sync::RwLock>, /// Optional override for the repos config path (used in tests to avoid env vars). config_path_override: Option, - /// Aliases currently being reindexed — prevents concurrent force reindex - /// on the same repo. The value is the `Instant` the entry was inserted so - /// that stale (leaked) entries can be detected and self-healed; see + /// Aliases currently being indexed, per owner. Owners exist so one flow's + /// completion marker cannot erase another's: the file-watcher callback + /// used to clear the whole entry with a plain bool, wiping a concurrent + /// force reindex's marker mid-run. The `Instant` is the last heartbeat so + /// stale (leaked) entries can be detected and self-healed; see /// [`Self::begin_indexing`] / [`Self::is_indexing`] and /// [`MAX_INDEXING_SECS`]. /// @@ -266,7 +285,7 @@ pub(crate) struct ServeState { /// capture a cheap clone that **shares** the underlying map (a bare /// `DashMap::clone()` is a deep copy and would silently disconnect the /// file-watcher callback from this field). - active_reindexes: Arc>, + active_reindexes: Arc>>, /// Per-repo change count since serve started (incremented by index/reindex operations). repo_changes: DashMap, /// Per-repo last tool call: (tool_name, timestamp). @@ -608,65 +627,77 @@ impl ServeState { /// /// The callback captures a clone of `active_reindexes` so it can be sent /// into the file-watcher background task. The watcher calls this closure to - /// insert/remove the alias around every reindex — branch-change refresh, - /// text-batch flush, and symbol rebuild — making "Indexing" visible in the - /// TUI status column. + /// add/remove the [`IndexingOwner::Watcher`] marker around every refresh — + /// branch-change refresh, text-batch flush, and symbol rebuild — making + /// "Indexing" visible in the TUI status column. Owner-scoped on purpose: + /// the completion half must not erase another flow's marker. fn make_indexing_status_callback(&self, alias: &str) -> IndexingStatusCallback { let reindexes = self.active_reindexes.clone(); let alias_key = alias.to_string(); Arc::new(move |active: bool| { if active { - reindexes.insert(alias_key.clone(), Instant::now()); + reindexes + .entry(alias_key.clone()) + .or_default() + .insert(IndexingOwner::Watcher, Instant::now()); } else { - reindexes.remove(&alias_key); + if let Some(mut owners) = reindexes.get_mut(&alias_key) { + owners.remove(&IndexingOwner::Watcher); + } + // Atomic empty-cleanup (see end_indexing): a concurrent + // begin_indexing insert must never be erased by this remove. + reindexes.remove_if(&alias_key, |_, m| m.is_empty()); } }) } - /// Mark `alias` as actively indexing, returning `true` if the caller may - /// proceed. Returns `false` only when a **non-stale** entry already exists - /// (i.e. another reindex is genuinely in progress) — in that case the - /// caller should return HTTP 409. Stale entries are silently overwritten - /// with a fresh timestamp. + /// Mark `alias` as actively indexing for `owner`, returning `true` if the + /// caller may proceed. Returns `false` when **any** owner holds a + /// non-stale marker (i.e. another indexing run is genuinely in progress) — + /// in that case the caller should return HTTP 409. Stale markers from + /// leaked/crashed tasks are silently dropped first. /// /// This is the guard used by `reindex_handler`, `add_repo_handler`, and /// `spawn_force_reindex` to reject concurrent reindexes. - fn begin_indexing(&self, alias: &str) -> bool { + fn begin_indexing(&self, alias: &str, owner: IndexingOwner) -> bool { let now = Instant::now(); let max = self.indexing_timeout(); - match self.active_reindexes.entry(alias.to_string()) { - dashmap::mapref::entry::Entry::Occupied(mut e) => { - if now.duration_since(*e.get()) < max { - // Genuinely in progress — reject. - false - } else { - // Stale entry from a leaked/crashed task — overwrite. - *e.get_mut() = now; - true - } - } - dashmap::mapref::entry::Entry::Vacant(e) => { - e.insert(now); - true - } + let mut owners = self.active_reindexes.entry(alias.to_string()).or_default(); + owners.retain(|_, ts| now.duration_since(*ts) < max); + if owners.is_empty() { + owners.insert(owner, now); + true + } else { + // Genuinely in progress — reject. + false } } - /// Remove the indexing marker for `alias`. Called when a background - /// indexing task finishes (success, error, or panic). - fn end_indexing(&self, alias: &str) { - self.active_reindexes.remove(alias); + /// Remove `owner`'s indexing marker for `alias`. Called when that flow's + /// background task finishes (success, error, or panic). Other owners' + /// markers are untouched — the file-watcher's completion must not end a + /// concurrent reindex's marker. + fn end_indexing(&self, alias: &str, owner: IndexingOwner) { + if let Some(mut owners) = self.active_reindexes.get_mut(alias) { + owners.remove(&owner); + } + // Atomic empty-cleanup: the `remove_if` predicate+removal holds the + // shard lock, so a `begin_indexing` that inserts a fresh marker + // between the inner removal and this call keeps its marker. + self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); } - /// Renew the indexing marker for `alias` — the per-batch heartbeat a long - /// refresh uses to stay alive past the stale-marker threshold + /// Renew `owner`'s indexing marker for `alias` — the per-batch heartbeat a + /// long refresh uses to stay alive past the stale-marker threshold /// (`is_indexing` would otherwise evict it mid-run and let the reaper and /// FSW race the live refresh). Upsert on purpose: if the marker was /// already evicted by a race, re-inserting still protects the remainder /// of the refresh. - fn renew_indexing(&self, alias: &str) { + fn renew_indexing(&self, alias: &str, owner: IndexingOwner) { self.active_reindexes - .insert(alias.to_string(), Instant::now()); + .entry(alias.to_string()) + .or_default() + .insert(owner, Instant::now()); } /// True iff an error chain indicates LMDB storage-format corruption — @@ -884,39 +915,52 @@ impl ServeState { Ok(()) } - /// Returns `true` if `alias` is currently (non-stale) indexing. + /// Returns `true` if `alias` is currently (non-stale) indexing for any + /// owner. /// - /// Stale entries — those older than [`MAX_INDEXING_SECS`] — are lazily + /// Stale markers — those older than [`MAX_INDEXING_SECS`] — are lazily /// evicted here. This is the self-healing mechanism: even if a /// fire-and-forget background task panics or is cancelled between - /// `begin_indexing` and `end_indexing`, the entry eventually expires and + /// `begin_indexing` and `end_indexing`, its marker eventually expires and /// the TUI returns to the correct state without a server restart. /// - /// The eviction uses the atomic `remove_if` primitive so that a concurrent - /// `begin_indexing` that inserts a fresh timestamp between the staleness - /// check and the removal cannot be wrongly evicted. + /// Eviction runs under the shard's write lock (`get_mut` + `retain`), so a + /// concurrent `begin_indexing`/`renew_indexing` that refreshed a marker in + /// the meantime is never wrongly evicted: its timestamp is fresh when the + /// predicate runs, or its insert lands before/after the atomic + /// `remove_if` empty-cleanup (which holds the same shard lock across + /// predicate + removal). pub(crate) fn is_indexing(&self, alias: &str) -> bool { let max = self.indexing_timeout(); - // Atomically evict a stale entry. `remove_if` holds the shard's write - // lock for the predicate check + removal, so a racing `begin_indexing` - // that refreshed the timestamp in the meantime will cause the predicate - // to return false and the entry to be kept. - if self - .active_reindexes - .remove_if(alias, |_, ts| ts.elapsed() >= max) - .is_some() - { + let mut evicted: Vec = Vec::new(); + let mut any_left = false; + if let Some(mut owners) = self.active_reindexes.get_mut(alias) { + let now = Instant::now(); + owners.retain(|owner, ts| { + let fresh = now.duration_since(*ts) < max; + if !fresh { + evicted.push(*owner); + } + fresh + }); + any_left = !owners.is_empty(); + } + if !any_left { + // Atomic empty-cleanup (see end_indexing): a begin_indexing that + // raced this eviction keeps its fresh marker. + self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); + } + if !evicted.is_empty() { tracing::warn!( - "🧹 Evicted stale indexing marker for '{}' (older than {}s) — \ - likely a leaked/crashed background task", + "🧹 Evicted stale indexing marker(s) for '{}' (older than {}s, owners \ + {:?}) — likely a leaked/crashed background task", alias, - max.as_secs() + max.as_secs(), + evicted ); self.cancel_stale_index_task(alias); - return false; } - // Entry is either absent or still within the active window. - self.active_reindexes.contains_key(alias) + any_left } /// Cancel the background index task still registered for `alias` after its @@ -2507,7 +2551,7 @@ impl ServeState { // Tracked like any reindex: status reports "indexing", the reaper // leaves the repo alone, and a concurrent reindex is not doubled. - if !self.begin_indexing(alias) { + if !self.begin_indexing(alias, IndexingOwner::Warmup) { info!( "Warmup '{}': another indexing run is active, skipping refresh", alias @@ -2524,7 +2568,7 @@ impl ServeState { let heartbeat: IndexingHeartbeat = { let state = Arc::clone(self); let alias_hb = alias.to_string(); - Arc::new(move || state.renew_indexing(&alias_hb)) + Arc::new(move || state.renew_indexing(&alias_hb, IndexingOwner::Warmup)) }; let refreshed = IndexManager::perform_incremental_refresh_with_stores( &path, @@ -2535,7 +2579,7 @@ impl ServeState { Some(&heartbeat), ) .await; - self.end_indexing(alias); + self.end_indexing(alias, IndexingOwner::Warmup); if let Err(e) = refreshed { tracing::warn!("Warmup '{}': incremental refresh failed: {}", alias, e); } @@ -4055,14 +4099,11 @@ async fn trigger_symbol_rebuild( // Mark as actively indexing so the TUI status column shows "Indexing" // (not just the C# indicator). This mirrors what reindex_handler does. // - // Known benign race: if the FSW-SCIP rebuild path (indexing_cb) fires for - // the same alias simultaneously, both paths insert into active_reindexes. - // Because the map key is the alias, there is no data corruption. - // However, whichever path finishes first will call remove(), which may - // briefly flip the TUI back to Warm/Open while the other path is still - // running. This is a cosmetic flash only — no state is corrupted. - // (Stale entries from a crashed task self-heal via `is_indexing`.) - state.begin_indexing(alias); + // Owner-scoped markers: if the FSW-SCIP rebuild path (indexing_cb) fires + // for the same alias simultaneously, each path ends only its own marker, + // so neither can flip the TUI back to Warm/Open while the other still + // runs. (Stale entries from a crashed task self-heal via `is_indexing`.) + state.begin_indexing(alias, IndexingOwner::Symbol); let rp = project_path.to_path_buf(); let dp = db_path.to_path_buf(); let alias_owned = alias.to_string(); @@ -4086,7 +4127,7 @@ async fn trigger_symbol_rebuild( summary.references_stored, summary.duration_ms ); - state.end_indexing(&alias_owned); + state.end_indexing(&alias_owned, IndexingOwner::Symbol); state .csharp_index_status .insert(alias_owned.clone(), CSharpIndexStatus::Ready); @@ -4102,7 +4143,7 @@ async fn trigger_symbol_rebuild( // SCIP puts wrap their errors (table + key size), and plain `{}` // would hide the `MDB_*` code the classifier below matches on. let msg = format!("{e:#}"); - state.end_indexing(&alias_owned); + state.end_indexing(&alias_owned, IndexingOwner::Symbol); state .csharp_index_error .insert(alias_owned.clone(), msg.clone()); @@ -4147,7 +4188,7 @@ async fn trigger_symbol_rebuild( alias_owned, e ); - state.end_indexing(&alias_owned); + state.end_indexing(&alias_owned, IndexingOwner::Symbol); state .csharp_index_error .insert(alias_owned.clone(), format!("Task panicked: {}", e)); @@ -4272,7 +4313,7 @@ async fn reindex_handler( let alias_bg = alias.clone(); // Concurrent reindex guard — reject if this alias is already being reindexed - if !state.begin_indexing(&alias_bg) { + if !state.begin_indexing(&alias_bg, IndexingOwner::Reindex) { return ( StatusCode::CONFLICT, axum::response::Json(json!({ @@ -4317,7 +4358,7 @@ async fn reindex_handler( } Ok(OpenedStores::Readonly(_)) => { // Cannot force-reindex against a readonly store. - state.end_indexing(&guard_alias); + state.end_indexing(&guard_alias, IndexingOwner::Reindex); return ( StatusCode::INTERNAL_SERVER_ERROR, axum::response::Json(json!({ @@ -4330,7 +4371,7 @@ async fn reindex_handler( ); } Err(e) => { - state.end_indexing(&guard_alias); + state.end_indexing(&guard_alias, IndexingOwner::Reindex); return ( StatusCode::INTERNAL_SERVER_ERROR, axum::response::Json(json!({ @@ -4381,7 +4422,7 @@ async fn reindex_handler( // would resurrect the removed alias with a fresh, // uncancellable task. tracing::info!("Reindex cancelled for '{}': {}", alias_bg, e); - g_state.end_indexing(&g_alias); + g_state.end_indexing(&g_alias, IndexingOwner::Reindex); return; } tracing::error!("Force reindex failed for '{}': {}", alias_bg, e); @@ -4404,7 +4445,7 @@ async fn reindex_handler( ); drop(stores); g_state.self_clean_if_unregistered(&g_alias, &db_path); - g_state.end_indexing(&g_alias); + g_state.end_indexing(&g_alias, IndexingOwner::Reindex); return; } @@ -4416,7 +4457,7 @@ async fn reindex_handler( trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; } - g_state.end_indexing(&g_alias); + g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); state .index_tasks @@ -4426,7 +4467,7 @@ async fn reindex_handler( let stores = match state.get_or_open_stores(&alias, true).await { Ok(s) => s, Err(e) => { - state.end_indexing(&guard_alias); + state.end_indexing(&guard_alias, IndexingOwner::Reindex); return ( StatusCode::INTERNAL_SERVER_ERROR, axum::response::Json(json!({ @@ -4481,7 +4522,7 @@ async fn reindex_handler( ); drop(stores); g_state.self_clean_if_unregistered(&g_alias, &db_path); - g_state.end_indexing(&g_alias); + g_state.end_indexing(&g_alias, IndexingOwner::Reindex); return; } @@ -4490,7 +4531,7 @@ async fn reindex_handler( trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; } - g_state.end_indexing(&g_alias); + g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); state .index_tasks @@ -4724,7 +4765,7 @@ async fn add_repo_handler( state.touch_access(&alias); // Guard against concurrent reindex for the same alias. - if !state.begin_indexing(&alias) { + if !state.begin_indexing(&alias, IndexingOwner::Reindex) { // Another reindex for this alias is already in progress. // We must undo *all* side-effects created so far: // 1. Cancel the token and remove from repos (releases the LMDB handle). @@ -4795,13 +4836,13 @@ async fn add_repo_handler( // destructive cleanup (repos.remove/unregister) here, just // release the indexing guard and let remove_repo finish. tracing::info!("Indexing cancelled for '{}': {}", alias_bg, e); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); return; } tracing::error!("Index creation failed for '{}': {}", alias_bg, e); // Clean up: remove from repos and config state_bg.repos.remove(&alias_bg); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); if let Ok(mut config) = state_bg.config.write() { config.unregister_alias(&alias_bg); if let Err(e) = state_bg.persist_config(&config) { @@ -4825,7 +4866,7 @@ async fn add_repo_handler( "Skipping build_index for '{}': repo removed or cancelled mid-index", alias_bg ); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); return; } @@ -4866,14 +4907,14 @@ async fn add_repo_handler( ); drop(stores); state_bg.self_clean_if_unregistered(&alias_bg, &db_path); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); return; } // Start FSW and transition to proper Write state with IndexManager state_bg.restart_fsw(&alias_bg, stores).await; - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); tracing::info!("Repo '{}' fully indexed and ready", alias_bg); }); diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 4be041ea..7766cb3f 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -2346,11 +2346,11 @@ fn freshness_for_path_reports_indexing_state() { // Mid-reindex (the exact state a branch-switch full refresh puts the // repo in — make_indexing_status_callback inserts around it). - state.begin_indexing("fresh"); + state.begin_indexing("fresh", IndexingOwner::Watcher); let (_, indexing) = state.freshness_for_path(&target.to_string_lossy()); assert!(indexing, "begin_indexing must surface as indexing=true"); - state.end_indexing("fresh"); + state.end_indexing("fresh", IndexingOwner::Watcher); let (_, indexing) = state.freshness_for_path(&target.to_string_lossy()); assert!(!indexing); @@ -3021,9 +3021,13 @@ async fn stale_indexing_marker_cancels_the_leaked_index_task() { .insert("leaky".to_string(), (handle, token.clone())); state.active_reindexes.insert( "leaky".to_string(), - Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("monotonic clock at least 2s old"), + std::iter::once(( + IndexingOwner::Reindex, + Instant::now() + .checked_sub(Duration::from_secs(2)) + .expect("monotonic clock at least 2s old"), + )) + .collect(), ); assert!( @@ -3060,7 +3064,7 @@ async fn fresh_indexing_marker_leaves_its_index_task_running() { state .index_tasks .insert("busy".to_string(), (handle, token.clone())); - state.begin_indexing("busy"); + state.begin_indexing("busy", IndexingOwner::Reindex); assert!( state.is_indexing("busy"), @@ -3124,9 +3128,9 @@ async fn own_lmdb_holder_and_indexing_count_as_held_in_process() { drop(store); assert!(!state.is_held_in_process("own", &db_path)); - state.begin_indexing("own"); + state.begin_indexing("own", IndexingOwner::Reindex); assert!(state.is_held_in_process("own", &db_path)); - state.end_indexing("own"); + state.end_indexing("own", IndexingOwner::Reindex); } #[tokio::test] @@ -3147,7 +3151,7 @@ async fn warm_query_does_not_start_fsw_while_warmup_indexes() { state .repos .insert("warming".to_string(), RepoState::Warm { stores }); - state.begin_indexing("warming"); + state.begin_indexing("warming", IndexingOwner::Warmup); let result = state.try_cached_stores("warming", true); assert!( @@ -3161,7 +3165,7 @@ async fn warm_query_does_not_start_fsw_while_warmup_indexes() { ), "a query during warmup indexing must not transition Warm to Write" ); - state.end_indexing("warming"); + state.end_indexing("warming", IndexingOwner::Warmup); state.repos.remove("warming"); } @@ -3177,7 +3181,7 @@ async fn indexing_marker_heartbeat_survives_past_max() { let state = Arc::new(ServeState::new(ReposConfig::default(), None)); // Control: without renewal the marker expires at the threshold. - state.begin_indexing("ctrl"); + state.begin_indexing("ctrl", IndexingOwner::Warmup); tokio::time::sleep(Duration::from_millis(2100)).await; assert!( !state.is_indexing("ctrl"), @@ -3185,17 +3189,101 @@ async fn indexing_marker_heartbeat_survives_past_max() { ); // Heartbeat: renewal inside the window keeps the marker alive past max. - state.begin_indexing("hb"); + state.begin_indexing("hb", IndexingOwner::Warmup); tokio::time::sleep(Duration::from_millis(1000)).await; - state.renew_indexing("hb"); + state.renew_indexing("hb", IndexingOwner::Warmup); tokio::time::sleep(Duration::from_millis(1000)).await; assert!( state.is_indexing("hb"), "a heartbeat-renewed marker must stay alive past the threshold" ); - state.end_indexing("hb"); + state.end_indexing("hb", IndexingOwner::Warmup); assert!( !state.is_indexing("hb"), "end_indexing must clear the marker" ); } + +#[tokio::test] +#[serial] +async fn fsw_callback_end_cannot_clear_a_concurrent_reindex_marker() { + // Regression: the FSW completion callback cleared the whole alias entry + // with a plain bool, so a watcher refresh finishing mid-run erased a + // concurrent force reindex's marker — the reaper could then evict the + // repo while the reindex still held its stores. Markers are owner-scoped + // now: the watcher ends only its own. + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "600")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + + assert!(state.begin_indexing("race", IndexingOwner::Reindex)); + let cb = state.make_indexing_status_callback("race"); + cb(true); + assert!( + state.is_indexing("race"), + "watcher marker stacks on reindex's" + ); + cb(false); + assert!( + state.is_indexing("race"), + "the watcher's completion must NOT clear the reindex marker" + ); + state.end_indexing("race", IndexingOwner::Reindex); + assert!(!state.is_indexing("race")); +} + +#[tokio::test] +#[serial] +async fn begin_indexing_rejects_while_any_owner_marker_is_fresh() { + // The exclusivity guard must survive owner-scoping: a force reindex is + // still rejected while warmup (or any other owner) is indexing, and + // allowed again once that marker ends. + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "600")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + + assert!(state.begin_indexing("excl", IndexingOwner::Warmup)); + assert!( + !state.begin_indexing("excl", IndexingOwner::Reindex), + "a fresh marker from another owner must reject a concurrent reindex" + ); + state.end_indexing("excl", IndexingOwner::Warmup); + assert!( + state.begin_indexing("excl", IndexingOwner::Reindex), + "ending the warmup marker must let the reindex proceed" + ); + state.end_indexing("excl", IndexingOwner::Reindex); + assert!(!state.is_indexing("excl")); +} + +#[tokio::test] +#[serial] +async fn stale_eviction_is_scoped_to_the_stale_owner() { + // A stale marker from a leaked task must not block (or cancel for) an + // owner whose marker is still fresh — and an eviction must not touch + // other owners' markers. + let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "1")]); + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + + // Fresh warmup marker; a stale reindex marker from a leaked task. + state.begin_indexing("scoped", IndexingOwner::Warmup); + let stale = Instant::now() + .checked_sub(Duration::from_secs(2)) + .expect("monotonic clock at least 2s old"); + state + .active_reindexes + .entry("scoped".to_string()) + .or_default() + .insert(IndexingOwner::Reindex, stale); + + assert!( + state.is_indexing("scoped"), + "the fresh warmup marker must survive the stale owner's eviction" + ); + + // Once warmup's marker ages out too, everything is evicted. + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(!state.is_indexing("scoped")); + assert!( + !state.active_reindexes.contains_key("scoped"), + "an empty owner map must be removed, not left behind" + ); +} diff --git a/src/serve/tui.rs b/src/serve/tui.rs index 6176bd32..aa6009be 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken; use super::tui_common::{ self, KeyAction, OverlayKeyAction, OverlayState, RemoteIndexStats, RemoteStatsState, RepoRow, }; -use super::ServeState; +use super::{IndexingOwner, ServeState}; use crate::cli::doctor; use crate::constants::{DB_DIR_NAME, LANG_CSHARP, LANG_TYPESCRIPT}; use crate::index::IndexManager; @@ -1073,7 +1073,7 @@ pub(crate) enum ReindexLaunch { /// Follows the same flow as the HTTP `reindex_handler`. pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> ReindexLaunch { // Guard against concurrent reindex - if !state.begin_indexing(&alias) { + if !state.begin_indexing(&alias, IndexingOwner::Reindex) { tracing::warn!( "Force reindex already in progress for '{}', skipping TUI request", alias @@ -1085,7 +1085,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei Ok(c) => c, Err(e) => { tracing::error!("Config lock poisoned: {}", e); - state.end_indexing(&alias); + state.end_indexing(&alias, IndexingOwner::Reindex); return ReindexLaunch::Failed; } }; @@ -1093,7 +1093,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei Some(p) => p, None => { tracing::error!("Cannot resolve alias '{}' for force reindex", alias); - state.end_indexing(&alias); + state.end_indexing(&alias, IndexingOwner::Reindex); return ReindexLaunch::Failed; } }; @@ -1107,7 +1107,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei owned by another writer", alias ); - state.end_indexing(&alias); + state.end_indexing(&alias, IndexingOwner::Reindex); return ReindexLaunch::Failed; } drop(config); // release read lock @@ -1138,12 +1138,12 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei "Repo {} opened read-only; cannot force-reindex from TUI", alias ); - state.end_indexing(&alias); + state.end_indexing(&alias, IndexingOwner::Reindex); return ReindexLaunch::Failed; } Err(e) => { tracing::error!("Cannot open stores for '{}': {}", alias, e); - state.end_indexing(&alias); + state.end_indexing(&alias, IndexingOwner::Reindex); return ReindexLaunch::Failed; } } @@ -1183,7 +1183,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei // the FSW, which would resurrect the removed alias with a // fresh, uncancellable task. tracing::info!("TUI: Reindex cancelled for '{}': {}", alias_bg, e); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); return; } tracing::error!("TUI: Force reindex failed for '{}': {}", alias_bg, e); @@ -1206,7 +1206,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei ); drop(stores); state_bg.self_clean_if_unregistered(&alias_bg, &db_path); - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); return; } @@ -1214,7 +1214,7 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei state_bg.restart_fsw(&alias_bg, stores).await; // Remove guard - state_bg.end_indexing(&alias_bg); + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); }); state.index_tasks.insert(alias, (handle, reindex_token)); From 99ec1001dd7f0f9f24fbe02631e321a1576af03b Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 22:50:54 +0200 Subject: [PATCH 08/77] Read group status stats via try_read so status never waits on a store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit index_status_impl's group fan-out still blocked on vector_store.read(): an indexing run holds the write lock for whole batches (build_index for minutes), so a group status call queued behind it. It now uses try_read like list_projects; a busy store counts as a failed stats read with a caller-facing warning naming the busy lock — never silently skipped. --- src/mcp/status.rs | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/mcp/status.rs b/src/mcp/status.rs index 6fbfba3c..0ac0aecf 100644 --- a/src/mcp/status.rs +++ b/src/mcp/status.rs @@ -171,9 +171,18 @@ impl CodesearchService { let mut failed_count = 0usize; for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; - match store.stats() { - Ok(stats) => { + // Never wait on the store lock: an indexing run holds it for + // whole batches and build_index() for minutes, and status must + // answer immediately — the same contract as list_projects. A + // busy store is attributed to THIS store, never skipped + // silently. + let stats_result = store_arc + .vector_store + .try_read() + .ok() + .map(|store| store.stats()); + match stats_result { + Some(Ok(stats)) => { total_chunks += stats.total_chunks; total_files += stats.total_files; if stats.max_chunk_id > max_chunk_id { @@ -192,11 +201,25 @@ impl CodesearchService { // store is down". This is the tool whose job is reporting index // health, so it must not stay silent on the one signal that // matters here: bind the error, carry it, never `Err(_)`. - Err(ref e) => { + Some(Err(ref e)) => { all_indexed = false; failed_count += 1; note_store_failure(&mut stats_warnings, aliases, i, "stats", e); } + None => { + all_indexed = false; + failed_count += 1; + let alias = aliases.get(i).map(|s| s.as_str()).unwrap_or("unknown"); + push_store_warning( + &mut stats_warnings, + &store_warning( + alias, + "stats", + "store lock busy (an indexing run holds it); live stats \ + unavailable", + ), + ); + } } } From c4e8eb9034df21b87b8e7af554d02c7a019583d3 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 23:15:41 +0200 Subject: [PATCH 09/77] Bound the vector-store read wait so a stuck indexer cannot hang every reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live indexer holds vector_store's write lock for the whole batch; every read-side caller blocked on it forever, so one slow/stuck reindex hung every query. New CODESEARCH_STORE_LOCK_WAIT_SECS-configurable bounded_vector_read() (default 300s) wraps every direct vector_store.read().await in mcp/mod.rs (with_vector_store_read_for, with_vector_store_read_multi), mcp/runtime.rs (require_prebuilt_index_ready) and every per-store fan-out site in mcp/find.rs, mcp/get_chunk.rs, mcp/graph.rs, mcp/literal_search.rs, mcp/search.rs, mcp/explore.rs — each site now attributes a timeout the same way it already attributes a store error (note_store_failure / push_store_warning, per its existing warnings channel), so a busy store degrades one caller's answer instead of hanging it. FTS store reads are left unbounded — short-duration ops, out of scope. --- src/constants.rs | 16 ++++++++++ src/mcp/explore.rs | 8 ++++- src/mcp/find.rs | 28 +++++++++++++++-- src/mcp/get_chunk.rs | 34 +++++++++++++++++---- src/mcp/graph.rs | 64 ++++++++++++++++++++++++++++++++++++--- src/mcp/literal_search.rs | 19 ++++++++++-- src/mcp/mod.rs | 43 ++++++++++++++++++++++++-- src/mcp/runtime.rs | 2 +- src/mcp/search.rs | 16 ++++++++-- 9 files changed, 208 insertions(+), 22 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index fe6b1e9b..6d783876 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -518,6 +518,22 @@ pub const MAX_INDEXING_SECS: u64 = 30 * 60; // 30 minutes /// Environment variable to override the maximum indexing duration. pub const MAX_INDEXING_SECS_ENV: &str = "CODESEARCH_MAX_INDEXING_SECS"; +/// How long an MCP handler may wait for the vector-store read lock before +/// giving up with a "store busy" error. +/// +/// A handler queued on this lock keeps its `Arc` — and with it +/// the repo's LMDB env and `.writer.lock` — alive for the whole wait, and a +/// cancelled rmcp request does not reliably drop its handler future. An +/// unbounded wait therefore pinned stores open long after the client gave +/// up, blocking idle eviction and writer-lock recovery. The bound also turns +/// a wait behind a long `build_index()` into a retryable error instead of a +/// hang. Generous by design: it must outlast any legitimate reader queue +/// behind a single batch or graph build, not the whole warmup. +pub const STORE_LOCK_WAIT_SECS: u64 = 300; + +/// Environment variable overriding [`STORE_LOCK_WAIT_SECS`]. +pub const STORE_LOCK_WAIT_SECS_ENV: &str = "CODESEARCH_STORE_LOCK_WAIT_SECS"; + /// Total number of attempts (initial request + retries) the federation client /// makes against a remote peer that answers with a transient HTTP status /// (502/503/504). Federated peers commonly run on scale-to-zero hosts (Azure diff --git a/src/mcp/explore.rs b/src/mcp/explore.rs index 88149237..72fd6e74 100644 --- a/src/mcp/explore.rs +++ b/src/mcp/explore.rs @@ -82,7 +82,13 @@ impl CodesearchService { let mut all_items: Vec = Vec::new(); let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(warnings, aliases, store_idx, "outline scan", &e); + continue; + } + }; match store.chunks_for_file(normalized) { Ok(metas) => { for c in metas { diff --git a/src/mcp/find.rs b/src/mcp/find.rs index 673f656c..797f4d97 100644 --- a/src/mcp/find.rs +++ b/src/mcp/find.rs @@ -159,7 +159,19 @@ impl CodesearchService { let mut items: Vec = Vec::new(); 'outer: for fts_result in &fts_results { for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut find_warnings, + aliases, + store_idx, + "chunk lookup", + &e, + ); + continue; + } + }; let looked_up = store.get_chunk(fts_result.chunk_id); if let Err(ref e) = looked_up { // `Ok(None)` = chunk not in this store (normal during @@ -354,7 +366,19 @@ impl CodesearchService { let mut items: Vec = Vec::new(); for fts_result in &fts_results { for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut find_warnings, + aliases, + store_idx, + "chunk lookup", + &e, + ); + continue; + } + }; let looked_up = store.get_chunk(fts_result.chunk_id); if let Err(ref e) = looked_up { // Same rule as find_definition: `Err` is a broken diff --git a/src/mcp/get_chunk.rs b/src/mcp/get_chunk.rs index c32cdd59..eaba0e6e 100644 --- a/src/mcp/get_chunk.rs +++ b/src/mcp/get_chunk.rs @@ -107,7 +107,13 @@ impl CodesearchService { let mut candidates: Vec<(&Arc, String)> = Vec::new(); let aliases = ctx.aliases(); for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", &e); + continue; + } + }; match store.get_chunk(request.chunk_id) { Ok(Some(_)) => { // A store that HAS the chunk stays a candidate even if @@ -153,16 +159,26 @@ impl CodesearchService { serve_state.record_tool_call(alias, "get_chunk"); serve_state.touch_access(alias); } - let store = store_arc.vector_store.read().await; - match store.get_chunk(request.chunk_id) { - Ok(c) => c, - Err(ref e) => { + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => Some(store), + Err(e) => { + push_store_warning( + &mut chunk_warnings, + &store_warning(alias, "chunk lookup", &format!("{e:#}")), + ); + None + } + }; + match store.as_ref().map(|s| s.get_chunk(request.chunk_id)) { + Some(Ok(c)) => c, + Some(Err(ref e)) => { push_store_warning( &mut chunk_warnings, &store_warning(alias, "chunk lookup", &format!("{e:#}")), ); None } + None => None, } } _ => { @@ -188,7 +204,13 @@ impl CodesearchService { let aliases = ctx.aliases(); let mut found = None; for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(&mut chunk_warnings, aliases, i, "chunk lookup", &e); + continue; + } + }; match store.get_chunk(request.chunk_id) { Ok(Some(c)) => { found = Some(c); diff --git a/src/mcp/graph.rs b/src/mcp/graph.rs index c1e5b190..cffca07b 100644 --- a/src/mcp/graph.rs +++ b/src/mcp/graph.rs @@ -58,7 +58,19 @@ impl CodesearchService { let mut all_items: Vec = Vec::new(); let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut import_warnings, + import_aliases, + store_idx, + "chunk lookup", + &e, + ); + continue; + } + }; match store.chunks_for_file(&normalized) { Ok(metas) => { for meta in metas { @@ -156,7 +168,19 @@ impl CodesearchService { let mut resolved: Vec = Vec::new(); for (chunk_id, _) in &all_hits { for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut import_warnings, + import_aliases, + store_idx, + "chunk lookup", + &e, + ); + continue; + } + }; match store.get_chunk(*chunk_id) { Ok(Some(chunk)) => { if crate::cache::normalize_path_str(&chunk.path) == normalized { @@ -372,7 +396,19 @@ impl CodesearchService { let mut out = Vec::new(); for f in &fts_results { for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut dep_warnings, + dep_aliases, + store_idx, + "chunk lookup", + &e, + ); + continue; + } + }; match store.get_chunk(f.chunk_id) { Ok(Some(chunk)) => { if !is_import_kind(&chunk.kind) { @@ -532,7 +568,19 @@ impl CodesearchService { let aliases = ctx.aliases(); let mut embedding: Option> = None; for (i, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut similar_warnings, + aliases, + i, + "embedding lookup", + &e, + ); + continue; + } + }; match store.get_embedding(request.chunk_id) { Ok(Some(emb)) => { embedding = Some(emb); @@ -571,7 +619,13 @@ impl CodesearchService { let mut all_results: Vec = Vec::new(); let mut seen_ids: std::collections::HashSet = std::collections::HashSet::new(); for (store_idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(&mut similar_warnings, aliases, store_idx, "search", &e); + continue; + } + }; match store.search(&embedding, limit + 1) { Ok(mut neighbors) => { neighbors.retain(|r| r.id != request.chunk_id); diff --git a/src/mcp/literal_search.rs b/src/mcp/literal_search.rs index b2edb2ce..f2903d8f 100644 --- a/src/mcp/literal_search.rs +++ b/src/mcp/literal_search.rs @@ -86,7 +86,10 @@ impl CodesearchService { // Multi-store scan let mut items: Vec = Vec::new(); for store_arc in sv { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(_) => continue, + }; let all_chunks = match store.iter_all_chunks() { Ok(chunks) => chunks, Err(_) => continue, @@ -270,7 +273,19 @@ impl CodesearchService { 'outer: for fts_result in &fts_results { let sa = ctx.store_aliases.as_ref().unwrap(); for (idx, store_arc) in sv.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure( + &mut literal_warnings, + sa, + idx, + "chunk lookup", + &e, + ); + continue; + } + }; let looked_up = store.get_chunk(fts_result.chunk_id); if let Err(ref e) = looked_up { note_store_failure(&mut literal_warnings, sa, idx, "chunk lookup", e); diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 9442ae28..9136b8b5 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -27,6 +27,37 @@ fn serve_url_from_env() -> String { format!("http://{}:{}", host, port) } +/// Configured vector-store lock wait for MCP handlers. +fn store_lock_wait() -> std::time::Duration { + std::env::var(crate::constants::STORE_LOCK_WAIT_SECS_ENV) + .ok() + .and_then(|s| s.parse::().ok()) + .filter(|&s| s > 0) + .map(std::time::Duration::from_secs) + .unwrap_or_else(|| std::time::Duration::from_secs(crate::constants::STORE_LOCK_WAIT_SECS)) +} + +/// Acquire the vector-store read lock with a bounded wait. +/// +/// A handler queued on this lock keeps its `Arc` — and with it +/// the repo's LMDB env and `.writer.lock` — alive for the whole wait, and a +/// cancelled rmcp request does not reliably drop its handler future. An +/// unbounded wait therefore pinned stores open long after the client gave +/// up; the bound turns that into a retryable "store busy" error and lets +/// eviction proceed. +pub(crate) async fn bounded_vector_read( + lock: &tokio::sync::RwLock, +) -> Result> { + let wait = store_lock_wait(); + tokio::time::timeout(wait, lock.read()).await.map_err(|_| { + anyhow::anyhow!( + "vector store still busy after {}s (an indexing run holds the write \ + lock) — retry shortly", + wait.as_secs() + ) + }) +} + use crate::db_discovery::{find_best_database, load_repos_config}; use crate::embed::{EmbeddingServicePool, ModelType}; use crate::file::Language; @@ -1263,13 +1294,13 @@ impl CodesearchService { { // Priority 1: explicit store override (from project/group routing) if let Some(stores) = store_override { - let store = stores.vector_store.read().await; + let store = bounded_vector_read(&stores.vector_store).await?; return action(&store).context("Error reading from project-routed vector store"); } // Priority 2: shared stores (set during IndexManager init) if let Some(ref stores) = self.shared_stores { - let store = stores.vector_store.read().await; + let store = bounded_vector_read(&stores.vector_store).await?; match action(&store) { Ok(result) => return Ok(result), Err(shared_err) => { @@ -1350,7 +1381,13 @@ impl CodesearchService { for (idx, store_arc) in stores.iter().enumerate() { let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + failures.push((alias.to_string(), format!("{e:#}"))); + continue; + } + }; match action(alias, &store) { Ok(results) => { for r in results { diff --git a/src/mcp/runtime.rs b/src/mcp/runtime.rs index 221e94e7..6104c7b6 100644 --- a/src/mcp/runtime.rs +++ b/src/mcp/runtime.rs @@ -116,7 +116,7 @@ pub(crate) async fn require_prebuilt_index_ready( db_path: &Path, ) -> Result<()> { let (total_chunks, vector_indexed) = { - let vector_store = stores.vector_store.read().await; + let vector_store = crate::mcp::bounded_vector_read(&stores.vector_store).await?; vector_store .index_health() .context("Failed to inspect prebuilt vector index")? diff --git a/src/mcp/search.rs b/src/mcp/search.rs index a00499a9..5ac0200a 100644 --- a/src/mcp/search.rs +++ b/src/mcp/search.rs @@ -841,7 +841,13 @@ impl CodesearchService { warnings: &mut Vec, ) -> Option { for (idx, store_arc) in stores.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(warnings, aliases, idx, "chunk lookup", &e); + continue; + } + }; let looked_up = store.get_chunk(chunk_id); if let Err(ref e) = looked_up { note_store_failure(warnings, aliases, idx, "chunk lookup", e); @@ -880,7 +886,13 @@ impl CodesearchService { let mut results = Vec::new(); for fts in fts_results.iter().take(limit) { for (idx, store_arc) in stores.iter().enumerate() { - let store = store_arc.vector_store.read().await; + let store = match bounded_vector_read(&store_arc.vector_store).await { + Ok(store) => store, + Err(e) => { + note_store_failure(warnings, aliases, idx, "chunk lookup", &e); + continue; + } + }; let looked_up = store.get_chunk(fts.chunk_id); if let Err(ref e) = looked_up { // `Ok(None)` means "this store does not hold that chunk" and From 6d21317abb90126e113369700bd278303827c44f Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 23:28:15 +0200 Subject: [PATCH 10/77] Bound the FTS-store read wait to mirror the vector-store fix The prior commit bounded vector_store reads but deliberately left fts_store.read().await unbounded. A live indexing batch holds both locks for its duration, so a stuck reindex could still hang every literal-search/find caller through the FTS side alone. New bounded_fts_read() (src/mcp/mod.rs, same CODESEARCH_STORE_LOCK_WAIT_SECS wait as bounded_vector_read) wraps the FTS lock at all four direct read sites: with_fts_store_read_for's two branches, the per-store loop in with_fts_store_read_multi (attributes the timeout into `failures`, same as its vector-store sibling), and runtime.rs's require_prebuilt_index_ready. --- src/mcp/mod.rs | 28 +++++++++++++++++++++++++--- src/mcp/runtime.rs | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index 9136b8b5..ac0a38a7 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -58,6 +58,22 @@ pub(crate) async fn bounded_vector_read( }) } +/// Acquire the FTS-store read lock with a bounded wait — the same rationale +/// as `bounded_vector_read`, mirrored for the other lock a live indexing +/// batch holds for its duration. +pub(crate) async fn bounded_fts_read( + lock: &tokio::sync::RwLock, +) -> Result> { + let wait = store_lock_wait(); + tokio::time::timeout(wait, lock.read()).await.map_err(|_| { + anyhow::anyhow!( + "full-text store still busy after {}s (an indexing run holds the write \ + lock) — retry shortly", + wait.as_secs() + ) + }) +} + use crate::db_discovery::{find_best_database, load_repos_config}; use crate::embed::{EmbeddingServicePool, ModelType}; use crate::file::Language; @@ -1337,13 +1353,13 @@ impl CodesearchService { { // Priority 1: explicit store override (from project/group routing) if let Some(stores) = store_override { - let fts = stores.fts_store.read().await; + let fts = bounded_fts_read(&stores.fts_store).await?; return action(&fts); } // Priority 2: shared stores if let Some(ref stores) = self.shared_stores { - let fts = stores.fts_store.read().await; + let fts = bounded_fts_read(&stores.fts_store).await?; return action(&fts); } @@ -1457,7 +1473,13 @@ impl CodesearchService { for (idx, store_arc) in stores.iter().enumerate() { let alias = aliases.get(idx).map(|s| s.as_str()).unwrap_or("unknown"); - let fts = store_arc.fts_store.read().await; + let fts = match bounded_fts_read(&store_arc.fts_store).await { + Ok(fts) => fts, + Err(e) => { + failures.push((alias.to_string(), format!("{e:#}"))); + continue; + } + }; match action(&fts) { Ok(results) => { for r in results { diff --git a/src/mcp/runtime.rs b/src/mcp/runtime.rs index 6104c7b6..61fe09a9 100644 --- a/src/mcp/runtime.rs +++ b/src/mcp/runtime.rs @@ -122,7 +122,7 @@ pub(crate) async fn require_prebuilt_index_ready( .context("Failed to inspect prebuilt vector index")? }; let fts_documents = { - let fts_store = stores.fts_store.read().await; + let fts_store = crate::mcp::bounded_fts_read(&stores.fts_store).await?; fts_store .stats() .context("Failed to inspect prebuilt full-text index")? From 7a1db6370a4fb683715e7d1f451137c6a9410d91 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 25 Sep 2026 23:58:23 +0200 Subject: [PATCH 11/77] Close the warmup-marker gap before FSW's gate check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final review of the 6-stage fallout fix found a narrow window: warmup registered RepoState::Warm before calling begin_indexing(Warmup), so a concurrent query landing in that gap could see is_indexing() == false and let FSW start a Watcher-owned refresh racing warmup's still-loading file_meta — the exact hazard the per-owner marker refactor exists to prevent. Fix: acquire the Warmup marker before the repo becomes visible in self.repos, so FSW's gate always sees it. --- src/serve/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/serve/mod.rs b/src/serve/mod.rs index ed168d5b..480d4656 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -2532,6 +2532,13 @@ impl ServeState { } } + // Begin the indexing marker BEFORE registering as Warm: FSW's gate + // (`!touch || self.is_indexing(alias)`) is checked by a concurrent + // query the instant the repo appears in `self.repos`, so the marker + // must already be visible or FSW can start and race warmup's + // unsaved file_meta. + let warmup_marker_acquired = self.begin_indexing(alias, IndexingOwner::Warmup); + // Register as Warm BEFORE the refresh and release the single-flight // lock: a refresh of a large repo takes minutes, and queries must be // answered from the existing data meanwhile instead of queueing on @@ -2551,7 +2558,7 @@ impl ServeState { // Tracked like any reindex: status reports "indexing", the reaper // leaves the repo alone, and a concurrent reindex is not doubled. - if !self.begin_indexing(alias, IndexingOwner::Warmup) { + if !warmup_marker_acquired { info!( "Warmup '{}': another indexing run is active, skipping refresh", alias From 6852772f511ebc2484ae0ccd9693e8d0608f5d30 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 00:14:46 +0200 Subject: [PATCH 12/77] Drop client codename from comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-push leak scan flagged 'BOIN' (client codename) in three rationale comments about warmup duration at scale. Reworded to 'large repos' / 'large-repo' — no behavior change. --- src/index/manager.rs | 2 +- src/serve/mod.rs | 2 +- src/serve/tests.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/index/manager.rs b/src/index/manager.rs index 301d2b2d..ff190f91 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -74,7 +74,7 @@ pub type IndexingStatusCallback = Arc; /// Heartbeat fired by long indexing loops (once per batch, plus before each /// slow phase) so the caller can renew its indexing marker before -/// `MAX_INDEXING_SECS` expires. BOIN-scale warmups run well past 30 minutes; +/// `MAX_INDEXING_SECS` expires. Large repos' warmups run well past 30 minutes; /// without renewal the stale-marker eviction drops the marker mid-refresh and /// the reaper/FSW can then race the live refresh (todo #131 shape). pub type IndexingHeartbeat = Arc; diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 480d4656..08db7708 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -2570,7 +2570,7 @@ impl ServeState { // to completion. A real user-initiated cancel routes through the // RepoState::Write token owned by the live task instead. let pool = self.embedding_pool(); - // BOIN-scale warmups run past MAX_INDEXING_SECS; the per-batch heartbeat + // Large-repo warmups run past MAX_INDEXING_SECS; the per-batch heartbeat // keeps the marker alive so the reaper and FSW keep honouring it. let heartbeat: IndexingHeartbeat = { let state = Arc::clone(self); diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 7766cb3f..1e71a301 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3172,7 +3172,7 @@ async fn warm_query_does_not_start_fsw_while_warmup_indexes() { #[tokio::test] #[serial] async fn indexing_marker_heartbeat_survives_past_max() { - // Regression: a BOIN-scale warmup runs longer than MAX_INDEXING_SECS, so + // Regression: a large-repo warmup runs longer than MAX_INDEXING_SECS, so // the lazy stale-marker eviction dropped its marker mid-refresh and the // reaper could evict the repo while the refresh still held its stores. // The per-batch heartbeat must keep the marker alive for as long as the From 0135e9229f1886b4d9bf1aa2946a93a194f85c8f Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 00:16:33 +0200 Subject: [PATCH 13/77] Accept 'symbol' field for edit-guard target attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit find(kind="usages") names its target via 'symbol', not file/path — edit-guard-post hooks now recognize it so both find_impact and find attribute to the same key. --- integrations/claude-code/hooks/edit-guard-post.ps1 | 2 +- integrations/claude-code/hooks/edit-guard-post.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integrations/claude-code/hooks/edit-guard-post.ps1 b/integrations/claude-code/hooks/edit-guard-post.ps1 index d1022ce5..5f90150b 100644 --- a/integrations/claude-code/hooks/edit-guard-post.ps1 +++ b/integrations/claude-code/hooks/edit-guard-post.ps1 @@ -37,7 +37,7 @@ switch ($tool) { } $p = $null -foreach ($field in @('file', 'path', 'file_path')) { +foreach ($field in @('file', 'path', 'file_path', 'symbol')) { if ($inp -and @($inp.PSObject.Properties.Name) -contains $field) { $candidate = [string]$inp.$field if (-not [string]::IsNullOrEmpty($candidate)) { $p = $candidate; break } diff --git a/integrations/claude-code/hooks/edit-guard-post.sh b/integrations/claude-code/hooks/edit-guard-post.sh index 381267c6..46e0b1a3 100644 --- a/integrations/claude-code/hooks/edit-guard-post.sh +++ b/integrations/claude-code/hooks/edit-guard-post.sh @@ -34,12 +34,12 @@ case "$tool" in *) exit 0 ;; esac -# find_impact names its target via `file`/`path`; find uses `path`. Accept -# any of them — FIRST NON-EMPTY (jq's `//` treats "" as truthy, so the -# empties are filtered explicitly; the ps1 twin does the same) — so both -# tools attribute to the same key. +# find_impact names its target via `file`/`path`; find(kind="usages") uses +# `symbol`. Accept any of them — FIRST NON-EMPTY (jq's `//` treats "" as +# truthy, so the empties are filtered explicitly; the ps1 twin does the +# same) — so both tools attribute to the same key. p=$(echo "$raw" | jq -r ' - [ .tool_input.file, .tool_input.path, .tool_input.file_path ] + [ .tool_input.file, .tool_input.path, .tool_input.file_path, .tool_input.symbol ] | map(select(. != null and . != "")) | .[0] // empty ' 2>/dev/null | jq_str) From f9b3b70739d795f1706b5d146ebd43c0570fd081 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:27:33 +0000 Subject: [PATCH 14/77] chore: bump version to 1.4.10 (auto, PR #288 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4268d66f..377d1980 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.9" +version = "1.4.10" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index f1d25926..9589f590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.9" +version = "1.4.10" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 041e4dd2baa1fb7c4d2054c5724ebf72e7a0b745 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 01:22:16 +0200 Subject: [PATCH 15/77] Fix cross-platform absolute-path check in prefix_path_with_alias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Path::has_root()` only recognizes a Windows drive-letter path (`C:/...`) as rooted on Windows. On the Linux CI runner it saw such a path as relative, so `prefix_path_with_alias` wrongly alias-prefixed an out-of-root Windows path instead of leaving it unprefixed — failing mcp::tests::test_path_prefix_preserves_path_outside_root in CI. Added `is_portable_absolute()`, a platform-independent textual check (leading `/` or drive-letter + `:`), used instead of `has_root()`. --- src/mcp/helpers.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/mcp/helpers.rs b/src/mcp/helpers.rs index 243e2aea..02fbdcd6 100644 --- a/src/mcp/helpers.rs +++ b/src/mcp/helpers.rs @@ -70,6 +70,25 @@ pub(crate) fn strip_alias_prefix(path: &str, alias: Option<&String>) -> String { } } +/// Is `path` absolute under either path convention (Unix `/...` or Windows +/// `C:/...` / `C:\...`), regardless of the platform this binary runs on. +/// +/// `Path::has_root()` is platform-scoped: on Linux it does not recognize a +/// Windows drive-letter path as rooted, so a CI run on a Linux runner sees +/// `"C:/other/src/main.rs"` as relative and wrongly alias-prefixes it. Test +/// data and stored paths cross platforms (a Windows client's paths get +/// tested on a Linux CI runner), so absoluteness must be checked textually. +fn is_portable_absolute(path: &str) -> bool { + if path.starts_with('/') { + return true; + } + let b = path.as_bytes(); + b.len() >= 2 + && b[0].is_ascii_alphabetic() + && b[1] == b':' + && (b.len() == 2 || b[2] == b'/' || b[2] == b'\\') +} + /// Prefix a result path with its repo alias for group queries, normalizing /// Windows backslashes to forward slashes in the process. When `alias` is /// None or empty, the path is still normalized (useful for stdio mode). @@ -88,7 +107,7 @@ pub(crate) fn prefix_path_with_alias( .to_string(); let relative = match normalized.strip_prefix(&normalized_root) { Some(rest) => Some(rest.trim_start_matches('/')), - None if !Path::new(&normalized).has_root() => Some(normalized.as_str()), + None if !is_portable_absolute(&normalized) => Some(normalized.as_str()), None => None, }; match relative { From 0b0a8d7b7ee4cdf976d67c72fc10b8750ec12a04 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 16:52:04 +0200 Subject: [PATCH 16/77] [worker] stage 1/1: rename Closed repo state to Idle; hub status reports ready MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small models misread "Closed"/indexed:false as "index unavailable" and wrongly fall back to grep. Idle = index on disk, stores evicted by the idle reaper, auto-reopen on next query — i.e. searchable, first query slightly slower. Aggregated index_status now reports indexed:true + status:"ready" when any repo has an index on disk, with an explicit "Indexed and searchable: N" hint; "no_index" only when nothing is on disk. Wire string becomes "idle" (TUIs keep tolerating legacy "closed"). Review-fixes: - [Important] aggregation branches untested → extracted pure helper aggregate_serve_index_status (responses.rs), index_status_impl calls it, 3 unit tests pin all-idle→ready/indexed, nothing-on-disk→no_index, empty hub→no_repos - [minor, pre-existing] all-Readonly hub still reports no_index → deferred, out of this change's scope (requirement enumerated open+warm+idle); follow-up candidate --- AGENTS.develop.md | 2 +- docker/entrypoint.sh | 4 ++-- src/federation/mod.rs | 2 +- src/mcp/responses.rs | 36 ++++++++++++++++++++++++++++++++++++ src/mcp/status.rs | 27 ++++++++++++--------------- src/mcp/tests.rs | 34 ++++++++++++++++++++++++++++++++++ src/serve/mod.rs | 20 ++++++++++---------- src/serve/tui.rs | 2 +- src/serve/tui_common.rs | 8 +++++--- 9 files changed, 102 insertions(+), 33 deletions(-) diff --git a/AGENTS.develop.md b/AGENTS.develop.md index 8e402dd9..9088733b 100644 --- a/AGENTS.develop.md +++ b/AGENTS.develop.md @@ -102,7 +102,7 @@ or per-group. - `Warm` — DB open, vector index ready, no FSW. State after background warmup or fan-out open. - `Write` — Warm + file system watcher running. Transitions from Warm on first explicit project query. - `Readonly` — Another process holds the write lock. -- `Closed` — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity. +- `Idle` (status string was `Closed`) — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity; index stays on disk, stores auto-reopen on the next query. **Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. All opens update `last_access` so the reaper can track every open. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 21bd98e7..a5eb6000 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -303,7 +303,7 @@ wait_healthz() { # the same LMDB env and fails with HTTP 500 "locked by another codesearch # process" (observed). So we let the warmup own the refresh and simply wait for # the repo to reach a ready ("warm") state. During warmup /status reports the -# repo as "closed"; it flips to "warm" only after the refresh completes, which is +# repo as "idle"; it flips to "warm" only after the refresh completes, which is # exactly the signal wait_repo_ready() blocks on — note that warmup never reports # "indexing", so the alias-specific status check is the ONLY signal here. # /reindex?force=true is also unused (returns 500 in this deployment). @@ -672,7 +672,7 @@ repo_status() { # # 2. Phase-1 STARTUP WARMUP — the path that actually runs for every vendor # restored from a snapshot ("already registered") — never sets "indexing". -# A warming repo is simply absent from the state map and reports "closed", +# A warming repo is simply absent from the state map and reports "idle", # flipping to "warm" only once the warmup has built AND committed its HNSW # graph. Waiting on signal 1 alone therefore returned after the initial 5s # sleep for every already-registered vendor ("build settled after ~5s" for diff --git a/src/federation/mod.rs b/src/federation/mod.rs index d16a56e4..e6456fc6 100644 --- a/src/federation/mod.rs +++ b/src/federation/mod.rs @@ -170,7 +170,7 @@ pub struct RemoteRepoStatus { #[serde(default)] pub alias: String, /// Repo lifecycle state reported by the server - /// (`open`/`warm`/`readonly`/`closed`/`indexing`/`error`/`no_index`). + /// (`open`/`warm`/`readonly`/`idle`/`indexing`/`error`/`no_index`; older peers may send `closed`). #[serde(default)] pub status: String, /// `write`/`read`/`-`. diff --git a/src/mcp/responses.rs b/src/mcp/responses.rs index 4507fef3..d2b4ca73 100644 --- a/src/mcp/responses.rs +++ b/src/mcp/responses.rs @@ -224,6 +224,42 @@ pub(crate) fn index_status_summary( } } +/// Decide `indexed`/`status`/`status_message` for the aggregated serve-mode +/// hub response (no project/group): pure arithmetic over lightweight repo +/// state counts. +/// +/// An `Idle` repo still has its index on disk (the missing-index case is +/// `NoIndex`) — it is searchable; the first query just reopens the evicted +/// stores. Reporting `indexed: false` for an all-idle hub made small-model +/// clients conclude "no index" and fall back to grep, so idle counts as +/// indexed and the message says so explicitly. +pub(crate) fn aggregate_serve_index_status( + repo_count: usize, + group_count: usize, + open: usize, + warm: usize, + idle: usize, +) -> (bool, String, String) { + let indexed_repos = open + warm + idle; + let status = if indexed_repos > 0 { + "ready".to_string() + } else if repo_count > 0 { + "no_index".to_string() + } else { + "no_repos".to_string() + }; + let status_message = if indexed_repos > 0 { + format!( + "{repo_count} repo(s) registered, {group_count} group(s). Indexed and searchable: {indexed_repos}. Open: {open}, Warm: {warm}, Idle: {idle} (idle = index on disk, stores auto-reopen on first query)." + ) + } else { + format!( + "{repo_count} repo(s) registered, {group_count} group(s). No index built yet — run 'codesearch index' per repo. Open: {open}, Warm: {warm}." + ) + }; + (indexed_repos > 0, status, status_message) +} + /// Status/message for a single routed store's index. /// /// `indexed` (the HNSW graph is built and committed) is load-bearing: a store diff --git a/src/mcp/status.rs b/src/mcp/status.rs index 0ac0aecf..72368231 100644 --- a/src/mcp/status.rs +++ b/src/mcp/status.rs @@ -87,26 +87,23 @@ impl CodesearchService { .iter() .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Warm)) .count(); - let closed_count = statuses + let idle_count = statuses .iter() - .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Closed)) + .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Idle)) .count(); - - let status = if open_count + warm_count > 0 { - "ready".to_string() - } else if repo_count > 0 { - "idle".to_string() - } else { - "no_repos".to_string() - }; - - let status_message = format!( - "{} repo(s) registered, {} group(s). Open: {}, Warm: {}, Closed: {}.", - repo_count, group_count, open_count, warm_count, closed_count + // Counting logic lives in `aggregate_serve_index_status` + // (responses.rs): idle repos still have their index on disk + // and are searchable, so they count as indexed. + let (indexed, status, status_message) = aggregate_serve_index_status( + repo_count, + group_count, + open_count, + warm_count, + idle_count, ); let response = IndexStatusResponse { - indexed: open_count + warm_count > 0, + indexed, status, status_message, total_chunks: 0, // Not available without opening DBs diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index e3541b40..79408b7d 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -2125,6 +2125,40 @@ fn index_status_summary_reports_error_when_every_store_failed() { assert!(message.contains("warnings"), "got: {message}"); } +// === aggregated serve-mode hub status (no project/group) ============== +// +// The all-idle case is the one that drove the fix: an idle repo has its +// index on disk (only the in-memory stores were evicted), so reporting +// `indexed: false` made small-model clients conclude "no index" and fall +// back to grep. + +#[test] +fn aggregate_serve_index_status_counts_idle_repos_as_indexed() { + let (indexed, status, message) = super::aggregate_serve_index_status(26, 1, 0, 0, 26); + assert!(indexed, "an all-idle hub is still searchable"); + assert_eq!(status, "ready"); + assert!( + message.contains("Indexed and searchable: 26") && message.contains("Idle: 26"), + "got: {message}" + ); +} + +#[test] +fn aggregate_serve_index_status_reports_no_index_when_nothing_is_on_disk() { + let (indexed, status, message) = super::aggregate_serve_index_status(3, 1, 0, 0, 0); + assert!(!indexed); + assert_eq!(status, "no_index"); + assert!(message.contains("No index built yet"), "got: {message}"); +} + +#[test] +fn aggregate_serve_index_status_reports_no_repos_on_an_empty_hub() { + let (indexed, status, message) = super::aggregate_serve_index_status(0, 0, 0, 0, 0); + assert!(!indexed); + assert_eq!(status, "no_repos"); + assert!(message.contains("0 repo(s)"), "got: {message}"); +} + #[test] fn repo_stats_from_result_carries_counts_and_no_error_on_success() { let stats = crate::vectordb::StoreStats { diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 08db7708..5ea6d09d 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -79,7 +79,7 @@ pub(crate) enum RepoStateLabel { Open, Warm, Readonly, - Closed, + Idle, Indexing, Error, NoIndex, @@ -116,7 +116,7 @@ impl RepoStateLabel { Self::Open => "Open".green().bold(), Self::Warm => "Warm".yellow(), Self::Readonly => "Readonly".cyan(), - Self::Closed => "Closed".dimmed(), + Self::Idle => "Idle".dimmed(), Self::Indexing => "Indexing".magenta().bold(), Self::Error => "Error".red().bold(), Self::NoIndex => "No Index".dimmed(), @@ -2046,7 +2046,7 @@ impl ServeState { self.clear_csharp_index_state(alias); if self.repos.remove(alias).is_some() { tracing::info!( - "Closed repo '{}' (dropped stores, released LMDB handles)", + "Idled repo '{}' (stores dropped, LMDB released; auto-reopens on next query)", alias ); } @@ -3254,7 +3254,7 @@ impl ServeState { if !db_exists { RepoStateLabel::NoIndex } else { - RepoStateLabel::Closed + RepoStateLabel::Idle } } } @@ -3414,7 +3414,7 @@ impl ServeState { RepoStateLabel::Open => "Open", RepoStateLabel::Warm => "Warm", RepoStateLabel::Readonly => "Readonly", - RepoStateLabel::Closed => "Closed", + RepoStateLabel::Idle => "Idle", RepoStateLabel::Indexing => "Indexing", RepoStateLabel::Error => "Error", RepoStateLabel::NoIndex => "No Index", @@ -3459,9 +3459,9 @@ impl ServeState { .iter() .filter(|(_, r)| matches!(r.status, RepoStateLabel::Warm)) .count(); - let closed_count = repos + let idle_count = repos .iter() - .filter(|(_, r)| matches!(r.status, RepoStateLabel::Closed | RepoStateLabel::NoIndex)) + .filter(|(_, r)| matches!(r.status, RepoStateLabel::Idle | RepoStateLabel::NoIndex)) .count(); eprintln!(); @@ -3473,8 +3473,8 @@ impl ServeState { format!("{}", open_count).green(), "Warm:".dimmed(), format!("{}", warm_count).yellow(), - "Closed:".dimmed(), - format!("{}", closed_count).dimmed(), + "Idle:".dimmed(), + format!("{}", idle_count).dimmed(), ); eprintln!( " {} {} {} {}", @@ -3753,7 +3753,7 @@ async fn status_handler( RepoStateLabel::Open => "open", RepoStateLabel::Warm => "warm", RepoStateLabel::Readonly => "readonly", - RepoStateLabel::Closed => "closed", + RepoStateLabel::Idle => "idle", RepoStateLabel::Indexing => "indexing", RepoStateLabel::Error => "error", RepoStateLabel::NoIndex => "no_index", diff --git a/src/serve/tui.rs b/src/serve/tui.rs index aa6009be..497051f7 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -435,7 +435,7 @@ fn map_repo_rows( super::RepoStateLabel::Open => "open", super::RepoStateLabel::Warm => "warm", super::RepoStateLabel::Readonly => "readonly", - super::RepoStateLabel::Closed => "closed", + super::RepoStateLabel::Idle => "idle", super::RepoStateLabel::Indexing => "indexing", super::RepoStateLabel::Error => "error", super::RepoStateLabel::NoIndex => "no_index", diff --git a/src/serve/tui_common.rs b/src/serve/tui_common.rs index e98cd29b..7ab7a3f1 100644 --- a/src/serve/tui_common.rs +++ b/src/serve/tui_common.rs @@ -52,7 +52,7 @@ use crossterm::event::{KeyCode, KeyEvent}; #[derive(Debug, Clone)] pub struct RepoRow { pub alias: String, - /// Human-readable status: "open", "warm", "readonly", "closed", "indexing", "error", "no_index" + /// Human-readable status: "open", "warm", "readonly", "idle", "indexing", "error", "no_index" pub status: String, /// C# index status: "none", "ready", "indexing", "error" (empty also treated as none) pub csharp_index: String, @@ -1197,7 +1197,9 @@ fn status_cell(status: &str, csharp: &str) -> Cell<'static> { } } } - "closed" => Cell::from("○ closed ".to_string()).style(Style::default().fg(Color::Gray)), + "idle" | "closed" => { + Cell::from("○ idle ".to_string()).style(Style::default().fg(Color::Gray)) + } "error" => Cell::from("✗ error ".to_string()) .style(Style::default().fg(Color::Red).add_modifier(Modifier::BOLD)), "no_index" => { @@ -1246,7 +1248,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { _ => ("Warm".to_string(), Color::Yellow), }, "readonly" => ("Readonly".to_string(), Color::Cyan), - "closed" => ("Closed".to_string(), Color::Gray), + "idle" | "closed" => ("Idle".to_string(), Color::Gray), "indexing" => match csharp { "indexing" => ( "Index C#…".to_string(), From 09770219e42ecc54c25217d27ac164b58e2401d7 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 17:41:19 +0200 Subject: [PATCH 17/77] [worker] final review: pin TUI idle/legacy-closed arms with tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit status_cell's string match now delegates to pure status_cell_spec -> (String, Style) so tests can assert text+style without a Cell style getter (ratatui 0.30 Cell::style is a setter). 3 tests: idle+closed gray cell, Idle label mapping, unknown-status fallback distinct from idle. Review-fixes: - [Important] status_cell/detail_status_style string arms (idle | closed) had no test — silent _ fallback could regress legacy-peer tolerance unnoticed → 3 tests in src/serve/tui_common.rs pin cell text/style and label mapping for idle, legacy closed, and the fallback arm --- src/serve/tui_common.rs | 167 ++++++++++++++++++++++++++++++---------- 1 file changed, 125 insertions(+), 42 deletions(-) diff --git a/src/serve/tui_common.rs b/src/serve/tui_common.rs index 7ab7a3f1..d098a3e7 100644 --- a/src/serve/tui_common.rs +++ b/src/serve/tui_common.rs @@ -1111,74 +1111,102 @@ pub fn render_centered_modal_with_border_color( // --------------------------------------------------------------------------- fn status_cell(status: &str, csharp: &str) -> Cell<'static> { + let (text, style) = status_cell_spec(status, csharp); + Cell::from(text).style(style) +} + +/// Pure (text, style) decision behind `status_cell`, kept separate so tests can +/// assert on it (0.30 `Cell::style` is a setter, not a getter). +fn status_cell_spec(status: &str, csharp: &str) -> (String, Style) { let bright = pulse_bright(); match status { "open" => match csharp { - "ready" => Cell::from("✓ ready C#· ".to_string()).style( + "ready" => ( + "✓ ready C#· ".to_string(), Style::default() .fg(Color::Green) .add_modifier(Modifier::BOLD), ), "indexing" => { if bright { - Cell::from("⟳ idx C#… ".to_string()).style( + ( + "⟳ idx C#… ".to_string(), Style::default() .fg(Color::Yellow) .add_modifier(Modifier::BOLD), ) } else { - Cell::from("⟳ idx C#… ".to_string()) - .style(Style::default().fg(Color::DarkGray)) + ( + "⟳ idx C#… ".to_string(), + Style::default().fg(Color::DarkGray), + ) } } - "error" => Cell::from("✓ ready C#! ".to_string()) - .style(Style::default().fg(Color::Red).add_modifier(Modifier::BOLD)), - _ => Cell::from("✓ ready ".to_string()).style( + "error" => ( + "✓ ready C#! ".to_string(), + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + _ => ( + "✓ ready ".to_string(), Style::default() .fg(Color::Green) .add_modifier(Modifier::BOLD), ), }, "warm" => match csharp { - "ready" => { - Cell::from("◐ warm C#· ".to_string()).style(Style::default().fg(Color::Yellow)) - } + "ready" => ( + "◐ warm C#· ".to_string(), + Style::default().fg(Color::Yellow), + ), "indexing" => { if bright { - Cell::from("⟳ idx C#… ".to_string()).style( + ( + "⟳ idx C#… ".to_string(), Style::default() .fg(Color::Yellow) .add_modifier(Modifier::BOLD), ) } else { - Cell::from("⟳ idx C#… ".to_string()) - .style(Style::default().fg(Color::DarkGray)) + ( + "⟳ idx C#… ".to_string(), + Style::default().fg(Color::DarkGray), + ) } } - "error" => { - Cell::from("◐ warm C#! ".to_string()).style(Style::default().fg(Color::Yellow)) - } - _ => Cell::from("◐ warm ".to_string()).style(Style::default().fg(Color::Yellow)), + "error" => ( + "◐ warm C#! ".to_string(), + Style::default().fg(Color::Yellow), + ), + _ => ( + "◐ warm ".to_string(), + Style::default().fg(Color::Yellow), + ), }, - "readonly" => { - Cell::from("◑ ro ".to_string()).style(Style::default().fg(Color::Cyan)) - } + "readonly" => ( + "◑ ro ".to_string(), + Style::default().fg(Color::Cyan), + ), "indexing" => { if bright { match csharp { - "ready" => Cell::from("⟳ idx… C#· ".to_string()).style( + "ready" => ( + "⟳ idx… C#· ".to_string(), Style::default() .fg(Color::Yellow) .add_modifier(Modifier::BOLD), ), - "indexing" => Cell::from("⟳ idx… C#… ".to_string()).style( + "indexing" => ( + "⟳ idx… C#… ".to_string(), Style::default() .fg(Color::Yellow) .add_modifier(Modifier::BOLD), ), - "error" => Cell::from("⟳ idx… C#! ".to_string()) - .style(Style::default().fg(Color::Red).add_modifier(Modifier::BOLD)), - _ => Cell::from("⟳ idx… ".to_string()).style( + "error" => ( + "⟳ idx… C#! ".to_string(), + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + _ => ( + "⟳ idx… ".to_string(), Style::default() .fg(Color::Yellow) .add_modifier(Modifier::BOLD), @@ -1186,26 +1214,38 @@ fn status_cell(status: &str, csharp: &str) -> Cell<'static> { } } else { match csharp { - "ready" => Cell::from("⟳ idx… C#· ".to_string()) - .style(Style::default().fg(Color::DarkGray)), - "indexing" => Cell::from("⟳ idx… C#… ".to_string()) - .style(Style::default().fg(Color::DarkGray)), - "error" => Cell::from("⟳ idx… C#! ".to_string()) - .style(Style::default().fg(Color::DarkGray)), - _ => Cell::from("⟳ idx… ".to_string()) - .style(Style::default().fg(Color::DarkGray)), + "ready" => ( + "⟳ idx… C#· ".to_string(), + Style::default().fg(Color::DarkGray), + ), + "indexing" => ( + "⟳ idx… C#… ".to_string(), + Style::default().fg(Color::DarkGray), + ), + "error" => ( + "⟳ idx… C#! ".to_string(), + Style::default().fg(Color::DarkGray), + ), + _ => ( + "⟳ idx… ".to_string(), + Style::default().fg(Color::DarkGray), + ), } } } - "idle" | "closed" => { - Cell::from("○ idle ".to_string()).style(Style::default().fg(Color::Gray)) - } - "error" => Cell::from("✗ error ".to_string()) - .style(Style::default().fg(Color::Red).add_modifier(Modifier::BOLD)), - "no_index" => { - Cell::from("— no idx ".to_string()).style(Style::default().fg(Color::Gray)) - } - _ => Cell::from(format!("{:<14}", status)).style(Style::default().fg(Color::White)), + "idle" | "closed" => ( + "○ idle ".to_string(), + Style::default().fg(Color::Gray), + ), + "error" => ( + "✗ error ".to_string(), + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), + "no_index" => ( + "— no idx ".to_string(), + Style::default().fg(Color::Gray), + ), + _ => (format!("{:<14}", status), Style::default().fg(Color::White)), } } @@ -1288,3 +1328,46 @@ pub fn restore_terminal( terminal.show_cursor()?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_cell_renders_idle_and_legacy_closed_as_gray() { + let expected = ( + "○ idle ".to_string(), + Style::default().fg(Color::Gray), + ); + assert_eq!(status_cell_spec("idle", "ready"), expected); + assert_eq!(status_cell_spec("closed", ""), expected); + } + + #[test] + fn detail_status_style_maps_idle_and_legacy_closed_to_idle_label() { + assert_eq!( + detail_status_style("idle", ""), + ("Idle".to_string(), Color::Gray) + ); + assert_eq!( + detail_status_style("closed", ""), + ("Idle".to_string(), Color::Gray) + ); + } + + #[test] + fn unknown_status_falls_back_without_idle_treatment() { + assert_ne!( + status_cell_spec("future_state", "").1, + Style::default().fg(Color::Gray) + ); + assert_eq!( + status_cell_spec("future_state", "").0, + "future_state ".to_string() + ); + assert_eq!( + detail_status_style("future_state", ""), + ("future_state".to_string(), Color::White) + ); + } +} From 8f546465f51ef2fcc5a73d0ea81208639f955bb4 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 26 Sep 2026 20:44:34 +0200 Subject: [PATCH 18/77] =?UTF-8?q?chore:=20bump=20to=201.4.11=20=E2=80=94?= =?UTF-8?q?=20changelog=20for=20Closed=E2=86=92Idle=20rename=20+=20hub=20r?= =?UTF-8?q?eady=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct push to develop bypassed bump-develop.yml (fires on PR merges only), so the patch bump is done by hand; CHANGELOG gains the pending [1.4.11] section (finalized with a date at tag time, per the file's convention note). --- CHANGELOG.md | 6 ++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a082c69..fa375c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.4.11] + +### Changed + +- **Repo state `Closed` renamed to `Idle`; the aggregated hub status now reports `indexed: true` + `status: "ready"` as soon as any repo has an index on disk.** After the idle reaper evicted a repo's stores (30 min inactivity by default), the repo reported state `Closed`, and when ALL repos were evicted the hub-level `index_status` answered `indexed: false` / `idle` — which small-model MCP clients read as "index unavailable" and fell back to raw grep, even though every index was intact on disk and the next query would reopen it (first query slightly slower). The state is now `Idle` (wire string `"idle"`; a peer still sending legacy `"closed"` renders identically), and the aggregated status distinguishes three cases: any on-disk index → `indexed: true`, `"ready"`, message naming Open/Warm/Idle counts with the auto-reopen hint; repos registered but nothing on disk → `no_index`; no repos registered → `no_repos`. Per-project statuses, reaper timing and behavior are unchanged. + ## [1.4.10] - 2026-09-26 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 377d1980..9c85609b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.10" +version = "1.4.11" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 9589f590..a7a732b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.10" +version = "1.4.11" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From d200a2a613c370ccd3e56760e8c54d7a27b8712a Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 27 Sep 2026 21:45:57 +0200 Subject: [PATCH 19/77] [worker] stage 1-2: debug-only builds + rust-lld linker (issue #192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .cargo/config.toml: [target.x86_64-pc-windows-msvc] linker = "rust-lld" Bin-crate rebuild+link: 2m56s -> 1m32s (x2 faster dev cycle) - build.ps1: -Release now throws (debug-only policy); release artifacts are built by CI (release.yml) only - sccache: REJECTED — caches compilation, not linking (linking is the bottleneck), and disables incremental compilation; same verdict as audit #191 - tests/*.rs consolidation (53-binaries item in #191): N/A here — codesearch has 5 integration test binaries, consolidation not warranted - README.md "cargo build --release" (end-user build-from-source) kept: end users want optimized binaries; debug-only applies to the dev loop (AGENTS.develop.md already mandates it, CI builds debug) --- .cargo/config.toml | 6 ++++++ build.ps1 | 22 +++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 57fad0d6..3942b26c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,9 @@ [build] # Place target directory outside the repository to keep source tree clean target-dir = "../target" + +# Fast debug builds: rust-lld links several times faster than link.exe. +# Debug-only policy (issue #192): release builds ship via release.yml and are +# built there explicitly, never locally. +[target.x86_64-pc-windows-msvc] +linker = "rust-lld" diff --git a/build.ps1 b/build.ps1 index 50acac0c..3e1fb376 100644 --- a/build.ps1 +++ b/build.ps1 @@ -4,7 +4,7 @@ Builds CodeSearch. .DESCRIPTION - Always runs `cargo build` (debug or release). Cargo's own incremental + Always runs `cargo build` (debug). Cargo's own incremental compilation decides what actually needs to be recompiled — this script no longer tries to second-guess it with git-diff heuristics. @@ -18,11 +18,7 @@ .EXAMPLE .\build.ps1 - Builds in debug mode - -.EXAMPLE - .\build.ps1 -Release - Builds in release mode + Builds in debug mode (the only supported mode) #> param( @@ -31,6 +27,10 @@ param( $ErrorActionPreference = "Stop" +if ($Release) { + throw "-Release is not supported (debug-only policy, issue #192). Release artifacts are built by CI (.github/workflows/release.yml)." +} + # Change to script directory (where Cargo.toml is located) $ScriptDir = $PSScriptRoot Set-Location $ScriptDir @@ -67,8 +67,8 @@ if ($orphans.Count -gt 0) { Write-Host " [warn] if this build hangs on 'Blocking waiting for file lock', terminate the stale process manually." -ForegroundColor Yellow } -# Determine build mode -$BuildMode = if ($Release) { "release" } else { "debug" } +# Debug only (see -Release guard above) +$BuildMode = "debug" Write-Host "Building in $BuildMode mode..." -ForegroundColor Yellow @@ -81,11 +81,7 @@ $TmpDir = Join-Path $ScriptDir ".tmp" New-Item -ItemType Directory -Force -Path $TmpDir | Out-Null $LogFile = Join-Path $TmpDir "build-$BuildMode.log" -if ($Release) { - & cargo build --release 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 -} else { - & cargo build 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 -} +& cargo build 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 $BuildExit = $LASTEXITCODE Get-Content $LogFile | ForEach-Object { Write-Host $_ } From 38531fb01c975c6e8c4d2180deebf276f5ccfe46 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 27 Sep 2026 21:50:30 +0200 Subject: [PATCH 20/77] [worker] stage 3: document debug-only build policy + rust-lld in AGENTS.develop.md; branch-local plan --- AGENTS.develop.md | 2 +- AGENTS.md | 259 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 AGENTS.md diff --git a/AGENTS.develop.md b/AGENTS.develop.md index 9088733b..b378a1f2 100644 --- a/AGENTS.develop.md +++ b/AGENTS.develop.md @@ -187,7 +187,7 @@ TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). ## Runtime locations - **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. -- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. +- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). - **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. - **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..3ab90b86 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,259 @@ +# AGENTS.develop.md + +This file is the **develop-branch reference** for all coding agents. +It is committed on `develop` and merged from every feature branch. + +`AGENTS.md` and `CLAUDE.md` are **branch-local working files**: committed on the +feature branch while building, dropped before the PR merges — they never appear +on `develop` or `master`. See *For agents starting a new feature branch* below. + +## Setup (once per machine) + +```bash +git config core.hooksPath .githooks +``` + +This enables the hooks: `post-checkout` auto-creates `AGENTS.md` (from this +file) and the `CLAUDE.md` pointer on feature branches, `pre-commit` runs +`cargo fmt` + the root-md allowlist guard (branch-aware, see below), `pre-push` +blocks direct pushes to `master`. See `.githooks/README.md`. + +## For agents starting a new feature branch + +If `AGENTS.md` was not auto-created by the hook, create it manually: + +```bash +cp AGENTS.develop.md AGENTS.md +``` + +Then append your work plan as a `## Plan` section at the end of `AGENTS.md`. +Leave the architecture sections intact — they provide context. + +**Commit `AGENTS.md`/`CLAUDE.md` on the feature branch as you work** — the plan +travels through the PR and survives branch switches. + +**At the end of the branch, before the PR (the changelog moment):** +1. Update `## Active feature branches` below (remove this branch) +2. Add one line to `## Changelog highlights` +3. Add the CHANGELOG.md entry under the pending-version heading (functionality changes only — pure tooling/docs churn gets the `no-changelog` PR label instead; workflow changes like this one never go in the changelog) +4. Drop the branch-local files and commit: + `git rm AGENTS.md CLAUDE.md` → `chore: drop branch-local agent files` + +Enforced mechanically (forgetting step 4 cannot leak the files): +- `pre-commit` rejects introducing root `AGENTS.md`/`CLAUDE.md` on + `develop`/`master` (direct commits and conflict resolutions). +- `pre-merge-commit` runs the same guard on the merge result — a clean + merge never invokes pre-commit. +- The `agent-files-check` CI workflow flags any PR into `develop`/`master` + whose head still carries them. + +**No active work plan lives here.** Feature branches carry their own `AGENTS.md`. +This file contains only architecture, conventions, and changelog. + +--- + +## ⚠️ Branching & PR workflow (READ FIRST) + +This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. + +- **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. +- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`. NEVER target `master` (releases only, cut at release time). +- **Merge style:** feature/fix → `develop` = merge commits (`--merge`); `develop` → `master` release PR = **squash**, with `--body "$(scripts/release-coauthors.sh)"` so contributors stay credited on the default branch. +- **Review requirement** is enforced by a repo ruleset. As repo owner, override with `gh pr merge --merge --admin --delete-branch`. +- Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. +- **Release squashes regress the merge-base** → a release PR may fail with "cannot be cleanly created" even when content is fine. Fix: cut `release/vX.Y.Z` off develop, run `git merge -s ours origin/master` in it, verify the diff vs master is the intended release delta, PR that into master. Never merge master into develop directly. + +Common mistake: a subagent creates a PR with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. + +--- + +## What codesearch is + +A fast, local, offline MCP server for semantic code search. Single Rust binary. +No Docker, no cloud, no external services. Designed for coding agents (OpenCode, +Claude Code, Claude Desktop) that need to search and navigate large codebases efficiently. + +Core stack: Tantivy (BM25 FTS) + arroy (HNSW vectors) + fastembed/ONNX (embeddings) + +tree-sitter (AST chunking) + LMDB (persistent storage) + rmcp 1.5.0 (MCP protocol). +Hybrid BM25 + vector search fused via RRF. + +## MCP tools + +Five tools exposed to agents: + +| Tool | Description | +|---|---| +| `search` | Hybrid semantic + BM25 search. Requires `project` or `group` in multi-repo mode. | +| `find` | Symbol navigation: definition, usages, imports, dependents. Requires scope. | +| `explore` | File outline or similar-chunk lookup. Requires scope. | +| `get_chunk` | Retrieve a chunk by ID with optional context lines. Requires `project` in multi-repo mode. | +| `status` | Index and project status. Lightweight (no DB open) when called without scope. | + +All tools return `scope_required` structured errors in multi-repo mode when no `project` +or `group` is specified, with `available_projects`, `available_groups`, and `hint_for_agent`. + +## Multi-repo serve mode + +`codesearch serve` starts an MCP HTTP server on `127.0.0.1:{port}` (default 39725). +Multiple repos register via `repos.json` with aliases. Agents route queries per-alias +or per-group. + +**Repo lifecycle:** +- `Warm` — DB open, vector index ready, no FSW. State after background warmup or fan-out open. +- `Write` — Warm + file system watcher running. Transitions from Warm on first explicit project query. +- `Readonly` — Another process holds the write lock. +- `Idle` (status string was `Closed`) — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity; index stays on disk, stores auto-reopen on the next query. + +**Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. +All opens update `last_access` so the reaper can track every open. + +**Fan-out rule:** `get_chunk` and group queries use `touch=false` → repos open as Warm only, +no FSW spawned. Only explicit `project=` queries use `touch=true` → Warm→Write transition. + +## TUI + +`codesearch serve` with a TTY starts an embedded ratatui TUI (repo table, status, CPU). +Without TTY: headless, logs only. + +- `codesearch serve --no-tui` — suppress TUI even with TTY +- `codesearch serve tui [--url http://...]` — standalone TUI via HTTP polling of `GET /status` + +## HTTP endpoints (serve mode) + +| Endpoint | Method | Description | +|---|---|---| +| `/health` | GET | Health check JSON | +| `/status` | GET | Lightweight repo state snapshot for TUI polling | +| `/repos` | POST | Register + index + warmup a new repo | +| `/repos/:alias` | DELETE | Stop FSW, evict, unregister, delete DB | +| `/repos/:alias/reindex` | POST | Incremental or force reindex (background) | +| `/mcp` | GET/POST | MCP streamable HTTP endpoint | + +## Supported languages + +17 tree-sitter grammars (table in README.md). `find_impact` has SCIP symbol +precision for C# (bundled `scip-csharp`, resident helper via its `serve` +subcommand) and TypeScript (`npx scip-typescript`). Protobuf is text-aware +chunking only. + +## Release artifacts + +GitHub Actions produces release binaries per tag (plain + `-with-csharp` +variants): `codesearch-{windows-x86_64.zip,linux-x86_64.tar.gz,macos-arm64.tar.gz}`. +Single-file native binaries, no runtime dependencies. macOS build is manual-trigger +only (expensive runners). + +--- + +## Current state + +- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (`.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness: `build.rs` `+`. See `RELEASING.md`. +- **Validation:** `cargo check` for iteration → `cargo clippy --all-targets -- -D warnings` → `cargo test --lib --bins` before a branch is done. No `--release` builds during the fix loop. +- **Deploy:** `..\copy-to-common.ps1` builds + copies both binaries to `~/.local/bin/`. Stop serve first — a running `codesearch.exe` is file-locked on Windows. + +## Implemented features (orientation only — narratives live in CHANGELOG.md) + +- **Federation** — `codesearch remote add/rm/list/mounts`; `@peer` group fan-out; mounted projects addressable as `project=/`; TUI mount management with live status. Remote write verbs (`add`, `reindex --force`) need a read-write peer; `rm` is not durable across a cold start. +- **Cloud indexer-job split** — heavy build job uploads a snapshot, light serve restores it. DOCS repos are `repo_read_only` in `repos.json`; only `custom-kb` stays writable with a bounded incremental reindex per KB `git pull`. +- **`find_impact`** — SCIP-precise transitive call-sites (C#, TypeScript), ambiguity envelope, partial-results warnings, freshness fields. On `busy: true` sleep `retry_after_seconds` and retry the SAME call (busy is progress, never a reason to fall back to text search); `index_head_sha` vs `current_head_sha` drift is surfaced, never auto-reindexed. +- **REST mirrors** — read-only HTTP mirrors of the MCP tools incl. `/find-impact`. + +## ⚠️ Design constraint — never poll a federated peer + +**A federated peer is NEVER contacted on a timer.** Each poll wakes the peer's +scale-to-zero replica, which then self-warms its full idle window (measured ~50% +duty cycle from zero searches). Shipped twice (PR #181/#184), reverted twice — +do not re-attempt. Local repos may be polled in the background; peers are +contacted only by a real federated tool call (activity poke) or the explicit +TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). + +## Open TODOs + +- **Low severity:** literal-mode snippets for markdown chunks sometimes show the chunk's opening line instead of the matched line (the `match_info.unwrap_or_else` fallback), making true hits look like false negatives. + +--- + +## Key conventions for agents + +- **Branch from develop**, never from master. Feature branches: `features/` (or `fix/`, `chore/`). +- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. +- **Always fix bugs you encounter — including pre-existing ones** the current branch did not introduce. Clean code and tests are the bar: a fix lands with a test that fails without it (reintroduce the defect, watch it fail — the proof rule under *Search errors*). +- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to `C:\WorkArea\AI\codesearch\instructions\`. +- **Root file hygiene (markdown)**: root markdown is allowlisted — `AGENTS.develop.md`, `README.md`, `README_CSharp.md`, `CHANGELOG.md`, `RELEASING.md` always; `AGENTS.md` and `CLAUDE.md` on **feature branches only** (branch-local, dropped at finalization — the pre-commit guard rejects them on `develop`/`master`). Anything else (diagnoses, plans, test scenarios, worklogs) goes into `.docs/` (gitignored, local-only). Enforced by the `pre-commit` root-md allowlist guard and the `agent-files-check` CI workflow. +- **Path normalization**: all path comparisons must go through a single normalize utility. Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree `.git` file resolution have each caused subtle bugs in the past. +- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. ~2GB memory during indexing is a known limitation. No local fix available until upstream fastembed exposes `OrtArenaCfg`. +- **Git worktrees**: `find_git_root` returns the worktree directory itself. Each worktree is a separate indexable repo. Groups can be used to search across worktrees of the same base repo. + +## Runtime locations + +- **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. +- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). +- **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. +- **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. + +## Deploying to runtime + +- `..\copy-to-common.ps1` — builds and copies **both** `codesearch.exe` and `scip-csharp.exe` to `~/.local/bin/` (the common execution dir). Use this to update the runtime binaries. **No `--release` builds — always dev/debug.** +- The C# helper is built via: `dotnet publish helpers/csharp/scip-csharp.csproj -r win-x64 --self-contained -c Release` +- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true`. +- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location. + +## Notes for agents + +- **Never use the bundled `codesearch` binary to investigate this repo** (it is the project under development). Use codesearch MCP tools first for discovery (this repo is indexed as `codesearch-git`); `grep`/`Read` for exact refs, other git refs, or when MCP returns nothing. +- **Tests live in sibling `_tests.rs` files**, table-driven preferred over near-duplicate per-case fns. +- **Tests that set env vars must be `#[serial]`** and set them via `crate::testing::EnvRestore` — cargo runs tests as parallel threads of one process, so an unserialised `set_var` races every reader. +- **Never call `.canonicalize()`** — use `safe_canonicalize()`. +- **Windows transient rename errors** (os error 5/32/33 from AV/Search-Indexer races): classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry. Never retry non-transient errors. +- **Counter-then-teardown races:** a background task tearing down state guarded by an in-flight counter must take the write lock BEFORE checking the counter and hold it across check + clear. Consumers increment the counter before acquiring the resource, so `counter == 0` under the write lock proves no consumer exists. + +### LMDB rules + +- **One `EnvOpenOptions::open()` per directory per process.** All access via `get_or_open_stores()` → `Arc`; SCIP opens share a per-directory env (`get_or_open_shared_env`). +- **Open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`) — heed refuses to reopen one path with different options. +- **Commit, never drop, a txn whose DB handle you keep** — an aborted txn's DBI is closed by LMDB; using it later yields a bare `EINVAL`. +- **A dropped `TrackedEnv` must close via `prepare_for_closing()`** — heed's `OPENED_ENV` cache keeps a clone, so a plain drop never runs `mdb_env_close` and Windows keeps the files locked for the process lifetime (the `index rm` os-error-32 bug). + +### Search errors must not become empty results + +Never `unwrap_or_default()` a store error on a search path — "no results" and "store down" must stay distinguishable. This defect was re-introduced across sibling handlers in seven review rounds; it is a class, not a site: + +- Render error chains with `{:#}`, never `{}`. Pass "not found" claims through `qualify_empty_result()`; never state a diagnosis you did not verify. +- The rule covers EVERY MCP handler: `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents`, and the single-store `project=` paths. +- **Warnings channels must terminate** on every path that writes to them, and the read must be reachable from the last write. +- **Every `for store in stores` fan-out opens its `*_warnings` channel before the loop.** `Err(_)` over a store result is banned: bind it, render `{e:#}`, carry it. `MultiReadOutcome` is `#[must_use]`. +- **Take the channel as a parameter** — use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. +- **New response shapes use the shared exits, not hand-rolled ones.** `serde_json::json!` renders `None` as an explicit `null`, so conditional keys must be *inserted*, not set. +- **Suppress `suggested_tool` when warnings are present.** +- **Verify a batch edit by re-running its detector over the whole file**, including the lines the edit added. +- **Before claiming a test pins a fix, reintroduce the defect and watch it fail.** +- **A caller-facing literal wrapped across lines needs a `\` continuation** — enforced by `tests/caller_facing_literals.rs`. + +--- + +## Active feature branches (not yet merged) + +| Branch | Description | +|---|---| +| *(none)* | | + +--- + +## Changelog highlights (recent) + +- **v1.4.4** — resident C# workspace pool no longer serves stale `find_impact` results after a rebuild: `WorkspacePool::evict` bumps a per-solution generation counter closing a spawn-in-flight race, and `scip_ref_cache` is now cleared unconditionally on both full and incremental rebuilds +- **v1.3.37** — per-index embedding models end-to-end: serve queries, `POST /repos` and CLI index/stats/status honour the model each index records in its `metadata.json`; `serve --model` sets the default for newly created indexes; unrecorded indexes are queried with the built-in model plus a caller-facing warning; mid-rebuild indexes no longer report ready (PR #248) +- **v1.3.23–v1.3.36** — dependency + platform wave: rmcp 3.3, fastembed 6.1 + ort rc.13, tantivy 0.26, axum 0.8, ratatui 0.30 + crossterm 0.29, thiserror 2, notify 8, tree-sitter 0.27, dirs/sha2/scip/sysinfo refresh + dependabot (weekly); clears the open Aikido/RUSTSEC advisories +- **v1.3.19** — `find_impact` ambiguity envelope + `resolved_symbol`; partial-results `warnings`; C# symbol-key uniqueness (index v2.0) +- **v1.3.16** — REST `/find-impact` endpoint +- **v1.3.3** — federation hardening release + +Older entries: see `CHANGELOG.md`. + +## Plan (issue #192: debug builds + rust-lld, per audit #191) + +- rust-lld in .cargo/config.toml (msvc host) — measured 2m56s -> 1m32s bin rebuild +- build.ps1 refuses -Release; CI (release.yml) builds release artifacts +- sccache rejected (caches compilation, disables incremental — same as #191) +- tests/ has 5 integration binaries — consolidation not warranted +- README build-from-source keeps --release (end users want optimized) From 17dfed6c73b7a1ff96317e16c4573cc90834b726 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 27 Sep 2026 22:08:14 +0200 Subject: [PATCH 21/77] chore: drop branch-local agent files --- AGENTS.md | 259 ------------------------------------------------------ 1 file changed, 259 deletions(-) delete mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 3ab90b86..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,259 +0,0 @@ -# AGENTS.develop.md - -This file is the **develop-branch reference** for all coding agents. -It is committed on `develop` and merged from every feature branch. - -`AGENTS.md` and `CLAUDE.md` are **branch-local working files**: committed on the -feature branch while building, dropped before the PR merges — they never appear -on `develop` or `master`. See *For agents starting a new feature branch* below. - -## Setup (once per machine) - -```bash -git config core.hooksPath .githooks -``` - -This enables the hooks: `post-checkout` auto-creates `AGENTS.md` (from this -file) and the `CLAUDE.md` pointer on feature branches, `pre-commit` runs -`cargo fmt` + the root-md allowlist guard (branch-aware, see below), `pre-push` -blocks direct pushes to `master`. See `.githooks/README.md`. - -## For agents starting a new feature branch - -If `AGENTS.md` was not auto-created by the hook, create it manually: - -```bash -cp AGENTS.develop.md AGENTS.md -``` - -Then append your work plan as a `## Plan` section at the end of `AGENTS.md`. -Leave the architecture sections intact — they provide context. - -**Commit `AGENTS.md`/`CLAUDE.md` on the feature branch as you work** — the plan -travels through the PR and survives branch switches. - -**At the end of the branch, before the PR (the changelog moment):** -1. Update `## Active feature branches` below (remove this branch) -2. Add one line to `## Changelog highlights` -3. Add the CHANGELOG.md entry under the pending-version heading (functionality changes only — pure tooling/docs churn gets the `no-changelog` PR label instead; workflow changes like this one never go in the changelog) -4. Drop the branch-local files and commit: - `git rm AGENTS.md CLAUDE.md` → `chore: drop branch-local agent files` - -Enforced mechanically (forgetting step 4 cannot leak the files): -- `pre-commit` rejects introducing root `AGENTS.md`/`CLAUDE.md` on - `develop`/`master` (direct commits and conflict resolutions). -- `pre-merge-commit` runs the same guard on the merge result — a clean - merge never invokes pre-commit. -- The `agent-files-check` CI workflow flags any PR into `develop`/`master` - whose head still carries them. - -**No active work plan lives here.** Feature branches carry their own `AGENTS.md`. -This file contains only architecture, conventions, and changelog. - ---- - -## ⚠️ Branching & PR workflow (READ FIRST) - -This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. - -- **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. -- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`. NEVER target `master` (releases only, cut at release time). -- **Merge style:** feature/fix → `develop` = merge commits (`--merge`); `develop` → `master` release PR = **squash**, with `--body "$(scripts/release-coauthors.sh)"` so contributors stay credited on the default branch. -- **Review requirement** is enforced by a repo ruleset. As repo owner, override with `gh pr merge --merge --admin --delete-branch`. -- Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. -- **Release squashes regress the merge-base** → a release PR may fail with "cannot be cleanly created" even when content is fine. Fix: cut `release/vX.Y.Z` off develop, run `git merge -s ours origin/master` in it, verify the diff vs master is the intended release delta, PR that into master. Never merge master into develop directly. - -Common mistake: a subagent creates a PR with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. - ---- - -## What codesearch is - -A fast, local, offline MCP server for semantic code search. Single Rust binary. -No Docker, no cloud, no external services. Designed for coding agents (OpenCode, -Claude Code, Claude Desktop) that need to search and navigate large codebases efficiently. - -Core stack: Tantivy (BM25 FTS) + arroy (HNSW vectors) + fastembed/ONNX (embeddings) + -tree-sitter (AST chunking) + LMDB (persistent storage) + rmcp 1.5.0 (MCP protocol). -Hybrid BM25 + vector search fused via RRF. - -## MCP tools - -Five tools exposed to agents: - -| Tool | Description | -|---|---| -| `search` | Hybrid semantic + BM25 search. Requires `project` or `group` in multi-repo mode. | -| `find` | Symbol navigation: definition, usages, imports, dependents. Requires scope. | -| `explore` | File outline or similar-chunk lookup. Requires scope. | -| `get_chunk` | Retrieve a chunk by ID with optional context lines. Requires `project` in multi-repo mode. | -| `status` | Index and project status. Lightweight (no DB open) when called without scope. | - -All tools return `scope_required` structured errors in multi-repo mode when no `project` -or `group` is specified, with `available_projects`, `available_groups`, and `hint_for_agent`. - -## Multi-repo serve mode - -`codesearch serve` starts an MCP HTTP server on `127.0.0.1:{port}` (default 39725). -Multiple repos register via `repos.json` with aliases. Agents route queries per-alias -or per-group. - -**Repo lifecycle:** -- `Warm` — DB open, vector index ready, no FSW. State after background warmup or fan-out open. -- `Write` — Warm + file system watcher running. Transitions from Warm on first explicit project query. -- `Readonly` — Another process holds the write lock. -- `Idle` (status string was `Closed`) — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity; index stays on disk, stores auto-reopen on the next query. - -**Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. -All opens update `last_access` so the reaper can track every open. - -**Fan-out rule:** `get_chunk` and group queries use `touch=false` → repos open as Warm only, -no FSW spawned. Only explicit `project=` queries use `touch=true` → Warm→Write transition. - -## TUI - -`codesearch serve` with a TTY starts an embedded ratatui TUI (repo table, status, CPU). -Without TTY: headless, logs only. - -- `codesearch serve --no-tui` — suppress TUI even with TTY -- `codesearch serve tui [--url http://...]` — standalone TUI via HTTP polling of `GET /status` - -## HTTP endpoints (serve mode) - -| Endpoint | Method | Description | -|---|---|---| -| `/health` | GET | Health check JSON | -| `/status` | GET | Lightweight repo state snapshot for TUI polling | -| `/repos` | POST | Register + index + warmup a new repo | -| `/repos/:alias` | DELETE | Stop FSW, evict, unregister, delete DB | -| `/repos/:alias/reindex` | POST | Incremental or force reindex (background) | -| `/mcp` | GET/POST | MCP streamable HTTP endpoint | - -## Supported languages - -17 tree-sitter grammars (table in README.md). `find_impact` has SCIP symbol -precision for C# (bundled `scip-csharp`, resident helper via its `serve` -subcommand) and TypeScript (`npx scip-typescript`). Protobuf is text-aware -chunking only. - -## Release artifacts - -GitHub Actions produces release binaries per tag (plain + `-with-csharp` -variants): `codesearch-{windows-x86_64.zip,linux-x86_64.tar.gz,macos-arm64.tar.gz}`. -Single-file native binaries, no runtime dependencies. macOS build is manual-trigger -only (expensive runners). - ---- - -## Current state - -- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (`.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness: `build.rs` `+`. See `RELEASING.md`. -- **Validation:** `cargo check` for iteration → `cargo clippy --all-targets -- -D warnings` → `cargo test --lib --bins` before a branch is done. No `--release` builds during the fix loop. -- **Deploy:** `..\copy-to-common.ps1` builds + copies both binaries to `~/.local/bin/`. Stop serve first — a running `codesearch.exe` is file-locked on Windows. - -## Implemented features (orientation only — narratives live in CHANGELOG.md) - -- **Federation** — `codesearch remote add/rm/list/mounts`; `@peer` group fan-out; mounted projects addressable as `project=/`; TUI mount management with live status. Remote write verbs (`add`, `reindex --force`) need a read-write peer; `rm` is not durable across a cold start. -- **Cloud indexer-job split** — heavy build job uploads a snapshot, light serve restores it. DOCS repos are `repo_read_only` in `repos.json`; only `custom-kb` stays writable with a bounded incremental reindex per KB `git pull`. -- **`find_impact`** — SCIP-precise transitive call-sites (C#, TypeScript), ambiguity envelope, partial-results warnings, freshness fields. On `busy: true` sleep `retry_after_seconds` and retry the SAME call (busy is progress, never a reason to fall back to text search); `index_head_sha` vs `current_head_sha` drift is surfaced, never auto-reindexed. -- **REST mirrors** — read-only HTTP mirrors of the MCP tools incl. `/find-impact`. - -## ⚠️ Design constraint — never poll a federated peer - -**A federated peer is NEVER contacted on a timer.** Each poll wakes the peer's -scale-to-zero replica, which then self-warms its full idle window (measured ~50% -duty cycle from zero searches). Shipped twice (PR #181/#184), reverted twice — -do not re-attempt. Local repos may be polled in the background; peers are -contacted only by a real federated tool call (activity poke) or the explicit -TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). - -## Open TODOs - -- **Low severity:** literal-mode snippets for markdown chunks sometimes show the chunk's opening line instead of the matched line (the `match_info.unwrap_or_else` fallback), making true hits look like false negatives. - ---- - -## Key conventions for agents - -- **Branch from develop**, never from master. Feature branches: `features/` (or `fix/`, `chore/`). -- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. -- **Always fix bugs you encounter — including pre-existing ones** the current branch did not introduce. Clean code and tests are the bar: a fix lands with a test that fails without it (reintroduce the defect, watch it fail — the proof rule under *Search errors*). -- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to `C:\WorkArea\AI\codesearch\instructions\`. -- **Root file hygiene (markdown)**: root markdown is allowlisted — `AGENTS.develop.md`, `README.md`, `README_CSharp.md`, `CHANGELOG.md`, `RELEASING.md` always; `AGENTS.md` and `CLAUDE.md` on **feature branches only** (branch-local, dropped at finalization — the pre-commit guard rejects them on `develop`/`master`). Anything else (diagnoses, plans, test scenarios, worklogs) goes into `.docs/` (gitignored, local-only). Enforced by the `pre-commit` root-md allowlist guard and the `agent-files-check` CI workflow. -- **Path normalization**: all path comparisons must go through a single normalize utility. Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree `.git` file resolution have each caused subtle bugs in the past. -- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. ~2GB memory during indexing is a known limitation. No local fix available until upstream fastembed exposes `OrtArenaCfg`. -- **Git worktrees**: `find_git_root` returns the worktree directory itself. Each worktree is a separate indexable repo. Groups can be used to search across worktrees of the same base repo. - -## Runtime locations - -- **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. -- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). -- **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. -- **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. - -## Deploying to runtime - -- `..\copy-to-common.ps1` — builds and copies **both** `codesearch.exe` and `scip-csharp.exe` to `~/.local/bin/` (the common execution dir). Use this to update the runtime binaries. **No `--release` builds — always dev/debug.** -- The C# helper is built via: `dotnet publish helpers/csharp/scip-csharp.csproj -r win-x64 --self-contained -c Release` -- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true`. -- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location. - -## Notes for agents - -- **Never use the bundled `codesearch` binary to investigate this repo** (it is the project under development). Use codesearch MCP tools first for discovery (this repo is indexed as `codesearch-git`); `grep`/`Read` for exact refs, other git refs, or when MCP returns nothing. -- **Tests live in sibling `_tests.rs` files**, table-driven preferred over near-duplicate per-case fns. -- **Tests that set env vars must be `#[serial]`** and set them via `crate::testing::EnvRestore` — cargo runs tests as parallel threads of one process, so an unserialised `set_var` races every reader. -- **Never call `.canonicalize()`** — use `safe_canonicalize()`. -- **Windows transient rename errors** (os error 5/32/33 from AV/Search-Indexer races): classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry. Never retry non-transient errors. -- **Counter-then-teardown races:** a background task tearing down state guarded by an in-flight counter must take the write lock BEFORE checking the counter and hold it across check + clear. Consumers increment the counter before acquiring the resource, so `counter == 0` under the write lock proves no consumer exists. - -### LMDB rules - -- **One `EnvOpenOptions::open()` per directory per process.** All access via `get_or_open_stores()` → `Arc`; SCIP opens share a per-directory env (`get_or_open_shared_env`). -- **Open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`) — heed refuses to reopen one path with different options. -- **Commit, never drop, a txn whose DB handle you keep** — an aborted txn's DBI is closed by LMDB; using it later yields a bare `EINVAL`. -- **A dropped `TrackedEnv` must close via `prepare_for_closing()`** — heed's `OPENED_ENV` cache keeps a clone, so a plain drop never runs `mdb_env_close` and Windows keeps the files locked for the process lifetime (the `index rm` os-error-32 bug). - -### Search errors must not become empty results - -Never `unwrap_or_default()` a store error on a search path — "no results" and "store down" must stay distinguishable. This defect was re-introduced across sibling handlers in seven review rounds; it is a class, not a site: - -- Render error chains with `{:#}`, never `{}`. Pass "not found" claims through `qualify_empty_result()`; never state a diagnosis you did not verify. -- The rule covers EVERY MCP handler: `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents`, and the single-store `project=` paths. -- **Warnings channels must terminate** on every path that writes to them, and the read must be reachable from the last write. -- **Every `for store in stores` fan-out opens its `*_warnings` channel before the loop.** `Err(_)` over a store result is banned: bind it, render `{e:#}`, carry it. `MultiReadOutcome` is `#[must_use]`. -- **Take the channel as a parameter** — use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. -- **New response shapes use the shared exits, not hand-rolled ones.** `serde_json::json!` renders `None` as an explicit `null`, so conditional keys must be *inserted*, not set. -- **Suppress `suggested_tool` when warnings are present.** -- **Verify a batch edit by re-running its detector over the whole file**, including the lines the edit added. -- **Before claiming a test pins a fix, reintroduce the defect and watch it fail.** -- **A caller-facing literal wrapped across lines needs a `\` continuation** — enforced by `tests/caller_facing_literals.rs`. - ---- - -## Active feature branches (not yet merged) - -| Branch | Description | -|---|---| -| *(none)* | | - ---- - -## Changelog highlights (recent) - -- **v1.4.4** — resident C# workspace pool no longer serves stale `find_impact` results after a rebuild: `WorkspacePool::evict` bumps a per-solution generation counter closing a spawn-in-flight race, and `scip_ref_cache` is now cleared unconditionally on both full and incremental rebuilds -- **v1.3.37** — per-index embedding models end-to-end: serve queries, `POST /repos` and CLI index/stats/status honour the model each index records in its `metadata.json`; `serve --model` sets the default for newly created indexes; unrecorded indexes are queried with the built-in model plus a caller-facing warning; mid-rebuild indexes no longer report ready (PR #248) -- **v1.3.23–v1.3.36** — dependency + platform wave: rmcp 3.3, fastembed 6.1 + ort rc.13, tantivy 0.26, axum 0.8, ratatui 0.30 + crossterm 0.29, thiserror 2, notify 8, tree-sitter 0.27, dirs/sha2/scip/sysinfo refresh + dependabot (weekly); clears the open Aikido/RUSTSEC advisories -- **v1.3.19** — `find_impact` ambiguity envelope + `resolved_symbol`; partial-results `warnings`; C# symbol-key uniqueness (index v2.0) -- **v1.3.16** — REST `/find-impact` endpoint -- **v1.3.3** — federation hardening release - -Older entries: see `CHANGELOG.md`. - -## Plan (issue #192: debug builds + rust-lld, per audit #191) - -- rust-lld in .cargo/config.toml (msvc host) — measured 2m56s -> 1m32s bin rebuild -- build.ps1 refuses -Release; CI (release.yml) builds release artifacts -- sccache rejected (caches compilation, disables incremental — same as #191) -- tests/ has 5 integration binaries — consolidation not warranted -- README build-from-source keeps --release (end users want optimized) From 61bb6e85e8756d74ee173bf15f9979fad97d6417 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:13:46 +0000 Subject: [PATCH 22/77] chore: bump version to 1.4.12 (auto, PR #291 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c85609b..3dc819f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.11" +version = "1.4.12" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index a7a732b9..4533ace3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.11" +version = "1.4.12" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 88afe28ca205d7c78d148531cb6610013e5ce51e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:36:46 +0000 Subject: [PATCH 23/77] chore(deps): bump the cargo-minor-and-patch group with 4 updates Bumps the cargo-minor-and-patch group with 4 updates: [thiserror](https://github.com/dtolnay/thiserror), [fastembed](https://github.com/Anush008/fastembed-rs), [rand](https://github.com/rust-random/rand) and [rmcp](https://github.com/modelcontextprotocol/rust-sdk). Updates `thiserror` from 2.0.20 to 2.0.21 - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/2.0.20...2.0.21) Updates `fastembed` from 7.0.1 to 7.1.0 - [Release notes](https://github.com/Anush008/fastembed-rs/releases) - [Commits](https://github.com/Anush008/fastembed-rs/compare/v7.0.1...v7.1.0) Updates `rand` from 0.10.2 to 0.10.3 - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.10.2...0.10.3) Updates `rmcp` from 3.4.0 to 3.4.1 - [Release notes](https://github.com/modelcontextprotocol/rust-sdk/releases) - [Changelog](https://github.com/modelcontextprotocol/rust-sdk/blob/main/release-plz.toml) - [Commits](https://github.com/modelcontextprotocol/rust-sdk/compare/rmcp-v3.4.0...rmcp-v3.4.1) --- updated-dependencies: - dependency-name: thiserror dependency-version: 2.0.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo-minor-and-patch - dependency-name: fastembed dependency-version: 7.1.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo-minor-and-patch - dependency-name: rand dependency-version: 0.10.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo-minor-and-patch - dependency-name: rmcp dependency-version: 3.4.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo-minor-and-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 90 +++++++++++++++++++++++++++--------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dc819f7..b3684ae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,11 +181,11 @@ dependencies = [ "nohash", "ordered-float 5.5.0", "page_size", - "rand 0.10.2", + "rand 0.10.3", "rayon", "roaring", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -245,7 +245,7 @@ dependencies = [ "num-traits", "pastey 0.1.1", "rayon", - "thiserror 2.0.20", + "thiserror 2.0.21", "v_frame", "y4m", ] @@ -743,7 +743,7 @@ dependencies = [ "ort", "pretty_assertions", "protobuf", - "rand 0.10.2", + "rand 0.10.3", "ratatui", "rayon", "regex", @@ -758,7 +758,7 @@ dependencies = [ "sysinfo 0.39.6", "tantivy", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -1600,9 +1600,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastembed" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c88b42d8347afbf09e3d2fac61fcc22b698b7b2f35a27a6e988bcdad5784b07" +checksum = "3850d724d559da51a4b88f68b0ba111a2627c32279bec6b3c256f073c71a3195" dependencies = [ "hf-hub 0.5.0", "image", @@ -1611,7 +1611,7 @@ dependencies = [ "safetensors", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokenizers", ] @@ -2080,7 +2080,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "ureq", "windows-sys 0.61.2", ] @@ -2105,7 +2105,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio-util", @@ -2127,7 +2127,7 @@ dependencies = [ "more-asserts", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -2638,7 +2638,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -2704,7 +2704,7 @@ checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" dependencies = [ "hashbrown 0.16.1", "portable-atomic", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -4064,7 +4064,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -4080,14 +4080,14 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -4149,9 +4149,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -4230,7 +4230,7 @@ dependencies = [ "palette", "serde", "strum", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-segmentation", "unicode-truncate", "unicode-width", @@ -4329,7 +4329,7 @@ dependencies = [ "rand 0.9.5", "rand_chacha", "simd_helpers", - "thiserror 2.0.20", + "thiserror 2.0.21", "v_frame", "wasm-bindgen", ] @@ -4427,7 +4427,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -4577,7 +4577,7 @@ dependencies = [ "async-trait", "http", "reqwest 0.13.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tower-service", ] @@ -4603,9 +4603,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "3.4.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23c62fe489ac1d401ab32688cfacac3737a8978dc3343e5361464c7724fd3cb" +checksum = "5b6317cd8c13e3ec9033cf2aa5aa92cd743f0f4f8a93cddc42ea0ae6ce3b8898" dependencies = [ "async-trait", "base64 0.23.1", @@ -4618,14 +4618,14 @@ dependencies = [ "indexmap", "pastey 0.2.3", "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "rmcp-macros", "schemars", "serde", "serde_json", "sse-stream", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -4636,9 +4636,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd740c45d66ceb87e5579082abc27bd771665e464e9660a17a048c721b2a6025" +checksum = "c7e66877136e969d5a2d561ee5841b17717a0619aebd64abb47a2a2f46e34683" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -5419,7 +5419,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "typetag", "uuid", @@ -5621,11 +5621,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -5641,9 +5641,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", @@ -5758,7 +5758,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -5809,7 +5809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "tokio", ] @@ -5939,7 +5939,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -7053,7 +7053,7 @@ dependencies = [ "http", "hyper", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "redb", "reqwest 0.13.5", "reqwest-middleware", @@ -7062,7 +7062,7 @@ dependencies = [ "serde_repr", "statrs", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio_with_wasm", @@ -7093,12 +7093,12 @@ dependencies = [ "itertools 0.14.0", "lz4_flex", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "regex", "safe-transmute", "serde", "static_assertions", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -7121,12 +7121,12 @@ dependencies = [ "http", "itertools 0.14.0", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", "sha2 0.11.0", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -7161,13 +7161,13 @@ dependencies = [ "more-asserts", "oneshot", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "serde", "serde_json", "shellexpand", "sysinfo 0.38.4", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", From 54e4def6013491ce665f0104674b03eb5a6fbe96 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 12:03:24 +0200 Subject: [PATCH 24/77] chore: branch-local plan for always-on symbol indexes --- AGENTS.md | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 1 + 2 files changed, 265 insertions(+) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..c915512c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,264 @@ +# AGENTS.develop.md + +This file is the **develop-branch reference** for all coding agents. +It is committed on `develop` and merged from every feature branch. + +`AGENTS.md` and `CLAUDE.md` are **branch-local working files**: committed on the +feature branch while building, dropped before the PR merges — they never appear +on `develop` or `master`. See *For agents starting a new feature branch* below. + +## Setup (once per machine) + +```bash +git config core.hooksPath .githooks +``` + +This enables the hooks: `post-checkout` auto-creates `AGENTS.md` (from this +file) and the `CLAUDE.md` pointer on feature branches, `pre-commit` runs +`cargo fmt` + the root-md allowlist guard (branch-aware, see below), `pre-push` +blocks direct pushes to `master`. See `.githooks/README.md`. + +## For agents starting a new feature branch + +If `AGENTS.md` was not auto-created by the hook, create it manually: + +```bash +cp AGENTS.develop.md AGENTS.md +``` + +Then append your work plan as a `## Plan` section at the end of `AGENTS.md`. +Leave the architecture sections intact — they provide context. + +**Commit `AGENTS.md`/`CLAUDE.md` on the feature branch as you work** — the plan +travels through the PR and survives branch switches. + +**At the end of the branch, before the PR (the changelog moment):** +1. Update `## Active feature branches` below (remove this branch) +2. Add one line to `## Changelog highlights` +3. Add the CHANGELOG.md entry under the pending-version heading (functionality changes only — pure tooling/docs churn gets the `no-changelog` PR label instead; workflow changes like this one never go in the changelog) +4. Drop the branch-local files and commit: + `git rm AGENTS.md CLAUDE.md` → `chore: drop branch-local agent files` + +Enforced mechanically (forgetting step 4 cannot leak the files): +- `pre-commit` rejects introducing root `AGENTS.md`/`CLAUDE.md` on + `develop`/`master` (direct commits and conflict resolutions). +- `pre-merge-commit` runs the same guard on the merge result — a clean + merge never invokes pre-commit. +- The `agent-files-check` CI workflow flags any PR into `develop`/`master` + whose head still carries them. + +**No active work plan lives here.** Feature branches carry their own `AGENTS.md`. +This file contains only architecture, conventions, and changelog. + +--- + +## ⚠️ Branching & PR workflow (READ FIRST) + +This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. + +- **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. +- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`. NEVER target `master` (releases only, cut at release time). +- **Merge style:** feature/fix → `develop` = merge commits (`--merge`); `develop` → `master` release PR = **squash**, with `--body "$(scripts/release-coauthors.sh)"` so contributors stay credited on the default branch. +- **Review requirement** is enforced by a repo ruleset. As repo owner, override with `gh pr merge --merge --admin --delete-branch`. +- Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. +- **Release squashes regress the merge-base** → a release PR may fail with "cannot be cleanly created" even when content is fine. Fix: cut `release/vX.Y.Z` off develop, run `git merge -s ours origin/master` in it, verify the diff vs master is the intended release delta, PR that into master. Never merge master into develop directly. + +Common mistake: a subagent creates a PR with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. + +--- + +## What codesearch is + +A fast, local, offline MCP server for semantic code search. Single Rust binary. +No Docker, no cloud, no external services. Designed for coding agents (OpenCode, +Claude Code, Claude Desktop) that need to search and navigate large codebases efficiently. + +Core stack: Tantivy (BM25 FTS) + arroy (HNSW vectors) + fastembed/ONNX (embeddings) + +tree-sitter (AST chunking) + LMDB (persistent storage) + rmcp 1.5.0 (MCP protocol). +Hybrid BM25 + vector search fused via RRF. + +## MCP tools + +Five tools exposed to agents: + +| Tool | Description | +|---|---| +| `search` | Hybrid semantic + BM25 search. Requires `project` or `group` in multi-repo mode. | +| `find` | Symbol navigation: definition, usages, imports, dependents. Requires scope. | +| `explore` | File outline or similar-chunk lookup. Requires scope. | +| `get_chunk` | Retrieve a chunk by ID with optional context lines. Requires `project` in multi-repo mode. | +| `status` | Index and project status. Lightweight (no DB open) when called without scope. | + +All tools return `scope_required` structured errors in multi-repo mode when no `project` +or `group` is specified, with `available_projects`, `available_groups`, and `hint_for_agent`. + +## Multi-repo serve mode + +`codesearch serve` starts an MCP HTTP server on `127.0.0.1:{port}` (default 39725). +Multiple repos register via `repos.json` with aliases. Agents route queries per-alias +or per-group. + +**Repo lifecycle:** +- `Warm` — DB open, vector index ready, no FSW. State after background warmup or fan-out open. +- `Write` — Warm + file system watcher running. Transitions from Warm on first explicit project query. +- `Readonly` — Another process holds the write lock. +- `Idle` (status string was `Closed`) — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity; index stays on disk, stores auto-reopen on the next query. + +**Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. +All opens update `last_access` so the reaper can track every open. + +**Fan-out rule:** `get_chunk` and group queries use `touch=false` → repos open as Warm only, +no FSW spawned. Only explicit `project=` queries use `touch=true` → Warm→Write transition. + +## TUI + +`codesearch serve` with a TTY starts an embedded ratatui TUI (repo table, status, CPU). +Without TTY: headless, logs only. + +- `codesearch serve --no-tui` — suppress TUI even with TTY +- `codesearch serve tui [--url http://...]` — standalone TUI via HTTP polling of `GET /status` + +## HTTP endpoints (serve mode) + +| Endpoint | Method | Description | +|---|---|---| +| `/health` | GET | Health check JSON | +| `/status` | GET | Lightweight repo state snapshot for TUI polling | +| `/repos` | POST | Register + index + warmup a new repo | +| `/repos/:alias` | DELETE | Stop FSW, evict, unregister, delete DB | +| `/repos/:alias/reindex` | POST | Incremental or force reindex (background) | +| `/mcp` | GET/POST | MCP streamable HTTP endpoint | + +## Supported languages + +17 tree-sitter grammars (table in README.md). `find_impact` has SCIP symbol +precision for C# (bundled `scip-csharp`, resident helper via its `serve` +subcommand) and TypeScript (`npx scip-typescript`). Protobuf is text-aware +chunking only. + +## Release artifacts + +GitHub Actions produces release binaries per tag (plain + `-with-csharp` +variants): `codesearch-{windows-x86_64.zip,linux-x86_64.tar.gz,macos-arm64.tar.gz}`. +Single-file native binaries, no runtime dependencies. macOS build is manual-trigger +only (expensive runners). + +--- + +## Current state + +- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (`.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness: `build.rs` `+`. See `RELEASING.md`. +- **Validation:** `cargo check` for iteration → `cargo clippy --all-targets -- -D warnings` → `cargo test --lib --bins` before a branch is done. No `--release` builds during the fix loop. +- **Deploy:** `..\copy-to-common.ps1` builds + copies both binaries to `~/.local/bin/`. Stop serve first — a running `codesearch.exe` is file-locked on Windows. + +## Implemented features (orientation only — narratives live in CHANGELOG.md) + +- **Federation** — `codesearch remote add/rm/list/mounts`; `@peer` group fan-out; mounted projects addressable as `project=/`; TUI mount management with live status. Remote write verbs (`add`, `reindex --force`) need a read-write peer; `rm` is not durable across a cold start. +- **Cloud indexer-job split** — heavy build job uploads a snapshot, light serve restores it. DOCS repos are `repo_read_only` in `repos.json`; only `custom-kb` stays writable with a bounded incremental reindex per KB `git pull`. +- **`find_impact`** — SCIP-precise transitive call-sites (C#, TypeScript), ambiguity envelope, partial-results warnings, freshness fields. On `busy: true` sleep `retry_after_seconds` and retry the SAME call (busy is progress, never a reason to fall back to text search); `index_head_sha` vs `current_head_sha` drift is surfaced, never auto-reindexed. +- **REST mirrors** — read-only HTTP mirrors of the MCP tools incl. `/find-impact`. + +## ⚠️ Design constraint — never poll a federated peer + +**A federated peer is NEVER contacted on a timer.** Each poll wakes the peer's +scale-to-zero replica, which then self-warms its full idle window (measured ~50% +duty cycle from zero searches). Shipped twice (PR #181/#184), reverted twice — +do not re-attempt. Local repos may be polled in the background; peers are +contacted only by a real federated tool call (activity poke) or the explicit +TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). + +## Open TODOs + +- **Low severity:** literal-mode snippets for markdown chunks sometimes show the chunk's opening line instead of the matched line (the `match_info.unwrap_or_else` fallback), making true hits look like false negatives. + +--- + +## Key conventions for agents + +- **Branch from develop**, never from master. Feature branches: `features/` (or `fix/`, `chore/`). +- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. +- **Always fix bugs you encounter — including pre-existing ones** the current branch did not introduce. Clean code and tests are the bar: a fix lands with a test that fails without it (reintroduce the defect, watch it fail — the proof rule under *Search errors*). +- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to `C:\WorkArea\AI\codesearch\instructions\`. +- **Root file hygiene (markdown)**: root markdown is allowlisted — `AGENTS.develop.md`, `README.md`, `README_CSharp.md`, `CHANGELOG.md`, `RELEASING.md` always; `AGENTS.md` and `CLAUDE.md` on **feature branches only** (branch-local, dropped at finalization — the pre-commit guard rejects them on `develop`/`master`). Anything else (diagnoses, plans, test scenarios, worklogs) goes into `.docs/` (gitignored, local-only). Enforced by the `pre-commit` root-md allowlist guard and the `agent-files-check` CI workflow. +- **Path normalization**: all path comparisons must go through a single normalize utility. Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree `.git` file resolution have each caused subtle bugs in the past. +- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. ~2GB memory during indexing is a known limitation. No local fix available until upstream fastembed exposes `OrtArenaCfg`. +- **Git worktrees**: `find_git_root` returns the worktree directory itself. Each worktree is a separate indexable repo. Groups can be used to search across worktrees of the same base repo. + +## Runtime locations + +- **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. +- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). +- **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. +- **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. + +## Deploying to runtime + +- `..\copy-to-common.ps1` — builds and copies **both** `codesearch.exe` and `scip-csharp.exe` to `~/.local/bin/` (the common execution dir). Use this to update the runtime binaries. **No `--release` builds — always dev/debug.** +- The C# helper is built via: `dotnet publish helpers/csharp/scip-csharp.csproj -r win-x64 --self-contained -c Release` +- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true`. +- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location. + +## Notes for agents + +- **Never use the bundled `codesearch` binary to investigate this repo** (it is the project under development). Use codesearch MCP tools first for discovery (this repo is indexed as `codesearch-git`); `grep`/`Read` for exact refs, other git refs, or when MCP returns nothing. +- **Tests live in sibling `_tests.rs` files**, table-driven preferred over near-duplicate per-case fns. +- **Tests that set env vars must be `#[serial]`** and set them via `crate::testing::EnvRestore` — cargo runs tests as parallel threads of one process, so an unserialised `set_var` races every reader. +- **Never call `.canonicalize()`** — use `safe_canonicalize()`. +- **Windows transient rename errors** (os error 5/32/33 from AV/Search-Indexer races): classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry. Never retry non-transient errors. +- **Counter-then-teardown races:** a background task tearing down state guarded by an in-flight counter must take the write lock BEFORE checking the counter and hold it across check + clear. Consumers increment the counter before acquiring the resource, so `counter == 0` under the write lock proves no consumer exists. + +### LMDB rules + +- **One `EnvOpenOptions::open()` per directory per process.** All access via `get_or_open_stores()` → `Arc`; SCIP opens share a per-directory env (`get_or_open_shared_env`). +- **Open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`) — heed refuses to reopen one path with different options. +- **Commit, never drop, a txn whose DB handle you keep** — an aborted txn's DBI is closed by LMDB; using it later yields a bare `EINVAL`. +- **A dropped `TrackedEnv` must close via `prepare_for_closing()`** — heed's `OPENED_ENV` cache keeps a clone, so a plain drop never runs `mdb_env_close` and Windows keeps the files locked for the process lifetime (the `index rm` os-error-32 bug). + +### Search errors must not become empty results + +Never `unwrap_or_default()` a store error on a search path — "no results" and "store down" must stay distinguishable. This defect was re-introduced across sibling handlers in seven review rounds; it is a class, not a site: + +- Render error chains with `{:#}`, never `{}`. Pass "not found" claims through `qualify_empty_result()`; never state a diagnosis you did not verify. +- The rule covers EVERY MCP handler: `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents`, and the single-store `project=` paths. +- **Warnings channels must terminate** on every path that writes to them, and the read must be reachable from the last write. +- **Every `for store in stores` fan-out opens its `*_warnings` channel before the loop.** `Err(_)` over a store result is banned: bind it, render `{e:#}`, carry it. `MultiReadOutcome` is `#[must_use]`. +- **Take the channel as a parameter** — use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. +- **New response shapes use the shared exits, not hand-rolled ones.** `serde_json::json!` renders `None` as an explicit `null`, so conditional keys must be *inserted*, not set. +- **Suppress `suggested_tool` when warnings are present.** +- **Verify a batch edit by re-running its detector over the whole file**, including the lines the edit added. +- **Before claiming a test pins a fix, reintroduce the defect and watch it fail.** +- **A caller-facing literal wrapped across lines needs a `\` continuation** — enforced by `tests/caller_facing_literals.rs`. + +--- + +## Active feature branches (not yet merged) + +| Branch | Description | +|---|---| +| *(none)* | | + +--- + +## Changelog highlights (recent) + +- **v1.4.4** — resident C# workspace pool no longer serves stale `find_impact` results after a rebuild: `WorkspacePool::evict` bumps a per-solution generation counter closing a spawn-in-flight race, and `scip_ref_cache` is now cleared unconditionally on both full and incremental rebuilds +- **v1.3.37** — per-index embedding models end-to-end: serve queries, `POST /repos` and CLI index/stats/status honour the model each index records in its `metadata.json`; `serve --model` sets the default for newly created indexes; unrecorded indexes are queried with the built-in model plus a caller-facing warning; mid-rebuild indexes no longer report ready (PR #248) +- **v1.3.23–v1.3.36** — dependency + platform wave: rmcp 3.3, fastembed 6.1 + ort rc.13, tantivy 0.26, axum 0.8, ratatui 0.30 + crossterm 0.29, thiserror 2, notify 8, tree-sitter 0.27, dirs/sha2/scip/sysinfo refresh + dependabot (weekly); clears the open Aikido/RUSTSEC advisories +- **v1.3.19** — `find_impact` ambiguity envelope + `resolved_symbol`; partial-results `warnings`; C# symbol-key uniqueness (index v2.0) +- **v1.3.16** — REST `/find-impact` endpoint +- **v1.3.3** — federation hardening release + +Older entries: see `CHANGELOG.md`. + +## Plan — always-on symbol indexes (features/always-on-symbol-indexes) + +**Doel:** elke geregistreerde repo heeft altijd een actuele SCIP-symbolenindex (C# en TypeScript), ongeacht het registratiepad. + +**Gap-analyse (geverifieerd tegen de bron):** runtime-add (`add_repo_handler`) bouwt tekst+vector+FSW maar geen symbolen; startup phase-2 is C#-only; TS krijgt nooit een initiele build; reindex alleen symbolen bij `?symbols=true`; find_impact antwoordt stilletjes leeg bij ontbrekende index. + +**Stappen:** +1. serve-kant: `trigger_symbol_rebuild` per taal + in-flight guard per (alias,lang) in `ServeState`; `evaluate_symbol_rebuild` (generic gate, C# delegeert naar `evaluate_csharp_rebuild`); phase-2 hernoemd naar `run_phase_2_symbols` itereert alle geinstalleerde talen; `maybe_rebuild_symbols` helper; reindex gebruikt die altijd (gate-gestuurd, `?symbols=true` forceert); add_repo_handler roept hem na `restart_fsw`. +2. find_impact self-heal: bij `!has_index` een fire-and-forget `heal_missing_symbol_index` (serve-mode, alias uit serve-config zodat remote mounts nooit lokaal healen) + warning in het antwoord; drift blijft onaangetast (no-thrash beleid). +3. Tests: in-flight claim/stale-takeover; heal/evaluate op niet-toepasselijke repo; existing suites groen. Validatie: fmt + clippy -D warnings + cargo test --lib --bins. + +**Buiten scope:** deploy naar ~/.local/bin en live verificatie op de draaiende hub (operator-stap: stop serve, copy-to-common.ps1, start serve). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..5d302681 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +Read AGENTS.md. From ab829bb69d18793cd0d50e5d49801d7ce1e05a50 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 12:29:28 +0200 Subject: [PATCH 25/77] =?UTF-8?q?[worker]=20stage=201-3/3:=20always-on=20s?= =?UTF-8?q?ymbol=20indexes=20=E2=80=94=20per-language=20serve=20pipeline,?= =?UTF-8?q?=20find=5Fimpact=20self-heal,=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - trigger_symbol_rebuild generalized per language with a per-(alias,lang) in-flight claim (DashMap, stale takeover past indexing timeout); C#-only TUI status / last_scip bookkeeping stays gated on the language - evaluate_symbol_rebuild: generic gate (applies_to before availability); C# delegates to evaluate_csharp_rebuild; non-C# rebuilds only a MISSING index (drift stays watcher/branch driven — no-thrash) - run_phase_2_symbols (was run_phase_2_csharp_scip) iterates all installed languages; TypeScript now gets its initial build at startup - reindex_handler: maybe_rebuild_symbols gate-driven for every language, symbols=true still forces (previous unconditional behaviour) - add_repo_handler: maybe_rebuild_symbols after restart_fsw — closes the runtime-add gap (registered while serve runs never saw the startup pass) - find_impact: missing definitions index (has_index false) starts a fire-and-forget heal_missing_symbol_index (serve mode, alias re-resolved from config so remote mounts are never healed locally) and the empty NotFound answer carries a warning marking it UNKNOWN; a successful resolution keeps its payload to the persisted-warnings channel - tests: claim reject/stale-takeover, gate applicability-before-availability, heal no-op outside remit, force skips non-applicable, never-built index warns on the empty answer (fails without the fix), phase-2 composition wires candidate through to trigger, serve-mode self-heal runs the rebuild Review-fixes: - [Test] no coverage reached run_phase_2_symbols wiring or the serve-mode heal spawn → added run_phase_2_symbols_wires_candidate_through_to_trigger and find_impact_self_heal_runs_the_rebuild_in_serve_mode (dummy helper via env override; rebuild failure observable as CSharpIndexStatus::Error) Validation: cargo fmt --check clean, clippy --all-targets -D warnings clean, cargo test --lib --bins: 761 passed / 0 failed. --- src/mcp/find_impact.rs | 53 ++++- src/mcp/find_impact_tests.rs | 65 +++++- src/serve/mod.rs | 436 ++++++++++++++++++++++++++--------- src/serve/tests.rs | 234 +++++++++++++++++++ 4 files changed, 678 insertions(+), 110 deletions(-) diff --git a/src/mcp/find_impact.rs b/src/mcp/find_impact.rs index 035a1957..93639bf5 100644 --- a/src/mcp/find_impact.rs +++ b/src/mcp/find_impact.rs @@ -93,6 +93,16 @@ fn dedupe_references( out } +/// Caller-facing warning for a find_impact answer against a project whose +/// symbol index was never built. The empty reference list such an answer +/// carries means UNKNOWN — stamping this on the payload is what keeps it from +/// passing for "no callers" while the self-heal rebuild runs. +pub(crate) fn missing_index_warning(lang: &str) -> String { + format!( + "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. A background rebuild was started; retry this call once it completes." + ) +} + #[tool_router(router = find_impact_router, vis = "pub(crate)")] impl CodesearchService { /// Symbol impact analysis — returns transitive call-sites of a symbol with file/line precision. @@ -249,6 +259,36 @@ impl CodesearchService { )])); } + // Self-heal + honesty: a project whose symbol index was never built + // answers EVERY query with NotFound/empty references — visually + // identical to "no callers", the exact trap that reads as "safe to + // remove" when it is not. Start a background rebuild (serve mode + // only) whenever the definitions index is absent; the *warning* + // below rides only on answers that actually come back empty + // (NotFound / resolution failure) — a successful resolution proves + // usable data exists and keeps its payload to the persisted + // warnings channel. Drift (index_head_sha vs current) is + // deliberately NOT healed — reindexing on every branch switch would + // thrash (see `SymbolIndexer::index_head_sha`). + let index_missing = !indexer.has_index(&db_path); + let heal_warning: Vec = if index_missing { + vec![missing_index_warning(indexer.language())] + } else { + Vec::new() + }; + if index_missing { + if let (Some(ref serve_state), Some(ref alias)) = + (&self.serve_state, &ctx.project_alias) + { + let st = std::sync::Arc::clone(serve_state); + let alias_heal = alias.clone(); + let lang_heal = indexer.language().to_string(); + tokio::spawn(async move { + crate::serve::heal_missing_symbol_index(&st, &alias_heal, &lang_heal).await; + }); + } + } + // Perform the lookup under an internal wall-clock budget. // // `find_references_for_key` may invoke `scip-csharp find-refs` on a cache miss @@ -367,10 +407,15 @@ impl CodesearchService { let canonical = match resolution { Err(e) => { - let failure = crate::symbols::SymbolLookupFailure::classify( + let mut failure = crate::symbols::SymbolLookupFailure::classify( format!("{e:#}"), indexer.index_age(&db_path), ); + if index_missing { + failure.hint_for_agent.push_str( + " A background rebuild of the missing index was started — retry this call once it completes.", + ); + } let json = serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); @@ -406,8 +451,10 @@ impl CodesearchService { } Ok(crate::symbols::KeyMatch::NotFound) => { // Preserve the historical contract for fuzzy queries: an - // unresolvable name/position answers empty references. - let impact = build_impact(Vec::new(), None, Vec::new()); + // unresolvable name/position answers empty references. With a + // missing index that answer also carries the self-heal + // warning — empty must not pass for "no callers". + let impact = build_impact(Vec::new(), None, heal_warning); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); } diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index a07b7927..a298e625 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -7,7 +7,10 @@ //! resolution is tested separately against `resolve_find_impact_budget_secs` //! with `#[serial]` + `EnvRestore` per the repo rule for env mutation. -use super::{find_impact_with_budget, resolve_find_impact_budget_secs, ImpactLookupOutcome}; +use super::{ + find_impact_with_budget, missing_index_warning, resolve_find_impact_budget_secs, + ImpactLookupOutcome, +}; use crate::constants::{DEFAULT_FIND_IMPACT_BUDGET_SECS, FIND_IMPACT_BUDGET_SECS_ENV}; use crate::symbols::{SymbolLookupBusy, SymbolReference}; use std::path::PathBuf; @@ -811,3 +814,63 @@ async fn clean_resolved_answer_omits_the_warnings_field() { "a clean answer must OMIT warnings (skip_serializing_if), got: {out}" ); } + +/// The missing-index warning is what keeps a never-built symbol index's +/// empty reference list from passing for "no callers": it must name the +/// language, state UNKNOWN explicitly, and tell the caller to retry after +/// the background rebuild. +#[test] +fn missing_index_warning_names_language_and_marks_answer_unknown() { + let warning = missing_index_warning("csharp"); + assert!( + warning.contains("csharp"), + "must name the language, got: {warning}" + ); + assert!( + warning.contains("UNKNOWN"), + "must mark the empty list as UNKNOWN, got: {warning}" + ); + assert!( + warning.contains("retry"), + "must tell the caller to retry after the rebuild, got: {warning}" + ); +} + +/// A never-built index must not answer "no references" silently: the empty +/// NotFound answer carries the self-heal warning. Fails without the fix — +/// the pre-fix answer is byte-identical to a genuine "no callers", the exact +/// trap that reads as "safe to remove" (observed live on a runtime-registered +/// worktree whose SCIP index was never built). +#[tokio::test] +#[serial_test::serial] +async fn never_built_index_warns_on_the_empty_answer() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + // No fixture population: a valid-but-empty store — neither symbols nor a + // rebuild timestamp. This is the never-built state. + let (service, _root) = build_service(); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("ResolveLinkedActivityAsync".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!( + v["references"].as_array().is_some_and(|a| a.is_empty()), + "the never-built index answers with empty references, got: {out}" + ); + let warnings = v["warnings"].as_array().expect("self-heal warning present"); + assert_eq!( + warnings.len(), + 1, + "exactly the missing-index warning, got: {out}" + ); + assert!( + warnings[0].as_str().is_some_and(|w| w.contains("UNKNOWN")), + "the warning must mark the answer UNKNOWN, got: {out}" + ); +} diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 5ea6d09d..edd98ce0 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -344,6 +344,12 @@ pub(crate) struct ServeState { /// Per-repo C# symbol index last error message (set when status is Error, cleared on success). /// Wrapped in `Arc` for the same reason as `csharp_index_status`. csharp_index_error: Arc>, + /// Symbol rebuilds in flight, keyed `"{lang}:{alias}"` → claim Instant. + /// Guards against double-triggered full rebuilds (find_impact self-heal + /// racing phase-2/reindex/watcher): two concurrent rebuilds interleave + /// their LMDB clear/write passes. Stale claims self-heal past the + /// indexing timeout, like `active_reindexes` markers. + symbol_rebuild_in_flight: DashMap, /// Debounced deadline for persisting repos config metadata (unix millis). persist_deadline_unix_ms: AtomicU64, /// Ensures only one debounce worker task runs. @@ -375,6 +381,9 @@ enum RebuildDecision { Fresh, /// No `.sln` file found; C# indexing is not applicable for this repo. NoSolutionFile, + /// The repo lacks this language's entrypoint (e.g. no `tsconfig.json`) — + /// the generic per-language gate's counterpart of `NoSolutionFile`. + NotApplicable, /// The scip-csharp helper binary is not available. HelperUnavailable, /// An indexing task for this alias is already running. @@ -425,6 +434,7 @@ impl ServeState { tool_call_counts: DashMap::new(), csharp_index_status: Arc::new(DashMap::new()), csharp_index_error: Arc::new(DashMap::new()), + symbol_rebuild_in_flight: DashMap::new(), persist_deadline_unix_ms: AtomicU64::new(0), persist_worker_started: AtomicBool::new(false), #[cfg(test)] @@ -1223,6 +1233,73 @@ impl ServeState { } } + /// Claim the `(alias, language)` symbol-rebuild slot. Returns `false` + /// when a fresh claim exists (a rebuild is genuinely running); a claim + /// older than the indexing timeout is taken over (leaked by a crashed + /// task, same self-heal as [`Self::begin_indexing`]). + fn begin_symbol_rebuild(&self, alias: &str, lang: &str) -> bool { + let now = std::time::Instant::now(); + let max = self.indexing_timeout(); + let key = format!("{lang}:{alias}"); + match self.symbol_rebuild_in_flight.entry(key) { + dashmap::mapref::entry::Entry::Occupied(mut occupied) => { + if now.duration_since(*occupied.get()) < max { + return false; + } + *occupied.get_mut() = now; + true + } + dashmap::mapref::entry::Entry::Vacant(vacant) => { + vacant.insert(now); + true + } + } + } + + /// Release the `(alias, language)` symbol-rebuild slot claimed by + /// [`Self::begin_symbol_rebuild`]. + fn end_symbol_rebuild(&self, alias: &str, lang: &str) { + self.symbol_rebuild_in_flight + .remove(&format!("{lang}:{alias}")); + } + + /// Generic per-language rebuild gate. C# keeps its dedicated evaluator + /// (last_scip/last_changed bookkeeping, TUI status map); every other + /// language rebuilds only a *missing* index — drift stays + /// watcher/branch-change driven, mirroring the deliberate no-thrash + /// policy documented on `SymbolIndexer::index_head_sha`. + fn evaluate_symbol_rebuild( + self: &Arc, + lang: &str, + alias: &str, + repo_path: &Path, + db_path: &Path, + ) -> RebuildDecision { + if lang.eq_ignore_ascii_case(LANG_CSHARP) { + return self.evaluate_csharp_rebuild(alias, repo_path, db_path); + } + let Some(indexer) = self.symbol_registry.get(lang) else { + return RebuildDecision::HelperUnavailable; + }; + // Applicability before availability: the entrypoint check is cheap and + // deterministic (mirrors `evaluate_csharp_rebuild`'s .sln-first order), + // while helper availability depends on the machine. + if !indexer.applies_to(repo_path) { + return RebuildDecision::NotApplicable; + } + if !indexer.is_available() { + return RebuildDecision::HelperUnavailable; + } + if !self.begin_symbol_rebuild(alias, lang) { + return RebuildDecision::AlreadyInFlight; + } + self.end_symbol_rebuild(alias, lang); + if !indexer.has_index(db_path) { + return RebuildDecision::NoIndex; + } + RebuildDecision::Fresh + } + pub(crate) fn schedule_persist_repos_config(self: &Arc) { let deadline = Self::now_unix_millis() + (PERSIST_DEBOUNCE_SECS * 1000); self.persist_deadline_unix_ms @@ -1411,10 +1488,15 @@ impl ServeState { } } - /// Phase 2: semaphore-bounded concurrent C# SCIP rebuilds, sorted by recency. - pub(crate) async fn run_phase_2_csharp_scip(self: &Arc) { + /// Phase 2: semaphore-bounded concurrent SCIP rebuilds across all + /// installed languages, sorted by recency. C# keeps its dedicated gate + /// (`evaluate_csharp_rebuild`); other languages rebuild only a missing + /// index (no-thrash: drift is watcher/branch-change driven). + pub(crate) async fn run_phase_2_symbols(self: &Arc) { let aliases = self.aliases(); - let mut candidates: Vec<(String, i64)> = Vec::new(); + let langs = self.symbol_registry.installed_languages(); + // (alias, lang, last_changed) + let mut candidates: Vec<(String, String, i64)> = Vec::new(); for alias in &aliases { let path = match self.config.read().ok().and_then(|c| c.resolve(alias)) { @@ -1422,70 +1504,75 @@ impl ServeState { None => continue, }; let db_path = path.join(DB_DIR_NAME); - // evaluate_csharp_rebuild may spawn a git subprocess and walk the - // filesystem — offload to the blocking pool so the async runtime - // stays responsive while processing all candidates. - let state2 = self.clone(); - let alias2 = alias.clone(); - let path2 = path.clone(); - let db_path2 = db_path.clone(); - let decision = match tokio::task::spawn_blocking(move || { - state2.evaluate_csharp_rebuild(&alias2, &path2, &db_path2) - }) - .await - { - Ok(d) => d, - Err(e) => { - warn!( - "phase-2: evaluate_csharp_rebuild panicked for '{}': {:?}", - alias, e - ); - continue; - } - }; - if !decision.needs_rebuild() { - // If the SCIP index exists and is fresh, mark C# status as Ready - // so the TUI shows the C# indicator (e.g. "C#·") instead of None. - if decision == RebuildDecision::Fresh { - let mut status = self - .csharp_index_status - .get(alias) - .map(|e| *e.value()) - .unwrap_or(CSharpIndexStatus::None); - if matches!(status, CSharpIndexStatus::None) { - status = CSharpIndexStatus::Ready; + for lang in &langs { + // evaluate_symbol_rebuild may spawn a git subprocess and walk + // the filesystem — offload to the blocking pool so the async + // runtime stays responsive while processing all candidates. + let state2 = self.clone(); + let alias2 = alias.clone(); + let lang2 = lang.clone(); + let path2 = path.clone(); + let db_path2 = db_path.clone(); + let decision = match tokio::task::spawn_blocking(move || { + state2.evaluate_symbol_rebuild(&lang2, &alias2, &path2, &db_path2) + }) + .await + { + Ok(d) => d, + Err(e) => { + warn!( + "phase-2: symbol evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + continue; } - self.csharp_index_status.insert(alias.to_string(), status); + }; + if !decision.needs_rebuild() { + // A fresh C# index flips the TUI C# indicator to Ready; + // other languages have no status map, nothing to update. + if decision == RebuildDecision::Fresh && lang.as_str() == LANG_CSHARP { + let mut status = self + .csharp_index_status + .get(alias) + .map(|e| *e.value()) + .unwrap_or(CSharpIndexStatus::None); + if matches!(status, CSharpIndexStatus::None) { + status = CSharpIndexStatus::Ready; + } + self.csharp_index_status.insert(alias.to_string(), status); + } + info!("phase-2: skip '{}' ({}) — {:?}", alias, lang, decision); + continue; } - info!("phase-2: skip '{}' — {:?}", alias, decision); - continue; + let last_changed = self + .config + .read() + .ok() + .and_then(|c| c.meta(alias).last_changed_unix) + .unwrap_or(0); + info!( + "phase-2: queued '{}' ({}) — {:?} (last_changed={})", + alias, lang, decision, last_changed + ); + candidates.push((alias.clone(), lang.clone(), last_changed)); } - let last_changed = self - .config - .read() - .ok() - .and_then(|c| c.meta(alias).last_changed_unix) - .unwrap_or(0); - info!( - "phase-2: queued '{}' — {:?} (last_changed={})", - alias, decision, last_changed - ); - candidates.push((alias.clone(), last_changed)); } - candidates.sort_by_key(|b| std::cmp::Reverse(b.1)); + candidates.sort_by_key(|c| std::cmp::Reverse(c.2)); if candidates.is_empty() { info!("phase-2 complete: 0 candidates"); return; } - // Pre-mark all queued candidates as C# Indexing so the TUI C# indicator + // Pre-mark queued C# candidates as Indexing so the TUI C# indicator // reflects pending rebuilds immediately, even before each repo acquires // its semaphore slot. trigger_symbol_rebuild will overwrite this with the // same value (no-op) and eventually with Ready or Error on completion. - for (alias, _) in &candidates { - self.csharp_index_status - .insert(alias.clone(), CSharpIndexStatus::Indexing); + for (alias, lang, _) in &candidates { + if lang.as_str() == LANG_CSHARP { + self.csharp_index_status + .insert(alias.clone(), CSharpIndexStatus::Indexing); + } } let concurrency = Self::csharp_scip_concurrency(); @@ -1497,7 +1584,7 @@ impl ServeState { let sem = Arc::new(Semaphore::new(concurrency)); let mut handles = Vec::with_capacity(candidates.len()); - for (alias, _) in candidates { + for (alias, lang, _) in candidates { let sem = sem.clone(); let state = self.clone(); handles.push(tokio::spawn(async move { @@ -1505,7 +1592,7 @@ impl ServeState { Ok(p) => p, Err(_) => return, }; - info!("phase-2: starting '{}'", alias); + info!("phase-2: starting '{}' ({})", alias, lang); let path = match state.config.read().ok().and_then(|c| c.resolve(&alias)) { Some(p) => p, None => { @@ -1526,7 +1613,7 @@ impl ServeState { return; } let db_path = path.join(DB_DIR_NAME); - trigger_symbol_rebuild(&alias, &path, &db_path, &state).await; + trigger_symbol_rebuild(&alias, &path, &db_path, &state, &lang).await; drop(permit); })); } @@ -4071,38 +4158,59 @@ async fn doctor_handler( AxumJson(json!({ "results": results })).into_response() } -/// Trigger a symbol index rebuild for a repo (C# etc.). +/// Trigger a symbol index rebuild for a repo in one language (C#, TypeScript). +/// +/// Reuses the shared `SymbolIndexerRegistry` from `ServeState`, looks up the +/// indexer for `lang`, and runs `rebuild()` in a blocking task. C#-only +/// bookkeeping (TUI status/error maps, `last_scip` timestamp) stays gated on +/// the language; `IndexingOwner::Symbol` markers apply to every language. /// -/// Reuses the shared `SymbolIndexerRegistry` from `ServeState`, looks up the C# indexer, -/// and runs `rebuild()` in a blocking task. Updates the C# index status on success/failure. +/// A per-`(alias, language)` in-flight slot prevents two concurrent full +/// rebuilds from interleaving their LMDB clear/write passes (find_impact +/// self-heal racing phase-2/reindex/watcher). Languages for one alias run +/// sequentially at multi-language call sites (see `maybe_rebuild_symbols`) — +/// the `IndexingOwner::Symbol` marker is not language-scoped. async fn trigger_symbol_rebuild( alias: &str, project_path: &Path, db_path: &Path, state: &Arc, + lang: &str, ) { + let is_csharp = lang.eq_ignore_ascii_case(LANG_CSHARP); // Skip non-applicable repos BEFORE touching status — otherwise the TUI // would flip C#-indicator red on Rust/Python repos that simply have no - // .sln. The phase-2 gate (`evaluate_csharp_rebuild`) already filters + // .sln. The phase-2 gate (`evaluate_symbol_rebuild`) already filters // these out, but other callers (POST /reindex?symbols=true, - // .cs watcher debounce, future paths) bypass that gate. + // .cs watcher debounce, find_impact self-heal) bypass that gate. let applies = state .symbol_registry - .get(LANG_CSHARP) + .get(lang) .map(|i| i.applies_to(project_path)) .unwrap_or(false); if !applies { tracing::info!( - "🔬 symbol reindex skipped for '{}': not applicable (no .sln)", + "🔬 symbol reindex skipped for '{}': not applicable (no .sln / tsconfig.json)", alias ); return; } - tracing::info!("🔬 symbol reindex triggered for '{}'", alias); - state - .csharp_index_status - .insert(alias.to_string(), CSharpIndexStatus::Indexing); + if !state.begin_symbol_rebuild(alias, lang) { + tracing::info!( + "🔬 symbol reindex skipped for '{}' ({}): already in flight", + alias, + lang + ); + return; + } + + tracing::info!("🔬 symbol reindex triggered for '{}' ({})", alias, lang); + if is_csharp { + state + .csharp_index_status + .insert(alias.to_string(), CSharpIndexStatus::Indexing); + } // Mark as actively indexing so the TUI status column shows "Indexing" // (not just the C# indicator). This mirrors what reindex_handler does. // @@ -4110,17 +4218,25 @@ async fn trigger_symbol_rebuild( // for the same alias simultaneously, each path ends only its own marker, // so neither can flip the TUI back to Warm/Open while the other still // runs. (Stale entries from a crashed task self-heal via `is_indexing`.) - state.begin_indexing(alias, IndexingOwner::Symbol); + // The return value is deliberately ignored: nested callers (the reindex + // and add-repo background tasks) legitimately hold the alias's Reindex + // marker while triggering this. + let _ = state.begin_indexing(alias, IndexingOwner::Symbol); let rp = project_path.to_path_buf(); let dp = db_path.to_path_buf(); let alias_owned = alias.to_string(); + let lang_owned = lang.to_string(); let registry = state.symbol_registry.clone(); match tokio::task::spawn_blocking(move || { - let Some(indexer) = registry.get(LANG_CSHARP) else { - return Err(anyhow::anyhow!("No C# symbol indexer registered")); + let Some(indexer) = registry.get(&lang_owned) else { + return Err(anyhow::anyhow!( + "No symbol indexer registered for '{lang_owned}'" + )); }; if !indexer.is_available() { - return Err(anyhow::anyhow!("scip-csharp helper not available")); + return Err(anyhow::anyhow!( + "symbol helper for '{lang_owned}' not available" + )); } indexer.rebuild(&rp, &dp, RebuildScope::Full) }) @@ -4135,15 +4251,17 @@ async fn trigger_symbol_rebuild( summary.duration_ms ); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_status - .insert(alias_owned.clone(), CSharpIndexStatus::Ready); - state.csharp_index_error.remove(&alias_owned); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned.clone(), CSharpIndexStatus::Ready); + state.csharp_index_error.remove(&alias_owned); - if let Ok(mut cfg) = state.config.write() { - cfg.touch_last_scip(&alias_owned, ServeState::now_unix_secs()); + if let Ok(mut cfg) = state.config.write() { + cfg.touch_last_scip(&alias_owned, ServeState::now_unix_secs()); + } + state.schedule_persist_repos_config(); } - state.schedule_persist_repos_config(); } Ok(Err(e)) => { // `{:#}` — the whole chain, not just the outermost context. The @@ -4151,9 +4269,11 @@ async fn trigger_symbol_rebuild( // would hide the `MDB_*` code the classifier below matches on. let msg = format!("{e:#}"); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_error - .insert(alias_owned.clone(), msg.clone()); + if is_csharp { + state + .csharp_index_error + .insert(alias_owned.clone(), msg.clone()); + } if ServeState::is_lmdb_format_corruption(&msg) && state.enqueue_format_recovery(&alias_owned) { @@ -4165,9 +4285,11 @@ async fn trigger_symbol_rebuild( ); // Recovery owns the outcome from here: show in-progress rather // than Error; it flips to Ready on success or Error on failure. - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Indexing); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Indexing); + } } else if ServeState::is_lmdb_format_corruption(&msg) { // A wipe already happened for this alias in this process, so the // data was written by the running binary: the error is write-side @@ -4179,14 +4301,18 @@ async fn trigger_symbol_rebuild( alias_owned, msg ); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } else { tracing::error!("❌ Symbol rebuild failed for '{}': {}", alias_owned, msg); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } } Err(e) => { @@ -4196,16 +4322,109 @@ async fn trigger_symbol_rebuild( e ); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_error - .insert(alias_owned.clone(), format!("Task panicked: {}", e)); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_error + .insert(alias_owned.clone(), format!("Task panicked: {}", e)); + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } + } + } + state.end_symbol_rebuild(alias, lang); +} + +/// Evaluate every installed symbol language for `alias` and rebuild the ones +/// whose gate says so. `force` (POST /reindex?symbols=true) bypasses the +/// gate, preserving the old unconditional-rebuild behaviour. +/// +/// Sequential per language: `IndexingOwner::Symbol` is not language-scoped, +/// so parallel languages would collide on the marker. Called from the +/// reindex and add-repo background tasks, where the alias already holds a +/// Reindex owner marker — `trigger_symbol_rebuild`'s own marker insert is +/// additive, not a guard, so nesting is safe. +async fn maybe_rebuild_symbols( + alias: &str, + project_path: &Path, + db_path: &Path, + state: &Arc, + force: bool, +) { + let langs = state.symbol_registry.installed_languages(); + for lang in langs { + let decision = if force { + RebuildDecision::ChangedSinceLastBuild + } else { + // evaluate may spawn a git subprocess — offload like phase-2. + let st = Arc::clone(state); + let a = alias.to_string(); + let l = lang.clone(); + let p = project_path.to_path_buf(); + let d = db_path.to_path_buf(); + match tokio::task::spawn_blocking(move || st.evaluate_symbol_rebuild(&l, &a, &p, &d)) + .await + { + Ok(d) => d, + Err(e) => { + warn!( + "symbol evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + continue; + } + } + }; + if decision.needs_rebuild() { + trigger_symbol_rebuild(alias, project_path, db_path, state, &lang).await; + } else { + info!("symbols fresh for '{}' ({}) — {:?}", alias, lang, decision); } } } +/// Fire-and-forget self-heal invoked by `find_impact` when the target index +/// does not exist: rebuild exactly that language's *missing* index — never on +/// drift (reindexing per query would thrash; see +/// `SymbolIndexer::index_head_sha`). The alias is re-resolved from the serve +/// config so remote-mounted projects (`peer/name`) are never healed locally. +pub(crate) async fn heal_missing_symbol_index(state: &Arc, alias: &str, lang: &str) { + let Some(path) = state.config.read().ok().and_then(|c| c.resolve(alias)) else { + return; + }; + if !path.exists() { + return; + } + let db_path = path.join(DB_DIR_NAME); + // evaluate may spawn a git subprocess (C# last_changed bootstrap) — + // offload so the heal task never blocks an async worker. + let st = Arc::clone(state); + let a = alias.to_string(); + let l = lang.to_string(); + let p = path.clone(); + let d = db_path.clone(); + let decision = + match tokio::task::spawn_blocking(move || st.evaluate_symbol_rebuild(&l, &a, &p, &d)).await + { + Ok(decision) => decision, + Err(e) => { + warn!( + "self-heal evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + return; + } + }; + if decision != RebuildDecision::NoIndex { + return; + } + info!( + "🔬 self-heal: '{}' has no {} symbol index — rebuilding in background (triggered by find_impact)", + alias, lang + ); + trigger_symbol_rebuild(alias, &path, &db_path, state, lang).await; +} + /// Reload repos config handler: POST /reload /// /// Forces a reload of repos.json from disk, even if the mtime hasn't changed. @@ -4334,6 +4553,8 @@ async fn reindex_handler( let guard_alias = alias_bg.clone(); let guard_state = state.clone(); + // `symbols=true` forces a symbol rebuild; without it the rebuild is + // gate-driven (missing/stale indexes are still healed). let do_symbols = symbols; if force { @@ -4459,10 +4680,9 @@ async fn reindex_handler( // 3. Restart FSW with fresh IndexManager. g_state.restart_fsw(&g_alias, stores).await; - // 4. Optional symbol index rebuild - if do_symbols { - trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; - } + // 4. Symbol index rebuild — gate-driven for every installed + // language; `symbols=true` forces it (previous behaviour). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &g_state, do_symbols).await; g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); @@ -4533,10 +4753,9 @@ async fn reindex_handler( return; } - // Optional symbol index rebuild - if do_symbols { - trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; - } + // Symbol index rebuild — gate-driven for every installed + // language; `symbols=true` forces it (previous behaviour). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &g_state, do_symbols).await; g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); @@ -4921,6 +5140,11 @@ async fn add_repo_handler( // Start FSW and transition to proper Write state with IndexManager state_bg.restart_fsw(&alias_bg, stores).await; + // Phase-2 equivalence for runtime adds: a repo registered while serve + // runs never sees the startup symbol pass, so its symbol indexes would + // not exist until a restart — build them now (gate-driven). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &state_bg, false).await; + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); tracing::info!("Repo '{}' fully indexed and ready", alias_bg); }); @@ -5901,7 +6125,7 @@ pub async fn run_serve( warn!("reconcile: spawn_blocking panicked: {:?}", e); } phase_state.run_phase_1_warmup_all().await; - phase_state.run_phase_2_csharp_scip().await; + phase_state.run_phase_2_symbols().await; phase_state.run_phase_3_prewarm().await; }); } diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 1e71a301..fe937812 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3287,3 +3287,237 @@ async fn stale_eviction_is_scoped_to_the_stale_owner() { "an empty owner map must be removed, not left behind" ); } + +// ── Always-on symbol indexes (features/always-on-symbol-indexes) ────────── + +/// The per-(alias, language) claim must reject a second fresh claim, accept +/// one per language independently, become claimable after release, and take +/// over a claim older than the indexing timeout (leaked by a crashed task). +#[test] +fn symbol_rebuild_claim_rejects_fresh_and_takes_over_stale() { + let state = ServeState::new(ReposConfig::default(), None); + + assert!( + state.begin_symbol_rebuild("repo", "csharp"), + "first claim must succeed" + ); + assert!( + !state.begin_symbol_rebuild("repo", "csharp"), + "a second fresh claim must be rejected" + ); + assert!( + state.begin_symbol_rebuild("repo", "typescript"), + "a different language must claim independently" + ); + + state.end_symbol_rebuild("repo", "csharp"); + assert!( + state.begin_symbol_rebuild("repo", "csharp"), + "a released slot must be claimable again" + ); + + // Stale takeover: a claim past the indexing timeout is taken over. + let max = state.indexing_timeout(); + state.symbol_rebuild_in_flight.insert( + "csharp:stale".to_string(), + std::time::Instant::now() - max - std::time::Duration::from_secs(1), + ); + assert!( + state.begin_symbol_rebuild("stale", "csharp"), + "a stale claim must be taken over" + ); +} + +/// The generic gate must answer NotApplicable for a repo without the +/// language's entrypoint (no tsconfig.json) — deterministically, before the +/// machine-dependent helper check — and route C# through its dedicated +/// evaluator (.sln check → NoSolutionFile). +#[test] +fn evaluate_symbol_rebuild_gates_applicability_before_availability() { + // `evaluate_symbol_rebuild` takes `self: &Arc` (it may be forwarded + // to spawn_blocking by callers), so the probe state lives in an Arc. + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + let tmp = tempfile::tempdir().unwrap(); + let db = tmp.path().join(DB_DIR_NAME); + + assert_eq!( + state.evaluate_symbol_rebuild("typescript", "x", tmp.path(), &db), + RebuildDecision::NotApplicable, + "a repo without tsconfig.json is not applicable, helper or not" + ); + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "x", tmp.path(), &db), + RebuildDecision::NoSolutionFile, + "C# must keep its dedicated .sln-first evaluator" + ); +} + +/// The find_impact self-heal must no-op quietly for an unknown alias (remote +/// mounts, typos) and for a registered repo the language does not apply to — +/// no claim taken, no C# status touched, no panic. +#[tokio::test] +async fn heal_missing_symbol_index_noops_outside_its_remit() { + // Unknown alias. + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + heal_missing_symbol_index(&state, "ghost", "csharp").await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "unknown alias must not take a claim" + ); + + // Registered alias, non-applicable language (no .sln / tsconfig.json). + let (_tmp, _path, bare) = state_with_repo("plainrepo"); + let state = std::sync::Arc::new(bare); + heal_missing_symbol_index(&state, "plainrepo", "csharp").await; + heal_missing_symbol_index(&state, "plainrepo", "typescript").await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "non-applicable repo must not take a claim" + ); + assert!( + !state.csharp_index_status.contains_key("plainrepo"), + "non-applicable repo must not flip the C# indicator" + ); +} + +/// `maybe_rebuild_symbols` on a repo no language applies to must neither +/// panic nor leave claims behind — with and without force (the force path +/// skips the gate but trigger_symbol_rebuild still skips non-applicable +/// repos before claiming). +#[tokio::test] +async fn maybe_rebuild_symbols_skips_non_applicable_repo_under_force() { + let (_tmp, _path, bare) = state_with_repo("plainrepo2"); + let state = std::sync::Arc::new(bare); + let path = state.config.read().unwrap().resolve("plainrepo2").unwrap(); + let db = path.join(DB_DIR_NAME); + maybe_rebuild_symbols("plainrepo2", &path, &db, &state, true).await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "force must not claim a slot for a non-applicable repo" + ); +} + +/// A dummy `scip-csharp` helper at the env-override path: available for +/// detection, fails on execution — which flips the C# status to Error while +/// proving a rebuild was actually attempted. The helper file lives in +/// `root`, which the caller keeps alive via its TempDir. +fn dummy_csharp_helper(root: &std::path::Path) -> crate::testing::EnvRestore { + let helper = root.join(if cfg!(windows) { + "scip-csharp.exe" + } else { + "scip-csharp" + }); + std::fs::write(&helper, b"dummy").expect("dummy helper file"); + crate::testing::EnvRestore::set(&[( + crate::constants::SCIP_CSHARP_HELPER_ENV, + helper.to_string_lossy().as_ref(), + )]) +} + +/// Phase-2 composition wiring: a registered repo whose language applies and +/// whose index is missing must flow gate → queue → trigger end-to-end. The +/// dummy helper makes the rebuild FAIL, observable as C# status Error once +/// `run_phase_2_symbols` completes — None would mean the candidate was never +/// queued (the composition gap this test pins). +#[tokio::test] +#[serial] +async fn run_phase_2_symbols_wires_candidate_through_to_trigger() { + let helper_root = tempfile::tempdir().unwrap(); + let _env = dummy_csharp_helper(helper_root.path()); + + let (_tmp, repo_path, bare) = state_with_repo("phase2repo"); + std::fs::write( + repo_path.join("test.sln"), + b"Microsoft Visual Studio Solution File", + ) + .unwrap(); + + let state = std::sync::Arc::new(bare); + state.run_phase_2_symbols().await; + + assert_eq!( + state + .csharp_index_status + .get("phase2repo") + .map(|e| *e.value()), + Some(CSharpIndexStatus::Error), + "the missing C# index must be queued and the rebuild attempted \ + (dummy helper → Error); None means never queued" + ); + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "the claim must be released after the failed rebuild" + ); +} + +/// The serve-mode find_impact self-heal spawn must reach +/// `trigger_symbol_rebuild` for a registered, applicable repo: after the +/// warned empty answer, the C# status eventually flips to Error (dummy +/// helper) in the serve state the heal captured. +#[tokio::test] +#[serial] +async fn find_impact_self_heal_runs_the_rebuild_in_serve_mode() { + let helper_root = tempfile::tempdir().unwrap(); + let _env = dummy_csharp_helper(helper_root.path()); + + let (_tmp, repo_path, bare) = state_with_repo("healrepo"); + std::fs::write( + repo_path.join("test.sln"), + b"Microsoft Visual Studio Solution File", + ) + .unwrap(); + // The routing layer requires an existing index dir before it will open + // the repo (seed like the find_impact fixtures do). + let db = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write( + db.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .unwrap(); + let state = std::sync::Arc::new(bare); + + let request = crate::mcp::types::FindImpactRequest { + symbol_name: Some("ResolveLinkedActivityAsync".to_string()), + file: None, + line: None, + symbol_key: None, + language: Some("csharp".to_string()), + project: Some("healrepo".to_string()), + group: None, + }; + // The REST mirror builds its own serve-mode service from the state — + // the same path the MCP tool takes for a project-scoped query. + let v = crate::mcp::rest_find_impact_handler( + axum::extract::State(state.clone()), + axum::Json(request), + ) + .await + .expect("rest mirror result") + .0; + let out = v.to_string(); + assert!( + v["warnings"] + .as_array() + .is_some_and(|w| !w.is_empty() && w[0].as_str().is_some_and(|s| s.contains("UNKNOWN"))), + "the never-built index answer must carry the UNKNOWN warning, got: {out}" + ); + + // The detached heal task runs the rebuild (dummy helper → Error). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + loop { + if state + .csharp_index_status + .get("healrepo") + .map(|e| *e.value()) + == Some(CSharpIndexStatus::Error) + { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the self-heal never flipped the C# status — spawn branch not wired" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } +} From 6e0afc3fb8b8026786af4c20d4cda5e240d53f28 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 13:03:23 +0200 Subject: [PATCH 26/77] [worker] final review: all stages complete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review-fixes (final full review, PASS): - [Debt] startup comment said phase-2 was C#-only → 'per-language' - [Debt] end_symbol_rebuild releases any claim in the slot, not just its own → deferred: mirrors the established begin_indexing pattern; the window needs a >30-min rebuild plus a stale takeover first, and a generation token is more machinery than the problem has needed --- src/serve/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serve/mod.rs b/src/serve/mod.rs index edd98ce0..25c83e17 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -6108,7 +6108,7 @@ pub async fn run_serve( // ── Startup phase orchestration ── // Phase 1 warms all repos sequentially (text/vector ready). - // Phase 2 runs gated C# SCIP rebuilds ordered by last_changed. + // Phase 2 runs gated per-language SCIP rebuilds ordered by last_changed. { let phase_state = serve_state.clone(); tokio::spawn(async move { From aa8954ccf89481983fc6e8a23c7ed25fe8cac279 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 21:45:16 +0200 Subject: [PATCH 27/77] [worker] find_impact: surface index-level rebuild warnings (partial-solution honesty) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defect (reported live on DRM-24427_Versioning): the SCIP rebuild survived MSBuild project-load failures (NuGet restore broken: SonarAnalyzer version mismatch) with exit 0 — an index with definitions but no cross-project references. Every find_impact answer read clean (references: [], no warnings), so an orphan-check read the empty lists as 'no callers'. - invoke_index_helper now captures warning-severity stderr lines (poison- safe) instead of only tracing them - summarize_index_warnings: normalize Msbuild-failure lines to ': ' (160-char cap), order-preserving dedup, cap 10 with an overflow note - rebuild persists them under scip_meta[index_warnings] (JSON array, with a stand-alone summary entry first); written or cleared on EVERY rebuild so meta always describes the newest build - SymbolIndexer::index_warnings(db_path) trait method (default empty; C# reads meta, corrupt/absent reads as clean) - find_impact merges index warnings into EVERY payload answer (resolved and empty NotFound alike) in the same spawn_blocking pass as resolution; the missing-index heal warning stays NotFound-only (a successful resolution proves usable data exists) Note: existing indexes built by the previous binary carry no meta until their next rebuild (any watcher debounce, reindex, or index symbol). Validation: fmt clean; clippy --all-targets -D warnings clean; test --lib --bins 765+760 passed, 1 pre-existing Windows-transient (os error 5, passes in isolation, unrelated module). --- src/constants.rs | 7 ++ src/mcp/find_impact.rs | 39 +++++++++-- src/mcp/find_impact_tests.rs | 83 ++++++++++++++++++++++ src/symbols/csharp.rs | 130 +++++++++++++++++++++++++++++++++-- src/symbols/csharp_tests.rs | 119 ++++++++++++++++++++++++++++++++ src/symbols/mod.rs | 18 ++++- 6 files changed, 384 insertions(+), 12 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 6d783876..968a6dcd 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -655,6 +655,13 @@ pub const SCIP_HEAD_SHA_KEY: &str = "head_sha"; /// old-format keys being served as fresh. pub const SCIP_KEY_FORMAT_KEY: &str = "key_format"; +/// LMDB metadata key holding a JSON array of index-level completeness +/// warnings from the last rebuild (workspace/MSBuild failures the helper +/// survived while still producing an index). Absent or empty = built clean. +/// Surfaced on every `find_impact` answer from that index — a reference +/// list out of a partially loaded solution may be missing callers. +pub const SCIP_INDEX_WARNINGS_KEY: &str = "index_warnings"; + /// Current value written for [`SCIP_KEY_FORMAT_KEY`]. Bump whenever the /// canonical SCIP symbol key format produced by a language helper changes /// shape (B4: C# generic arity / containing-type path / fully qualified diff --git a/src/mcp/find_impact.rs b/src/mcp/find_impact.rs index 93639bf5..904d0d87 100644 --- a/src/mcp/find_impact.rs +++ b/src/mcp/find_impact.rs @@ -381,6 +381,10 @@ impl CodesearchService { // helper invocation — as a typed candidates answer; the server never // silently picks among the query's matches (the old behaviour took // the shortest fuzzy candidate and answered about the wrong symbol). + // The same read pass collects the index-level rebuild warnings: a + // partially loaded solution produces an index whose reference lists + // are incomplete EVEN WHEN the lookup succeeds, and that honesty + // rides on every answer from this index. let registry_for_resolve = self.symbol_registry.clone(); let language_for_resolve = language_for_lookup.clone(); let db_path_for_resolve = db_path.clone(); @@ -394,7 +398,9 @@ impl CodesearchService { language_for_resolve ) })?; - indexer.resolve_query(&db_path_for_resolve, &query_for_resolve) + let resolution = indexer.resolve_query(&db_path_for_resolve, &query_for_resolve)?; + let index_warnings = indexer.index_warnings(&db_path_for_resolve); + Ok((resolution, index_warnings)) }) .await { @@ -404,6 +410,21 @@ impl CodesearchService { // (classified stale/failed by the index age) instead of `?`. Err(e) => Err(anyhow::anyhow!("symbol resolve task failed: {e:#}")), }; + // Both are read from the SAME env/txn pass above; on a JoinError the + // warnings are unknown and the failure path below speaks for itself. + let (resolution, index_warnings) = match resolution { + Ok((resolution, index_warnings)) => (Ok(resolution), index_warnings), + Err(e) => (Err(e), Vec::new()), + }; + // Warnings assembled per answer path: persisted per-symbol warnings + // (set at the lookup sites) + index-level rebuild warnings. The + // missing-index self-heal warning does NOT belong here — a successful + // resolution proves usable data exists (see the NotFound arm, the + // only payload that carries it). + let answer_warnings = |mut persisted: Vec| -> Vec { + persisted.extend(index_warnings.iter().cloned()); + persisted + }; let canonical = match resolution { Err(e) => { @@ -451,10 +472,12 @@ impl CodesearchService { } Ok(crate::symbols::KeyMatch::NotFound) => { // Preserve the historical contract for fuzzy queries: an - // unresolvable name/position answers empty references. With a - // missing index that answer also carries the self-heal + // unresolvable name/position answers empty references. With + // a missing index that answer also carries the self-heal // warning — empty must not pass for "no callers". - let impact = build_impact(Vec::new(), None, heal_warning); + let mut warnings = answer_warnings(Vec::new()); + warnings.extend(heal_warning.iter().cloned()); + let impact = build_impact(Vec::new(), None, warnings); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); } @@ -497,7 +520,11 @@ impl CodesearchService { // The warm retry must carry the same honesty as a fresh // answer: read the persisted warnings for this key. let warnings = indexer.lookup_warnings(&db_path, &canonical); - let impact = build_impact(references, Some(canonical.clone()), warnings); + let impact = build_impact( + references, + Some(canonical.clone()), + answer_warnings(warnings), + ); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); } @@ -553,7 +580,7 @@ impl CodesearchService { // Surface what the lookup survived: a partial answer must // say so in its own payload, not only in a log line. let warnings = indexer.lookup_warnings(&db_path, &canonical); - let impact = build_impact(references, Some(canonical), warnings); + let impact = build_impact(references, Some(canonical), answer_warnings(warnings)); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); Ok(CallToolResult::success(vec![ContentBlock::text(json)])) } diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index a298e625..15efbcd4 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -715,6 +715,24 @@ fn warnings_bytes(warnings: &[&str]) -> Vec { bytes } +/// Persist index-level rebuild warnings the way a C# rebuild does (meta +/// key, JSON array) — the fixture for answers served from an index built +/// out of a partially loaded solution. +fn write_index_warnings_meta(db_path: &std::path::Path, warnings: &[String]) { + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + serde_json::to_string(warnings).unwrap().as_str(), + ) + .unwrap(); + wtxn.commit().unwrap(); +} + /// One resolved symbol with persisted warnings, one without. fn populate_warnings_fixture(db_path: &std::path::Path) -> (String, String) { let warned_key = "csharp Ns . W#Warned().".to_string(); @@ -874,3 +892,68 @@ async fn never_built_index_warns_on_the_empty_answer() { "the warning must mark the answer UNKNOWN, got: {out}" ); } + +/// Index-level rebuild warnings must ride on EVERY answer from an index +/// built out of a partially loaded solution — resolved answers AND empty +/// NotFound answers. This pins the DRM-24427 defect: MSBuild project-load +/// failures (broken NuGet restore) produced an index with definitions but +/// no cross-project references, and every answer passed as clean while an +/// orphan-check read the empty lists as "no callers". +#[tokio::test] +#[serial_test::serial] +async fn index_level_rebuild_warnings_ride_on_resolved_and_empty_answers() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + let db = project.path().join(".codesearch.db"); + let (_warned_key, clean_key) = populate_warnings_fixture(&db); + let index_warnings = vec![ + "The symbol index was built while the C# workspace reported 2 distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex." + .to_string(), + "Dlw.Aprimo.Dam.csproj: depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but it was not found".to_string(), + ]; + write_index_warnings_meta(&db, &index_warnings); + + // A RESOLVED answer from the partial index carries the index warnings. + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some(clean_key.clone()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["resolved_symbol"], clean_key, + "the fixture query must resolve, got: {out}" + ); + let warnings = v["warnings"].as_array().expect("warnings present"); + assert_eq!( + serde_json::to_value(warnings).unwrap(), + serde_json::to_value(&index_warnings).unwrap(), + "a resolved answer from a partially built index carries exactly the index warnings: {out}" + ); + + // The EMPTY NotFound answer carries them too — an empty reference list + // out of a partial index must never pass for "no callers". + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("NotARealSymbolAnywhere".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!( + v["references"].as_array().is_some_and(|a| a.is_empty()), + "unknown symbol answers empty, got: {out}" + ); + assert!( + v["warnings"].as_array().is_some_and( + |w| !w.is_empty() && w[0].as_str().is_some_and(|s| s.contains("cross-project")) + ), + "the empty answer must carry the index-level warning, got: {out}" + ); +} diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 8ad33fa2..ce4180c6 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -61,6 +61,55 @@ pub(crate) fn is_helper_warning_line(line: &str) -> bool { line.contains("[WARN]") || line.contains("[Failure]") } +/// Normalize one warning-severity helper stderr line into a concise, +/// index-level completeness warning entry. MSBuild project-load failures +/// (the class that silently strips cross-project references from the index) +/// collapse to `: `; anything else passes through +/// trimmed. `None` for non-warning lines. +pub(crate) fn normalize_index_warning_line(line: &str) -> Option { + if !is_helper_warning_line(line) { + return None; + } + // "[WARN] Workspace error: [Failure] Msbuild failed when processing the + // file '' with message: " → ": " + if let Some((_, rest)) = line.split_once("Msbuild failed when processing the file '") { + if let Some((path, tail)) = rest.split_once('\'') { + let file = path.rsplit(['/', '\\']).next().unwrap_or(path); + let msg = tail + .split_once("with message: ") + .map(|(_, m)| m) + .unwrap_or(tail) + .trim(); + let msg: String = msg.chars().take(160).collect(); + return Some(format!("{file}: {msg}")); + } + } + let trimmed = line.trim(); + if trimmed.is_empty() { + return None; + } + Some(trimmed.chars().take(220).collect()) +} + +/// Collapse the raw warning lines of one helper run into the capped, +/// deduplicated list persisted as `index_warnings` meta. Order-preserving +/// dedup because MSBuild repeats the same failure per project and per +/// pass; capped because the list rides on every find_impact answer. +pub(crate) fn summarize_index_warnings(lines: &[String], cap: usize) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut out: Vec = lines + .iter() + .filter_map(|l| normalize_index_warning_line(l)) + .filter(|w| seen.insert(w.clone())) + .collect(); + if out.len() > cap { + let dropped = out.len() - cap; + out.truncate(cap); + out.push(format!("… and {dropped} more distinct warning(s)")); + } + out +} + /// Route one scip-csharp stderr line into tracing at the right severity. /// This is the ONLY sanctioned path for helper stderr: spawn helpers with /// `Stdio::piped()` and drain through here — never `Stdio::inherit()`, @@ -132,6 +181,15 @@ const META_HEAD_SHA: &str = crate::constants::SCIP_HEAD_SHA_KEY; /// built with (see [`crate::constants::SCIP_KEY_FORMAT`]). const META_KEY_FORMAT: &str = crate::constants::SCIP_KEY_FORMAT_KEY; +/// Key in the meta database holding the JSON array of index-level +/// completeness warnings from the last rebuild (see +/// [`summarize_index_warnings`]). +const META_INDEX_WARNINGS: &str = crate::constants::SCIP_INDEX_WARNINGS_KEY; + +/// How many distinct index-level warnings are persisted (and thus surfaced +/// per find_impact answer) before the list is capped with an overflow note. +const INDEX_WARNINGS_CAP: usize = 10; + /// Key in the meta database storing the count of indexed symbols. #[allow(dead_code)] const META_SYMBOL_COUNT: &str = "symbol_count"; @@ -518,13 +576,20 @@ impl CSharpSymbolIndexer { // ── Helper invocation ────────────────────────────────────────── /// Invoke `scip-csharp index` and stream stderr to tracing. + /// + /// Returns the raw warning-severity stderr lines of the run — the input + /// to [`summarize_index_warnings`]. A helper that survives MSBuild + /// project-load failures exits 0 with a syntactically valid but + /// semantically partial index (definitions without cross-project + /// references); those failures land HERE so the rebuild can persist them + /// as index-level warnings instead of them living only in the log. fn invoke_index_helper( &self, helper: &Path, solution: &Path, output_path: &Path, project_filter: Option<&Path>, - ) -> Result<()> { + ) -> Result> { let solution_short = solution .file_name() .map(|s| s.to_string_lossy().into_owned()) @@ -549,11 +614,17 @@ impl CSharpSymbolIndexer { .spawn() .with_context(|| format!("Failed to execute scip-csharp at {}", helper.display()))?; + let captured_warnings: std::sync::Arc>> = + std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); let stderr_handle = child.stderr.take().map(|stderr| { let label = solution_short.clone(); + let sink = std::sync::Arc::clone(&captured_warnings); thread::spawn(move || { drain_pipe_to_tracing(stderr, |line| { if !line.is_empty() { + if is_helper_warning_line(line) { + sink.lock().map(|mut v| v.push(line.to_string())).ok(); + } emit_helper_stderr_line("scip-csharp", &label, line); } }); @@ -591,7 +662,13 @@ impl CSharpSymbolIndexer { // Don't bail — partial output is acceptable per AGENTS.md spec } - Ok(()) + // A poisoned lock still holds every line pushed before the panic — + // recover rather than drop honesty signals on the floor. + let raw = match captured_warnings.lock() { + Ok(mut v) => std::mem::take(&mut *v), + Err(poisoned) => std::mem::take(&mut *poisoned.into_inner()), + }; + Ok(raw) } /// Invoke `scip-csharp find-refs` for a single symbol and return its @@ -1350,8 +1427,11 @@ impl SymbolIndexer for CSharpSymbolIndexer { )); let _output_guard = TempFileGuard(output_path.clone()); - // Invoke helper with stderr streaming - self.invoke_index_helper(&helper, &solution, &output_path, project.as_deref())?; + // Invoke helper with stderr streaming; warning-severity lines become + // the index-level completeness warnings persisted below. + let raw_index_warnings = + self.invoke_index_helper(&helper, &solution, &output_path, project.as_deref())?; + let index_warnings = summarize_index_warnings(&raw_index_warnings, INDEX_WARNINGS_CAP); // Parse the JSON output let index_data = std::fs::read(&output_path) @@ -1696,6 +1776,21 @@ impl SymbolIndexer for CSharpSymbolIndexer { META_KEY_FORMAT, crate::constants::SCIP_KEY_FORMAT, )?; + // Index-level completeness warnings from this run — written or + // cleared on EVERY rebuild (full or incremental) so meta always + // describes the newest build. The summary entry leads because the + // list rides on every find_impact answer and must stand alone. + if index_warnings.is_empty() { + meta_db.delete(&mut wtxn, META_INDEX_WARNINGS)?; + } else { + let mut stored = vec![format!( + "The symbol index was built while the C# workspace reported {} distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex.", + index_warnings.len() + )]; + stored.extend(index_warnings.iter().cloned()); + let json = serde_json::to_string(&stored).unwrap_or_else(|_| "[]".to_string()); + meta_db.put(&mut wtxn, META_INDEX_WARNINGS, json.as_str())?; + } wtxn.commit()?; @@ -1846,6 +1941,33 @@ impl SymbolIndexer for CSharpSymbolIndexer { (!sha.is_empty()).then_some(sha) } + /// Index-level completeness warnings from the last rebuild — the + /// workspace/MSBuild failures the helper survived. Non-empty means + /// reference lists from this index may be incomplete even when a + /// lookup succeeds (observed live: a worktree whose NuGet restore was + /// broken indexed every definition but zero cross-project references, + /// and every find_impact answer passed as clean). Plain LMDB read; + /// unreadable or absent metadata reads as clean (never blocks an answer). + fn index_warnings(&self, db_path: &Path) -> Vec { + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return Vec::new(), + }; + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return Vec::new(), + }; + let meta_db: Database = match env.open_database(&rtxn, Some(SCIP_META_DB_NAME)) { + Ok(Some(db)) => db, + _ => return Vec::new(), + }; + let json = match meta_db.get(&rtxn, META_INDEX_WARNINGS) { + Ok(Some(s)) => s, + _ => return Vec::new(), + }; + serde_json::from_str(json).unwrap_or_default() + } + /// Whether a SCIP index exists AND was built with the current key format. /// An index that is fresh by timestamp but stamped with another (or no) /// key-format generation would serve old-shaped canonical keys as truth, diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs index 31991aa0..15a99c15 100644 --- a/src/symbols/csharp_tests.rs +++ b/src/symbols/csharp_tests.rs @@ -124,3 +124,122 @@ fn helper_warning_classification_table() { ); } } + +// ── Index-level completeness warnings (features/always-on-symbol-indexes) ── + +/// The live MSBuild project-load failure format observed on the +/// DRM-24427_Versioning worktree (NuGet restore broken: SonarAnalyzer +/// version mismatch) — the exact line class whose warnings silently +/// disappeared into the log while every find_impact answer read clean. +const LIVE_MSBUILD_FAILURE: &str = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\BOIN.Aprimo.worktrees\\DRM-24427_Versioning\\src\\Dlw.Aprimo.Dam\\Dlw.Aprimo.Dam.csproj' with message: Dlw.Aprimo.Dam depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but SonarAnalyzer.CSharp 10.18.0.128626 was not found. SonarAnalyzer.CSharp 10.18.0.131500 was resolved instead."; + +/// Normalization collapses the project-load failure to `: `; +/// duplicates (MSBuild repeats per project and per pass) dedupe; plain +/// warning lines pass through trimmed; non-warning lines are dropped. +#[test] +fn summarize_index_warnings_normalizes_dedupes_and_skips_noise() { + let msbuild_second = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\other\\src\\Dlw.Aprimo.Web\\Dlw.Aprimo.Web.csproj' with message: Dlw.Aprimo.Web depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but SonarAnalyzer.CSharp 10.18.0.128626 was not found. SonarAnalyzer.CSharp 10.18.0.131500 was resolved instead."; + let lines = vec![ + "MSBuild: registering '.NET Core SDK' v10.0.401".to_string(), // info, dropped + LIVE_MSBUILD_FAILURE.to_string(), + LIVE_MSBUILD_FAILURE.to_string(), // duplicate, deduped + msbuild_second.to_string(), + "[WARN] Grpc.Net.ClientFactory 2.63.0 or earlier could cause issues".to_string(), + "".to_string(), + ]; + + let out = super::csharp::summarize_index_warnings(&lines, 10); + + assert_eq!(out.len(), 3, "expected 3 distinct warnings, got: {out:?}"); + assert!( + out[0].starts_with("Dlw.Aprimo.Dam.csproj: Dlw.Aprimo.Dam depends on SonarAnalyzer"), + "first entry must be the normalized project failure, got: {}", + out[0] + ); + assert!( + out[1].starts_with("Dlw.Aprimo.Web.csproj: "), + "second entry must be the second project, got: {}", + out[1] + ); + assert!( + out[2].starts_with("[WARN] Grpc"), + "unrecognized warning format passes through verbatim, got: {}", + out[2] + ); + // The SonarAnalyzer message is longer than the 160-char cap — pinned so + // a persisted entry can never blow up the warnings array on answers. + let msg = out[0].split_once(": ").expect("normalized shape").1; + assert!(msg.chars().count() <= 160, "message must be capped"); +} + +/// The cap kicks in with an explicit overflow entry — the list rides on +/// every find_impact answer and must stay bounded. +#[test] +fn summarize_index_warnings_caps_with_overflow_note() { + let lines: Vec = (0..5) + .map(|i| format!("[WARN] distinct failure number {i}")) + .collect(); + let out = super::csharp::summarize_index_warnings(&lines, 3); + assert_eq!(out.len(), 4, "3 capped entries + overflow note: {out:?}"); + assert_eq!(out[3], "… and 2 more distinct warning(s)"); +} + +/// The meta roundtrip: what rebuild persists is what `index_warnings` +/// returns; absent key and corrupt JSON both read as clean (never block an +/// answer), and the TypeScript adapter inherits the empty default. +#[test] +fn index_warnings_roundtrip_and_clean_absence() { + use crate::symbols::SymbolIndexer as _; + + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("codesearch.db"); + + let csharp = super::csharp::CSharpSymbolIndexer::new(); + assert!( + csharp.index_warnings(&db_path).is_empty(), + "absent meta must read as clean" + ); + + let env = crate::symbols::get_shared_scip_env(&db_path).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + let stored = vec![ + "summary line".to_string(), + "Dlw.Aprimo.Dam.csproj: SonarAnalyzer.CSharp not found".to_string(), + ]; + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + serde_json::to_string(&stored).unwrap().as_str(), + ) + .unwrap(); + wtxn.commit().unwrap(); + + assert_eq!( + csharp.index_warnings(&db_path), + stored, + "persisted warnings must round-trip verbatim" + ); + + // Corrupt JSON reads as clean, not as an error. + let mut wtxn = env.write_txn().unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + "not json [", + ) + .unwrap(); + wtxn.commit().unwrap(); + assert!( + csharp.index_warnings(&db_path).is_empty(), + "corrupt meta must read as clean, never fail the answer" + ); + + let ts = super::typescript::TypeScriptSymbolIndexer::new(); + assert!( + ts.index_warnings(&db_path).is_empty(), + "non-tracking adapters inherit the empty default" + ); +} diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index c3542077..09c5968b 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -69,8 +69,11 @@ pub struct FindImpactResult { /// reference list may be INCOMPLETE — a helper failure was survived /// rather than fatal (a project that would not compile, an exception /// during reference resolution, a non-zero scip-typescript exit) and - /// the partial result was kept. Entries name what failed. Omitted when - /// empty: absent means the answer is as complete as the index knows. + /// the partial result was kept, OR the index itself was built out of a + /// partially loaded solution (index-level rebuild warnings, e.g. + /// MSBuild project-load failures; see `SymbolIndexer::index_warnings`). + /// Entries name what failed. Omitted when empty: absent means the + /// answer is as complete as the index knows. #[serde(skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, /// Seconds since the symbol index was last rebuilt. @@ -448,6 +451,17 @@ pub trait SymbolIndexer: Send + Sync { /// How old is the current symbol index (seconds since last rebuild)? fn index_age(&self, db_path: &Path) -> u64; + /// Index-level completeness warnings from the last rebuild — build + /// environment failures the language helper survived while still + /// producing an index (e.g. MSBuild project-load failures for C#). + /// Non-empty means reference lists from this index may be incomplete + /// EVEN WHEN a lookup succeeds, and callers should attach these + /// warnings to answers served from this index. Plain read; empty is + /// the default for adapters that do not track build-environment state. + fn index_warnings(&self, _db_path: &Path) -> Vec { + Vec::new() + } + /// The git HEAD sha the current symbol index was built for, when /// recorded. `None` means unknown (pre-fingerprint index, or git was /// unreadable at build time). Compare with the repository's current From 1c064874a95992ea876f73b509c79d3f86a3a682 Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 22:00:43 +0200 Subject: [PATCH 28/77] chore: anonymize customer identifiers in test fixtures (QC gate) The new index-warning fixtures embedded real project/package names from the live incident. Replaced with shape-identical generic identifiers; assertions and coverage unchanged. --- src/mcp/find_impact_tests.rs | 11 ++++++----- src/symbols/csharp_tests.rs | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index 15efbcd4..72ef5139 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -895,10 +895,11 @@ async fn never_built_index_warns_on_the_empty_answer() { /// Index-level rebuild warnings must ride on EVERY answer from an index /// built out of a partially loaded solution — resolved answers AND empty -/// NotFound answers. This pins the DRM-24427 defect: MSBuild project-load -/// failures (broken NuGet restore) produced an index with definitions but -/// no cross-project references, and every answer passed as clean while an -/// orphan-check read the empty lists as "no callers". +/// NotFound answers. This pins the live defect reported on a customer +/// worktree: MSBuild project-load failures (broken NuGet restore) produced +/// an index with definitions but no cross-project references, and every +/// answer passed as clean while an orphan-check read the empty lists as +/// "no callers". #[tokio::test] #[serial_test::serial] async fn index_level_rebuild_warnings_ride_on_resolved_and_empty_answers() { @@ -910,7 +911,7 @@ async fn index_level_rebuild_warnings_ride_on_resolved_and_empty_answers() { let index_warnings = vec![ "The symbol index was built while the C# workspace reported 2 distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex." .to_string(), - "Dlw.Aprimo.Dam.csproj: depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but it was not found".to_string(), + "App.Dam.csproj: depends on Analyzer.X (>= 1.2.3) but it was not found".to_string(), ]; write_index_warnings_meta(&db, &index_warnings); diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs index 15a99c15..19b68bb0 100644 --- a/src/symbols/csharp_tests.rs +++ b/src/symbols/csharp_tests.rs @@ -127,18 +127,19 @@ fn helper_warning_classification_table() { // ── Index-level completeness warnings (features/always-on-symbol-indexes) ── -/// The live MSBuild project-load failure format observed on the -/// DRM-24427_Versioning worktree (NuGet restore broken: SonarAnalyzer -/// version mismatch) — the exact line class whose warnings silently -/// disappeared into the log while every find_impact answer read clean. -const LIVE_MSBUILD_FAILURE: &str = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\BOIN.Aprimo.worktrees\\DRM-24427_Versioning\\src\\Dlw.Aprimo.Dam\\Dlw.Aprimo.Dam.csproj' with message: Dlw.Aprimo.Dam depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but SonarAnalyzer.CSharp 10.18.0.128626 was not found. SonarAnalyzer.CSharp 10.18.0.131500 was resolved instead."; +/// The live MSBuild project-load failure format observed on a customer +/// worktree (NuGet restore broken: analyzer package version mismatch) — +/// the exact line class whose warnings silently disappeared into the log +/// while every find_impact answer read clean. Identifiers anonymized; the +/// shape is what the parser must match. +const LIVE_MSBUILD_FAILURE: &str = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\demo-worktree\\src\\App.Dam\\App.Dam.csproj' with message: App.Dam depends on Analyzer.X (>= 1.2.3) but Analyzer.X 1.2.3 was not found. Analyzer.X 1.3.0 was resolved instead."; /// Normalization collapses the project-load failure to `: `; /// duplicates (MSBuild repeats per project and per pass) dedupe; plain /// warning lines pass through trimmed; non-warning lines are dropped. #[test] fn summarize_index_warnings_normalizes_dedupes_and_skips_noise() { - let msbuild_second = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\other\\src\\Dlw.Aprimo.Web\\Dlw.Aprimo.Web.csproj' with message: Dlw.Aprimo.Web depends on SonarAnalyzer.CSharp (>= 10.18.0.128626) but SonarAnalyzer.CSharp 10.18.0.128626 was not found. SonarAnalyzer.CSharp 10.18.0.131500 was resolved instead."; + let msbuild_second = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\other\\src\\App.Web\\App.Web.csproj' with message: App.Web depends on Analyzer.X (>= 1.2.3) but Analyzer.X 1.2.3 was not found. Analyzer.X 1.3.0 was resolved instead."; let lines = vec![ "MSBuild: registering '.NET Core SDK' v10.0.401".to_string(), // info, dropped LIVE_MSBUILD_FAILURE.to_string(), @@ -152,12 +153,12 @@ fn summarize_index_warnings_normalizes_dedupes_and_skips_noise() { assert_eq!(out.len(), 3, "expected 3 distinct warnings, got: {out:?}"); assert!( - out[0].starts_with("Dlw.Aprimo.Dam.csproj: Dlw.Aprimo.Dam depends on SonarAnalyzer"), + out[0].starts_with("App.Dam.csproj: App.Dam depends on Analyzer.X"), "first entry must be the normalized project failure, got: {}", out[0] ); assert!( - out[1].starts_with("Dlw.Aprimo.Web.csproj: "), + out[1].starts_with("App.Web.csproj: "), "second entry must be the second project, got: {}", out[1] ); @@ -166,8 +167,8 @@ fn summarize_index_warnings_normalizes_dedupes_and_skips_noise() { "unrecognized warning format passes through verbatim, got: {}", out[2] ); - // The SonarAnalyzer message is longer than the 160-char cap — pinned so - // a persisted entry can never blow up the warnings array on answers. + // The failure message is longer than the 160-char cap — pinned so a + // persisted entry can never blow up the warnings array on answers. let msg = out[0].split_once(": ").expect("normalized shape").1; assert!(msg.chars().count() <= 160, "message must be capped"); } @@ -207,7 +208,7 @@ fn index_warnings_roundtrip_and_clean_absence() { .unwrap(); let stored = vec![ "summary line".to_string(), - "Dlw.Aprimo.Dam.csproj: SonarAnalyzer.CSharp not found".to_string(), + "App.Dam.csproj: Analyzer.X not found".to_string(), ]; meta.put( &mut wtxn, From ef3066fc5fb3e372a46c327eb296ebced051091b Mon Sep 17 00:00:00 2001 From: flupkede Date: Thu, 1 Oct 2026 22:49:26 +0200 Subject: [PATCH 29/77] =?UTF-8?q?[worker]=20TUI:=20degraded=20C#=20index?= =?UTF-8?q?=20renders=20red=20(C#=E2=9A=A0)=20+=20info=20surfaces=20the=20?= =?UTF-8?q?failures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User ask: when a rebuild survives workspace/MSBuild failures (the partial-solution class behind the empty-references defect), the C# indicator in the TUI index table must turn red, and the info views must show the failures present on a repo. - CSharpIndexStatus::Partial: a successful rebuild whose RebuildSummary carries index warnings. csharp_success_outcome (pure, takes &[String]) derives status + bounded detail (first three entries joined) - trigger_symbol_rebuild stores the detail in csharp_index_error so existing display paths carry it - watcher path included: SymbolRebuildSignal::Succeeded now carries index_warnings (full site passes the summary's; the incremental site accumulates + dedupes across project groups) and the notifier arm routes through csharp_success_outcome — a degraded watcher rebuild can no longer render green or erase a prior Partial - TUI table: 'partial' renders alias 'C#⚠' red bold ('TS⚠' red for the shared vocabulary); indicator_cols() budgets 6 display cols for the double-width ⚠ glyph (EAW=W) instead of the 4-col 1-col-glyph family; detail_status_style knows the label; the detail panel's ⚠ line shows for partial too - Info overlays (embedded 'i' key AND serve-tui /repos/:alias/info): new symbol_warnings read DURABLY from scip_meta[index_warnings] via ServeState::collect_symbol_warnings (single mechanism for both surfaces), language-prefixed, rendered as a red list. Survives a serve restart. Not filtered by installed_languages — meta needs no helper, and a helper vanishing after a degraded rebuild must not hide the record - RebuildSummary.index_warnings (TS: empty) carries warnings from rebuild to the trigger without re-reading meta Review-fixes: - [Important] watcher-triggered rebuilds rendered green on degraded rebuilds and erased prior Partial → Succeeded carries warnings, both emission sites + notifier arm route through csharp_success_outcome - [Important] 'C#⚠' is 6 display cols vs the 4-col budget → indicator_cols() per-variant width budget - [Debt] warning-collection loop duplicated across info surfaces → ServeState::collect_symbol_warnings, both surfaces call it - [Test, round 2] watcher notifier arm + indicator_cols budget had no direct regression test → csharp_notifier_partial_on_succeeded_with_ warnings drives Started/degraded-Succeeded/clean-Succeeded through the notifier; indicator_cols pins the double-width budget Validation: fmt clean; clippy --all-targets -D warnings clean; test --lib --bins 769+765 passed / 0 failed. --- src/index/manager.rs | 26 ++++++- src/serve/mod.rs | 83 ++++++++++++++++++-- src/serve/tests.rs | 160 ++++++++++++++++++++++++++++++++++++++ src/serve/tui.rs | 10 ++- src/serve/tui_common.rs | 102 ++++++++++++++++++++---- src/serve/tui_remote.rs | 5 ++ src/symbols/csharp.rs | 29 ++++--- src/symbols/mod.rs | 4 + src/symbols/typescript.rs | 1 + 9 files changed, 385 insertions(+), 35 deletions(-) diff --git a/src/index/manager.rs b/src/index/manager.rs index ff190f91..5f549219 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -48,8 +48,12 @@ use super::Result; pub enum SymbolRebuildSignal { /// A rebuild is about to run (helper available and project applies). Started, - /// Rebuild finished successfully. - Succeeded, + /// Rebuild finished successfully. Carries the index-level warnings the + /// rebuild survived (empty = clean) so the serve layer can render a + /// degraded index red instead of green — the watcher path is the common + /// path during active development, and a green indicator on an index + /// with missing cross-project references reads as "safe". + Succeeded { index_warnings: Vec }, /// Rebuild failed with the given message. Failed(String), } @@ -1338,7 +1342,9 @@ impl IndexManager { summary.duration_ms ); if let Some(n) = notifier { - n(SymbolRebuildSignal::Succeeded); + n(SymbolRebuildSignal::Succeeded { + index_warnings: summary.index_warnings.clone(), + }); } } Err(e) => { @@ -1786,6 +1792,11 @@ impl IndexManager { // cleaned up from LMDB regardless of origin project. let total_groups = groups.len(); let mut last_error: Option = None; + // Index-level warnings survived by any + // group's rebuild — deduped across groups + // (MSBuild repeats per project). + let mut index_warnings: Vec = Vec::new(); + let mut seen_warnings = std::collections::HashSet::new(); for (i, (csproj, files)) in groups.into_iter().enumerate() { let csproj_name = csproj .file_name() @@ -1806,6 +1817,11 @@ impl IndexManager { }; match indexer.rebuild(&rp, &dp, scope) { Ok(summary) => { + for w in &summary.index_warnings { + if seen_warnings.insert(w.clone()) { + index_warnings.push(w.clone()); + } + } if summary.symbols_indexed == 0 { warn!( "⚠️ [{}/{}] Symbol rebuild returned 0 symbols for '{}' — \ @@ -1844,7 +1860,9 @@ impl IndexManager { // Notify serve layer about overall outcome if let Some(ref n) = notifier { match last_error { - None => n(SymbolRebuildSignal::Succeeded), + None => { + n(SymbolRebuildSignal::Succeeded { index_warnings }) + } Some(msg) => n(SymbolRebuildSignal::Failed(msg)), } } diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 25c83e17..2a69884d 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -92,12 +92,35 @@ pub(crate) enum CSharpIndexStatus { None, /// Helper available and index built successfully. Ready, + /// Index built, but the helper survived workspace/MSBuild failures — + /// definitions are present while cross-project references may be + /// missing. Rendered red (`C#⚠`) in the TUI; the failures are in the + /// info panel and on every find_impact answer from this index. + Partial, /// Index exists but had errors or is stale. Error, /// Symbol index is currently being built. Indexing, } +/// Outcome of a SUCCESSFUL C# symbol rebuild, derived from the rebuild's +/// index warnings: clean → Ready (error cleared), survived failures → +/// Partial with a bounded detail line for the TUI info panel. Pure so the +/// status contract is testable without running a helper. +fn csharp_success_outcome(index_warnings: &[String]) -> (CSharpIndexStatus, Option) { + if index_warnings.is_empty() { + (CSharpIndexStatus::Ready, None) + } else { + let detail = index_warnings + .iter() + .take(3) + .cloned() + .collect::>() + .join(" | "); + (CSharpIndexStatus::Partial, Some(detail)) + } +} + /// Lightweight repo status derived from DashMap state only (no DB opens). pub(crate) struct RepoStatusInfo { pub(crate) status: RepoStateLabel, @@ -622,9 +645,17 @@ impl ServeState { // watcher-triggered rebuild, matching `trigger_symbol_rebuild`. status_map.insert(alias_key.clone(), CSharpIndexStatus::Indexing); } - SymbolRebuildSignal::Succeeded => { - status_map.insert(alias_key.clone(), CSharpIndexStatus::Ready); - error_map.remove(&alias_key); + SymbolRebuildSignal::Succeeded { index_warnings } => { + // Same contract as trigger_symbol_rebuild's success arm: a + // rebuild that survived workspace failures is Partial (red + // C#⚠) with the failures as the info detail — never green. + let (status, detail) = csharp_success_outcome(&index_warnings); + status_map.insert(alias_key.clone(), status); + if let Some(detail) = detail { + error_map.insert(alias_key.clone(), detail); + } else { + error_map.remove(&alias_key); + } } SymbolRebuildSignal::Failed(msg) => { error_map.insert(alias_key.clone(), msg); @@ -3314,6 +3345,29 @@ impl ServeState { self.active_sessions.load(Ordering::Relaxed) } + /// Durable symbol-index warnings for the info surfaces: read fresh from + /// the on-disk meta of every symbol language, language-prefixed. + /// Deliberately NOT filtered by installed languages — reading meta needs + /// no helper, and a helper that vanished after a degraded rebuild must + /// not hide the record. Unlike the in-memory C# status this survives a + /// serve restart. + pub(crate) fn collect_symbol_warnings(&self, db_path: &Path) -> Vec { + let mut warnings: Vec = Vec::new(); + for lang in [LANG_CSHARP, LANG_TYPESCRIPT] { + if let Some(indexer) = self.symbol_registry.get(lang) { + let lang_warnings = indexer.index_warnings(db_path); + if !lang_warnings.is_empty() { + warnings.extend( + lang_warnings + .into_iter() + .map(|w| format!("[{}] {w}", indexer.language())), + ); + } + } + } + warnings + } + /// Get lightweight repo statuses WITHOUT opening any databases. /// Returns a list of (alias, status_info) where status is derived from DashMap state only. pub(crate) fn repo_statuses_lightweight(&self) -> Vec<(String, RepoStatusInfo)> { @@ -3394,7 +3448,10 @@ impl ServeState { } }); - let csharp_error = if matches!(csharp_index, CSharpIndexStatus::Error) { + let csharp_error = if matches!( + csharp_index, + CSharpIndexStatus::Error | CSharpIndexStatus::Partial + ) { self.csharp_index_error .get(alias) .map(|e: dashmap::mapref::one::Ref| e.value().clone()) @@ -3853,12 +3910,14 @@ async fn status_handler( let csharp_str = match info.csharp_index { CSharpIndexStatus::None => "none", CSharpIndexStatus::Ready => "ready", + CSharpIndexStatus::Partial => "partial", CSharpIndexStatus::Error => "error", CSharpIndexStatus::Indexing => "indexing", }; let ts_str = match info.typescript_index { CSharpIndexStatus::None => "none", CSharpIndexStatus::Ready => "ready", + CSharpIndexStatus::Partial => "partial", CSharpIndexStatus::Error => "error", CSharpIndexStatus::Indexing => "indexing", }; @@ -4082,6 +4141,10 @@ async fn info_handler( let db_size_human = tui::dir_size_human(&db_path); + // Durable symbol-index warnings, read fresh from the on-disk meta + // (shared with the TUI info overlay so the two cannot drift). + let symbol_warnings = state.collect_symbol_warnings(&db_path); + AxumJson(json!({ "path": db_path.display().to_string(), "chunks": chunks, @@ -4092,6 +4155,9 @@ async fn info_handler( "dims": dims, "lock": lock, "index_age": index_age, + // Durable record of a degraded symbol index (see the TUI's C#⚠): + // build-environment failures the helper survived during rebuild. + "symbol_warnings": symbol_warnings, // Is the HNSW graph built and committed? A non-zero `chunks` with // `indexed: false` is a searchable-looking but silently dead index: // `VectorStore::search` refuses to run without the graph. The cloud @@ -4252,10 +4318,15 @@ async fn trigger_symbol_rebuild( ); state.end_indexing(&alias_owned, IndexingOwner::Symbol); if is_csharp { + let (status, detail) = csharp_success_outcome(&summary.index_warnings); state .csharp_index_status - .insert(alias_owned.clone(), CSharpIndexStatus::Ready); - state.csharp_index_error.remove(&alias_owned); + .insert(alias_owned.clone(), status); + if let Some(detail) = detail { + state.csharp_index_error.insert(alias_owned.clone(), detail); + } else { + state.csharp_index_error.remove(&alias_owned); + } if let Ok(mut cfg) = state.config.write() { cfg.touch_last_scip(&alias_owned, ServeState::now_unix_secs()); diff --git a/src/serve/tests.rs b/src/serve/tests.rs index fe937812..6118b6aa 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3450,6 +3450,166 @@ async fn run_phase_2_symbols_wires_candidate_through_to_trigger() { ); } +/// A successful rebuild WITHOUT survived failures is Ready with the error +/// cleared; one WITH survived workspace failures is Partial (red `C#⚠` in +/// the TUI) with a bounded detail line for the info panel — capped at the +/// first three entries so the detail row stays readable. +#[test] +fn csharp_success_outcome_partial_when_warnings_survived() { + assert_eq!( + csharp_success_outcome(&[]), + (CSharpIndexStatus::Ready, None), + "a clean rebuild must read Ready with no error detail" + ); + + let degraded = [ + "summary".to_string(), + "A.csproj: failure one".to_string(), + "B.csproj: failure two".to_string(), + "C.csproj: failure three".to_string(), + "D.csproj: failure four".to_string(), + ]; + let (status, detail) = csharp_success_outcome(°raded); + assert_eq!(status, CSharpIndexStatus::Partial); + let detail = detail.expect("partial carries detail"); + assert!( + detail.contains("summary") && detail.contains("B.csproj"), + "detail joins the leading entries, got: {detail}" + ); + assert!( + !detail.contains("D.csproj"), + "detail is capped at three entries, got: {detail}" + ); +} + +/// The watcher notifier must apply the SAME status contract as +/// trigger_symbol_rebuild's success arm: Succeeded-with-warnings → Partial +/// (red C#⚠) with bounded detail — never green — and a clean Succeeded → +/// Ready with the detail cleared. Reverting the arm to insert(Ready) ships +/// the round-1 defect (green indicator on a degraded index) with a green +/// suite. +#[test] +fn csharp_notifier_partial_on_succeeded_with_warnings() { + let state = ServeState::new(ReposConfig::default(), None); + let notifier = state.make_csharp_notifier("wrepo"); + + notifier(SymbolRebuildSignal::Started); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Indexing) + ); + + notifier(SymbolRebuildSignal::Succeeded { + index_warnings: vec![ + "summary".to_string(), + "A.csproj: boom".to_string(), + "B.csproj: boom2".to_string(), + "C.csproj: boom3".to_string(), + ], + }); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Partial), + "a degraded watcher rebuild must render Partial, never green" + ); + let detail = state + .csharp_index_error + .get("wrepo") + .map(|e| e.value().clone()) + .expect("detail stored"); + assert!(detail.contains("summary") && !detail.contains("C.csproj")); + + notifier(SymbolRebuildSignal::Succeeded { + index_warnings: Vec::new(), + }); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Ready) + ); + assert!( + !state.csharp_index_error.contains_key("wrepo"), + "a clean rebuild must clear the detail" + ); +} + +/// `repo_statuses_lightweight()` must carry the Partial status AND its +/// detail message — that pair is what renders the red `C#⚠` and the info +/// panel's failure line. +#[test] +fn repo_statuses_lightweight_carries_partial_with_detail() { + let (_tmp, _path, bare) = state_with_repo("partialrepo"); + let state = std::sync::Arc::new(bare); + state + .csharp_index_status + .insert("partialrepo".to_string(), CSharpIndexStatus::Partial); + state.csharp_index_error.insert( + "partialrepo".to_string(), + "summary | A.csproj: failure one".to_string(), + ); + + let statuses = state.repo_statuses_lightweight(); + let (_, info) = statuses + .iter() + .find(|(alias, _)| alias == "partialrepo") + .expect("partialrepo row"); + assert_eq!(info.csharp_index, CSharpIndexStatus::Partial); + assert_eq!( + info.csharp_error.as_deref(), + Some("summary | A.csproj: failure one"), + "the detail message must ride on the status row" + ); +} + +/// The serve-tui `i` overlay reads the DURABLE warnings from the on-disk +/// meta (survives a restart, unlike the in-memory status) and prefixes them +/// with the language. A repo whose meta carries no warnings yields none. +#[test] +fn build_info_overlay_reads_symbol_warnings_from_meta() { + use crate::serve::tui::build_info_overlay; + + let (_tmp, repo_path, bare) = state_with_repo("inforepo"); + let db = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write( + db.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .unwrap(); + // Seed the meta the way a degraded rebuild does (see the roundtrip test + // in csharp_tests for the read side). + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["summary","A.csproj: failure one"]"#, + ) + .unwrap(); + wtxn.commit().unwrap(); + + let state = std::sync::Arc::new(bare); + let statuses = state.repo_statuses_lightweight(); + let idx = statuses + .iter() + .position(|(alias, _)| alias == "inforepo") + .expect("inforepo row"); + let overlay = build_info_overlay(idx, &statuses, &state).expect("overlay"); + match overlay { + crate::serve::tui_common::OverlayState::Info { + symbol_warnings, .. + } => { + assert!( + symbol_warnings.len() == 2 && symbol_warnings[0].starts_with("[csharp] summary"), + "warnings must be language-prefixed from meta, got: {symbol_warnings:?}" + ); + } + other => panic!("expected Info overlay, got {other:?}"), + } +} + /// The serve-mode find_impact self-heal spawn must reach /// `trigger_symbol_rebuild` for a registered, applicable repo: after the /// warned empty answer, the C# status eventually flips to Error (dummy diff --git a/src/serve/tui.rs b/src/serve/tui.rs index 497051f7..3b7af57e 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -444,6 +444,7 @@ fn map_repo_rows( let csharp_str = match info.csharp_index { super::CSharpIndexStatus::Ready => "ready", + super::CSharpIndexStatus::Partial => "partial", super::CSharpIndexStatus::Indexing => "indexing", super::CSharpIndexStatus::Error => "error", super::CSharpIndexStatus::None => "none", @@ -452,6 +453,7 @@ fn map_repo_rows( let ts_str = match info.typescript_index { super::CSharpIndexStatus::Ready => "ready", + super::CSharpIndexStatus::Partial => "partial", super::CSharpIndexStatus::Indexing => "indexing", super::CSharpIndexStatus::Error => "error", super::CSharpIndexStatus::None => "none", @@ -737,7 +739,8 @@ fn cpu_usage_str(sys_system: &mut Option) -> String { // --------------------------------------------------------------------------- /// Build an `OverlayState::Info` by gathering live stats from SharedStores or metadata. -fn build_info_overlay( +/// pub(crate) so the serve test module can pin the symbol-warnings wiring. +pub(crate) fn build_info_overlay( idx: usize, repos: &[(String, super::RepoStatusInfo)], state: &Arc, @@ -808,6 +811,10 @@ fn build_info_overlay( // DB size on disk let db_size_human = dir_size_human(&db_path); + // Durable symbol-index warnings, read fresh from the on-disk meta + // (shared with the REST info surface so the two cannot drift). + let symbol_warnings = state.collect_symbol_warnings(&db_path); + Some(OverlayState::Info { alias: alias.clone(), path: db_path.display().to_string(), @@ -819,6 +826,7 @@ fn build_info_overlay( dims, lock, index_age, + symbol_warnings, }) } diff --git a/src/serve/tui_common.rs b/src/serve/tui_common.rs index d098a3e7..070f1498 100644 --- a/src/serve/tui_common.rs +++ b/src/serve/tui_common.rs @@ -148,6 +148,11 @@ pub enum OverlayState { dims: usize, lock: String, index_age: String, + /// Index-level symbol warnings read fresh from the on-disk meta + /// (`scip_meta[index_warnings]`): build-environment failures the + /// helper survived. Unlike the in-memory C# status these survive a + /// serve restart — the durable record of a degraded index. + symbol_warnings: Vec, }, /// Info modal for a *mounted remote project* (federation peer). Remote /// mounts have no local on-disk index, so the chunk/file/db-size/model stats @@ -341,14 +346,10 @@ pub fn render_table( let max_alias_w = repos .iter() .map(|r| { - // Each indicator (" C#·" / " TS·") is 4 display cols; account for both. - let mut extra = 0usize; - if matches!(r.csharp_index.as_str(), "ready" | "error" | "indexing") { - extra += 4; - } - if matches!(r.typescript_index.as_str(), "ready" | "error" | "indexing") { - extra += 4; - } + // Each indicator is a suffix like " C#·"; ⚠ (the partial glyph) + // is double-width (unicode-width EAW=W) so its variant budgets + // 6 display cols instead of 4. + let extra = indicator_cols(&r.csharp_index) + indicator_cols(&r.typescript_index); r.alias.len() + extra }) .max() @@ -390,6 +391,12 @@ pub fn render_table( format!("{} C#·", repo.alias), Style::default().fg(Color::White), ), + // Degraded index: built despite survived workspace failures — + // red and loud, the detail panel carries the failures. + "partial" => ( + format!("{} C#⚠", repo.alias), + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ), "error" => ( format!("{} C#!", repo.alias), Style::default().fg(Color::Red), @@ -416,6 +423,10 @@ pub fn render_table( alias_text.push_str(" TS!"); alias_style = alias_style.fg(Color::Red); } + "partial" => { + alias_text.push_str(" TS⚠"); + alias_style = alias_style.fg(Color::Red); + } "indexing" => alias_text.push_str(" TS…"), _ => {} } @@ -592,8 +603,8 @@ pub fn render_detail( let info_line = Line::from(info_spans); - // Optional error line for C# errors - let error_line = if repo.csharp_index == "error" { + // Optional error line for C# errors and degraded (partial) indexes + let error_line = if matches!(repo.csharp_index.as_str(), "error" | "partial") { let err_msg = repo.csharp_error.as_deref().unwrap_or("Unknown error"); const ERR_PREFIX_COLS: usize = 7; let max_err_chars = (area.width as usize).saturating_sub(ERR_PREFIX_COLS); @@ -806,6 +817,7 @@ pub fn render_overlay(f: &mut ratatui::Frame, area: Rect, overlay: &OverlayState dims, lock, index_age, + symbol_warnings, } => { let title = format!(" {} — Index Info ", alias); let lines = vec![ @@ -855,12 +867,28 @@ pub fn render_overlay(f: &mut ratatui::Frame, area: Rect, overlay: &OverlayState Span::styled(" Index age: ", Style::default().fg(Color::DarkGray)), Span::styled(index_age.clone(), Style::default().fg(Color::White)), ]), - Line::from(""), - Line::from(Span::styled( - " [Esc] close", - Style::default().fg(Color::DarkGray), - )), ]; + let mut lines = lines; + // Durable record of a degraded symbol index: the build problems + // the helper survived. Red, like the table's `C#⚠` indicator. + if !symbol_warnings.is_empty() { + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + " ⚠ Symbol index warnings:", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ))); + for warning in symbol_warnings { + lines.push(Line::from(Span::styled( + format!(" {warning}"), + Style::default().fg(Color::Red), + ))); + } + } + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + " [Esc] close", + Style::default().fg(Color::DarkGray), + ))); render_centered_modal(f, area, &title, lines); } OverlayState::RemoteInfo { @@ -1272,6 +1300,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Open C#!".to_string(), Color::Red), + "partial" => ("Open C#⚠".to_string(), Color::Red), _ => ("Open".to_string(), Color::Green), }, "warm" => match csharp { @@ -1285,6 +1314,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Warm C#!".to_string(), Color::Red), + "partial" => ("Warm C#⚠".to_string(), Color::Red), _ => ("Warm".to_string(), Color::Yellow), }, "readonly" => ("Readonly".to_string(), Color::Cyan), @@ -1313,6 +1343,17 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { } } +/// Display width of one alias-column indicator suffix (`" C#·"` family). +/// The partial glyph ⚠ is double-width (unicode-width EAW=W), so that +/// variant budgets 6 display cols; the 1-col glyphs budget 4. +fn indicator_cols(status: &str) -> usize { + match status { + "ready" | "error" | "indexing" => 4, + "partial" => 6, + _ => 0, + } +} + // --------------------------------------------------------------------------- // Terminal helpers // --------------------------------------------------------------------------- @@ -1355,6 +1396,37 @@ mod tests { ); } + /// A degraded (partial) C# index renders red with the ⚠ glyph in the + /// detail panel — same urgency class as an error, distinct label. + #[test] + fn detail_status_style_maps_partial_to_red_warning() { + assert_eq!( + detail_status_style("open", "partial"), + ("Open C#⚠".to_string(), Color::Red) + ); + assert_eq!( + detail_status_style("warm", "partial"), + ("Warm C#⚠".to_string(), Color::Red) + ); + } + + /// The alias-column width budget must cover the actually rendered + /// suffixes: the partial glyph ⚠ is double-width (EAW=W), so " C#⚠" is + /// 5 display cols — budgeting it at 4 (the 1-col glyph family) would + /// clip the ⚠ exactly when a partial row defines the widest alias. + #[test] + fn indicator_cols_budgets_double_width_partial_glyph() { + assert_eq!(indicator_cols("ready"), 4); + assert_eq!(indicator_cols("error"), 4); + assert_eq!(indicator_cols("indexing"), 4); + assert_eq!( + indicator_cols("partial"), + 6, + "double-width glyph needs more than the 4-col family budget" + ); + assert_eq!(indicator_cols("none"), 0); + } + #[test] fn unknown_status_falls_back_without_idle_treatment() { assert_ne!( diff --git a/src/serve/tui_remote.rs b/src/serve/tui_remote.rs index 6ca26da5..a14335b0 100644 --- a/src/serve/tui_remote.rs +++ b/src/serve/tui_remote.rs @@ -358,6 +358,7 @@ async fn run_remote_tui_loop( dims: info.dims, lock: info.lock, index_age: info.index_age, + symbol_warnings: info.symbol_warnings, }, Err(e) => OverlayState::Doctor { alias, @@ -503,6 +504,10 @@ struct InfoResponse { dims: usize, lock: String, index_age: String, + /// Index-level symbol warnings from the peer's on-disk meta. `default` for + /// the same older-serve compatibility as `path`. + #[serde(default)] + symbol_warnings: Vec, } #[derive(Debug, Deserialize)] diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index ce4180c6..2f4a14b4 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -110,6 +110,21 @@ pub(crate) fn summarize_index_warnings(lines: &[String], cap: usize) -> Vec Vec { + if entries.is_empty() { + return Vec::new(); + } + let mut stored = vec![format!( + "The symbol index was built while the C# workspace reported {} distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex.", + entries.len() + )]; + stored.extend(entries.iter().cloned()); + stored +} + /// Route one scip-csharp stderr line into tracing at the right severity. /// This is the ONLY sanctioned path for helper stderr: spawn helpers with /// `Stdio::piped()` and drain through here — never `Stdio::inherit()`, @@ -1778,17 +1793,12 @@ impl SymbolIndexer for CSharpSymbolIndexer { )?; // Index-level completeness warnings from this run — written or // cleared on EVERY rebuild (full or incremental) so meta always - // describes the newest build. The summary entry leads because the - // list rides on every find_impact answer and must stand alone. - if index_warnings.is_empty() { + // describes the newest build. + let stored_warnings = index_warnings_stored(&index_warnings); + if stored_warnings.is_empty() { meta_db.delete(&mut wtxn, META_INDEX_WARNINGS)?; } else { - let mut stored = vec![format!( - "The symbol index was built while the C# workspace reported {} distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex.", - index_warnings.len() - )]; - stored.extend(index_warnings.iter().cloned()); - let json = serde_json::to_string(&stored).unwrap_or_else(|_| "[]".to_string()); + let json = serde_json::to_string(&stored_warnings).unwrap_or_else(|_| "[]".to_string()); meta_db.put(&mut wtxn, META_INDEX_WARNINGS, json.as_str())?; } @@ -1817,6 +1827,7 @@ impl SymbolIndexer for CSharpSymbolIndexer { symbols_indexed: total_symbols, references_stored: total_defs, // definitions only; refs resolved lazily duration_ms, + index_warnings: stored_warnings, }) } diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 09c5968b..6d757e7a 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -380,6 +380,10 @@ pub struct RebuildSummary { pub references_stored: usize, /// Wall-clock duration in milliseconds. pub duration_ms: u64, + /// Index-level completeness warnings from this build (build-environment + /// failures the helper survived; see [`SymbolIndexer::index_warnings`]). + /// Empty = built clean. Drives the TUI's degraded indicator. + pub index_warnings: Vec, } /// Summary returned after a Phase 3 pre-warm completes. diff --git a/src/symbols/typescript.rs b/src/symbols/typescript.rs index 1cc11b24..2707b6fe 100644 --- a/src/symbols/typescript.rs +++ b/src/symbols/typescript.rs @@ -611,6 +611,7 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { symbols_indexed: total_symbols, references_stored: total_refs, duration_ms, + index_warnings: Vec::new(), }) } From 886ac332688fb6e0539ecf06a5a192ba6b8ae390 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 2 Oct 2026 11:44:29 +0200 Subject: [PATCH 30/77] helper: never escalate warnings to errors during the workspace load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User ask: the broken-restore projects (NU1603 class) must still load with their own settings — the symbol load must simply refuse warnings-as-errors, because repos that escalate warnings (own project settings or Directory.Build.props) turn those benign load-time diagnostics into MSBuild project-load failures. Each dropped project also silently drops every cross-project reference to it — the empty-references defect. WorkspaceLoad.GlobalProperties() (own file, compiled into the helper and the test assembly) adds TreatWarningsAsErrors=false, WarningsAsErrors='' and MSBuildTreatWarningsAsErrors=false next to the existing DesignTimeBuild/SkipCompilerExecution contract; tests pin all six keys. Validation: dotnet test (helper suite) 14/14 passed; dotnet build clean. --- helpers/csharp/Program.cs | 13 +------ helpers/csharp/WorkspaceLoad.cs | 32 +++++++++++++++++ helpers/csharp/tests/ScipCsharp.Tests.csproj | 1 + .../tests/WorkspaceLoadPropertiesTests.cs | 35 +++++++++++++++++++ 4 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 helpers/csharp/WorkspaceLoad.cs create mode 100644 helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index e8539efc..334013a9 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -309,18 +309,7 @@ await Console.Error.WriteLineAsync( /// private static MSBuildWorkspace CreateTolerantWorkspace() { - var properties = new Dictionary - { - // Tell Roslyn to skip projects it cannot load instead of crashing. - { "BuildingInsideVisualStudio", "true" }, - // Design-time build: prevents auto-generated files in obj/ (e.g. - // .AssemblyAttributes.cs, .AssemblyInfo.cs) from being included as - // explicit Compile items. Without this, SDK-style projects produce - // duplicate Compile items (auto-include + obj/ generated), which - // causes MSBuildWorkspace to fail loading the project. - { "DesignTimeBuild", "true" }, - { "SkipCompilerExecution", "true" }, - }; + var properties = WorkspaceLoad.GlobalProperties(); // If MSBUILD_EXE_PATH is set, pass it to the workspace so the BuildHost // uses the same MSBuild we registered via MSBuildLocator (typically the .NET SDK). diff --git a/helpers/csharp/WorkspaceLoad.cs b/helpers/csharp/WorkspaceLoad.cs new file mode 100644 index 00000000..e22dc096 --- /dev/null +++ b/helpers/csharp/WorkspaceLoad.cs @@ -0,0 +1,32 @@ +namespace ScipCsharp; + +/// +/// Global MSBuild properties for every workspace load. Non-negotiables: +/// DesignTimeBuild + SkipCompilerExecution and NO warnings-as-errors — a +/// repo that escalates warnings (own settings or Directory.Build.props) +/// would turn benign load-time diagnostics (the NU1603 version-unification +/// class) into MSBuild project-load failures, dropping the project — and +/// every cross-project reference to it — from the index. Everything else +/// about the project keeps its own settings; we only refuse the escalation. +/// Lives outside Program.cs so the test assembly can compile it in +/// (Program carries the Main entrypoint and is deliberately excluded there). +/// +internal static class WorkspaceLoad +{ + internal static System.Collections.Generic.Dictionary GlobalProperties() => new() + { + // Tell Roslyn to skip projects it cannot load instead of crashing. + { "BuildingInsideVisualStudio", "true" }, + // Design-time build: prevents auto-generated files in obj/ (e.g. + // .AssemblyAttributes.cs, .AssemblyInfo.cs) from being included as + // explicit Compile items. Without this, SDK-style projects produce + // duplicate Compile items (auto-include + obj/ generated), which + // causes MSBuildWorkspace to fail loading the project. + { "DesignTimeBuild", "true" }, + { "SkipCompilerExecution", "true" }, + // Warnings stay warnings during the load — never fatal (NU1603 etc.). + { "TreatWarningsAsErrors", "false" }, + { "WarningsAsErrors", "" }, + { "MSBuildTreatWarningsAsErrors", "false" }, + }; +} diff --git a/helpers/csharp/tests/ScipCsharp.Tests.csproj b/helpers/csharp/tests/ScipCsharp.Tests.csproj index 96712ece..868ea409 100644 --- a/helpers/csharp/tests/ScipCsharp.Tests.csproj +++ b/helpers/csharp/tests/ScipCsharp.Tests.csproj @@ -28,6 +28,7 @@ + diff --git a/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs b/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs new file mode 100644 index 00000000..60cda4ff --- /dev/null +++ b/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs @@ -0,0 +1,35 @@ +using Xunit; +using ScipCsharp; + +namespace ScipCsharp.Tests; + +/// +/// Pins the workspace load properties. The contract that matters: the +/// symbol load NEVER escalates warnings to errors — a repo with +/// TreatWarningsAsErrors (own settings or Directory.Build.props) must still +/// load its projects, because benign load-time diagnostics (NU1603 class) +/// would otherwise drop the project and every cross-project reference to it. +/// Reverting any of the three demotion properties must fail this test. +/// +public class WorkspaceLoadPropertiesTests +{ + [Fact] + public void Load_NeverEscalatesWarningsToErrors() + { + var properties = WorkspaceLoad.GlobalProperties(); + + Assert.Equal("false", properties["TreatWarningsAsErrors"]); + Assert.Equal("", properties["WarningsAsErrors"]); + Assert.Equal("false", properties["MSBuildTreatWarningsAsErrors"]); + } + + [Fact] + public void Load_KeepsDesignTimeBuildContract() + { + var properties = WorkspaceLoad.GlobalProperties(); + + Assert.Equal("true", properties["BuildingInsideVisualStudio"]); + Assert.Equal("true", properties["DesignTimeBuild"]); + Assert.Equal("true", properties["SkipCompilerExecution"]); + } +} From 78f6e6b6e156671e37ff342c5adcb0cff3f79e5f Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 2 Oct 2026 12:52:56 +0200 Subject: [PATCH 31/77] startup rebuilds symbol indexes it did not produce: builder-version stamp + degraded retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User ask: codesearch itself must do this at startup — after a deploy the fresh-by-timestamp gate kept serving an index an older binary had built (the DRM-24427_Versioning loop: every test hit the same stale index until someone ran a manual command). - scip_meta[index_builder_version] = full version (version+build count), stamped on every C# rebuild next to index_warnings (extracted into write_rebuild_meta, the one fn every scope's meta txn calls) - gate: stamp absent or != this binary → ChangedSinceLastBuild — a deployed binary rebuilds the inherited indexes itself during phase-2, bounded by one rebuild per binary change - gate: index_warnings non-empty → one automatic retry per serve process (DashSet guard) — a still-broken environment cannot loop; the next serve start retries once and the TUI keeps showing red between restarts - tests: 3 gate tests with a seeded-meta fixture (fresh/stale-by-version/ degraded, #[serial]) + the producer test pinning write_rebuild_meta (stamp written, clean clears, degraded persists with summary lead) Review-fixes: - [Important] stamp write reached by no test (deleting it kept the suite green; every startup would churn all inherited indexes) → producer test drives write_rebuild_meta directly; a fake-helper rebuild test is not executable on Windows (validated filename must be a real PE) Validation: fmt clean, clippy -D warnings clean, cargo test --lib --bins 775+771 passed / 0 failed. --- src/constants.rs | 10 +++ src/serve/mod.rs | 27 ++++++- src/serve/tests.rs | 143 ++++++++++++++++++++++++++++++++++++ src/symbols/csharp.rs | 80 +++++++++++++++----- src/symbols/csharp_tests.rs | 90 +++++++++++++++++++++++ src/symbols/mod.rs | 9 +++ 6 files changed, 340 insertions(+), 19 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 968a6dcd..f22fcd04 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -662,6 +662,16 @@ pub const SCIP_KEY_FORMAT_KEY: &str = "key_format"; /// list out of a partially loaded solution may be missing callers. pub const SCIP_INDEX_WARNINGS_KEY: &str = "index_warnings"; +/// LMDB metadata key stamping which codesearch build produced the symbol +/// index (full `CARGO_PKG_VERSION_FULL`, so every rebuilt binary differs). +/// The C# rebuild gate treats an index whose stamp is absent or differs from +/// [`INDEX_BUILDER_VERSION`] as stale: after a deploy, serve rebuilds the +/// indexes itself at startup instead of waiting for a manual command. +pub const SCIP_INDEX_BUILDER_VERSION_KEY: &str = "index_builder_version"; + +/// The value written for [`SCIP_INDEX_BUILDER_VERSION_KEY`] by this binary. +pub const INDEX_BUILDER_VERSION: &str = env!("CARGO_PKG_VERSION_FULL"); + /// Current value written for [`SCIP_KEY_FORMAT_KEY`]. Bump whenever the /// canonical SCIP symbol key format produced by a language helper changes /// shape (B4: C# generic arity / containing-type path / fully qualified diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 2a69884d..3d5a06c5 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use axum::response::{IntoResponse, Json as AxumJson}; use colored::Colorize; -use dashmap::DashMap; +use dashmap::{DashMap, DashSet}; use rmcp::transport::{ streamable_http_server::session::local::LocalSessionManager, StreamableHttpServerConfig, StreamableHttpService, @@ -373,6 +373,11 @@ pub(crate) struct ServeState { /// their LMDB clear/write passes. Stale claims self-heal past the /// indexing timeout, like `active_reindexes` markers. symbol_rebuild_in_flight: DashMap, + /// Aliases whose degraded index (index_warnings in meta) already got its + /// automatic retry this serve process. Without this, a still-broken + /// build environment would rebuild on every evaluation — the next serve + /// start retries once more, which is the operator-visible bound. + degraded_rebuild_attempted: DashSet, /// Debounced deadline for persisting repos config metadata (unix millis). persist_deadline_unix_ms: AtomicU64, /// Ensures only one debounce worker task runs. @@ -458,6 +463,7 @@ impl ServeState { csharp_index_status: Arc::new(DashMap::new()), csharp_index_error: Arc::new(DashMap::new()), symbol_rebuild_in_flight: DashMap::new(), + degraded_rebuild_attempted: DashSet::new(), persist_deadline_unix_ms: AtomicU64::new(0), persist_worker_started: AtomicBool::new(false), #[cfg(test)] @@ -1257,6 +1263,25 @@ impl ServeState { return RebuildDecision::NoIndex; } + // Fresh-by-timestamp is not correct: an index built by an older + // binary predates both the warning capture and the helper's + // load-behaviour fixes (the deployed binary must never serve an + // index it did not produce), and an index built WITH surviving + // workspace failures carries those warnings in its meta. Both + // rebuild automatically — the version path on every binary change, + // the degraded path once per serve process so a still-broken + // environment cannot loop; the next serve start retries once. + if indexer.index_builder_version(db_path).as_deref() + != Some(crate::constants::INDEX_BUILDER_VERSION) + { + return RebuildDecision::ChangedSinceLastBuild; + } + if !indexer.index_warnings(db_path).is_empty() + && self.degraded_rebuild_attempted.insert(alias.to_string()) + { + return RebuildDecision::ChangedSinceLastBuild; + } + if last_changed > last_scip { RebuildDecision::ChangedSinceLastBuild } else { diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 6118b6aa..199fcab0 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -3681,3 +3681,146 @@ async fn find_impact_self_heal_runs_the_rebuild_in_serve_mode() { tokio::time::sleep(std::time::Duration::from_millis(50)).await; } } + +// ── Rebuild gate: builder-version stamp + degraded retry ───────────────── + +/// Seeds the SCIP meta for `db` exactly like `rebuild` stamps it (rebuild +/// timestamp + key format), plus the caller's extra key/values — so gate +/// tests can shape a fresh, stale or degraded index without the helper. +fn seed_scip_meta(db: &std::path::Path, extra: &[(&str, &str)]) { + let env = crate::symbols::get_shared_scip_env(db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + meta.put(&mut wtxn, crate::constants::SCIP_REBUILD_TIMESTAMP_KEY, "0") + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_KEY_FORMAT_KEY, + crate::constants::SCIP_KEY_FORMAT, + ) + .unwrap(); + for (key, value) in extra { + meta.put(&mut wtxn, *key, *value).unwrap(); + } + wtxn.commit().unwrap(); +} + +/// Shared arrangement: applicable repo (`.sln`), dummy helper, seeded SCIP +/// meta, config timestamps showing a fresh build (last_scip ≥ last_changed) +/// so ONLY the builder-version / degraded-retry checks can move the gate. +/// The helper-env guard and both tempdirs are returned so they outlive the +/// fixture. +#[allow(clippy::type_complexity)] +fn gate_fixture( + alias: &str, + meta_extra: &[(&str, &str)], +) -> ( + crate::testing::EnvRestore, + tempfile::TempDir, + tempfile::TempDir, + std::path::PathBuf, + std::sync::Arc, +) { + let helper_root = tempfile::tempdir().unwrap(); + let env_guard = dummy_csharp_helper(helper_root.path()); + let (tmp, repo_path, bare) = state_with_repo(alias); + std::fs::write(repo_path.join("test.sln"), b"Solution File").unwrap(); + let db = repo_path.join(DB_DIR_NAME); + seed_scip_meta(&db, meta_extra); + let state = std::sync::Arc::new(bare); + let now = ServeState::now_unix_secs(); + { + let mut cfg = state.config.write().unwrap(); + cfg.touch_last_changed(alias, now - 100); + cfg.touch_last_scip(alias, now); + } + (env_guard, helper_root, tmp, repo_path, state) +} + +/// An index this binary never produced (no builder stamp — anything built +/// before the stamp existed) must read as stale: a deployed binary rebuilds +/// the indexes it inherited itself at startup, never waits for a manual +/// command. This is the DRM-24427_Versioning defect class. +#[test] +#[serial] +fn evaluate_rebuilds_an_index_from_an_older_binary() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture("oldbinary", &[]); + + assert_eq!( + state.evaluate_symbol_rebuild( + "csharp", + "oldbinary", + &repo_path, + &repo_path.join(DB_DIR_NAME) + ), + RebuildDecision::ChangedSinceLastBuild, + "absent builder stamp must read as stale" + ); +} + +/// A degraded index (warnings in meta, current builder version) rebuilds +/// exactly once per serve process: the second evaluation is Fresh (no loop +/// while the environment stays broken); a fresh process gets a fresh set. +#[test] +#[serial] +fn evaluate_retries_a_degraded_index_once_per_process() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture( + "degraded", + &[( + crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY, + crate::constants::INDEX_BUILDER_VERSION, + )], + ); + let db = repo_path.join(DB_DIR_NAME); + // Re-seed WITH warnings (the fixture seeded only the version stamp). + seed_scip_meta( + &db, + &[( + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["Broken.csproj: Msbuild failed when processing the file"]"#, + )], + ); + + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "degraded", &repo_path, &db), + RebuildDecision::ChangedSinceLastBuild, + "a degraded index must get its one automatic retry" + ); + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "degraded", &repo_path, &db), + RebuildDecision::Fresh, + "the retry must not loop within one serve process" + ); +} + +/// A clean index with the current builder stamp and fresh timestamps stays +/// Fresh — the gate must not churn indexes that are actually fine. +#[test] +#[serial] +fn evaluate_stays_fresh_when_index_is_current_and_clean() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture( + "cleanidx", + &[( + crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY, + crate::constants::INDEX_BUILDER_VERSION, + )], + ); + + assert_eq!( + state.evaluate_symbol_rebuild( + "csharp", + "cleanidx", + &repo_path, + &repo_path.join(DB_DIR_NAME) + ), + RebuildDecision::Fresh, + "current + clean + fresh timestamps must not rebuild" + ); + assert!( + !state.degraded_rebuild_attempted.contains("cleanidx"), + "a clean index must not consume the degraded retry" + ); +} diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 2f4a14b4..9df5a929 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -200,6 +200,7 @@ const META_KEY_FORMAT: &str = crate::constants::SCIP_KEY_FORMAT_KEY; /// completeness warnings from the last rebuild (see /// [`summarize_index_warnings`]). const META_INDEX_WARNINGS: &str = crate::constants::SCIP_INDEX_WARNINGS_KEY; +const META_BUILDER_VERSION: &str = crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY; /// How many distinct index-level warnings are persisted (and thus surfaced /// per find_impact answer) before the list is capped with an overflow note. @@ -438,6 +439,41 @@ impl CSharpSymbolIndexer { resolved } + /// Writes the rebuild bookkeeping shared by every scope (full and + /// incremental): key-format stamp, index warnings (written or cleared) + /// and the builder-version stamp. Called from `rebuild`'s meta txn, so + /// meta always describes the newest build. `pub(crate)` so the producer + /// test drives the exact write the pipeline runs — a fake helper cannot + /// execute on Windows, where the validated filename must be a real PE. + pub(crate) fn write_rebuild_meta( + meta_db: &Database, + wtxn: &mut heed::RwTxn, + index_warnings: &[String], + ) -> Result<()> { + // Unconditional: has_index refuses an index whose key-format stamp is + // absent or stale, so a key-format change forces exactly one rebuild. + meta_db.put(wtxn, META_KEY_FORMAT, crate::constants::SCIP_KEY_FORMAT)?; + // Index-level completeness warnings from this run — written or + // cleared on EVERY rebuild (full or incremental) so meta always + // describes the newest build. + let stored_warnings = index_warnings_stored(index_warnings); + if stored_warnings.is_empty() { + meta_db.delete(wtxn, META_INDEX_WARNINGS)?; + } else { + let json = serde_json::to_string(&stored_warnings).unwrap_or_else(|_| "[]".to_string()); + meta_db.put(wtxn, META_INDEX_WARNINGS, json.as_str())?; + } + // Stamp the producing build: the rebuild gate treats an index from + // another (or an unknown) build as stale, so a deployed binary + // rebuilds the indexes it inherited itself at startup. + meta_db.put( + wtxn, + META_BUILDER_VERSION, + crate::constants::INDEX_BUILDER_VERSION, + )?; + Ok(()) + } + fn resolve_helper_path(&self) -> Option { // 1. Environment variable override if let Ok(path) = std::env::var(HELPER_ENV_VAR) { @@ -1784,23 +1820,7 @@ impl SymbolIndexer for CSharpSymbolIndexer { if let Some(sha) = super::current_git_head(repo_path) { meta_db.put(&mut wtxn, META_HEAD_SHA, sha.as_str())?; } - // Unconditional: has_index refuses an index whose key-format stamp is - // absent or stale, so a key-format change forces exactly one rebuild. - meta_db.put( - &mut wtxn, - META_KEY_FORMAT, - crate::constants::SCIP_KEY_FORMAT, - )?; - // Index-level completeness warnings from this run — written or - // cleared on EVERY rebuild (full or incremental) so meta always - // describes the newest build. - let stored_warnings = index_warnings_stored(&index_warnings); - if stored_warnings.is_empty() { - meta_db.delete(&mut wtxn, META_INDEX_WARNINGS)?; - } else { - let json = serde_json::to_string(&stored_warnings).unwrap_or_else(|_| "[]".to_string()); - meta_db.put(&mut wtxn, META_INDEX_WARNINGS, json.as_str())?; - } + Self::write_rebuild_meta(&meta_db, &mut wtxn, &index_warnings)?; wtxn.commit()?; @@ -1827,7 +1847,7 @@ impl SymbolIndexer for CSharpSymbolIndexer { symbols_indexed: total_symbols, references_stored: total_defs, // definitions only; refs resolved lazily duration_ms, - index_warnings: stored_warnings, + index_warnings: index_warnings_stored(&index_warnings), }) } @@ -1979,6 +1999,30 @@ impl SymbolIndexer for CSharpSymbolIndexer { serde_json::from_str(json).unwrap_or_default() } + /// Which codesearch build produced this index — `None` when the stamp is + /// absent (pre-stamp index) or the meta is unreadable. The rebuild gate + /// compares against [`crate::constants::INDEX_BUILDER_VERSION`] and + /// rebuilds on mismatch: this binary never produced that index. + fn index_builder_version(&self, db_path: &Path) -> Option { + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return None, + }; + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return None, + }; + let meta_db: Database = match env.open_database(&rtxn, Some(SCIP_META_DB_NAME)) { + Ok(Some(db)) => db, + _ => return None, + }; + meta_db + .get(&rtxn, META_BUILDER_VERSION) + .ok() + .flatten() + .map(|s| s.to_string()) + } + /// Whether a SCIP index exists AND was built with the current key format. /// An index that is fresh by timestamp but stamped with another (or no) /// key-format generation would serve old-shaped canonical keys as truth, diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs index 19b68bb0..662d29c3 100644 --- a/src/symbols/csharp_tests.rs +++ b/src/symbols/csharp_tests.rs @@ -244,3 +244,93 @@ fn index_warnings_roundtrip_and_clean_absence() { "non-tracking adapters inherit the empty default" ); } + +// ── Builder-version stamp: the rebuild's meta-write site ───────────────── +// +// The gate tests hand-seed meta (consumer half); this one drives the +// producer. The write lives in `write_rebuild_meta` — the exact fn the +// rebuild pipeline calls from its meta txn — because a fake helper cannot +// execute on Windows: the validated filename must be a real PE, not a +// script. Deleting the stamp write must fail this test, or every serve +// start churns all inherited C# indexes (absent stamp reads as stale). + +#[test] +fn rebuild_meta_write_stamps_version_and_writes_or_clears_warnings() { + use crate::constants::{INDEX_BUILDER_VERSION, SCIP_INDEX_BUILDER_VERSION_KEY}; + + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let indexer = super::csharp::CSharpSymbolIndexer::new(); + + // Degraded previous build: stale warnings + an old binary's stamp. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta_db + .put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["Old.csproj: Msbuild failed when processing the file"]"#, + ) + .unwrap(); + meta_db + .put(&mut wtxn, SCIP_INDEX_BUILDER_VERSION_KEY, "v0.0.0+1") + .unwrap(); + wtxn.commit().unwrap(); + + // A clean rebuild's meta write: stamp this build, clear the warnings. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + super::csharp::CSharpSymbolIndexer::write_rebuild_meta(&meta_db, &mut wtxn, &[]) + .expect("meta write must succeed"); + wtxn.commit().unwrap(); + + use crate::symbols::SymbolIndexer as _; + assert_eq!( + indexer.index_builder_version(&db).as_deref(), + Some(INDEX_BUILDER_VERSION), + "the rebuild meta write must stamp the producing build" + ); + assert!( + indexer.index_warnings(&db).is_empty(), + "a clean rebuild must clear the previous run's warnings" + ); + + // A degraded rebuild's meta write persists its warnings instead. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + super::csharp::CSharpSymbolIndexer::write_rebuild_meta( + &meta_db, + &mut wtxn, + &["Broken.csproj: Msbuild failed when processing the file".to_string()], + ) + .expect("meta write must succeed"); + wtxn.commit().unwrap(); + + // index_warnings_stored prepends the summary entry (summary first). + let warnings = indexer.index_warnings(&db); + assert_eq!( + warnings.len(), + 2, + "summary entry + the raw failed-project line, got: {warnings:?}" + ); + assert!( + warnings[0].contains("1 distinct failure(s)"), + "the summary entry must lead, got: {warnings:?}" + ); + assert_eq!( + warnings[1], + "Broken.csproj: Msbuild failed when processing the file".to_string(), + "a degraded rebuild's warnings must be persisted for the answer path" + ); +} diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 6d757e7a..d662033d 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -466,6 +466,15 @@ pub trait SymbolIndexer: Send + Sync { Vec::new() } + /// Which codesearch build produced the current symbol index + /// (`scip_meta[index_builder_version]`). `None` means absent or + /// unreadable — the C# rebuild gate treats that as stale, so a newly + /// deployed binary rebuilds the indexes it inherited itself at startup + /// instead of serving an index it never produced. + fn index_builder_version(&self, _db_path: &Path) -> Option { + None + } + /// The git HEAD sha the current symbol index was built for, when /// recorded. `None` means unknown (pre-fingerprint index, or git was /// unreadable at build time). Compare with the repository's current From 5352f7e33aef7fd2f42263bd2e60ba85995e72b5 Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 2 Oct 2026 14:52:53 +0200 Subject: [PATCH 32/77] =?UTF-8?q?tui:=20severity=20language=20for=20symbol?= =?UTF-8?q?-index=20indicators=20=E2=80=94=20alias=20text=20stays=20neutra?= =?UTF-8?q?l?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User ask (screenshot: half the table red): warnings-degraded indexes must not read as half the fleet being down. Color lives only on the language indicators, extracted into csharp_indicator()/ts_indicator(): - alias text: neutral (white; cyan+italic for remote mounts) — the old red/red-bold alias styling for partial and repo-status error is gone, in the table AND the detail panel (remote error alias was still red) - degraded (warnings): yellow C#⚠ / TS⚠ — a warning, not an error - general error: fully red C#! / TS! - detail panel status line follows: partial yellow, error stays red The AprimoBase rows that triggered this: one root cause, not 31k problems (netcoreapp2.2 reference resolution — CS0518/CS0246 cascades), now at least no longer shouting error-red for it. Tests: indicator severity table (degraded yellow, error red, indexing dim on both languages) + detail-panel partial→yellow (error→red pinned), tui_common suite 6/6 green. Review-fixes: - [Important] detail-panel alias still went red for remote+status-error with a stale matches-the-table comment → now cyan-italic always, comment states the neutrality rule - [Test] ts_indicator("indexing")/csharp_indicator("indexing", false) dim arms unpinned → pinned in the severity test --- src/serve/tui_common.rs | 178 ++++++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 73 deletions(-) diff --git a/src/serve/tui_common.rs b/src/serve/tui_common.rs index 070f1498..e75e80fb 100644 --- a/src/serve/tui_common.rs +++ b/src/serve/tui_common.rs @@ -10,7 +10,7 @@ use ratatui::layout::{Constraint, Layout, Margin, Rect}; use ratatui::style::{Color, Modifier, Style}; -use ratatui::text::{Line, Span}; +use ratatui::text::{Line, Span, Text}; use ratatui::widgets::{Block, Borders, Cell, Row, Table, TableState}; /// Format elapsed time since `started_at` as "Up xxd xxh xxm xxs". @@ -321,6 +321,48 @@ pub fn render_header( f.render_widget(ratatui::widgets::Paragraph::new(title_line), centered[0]); } +/// The C# alias-column indicator: `(glyph, style)` when the status carries +/// one. Color belongs to the indicator only — degraded (warnings) reads as +/// yellow ⚠, a general error as fully red `!` — so a table full of +/// warnings-degraded repos does not read as half the fleet being down. +pub fn csharp_indicator(status: &str, pulse: bool) -> Option<(&'static str, Style)> { + match status { + "ready" => Some((" C#·", Style::default().fg(Color::White))), + "partial" => Some((" C#⚠", Style::default().fg(Color::Yellow))), + "error" => Some(( + " C#!", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + )), + "indexing" => { + let style = if pulse { + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(Color::DarkGray) + }; + Some((" C#…", style)) + } + _ => None, + } +} + +/// The TypeScript alias-column indicator — same severity language as the C# +/// one (`ts_indicator("partial")` is yellow; TS has no live Partial source +/// yet, but the shared vocabulary must already agree). +pub fn ts_indicator(status: &str) -> Option<(&'static str, Style)> { + match status { + "ready" => Some((" TS·", Style::default().fg(Color::White))), + "partial" => Some((" TS⚠", Style::default().fg(Color::Yellow))), + "error" => Some(( + " TS!", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + )), + "indexing" => Some((" TS…", Style::default().fg(Color::DarkGray))), + _ => None, + } +} + pub fn render_table( f: &mut ratatui::Frame, area: Rect, @@ -385,67 +427,30 @@ pub fn render_table( }; let lock_cell = lock_cell(&repo.lock_mode); - // Alias text with optional C# indicator suffix, plus its base style. - let (mut alias_text, mut alias_style) = match repo.csharp_index.as_str() { - "ready" => ( - format!("{} C#·", repo.alias), - Style::default().fg(Color::White), - ), - // Degraded index: built despite survived workspace failures — - // red and loud, the detail panel carries the failures. - "partial" => ( - format!("{} C#⚠", repo.alias), - Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), - ), - "error" => ( - format!("{} C#!", repo.alias), - Style::default().fg(Color::Red), - ), - "indexing" => { - let s = if pulse_bright() { - Style::default() - .fg(Color::Yellow) - .add_modifier(Modifier::BOLD) - } else { - Style::default().fg(Color::DarkGray) - }; - (format!("{} C#…", repo.alias), s) - } - _ => (repo.alias.clone(), Style::default().fg(Color::White)), + // Alias text stays NEUTRAL — color lives only on the language + // indicators: yellow ⚠ for a degraded (warnings) index, fully red + // ! for a general error. A table with red aliases reads as half + // the fleet being down, which a warnings-degraded index is not. + let base_alias_style = if repo.is_remote { + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::ITALIC) + } else { + Style::default().fg(Color::White) }; + let mut alias_spans: Vec = + vec![Span::styled(repo.alias.clone(), base_alias_style)]; - // Append the TypeScript indicator alongside the C# one when a TS - // index exists. The alias column is the canonical multi-language - // symbol-index indicator (the status cell only carries C#). - match repo.typescript_index.as_str() { - "ready" => alias_text.push_str(" TS·"), - "error" => { - alias_text.push_str(" TS!"); - alias_style = alias_style.fg(Color::Red); - } - "partial" => { - alias_text.push_str(" TS⚠"); - alias_style = alias_style.fg(Color::Red); - } - "indexing" => alias_text.push_str(" TS…"), - _ => {} + if let Some((glyph, style)) = + csharp_indicator(repo.csharp_index.as_str(), pulse_bright()) + { + alias_spans.push(Span::styled(glyph, style)); } - - // Red bold alias if the repo is in an error state. - if repo.status == "error" { - alias_style = Style::default().fg(Color::Red).add_modifier(Modifier::BOLD); + if let Some((glyph, style)) = ts_indicator(repo.typescript_index.as_str()) { + alias_spans.push(Span::styled(glyph, style)); } - // Mounted remote projects render italic to signal they live on a - // peer (cyan unless an error already claimed the color). - if repo.is_remote { - alias_style = alias_style.add_modifier(Modifier::ITALIC); - if repo.status != "error" { - alias_style = alias_style.fg(Color::Cyan); - } - } - - let alias_cell = Cell::from(alias_text).style(alias_style); + let alias_cell = Cell::from(Text::from(Line::from(alias_spans))); Row::new(vec![ alias_cell, @@ -526,17 +531,12 @@ pub fn render_detail( repo.path.clone() }; - // Mounted remote projects show italic here too (matches the table): cyan - // normally, red when in error state so the color stays consistent with the - // table's error highlight. Local rows are unchanged (white bold). + // Mounted remote projects show italic here too (matches the table): cyan, + // never red — alias text stays neutral on every surface; the status line + // below carries the error severity. Local rows are unchanged (white bold). let alias_style = if repo.is_remote { - let color = if repo.status == "error" { - Color::Red - } else { - Color::Cyan - }; Style::default() - .fg(color) + .fg(Color::Cyan) .add_modifier(Modifier::BOLD | Modifier::ITALIC) } else { Style::default() @@ -1300,7 +1300,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Open C#!".to_string(), Color::Red), - "partial" => ("Open C#⚠".to_string(), Color::Red), + "partial" => ("Open C#⚠".to_string(), Color::Yellow), _ => ("Open".to_string(), Color::Green), }, "warm" => match csharp { @@ -1314,7 +1314,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Warm C#!".to_string(), Color::Red), - "partial" => ("Warm C#⚠".to_string(), Color::Red), + "partial" => ("Warm C#⚠".to_string(), Color::Yellow), _ => ("Warm".to_string(), Color::Yellow), }, "readonly" => ("Readonly".to_string(), Color::Cyan), @@ -1396,20 +1396,52 @@ mod tests { ); } - /// A degraded (partial) C# index renders red with the ⚠ glyph in the - /// detail panel — same urgency class as an error, distinct label. + /// A degraded (partial) C# index renders YELLOW with the ⚠ glyph in the + /// detail panel — a warning, not an error; only a general error is red. #[test] - fn detail_status_style_maps_partial_to_red_warning() { + fn detail_status_style_maps_partial_to_yellow_warning() { assert_eq!( detail_status_style("open", "partial"), - ("Open C#⚠".to_string(), Color::Red) + ("Open C#⚠".to_string(), Color::Yellow) ); assert_eq!( detail_status_style("warm", "partial"), - ("Warm C#⚠".to_string(), Color::Red) + ("Warm C#⚠".to_string(), Color::Yellow) + ); + assert_eq!( + detail_status_style("warm", "error").1, + Color::Red, + "a general error stays fully red" ); } + /// Alias-column indicators: the alias text is styled by the caller and + /// stays neutral; degraded reads yellow, only a general error is red. + #[test] + fn alias_indicators_use_yellow_for_degraded_and_red_only_for_errors() { + let (glyph, style) = csharp_indicator("partial", false).unwrap(); + assert_eq!(glyph, " C#⚠"); + assert_eq!(style, Style::default().fg(Color::Yellow)); + + let (glyph, style) = csharp_indicator("error", false).unwrap(); + assert_eq!(glyph, " C#!"); + assert_eq!( + style, + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD) + ); + + let (glyph, style) = ts_indicator("partial").unwrap(); + assert_eq!(glyph, " TS⚠"); + assert_eq!(style, Style::default().fg(Color::Yellow)); + + // The transient dim/pulse phase: indexing indicators must stay dim — + // an inherited bright style here once turned the whole alias red. + let (_, style) = csharp_indicator("indexing", false).unwrap(); + assert_eq!(style, Style::default().fg(Color::DarkGray)); + let (_, style) = ts_indicator("indexing").unwrap(); + assert_eq!(style, Style::default().fg(Color::DarkGray)); + } + /// The alias-column width budget must cover the actually rendered /// suffixes: the partial glyph ⚠ is double-width (EAW=W), so " C#⚠" is /// 5 display cols — budgeting it at 4 (the 1-col glyph family) would From 3b5614c2a1a4cffb0c65cad2441eac6c8249cbbd Mon Sep 17 00:00:00 2001 From: flupkede Date: Fri, 2 Oct 2026 15:29:56 +0200 Subject: [PATCH 33/77] helper: aggregate per-project compilation errors into one warnings line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User ask: 'welk project build errors heeft en dan een paar errors' — a broken reference resolution (the AprimoBase netcoreapp2.2 class) cascades into 26k+ Roslyn diagnostics for ONE project, and the index-warnings channel carried every one of them as a distinct entry (31421 distinct warnings in one index's meta, none readable). CompilationErrorSummaryLine emits one line per erroring project: count + dominant error code + first specimen (capped at 160 chars). The per- diagnostic lines are gone. Symbols are unaffected — a Roslyn compilation with errors yields a complete symbol tree (same reason IntelliSense works while the build is red); the warnings channel now says WHY refs may be incomplete instead of drowning the reader. Lines are emitted after the loop, loudest project first, via the tested WarningLineFor (the [WARN] prefix the Rust capture keys on is load- bearing — the emitted form IS the tested form): the channel cap must never crowd out the worst project. Tests: aggregation (count/dominant/first) + [WARN]-prefix contract + specimen cap; helper suite 16/16 green, build clean. Review-fixes: - [Important] emission glue untested (dropped [WARN] prefix = silent empty channel) → WarningLineFor wraps marker + summary; test pins prefix; emitter now calls WarningLineFor so the tested form is the emitted form - [Design] solution-order emission could crowd the 26k-error project out of the cap → emitted loudest-first after the loop --- helpers/csharp/SymbolIndexer.cs | 52 ++++++++++++++++++++++++-- helpers/csharp/tests/IndexerTests.cs | 55 ++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/helpers/csharp/SymbolIndexer.cs b/helpers/csharp/SymbolIndexer.cs index 8a49e4e9..1a34b2df 100644 --- a/helpers/csharp/SymbolIndexer.cs +++ b/helpers/csharp/SymbolIndexer.cs @@ -48,6 +48,10 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj Console.Error.WriteLine($"Compiling {totalProjects} project(s)..."); var compileSw = System.Diagnostics.Stopwatch.StartNew(); + // (error count, summary line) per erroring project — emitted after + // the loop, loudest first, so the warnings-channel cap can never + // crowd out the worst project: that is the one the user needs named. + var erroringProjects = new List<(int Count, string Line)>(); for (int i = 0; i < totalProjects; i++) { var project = projectList[i]; @@ -59,17 +63,31 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj continue; } - // Report diagnostics but don't abort + // Report diagnostics but don't abort. A single broken reference + // resolution cascades into tens of thousands of errors per + // project (CS0518 "System.Object not defined" turns every type + // use into CS0246) — that is ONE root cause, not N problems, so + // the warnings channel carries ONE aggregated line per project + // (count + dominant code + first specimen). Symbols below are + // still collected: a Roslyn compilation with errors yields a + // complete symbol tree — the same reason IntelliSense works + // while the build is red. var diagnostics = compilation.GetDiagnostics() - .Where(d => d.Severity == DiagnosticSeverity.Error); - foreach (var diag in diagnostics) + .Where(d => d.Severity == DiagnosticSeverity.Error) + .ToList(); + if (diagnostics.Count > 0) { - Console.Error.WriteLine($"[WARN] Compilation error in {project.Name}: {diag}"); + erroringProjects.Add((diagnostics.Count, + WarningLineFor(project.Name, diagnostics))); } CollectSymbols(compilation.GlobalNamespace, symbolMap); } compileSw.Stop(); + foreach (var (_, line) in erroringProjects.OrderByDescending(e => e.Count)) + { + Console.Error.WriteLine(line); + } Console.Error.WriteLine($"Compiled {totalProjects} project(s) in {compileSw.Elapsed.TotalSeconds:F1}s"); Console.Error.WriteLine($"Collected {symbolMap.Count} project-internal symbols — building definition index..."); @@ -130,6 +148,32 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj return index; } + /// One-line aggregation of a project's compilation errors for the + /// index-level warnings channel: count, dominant error code, first + /// specimen (capped — the full diagnostic can run very long). Wrapped + /// with the [WARN] marker the Rust capture keys on by + /// . One broken reference resolution + /// cascades into tens of thousands of diagnostics; the channel must + /// carry the root cause, not the cascade. + /// + internal static string CompilationErrorSummaryLine( + string project, IReadOnlyList errors) + { + var byCode = errors.GroupBy(d => d.Id) + .OrderByDescending(g => g.Count()).ToList(); + var first = errors[0].ToString(); + if (first.Length > 160) first = first[..160] + "…"; + return + $"Compilation errors in {project}: {errors.Count} error(s) — dominant " + + $"{byCode[0].Key} x{byCode[0].Count()}; first: {first}"; + } + + /// The channel-emittable form. The `[WARN] ` prefix is LOAD-BEARING: + /// the Rust capture (`is_helper_warning_line`) keys on it, so a dropped + /// prefix does not flood the channel — it silently empties it. + internal static string WarningLineFor(string project, IReadOnlyList errors) => + "[WARN] " + CompilationErrorSummaryLine(project, errors); + internal static void CollectSymbols(INamespaceSymbol ns, Dictionary map) { foreach (var child in ns.GetMembers()) diff --git a/helpers/csharp/tests/IndexerTests.cs b/helpers/csharp/tests/IndexerTests.cs index fa4e93bb..b7a5cc52 100644 --- a/helpers/csharp/tests/IndexerTests.cs +++ b/helpers/csharp/tests/IndexerTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Microsoft.CodeAnalysis; using Xunit; using ScipCsharp; @@ -214,4 +215,58 @@ public void OutputWriter_ProducesValidJson() // Verify indented output (has newlines) Assert.Contains('\n', json); } + + /// + /// A broken reference resolution cascades into tens of thousands of + /// compilation errors per project (one missing assembly → every type use + /// errors). The warnings channel must carry ONE aggregated line per + /// project — count + dominant code + first specimen — not the cascade. + /// + [Fact] + public void CompilationErrorSummary_AggregatesCascadePerProject() + { + static Diagnostic Err(string id, string message) + { + var descriptor = new DiagnosticDescriptor( + id, title: message, messageFormat: message, category: "Test", + DiagnosticSeverity.Error, isEnabledByDefault: true); + return Diagnostic.Create(descriptor, Location.None); + } + + var errors = new List + { + Err("CS0518", "Predefined type 'System.Object' is not defined or referenced"), + Err("CS0518", "Predefined type 'System.Object' is not defined or referenced"), + Err("CS0246", "The type or namespace name 'Dam' could not be found"), + }; + + var line = SymbolIndexer.CompilationErrorSummaryLine("Acme.Catalog", errors); + + Assert.StartsWith("Compilation errors in Acme.Catalog:", line); + Assert.Contains("3 error(s)", line); + Assert.Contains("dominant CS0518 x2", line); + Assert.Contains("first:", line); + + // The channel-emittable form carries the [WARN] marker the Rust + // capture keys on — a dropped prefix would silently EMPTY the + // index-warnings channel instead of flooding it. + var warning = SymbolIndexer.WarningLineFor("Acme.Catalog", errors); + Assert.StartsWith("[WARN] Compilation errors in", warning); + } + + /// The specimen is capped: a full Roslyn diagnostic string can run + /// hundreds of chars, and the warnings channel caps the whole list. + [Fact] + public void CompilationErrorSummary_CapsTheSpecimen() + { + var descriptor = new DiagnosticDescriptor( + "CS0001", title: new string('x', 400), messageFormat: new string('x', 400), + category: "Test", DiagnosticSeverity.Error, isEnabledByDefault: true); + var err = Diagnostic.Create(descriptor, Location.None); + + var line = SymbolIndexer.CompilationErrorSummaryLine("Proj", [err]); + + Assert.True(line.Length < 300, $"summary must be capped, got {line.Length} chars"); + Assert.Contains("…", line); + } } From 8462fec968de6d3bd709c10eb5670be48291a090 Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Fri, 2 Oct 2026 19:01:45 -0400 Subject: [PATCH 34/77] fix(tests): isolate watch tests from the global codesearchignore build_gitignore always layers ~/.codesearch/.codesearchignore in when it exists, so watchability assertions depend on the machine's home directory: test_is_watchable fails when the global file ignores *.json, and test_no_ignore_files_returns_none fails whenever a global file exists. CI has none, which is why this only surfaced in a local QC run. Point HOME/USERPROFILE at an empty temp dir during construction via EnvRestore + #[serial], for every watcher test in the module. --- src/watch/mod.rs | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/watch/mod.rs b/src/watch/mod.rs index 42ddf268..b10d4229 100644 --- a/src/watch/mod.rs +++ b/src/watch/mod.rs @@ -633,6 +633,8 @@ impl GitHeadWatcher { #[cfg(test)] mod tests { use super::*; + use crate::testing::EnvRestore; + use serial_test::serial; use std::fs; use std::process::Command; use tempfile::tempdir; @@ -672,9 +674,21 @@ mod tests { Ok(()) } + /// Build a `FileWatcher` with the home directory pointed at an empty + /// temporary directory, so the developer's real + /// `~/.codesearch/.codesearchignore` cannot leak into the assertions. + /// CI has no such file, which is why the ambient dependency went unnoticed. + fn watcher_with_isolated_home(root: PathBuf) -> FileWatcher { + let home = tempdir().unwrap(); + let home = home.path().to_string_lossy().into_owned(); + let _guard = EnvRestore::set(&[("HOME", &home), ("USERPROFILE", &home)]); + FileWatcher::new(root) + } + #[test] + #[serial] fn test_is_watchable() { - let watcher = FileWatcher::new(PathBuf::from("/tmp")); + let watcher = watcher_with_isolated_home(PathBuf::from("/tmp")); // Should NOT watch (ignored dirs) assert!(!watcher.is_watchable(Path::new("/tmp/.git/config"))); @@ -706,6 +720,7 @@ mod tests { } #[test] + #[serial] fn test_gitignore_rules_respected() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -717,7 +732,7 @@ mod tests { ) .unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!(watcher.gitignore.is_some(), "Should have loaded .gitignore"); // Should NOT watch (gitignored patterns) @@ -751,9 +766,10 @@ mod tests { #[test] #[ignore] // Requires actual filesystem events + #[serial] fn test_file_watcher() { let dir = tempdir().unwrap(); - let mut watcher = FileWatcher::new(dir.path().to_path_buf()); + let mut watcher = watcher_with_isolated_home(dir.path().to_path_buf()); watcher.start(100).unwrap(); @@ -769,6 +785,7 @@ mod tests { } #[test] + #[serial] fn test_codesearchignore_loaded_and_respected() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -776,7 +793,7 @@ mod tests { // Create repo-local .codesearchignore excluding tests/ fs::write(root.join(".codesearchignore"), "tests/\n").unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!( watcher.gitignore.is_some(), "Should have loaded .codesearchignore" @@ -793,6 +810,7 @@ mod tests { } #[test] + #[serial] fn test_codesearchignore_overrides_gitignore() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -802,7 +820,7 @@ mod tests { fs::write(root.join(".gitignore"), "logs/\n").unwrap(); fs::write(root.join(".codesearchignore"), "src/generated/\n").unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!(watcher.gitignore.is_some()); // .gitignore pattern takes effect @@ -823,11 +841,12 @@ mod tests { } #[test] + #[serial] fn test_no_ignore_files_returns_none() { let dir = tempdir().unwrap(); let root = dir.path(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!( watcher.gitignore.is_none(), "Should have no gitignore matcher when no ignore files exist" From 375a5c2aea5225d632cc375b4a36171a7343c0be Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Fri, 2 Oct 2026 20:09:41 -0400 Subject: [PATCH 35/77] feat: bound concurrent indexing work with configurable limits A serve hub started one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass git pull, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once and held many file/chunk windows in memory while ONNX inference (already serialised per model) queued behind all of them. Add a process-wide job gate plus settable per-inference and per-batch limits: - CODESEARCH_INDEX_JOBS (default 1): max concurrent heavy repo jobs. Attached to SharedStores in serve mode; queries never take it. Gated: incremental refresh, force reindex (gate-free inner body prevents a re-acquire deadlock), FSW batches and branch refreshes, and the HNSW builds in warmup / cold open / add-repo. - CODESEARCH_EMBED_THREADS: ONNX intra-op threads per session (unset keeps the previous all-cores default). - CODESEARCH_EMBED_PAUSE_MS: duty-cycle pause between background embedding mini-batches. - CODESEARCH_MAX_CHUNKS_PER_BATCH: split a refresh batch by accumulated chunks, not just file count. - CODESEARCH_MIN_FREE_MB: bounded free-memory floor before a job starts. The pool now releases the per-model lock once per ONNX mini-batch (embed_chunks_yielding) instead of holding it for a whole 200-file batch, so interactive query embeddings interleave with background refreshes. serve gains --index-jobs, --embed-threads, --embed-pause-ms and --min-free-mb, which override the matching variables. Standalone codesearch index / mcp runs attach no gate and are unchanged. --- CHANGELOG.md | 10 ++ README.md | 47 ++++++ src/cli/mod.rs | 94 ++++++++++++ src/embed/embedder.rs | 45 ++++-- src/embed/mod.rs | 82 +++++++++++ src/index/manager.rs | 334 +++++++++++++++++++++++++++++++++++------- src/lib.rs | 1 + src/limits.rs | 219 +++++++++++++++++++++++++++ src/limits_tests.rs | 125 ++++++++++++++++ src/main.rs | 1 + src/serve/mod.rs | 40 ++++- src/serve/tests.rs | 27 ++++ 12 files changed, 958 insertions(+), 67 deletions(-) create mode 100644 src/limits.rs create mode 100644 src/limits_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fa375c13..0c3ce564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,16 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.4.12] + +### Added + +- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. + +### Changed + +- **`codesearch serve` runs heavy repo jobs one at a time by default (`CODESEARCH_INDEX_JOBS=1`).** Warmup, cold opens, file-watcher batches and reindexes now queue behind a running refresh instead of all running at once; raise the value deliberately on machines with cores and memory to spare. Standalone `codesearch index` / `codesearch mcp` runs are unchanged (no gate is attached). `CODESEARCH_EMBED_THREADS` unset keeps the previous ONNX default (all cores), so single-repo index throughput is unchanged unless the variable is set. + ## [1.4.11] ### Changed diff --git a/README.md b/README.md index a09b30db..f5bf89d7 100644 --- a/README.md +++ b/README.md @@ -578,10 +578,57 @@ In the `codesearch serve` TUI, mounts appear in **italic/cyan**, distinguishing | `CODESEARCH_CACHE_MAX_MEMORY` | Embedding cache MB (default: 500) | | `CODESEARCH_MAX_LMDB_MAP_SIZE_MB` | Hard cap (MB) for LMDB auto-resize on `MDB_MAP_FULL`, applied to both the vector store and the persistent embedding cache (default: 16384 = 16GB; clamped to at least 1024). Raise this for very large corpora (millions of chunks) that legitimately exceed the default cap — see #189. | | `CODESEARCH_BATCH_SIZE` | Embedding batch size | +| `CODESEARCH_INCREMENTAL_BATCH_SIZE` | Files read/chunked per incremental-refresh batch (default: 200) | +| `CODESEARCH_INDEX_JOBS` | Max concurrent heavy repo jobs — refresh / reindex / HNSW build — in one serve process (default: 1; queries are never gated). See [Indexing limits](#indexing-limits) | +| `CODESEARCH_EMBED_THREADS` | ONNX intra-op threads per embedding session (unset = ONNX default, all cores) | +| `CODESEARCH_EMBED_PAUSE_MS` | Pause (ms) between background embedding mini-batches, a duty cycle for shared machines (default: 0) | +| `CODESEARCH_MIN_FREE_MB` | Free-memory floor (MB) a heavy job waits for (max 30 s, then proceeds with a warning) before starting (default: 0 = disabled) | +| `CODESEARCH_MAX_CHUNKS_PER_BATCH` | Hard cap on chunks held in memory per refresh batch; splits the batch early (default: 0 = disabled) | | `CODESEARCH_SCIP_CSHARP` | Override path to `scip-csharp` helper | | `CODESEARCH_EXTENSION_MAP` | Path to the extension→language map (default: `~/.codesearch/extensions.json`) — see [Extension map](#extension-map) | | `RUST_LOG` | Log level (e.g. `codesearch=debug`) | +### Indexing limits + +A serve hub with many repos used to start one full refresh/reindex/HNSW-build +pass per repo as soon as that repo had work. A mass `git pull`, a burst of +first queries, or several `POST /repos/:alias/reindex` calls therefore ran +many CPU-heavy passes at once, held many file/chunk windows in memory, and +queued the (already serialised) ONNX inference behind all of them. These knobs +bound that work; they are read at startup, so a shell or systemd unit can set +them without editing anything: + +| Knob | Default | Effect | +|------|---------|--------| +| `CODESEARCH_INDEX_JOBS` | `1` | Max repos doing a refresh / reindex / HNSW build at once. Queries never take this slot. | +| `CODESEARCH_EMBED_THREADS` | unset (all cores) | ONNX intra-op threads per embedding session. | +| `CODESEARCH_INCREMENTAL_BATCH_SIZE` | `200` | Files read/chunked per refresh batch. | +| `CODESEARCH_MAX_CHUNKS_PER_BATCH` | `0` (off) | Split a batch early once it reaches this many chunks (a few huge files can produce more memory than many small ones). | +| `CODESEARCH_EMBED_PAUSE_MS` | `0` | Sleep this long between background embedding mini-batches (duty cycle). | +| `CODESEARCH_MIN_FREE_MB` | `0` (off) | A job waits (bounded to 30 s, then proceeds with a warning) until this much memory is free. | + +`codesearch serve` also accepts `--index-jobs`, `--embed-threads`, +`--embed-pause-ms` and `--min-free-mb`, which set the corresponding variable +for that process (flags win over the environment). Standalone `codesearch +index` and `codesearch mcp` runs have no job gate and keep their previous +behaviour; the embedding and batching variables still apply to them. + +Example for a large shared machine (128 cores, ~30 repos, dozens of repos +indexed at once are the failure mode): + +```ini +# ~/.config/systemd/user/codesearch.service.d/limits.conf +[Service] +Environment=CODESEARCH_INDEX_JOBS=1 +Environment=CODESEARCH_EMBED_THREADS=8 +Environment=CODESEARCH_INCREMENTAL_BATCH_SIZE=100 +Environment=CODESEARCH_MAX_CHUNKS_PER_BATCH=5000 +Environment=CODESEARCH_EMBED_PAUSE_MS=25 +Environment=CODESEARCH_MIN_FREE_MB=4096 +# C# SCIP helpers are bounded separately; lower it for the same reason. +Environment=CSHARP_SCIP_CONCURRENCY=1 +``` + ### `.codesearchignore` Place in repo root. Gitignore syntax. Excludes paths from indexing: diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 64a052e0..5b7a007b 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -460,6 +460,26 @@ pub enum Commands { #[arg(long)] idle_suspend_secs: Option, + /// Max concurrent repo indexing jobs (refresh / reindex / HNSW build) + /// this serve runs. Overrides CODESEARCH_INDEX_JOBS (default 1). + #[arg(long)] + index_jobs: Option, + + /// ONNX intra-op threads per embedding session. Overrides + /// CODESEARCH_EMBED_THREADS (unset = ONNX default, all cores). + #[arg(long)] + embed_threads: Option, + + /// Milliseconds to pause between background embedding mini-batches. + /// Overrides CODESEARCH_EMBED_PAUSE_MS (default 0). + #[arg(long)] + embed_pause_ms: Option, + + /// Minimum free memory (MB) before a heavy indexing job may start. + /// Overrides CODESEARCH_MIN_FREE_MB (default 0 = disabled). + #[arg(long)] + min_free_mb: Option, + /// For `tui` action: serve URL to connect to #[arg(long, default_value = DEFAULT_SERVE_URL)] url: String, @@ -1234,6 +1254,10 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { no_tui, keep_warm_url, idle_suspend_secs, + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, url, api_key, } => { @@ -1258,6 +1282,24 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Some(mt) = model_type { warn_if_heavier_model(mt); } + // CLI overrides for the heavy-work limits. Serve is a single + // long-lived process, so exporting the overrides here keeps a + // single source of truth (the environment) for every + // component that reads them (job gate, embedder, refresh + // batching) without threading a config struct through each + // constructor. Unset flags change nothing. + if let Some(n) = index_jobs { + std::env::set_var(crate::limits::INDEX_JOBS_ENV, n.to_string()); + } + if let Some(n) = embed_threads { + std::env::set_var(crate::limits::EMBED_THREADS_ENV, n.to_string()); + } + if let Some(ms) = embed_pause_ms { + std::env::set_var(crate::limits::EMBED_PAUSE_MS_ENV, ms.to_string()); + } + if let Some(mb) = min_free_mb { + std::env::set_var(crate::limits::MIN_FREE_MB_ENV, mb.to_string()); + } crate::serve::run_serve( host, port, @@ -2094,6 +2136,58 @@ mod tests { assert_eq!(after_second.matches(HOOK_BEGIN).count(), 1); } + #[test] + fn test_serve_heavy_work_limit_flags_parse() { + let cli = Cli::try_parse_from([ + "codesearch", + "serve", + "--index-jobs", + "2", + "--embed-threads", + "4", + "--embed-pause-ms", + "25", + "--min-free-mb", + "4096", + ]) + .expect("cli parse should succeed"); + match cli.command { + Commands::Serve { + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, + .. + } => { + assert_eq!(index_jobs, Some(2)); + assert_eq!(embed_threads, Some(4)); + assert_eq!(embed_pause_ms, Some(25)); + assert_eq!(min_free_mb, Some(4096)); + } + _ => panic!("expected Serve command"), + } + } + + #[test] + fn test_serve_heavy_work_limit_flags_default_to_none() { + let cli = Cli::try_parse_from(["codesearch", "serve"]).expect("cli parse should succeed"); + match cli.command { + Commands::Serve { + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, + .. + } => { + assert_eq!(index_jobs, None, "unset flags must leave env/defaults"); + assert_eq!(embed_threads, None); + assert_eq!(embed_pause_ms, None); + assert_eq!(min_free_mb, None); + } + _ => panic!("expected Serve command"), + } + } + #[test] fn test_mcp_create_index_defaults_to_true() { let cli = Cli::try_parse_from(["codesearch", "mcp"]).expect("cli parse should succeed"); diff --git a/src/embed/embedder.rs b/src/embed/embedder.rs index 75472d15..5434902c 100644 --- a/src/embed/embedder.rs +++ b/src/embed/embedder.rs @@ -332,31 +332,50 @@ impl FastEmbedder { // Arena allocator provides fast memory reuse during inference. let cpu_ep = CPU::default().with_arena_allocator(true).build(); - let model = TextEmbedding::try_new( - TextInitOptions::new(model_type.to_fastembed_model()) - .with_show_download_progress(false) - .with_execution_providers(vec![cpu_ep]), - ) - .map_err(|e| anyhow!("Failed to initialize embedding model: {}", e))?; + let options = TextInitOptions::new(model_type.to_fastembed_model()) + .with_show_download_progress(false) + .with_execution_providers(vec![cpu_ep]); + + // Cap ONNX intra-op threads when CODESEARCH_EMBED_THREADS is set. + // Unset leaves fastembed/ONNX Runtime at its default (all cores): one + // inference alone can saturate a large machine, which starves every + // other task on a shared box. See `crate::limits`. + let options = match crate::limits::embed_threads() { + Some(threads) => options.with_intra_threads(threads), + None => options, + }; + + let model = TextEmbedding::try_new(options) + .map_err(|e| anyhow!("Failed to initialize embedding model: {}", e))?; Ok(Self { model, model_type }) } - /// Embed a batch of texts (processes in mini-batches to avoid OOM) - /// Uses adaptive batch size based on model dimensions - /// Can be overridden with CODESEARCH_BATCH_SIZE environment variable - pub fn embed_batch(&mut self, texts: Vec) -> Result>> { + /// Mini-batch size for a single ONNX `embed` call: `CODESEARCH_BATCH_SIZE` + /// when set, otherwise adaptive by model dimensions. + /// + /// Shared by [`Self::embed_batch`] and the serve pool's interleaved batch + /// loop (`EmbeddingServicePool::embed_chunks_yielding`), so both agree on + /// how long one ONNX call (and therefore one lock acquisition) lasts. + pub fn effective_batch_size(model_type: ModelType) -> usize { // Check for env var override (tune with CODESEARCH_BATCH_SIZE=N) - let batch_size = if let Ok(env_size) = std::env::var("CODESEARCH_BATCH_SIZE") { + if let Ok(env_size) = std::env::var("CODESEARCH_BATCH_SIZE") { env_size.parse().unwrap_or(256) } else { // Adaptive batch size: without arena allocator, ONNX frees buffers after each batch // so larger batches are faster without accumulating memory. - match self.model_type.dimensions() { + match model_type.dimensions() { d if d <= 384 => 256, // Small models (MiniLM etc.) d if d <= 768 => 128, // Medium models (BGE-base, Jina etc.) _ => 64, // Large models (BGE-large, MxBai etc.) } - }; + } + } + + /// Embed a batch of texts (processes in mini-batches to avoid OOM) + /// Uses adaptive batch size based on model dimensions + /// Can be overridden with CODESEARCH_BATCH_SIZE environment variable + pub fn embed_batch(&mut self, texts: Vec) -> Result>> { + let batch_size = Self::effective_batch_size(self.model_type); self.embed_batch_chunked(texts, batch_size) } diff --git a/src/embed/mod.rs b/src/embed/mod.rs index bc0a497f..b653e7ba 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -341,6 +341,47 @@ impl EmbeddingServicePool { guard.insert(model, arc.clone()); Ok(arc) } + + /// Embed `chunks`, releasing the service lock between ONNX mini-batches. + /// + /// The per-model `Mutex` is what serialises embedding + /// between repos and queries. Holding it for a whole refresh batch (one + /// `CODESEARCH_INCREMENTAL_BATCH_SIZE` window can hold thousands of + /// chunks) made an interactive query embedding queue behind minutes of + /// background inference. Each lock acquisition here covers exactly one + /// ONNX mini-batch ([`FastEmbedder::effective_batch_size`]), followed by + /// the optional `pause` duty cycle, so a long background pass yields the + /// model between mini-batches. + pub fn embed_chunks_yielding( + &self, + model: ModelType, + chunks: Vec, + pause: std::time::Duration, + ) -> Result> { + if chunks.is_empty() { + return Ok(Vec::new()); + } + let service = self.get(model)?; + let group_size = FastEmbedder::effective_batch_size(model).max(1); + let mut out: Vec = Vec::with_capacity(chunks.len()); + for group in chunks.chunks(group_size) { + { + // Recover from poisoning: a panicked embed service still yields + // a valid guard; the batch result carries correctness, and a + // poison here must not hard-fail every later batch for the + // process lifetime. + let mut guard = service + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut embedded = guard.embed_chunks(group.to_vec())?; + out.append(&mut embedded); + } + if !pause.is_zero() { + std::thread::sleep(pause); + } + } + Ok(out) + } } #[cfg(test)] @@ -353,6 +394,47 @@ mod tests { assert_eq!(model.dimensions(), 384); } + /// Mini-batch size drives how long one lock acquisition lasts; pin the + /// dimension-adaptive default and the env override. + #[test] + #[serial_test::serial] + fn effective_batch_size_defaults_by_dimensions_and_env_overrides() { + let _unset = crate::testing::EnvRestore::remove(&["CODESEARCH_BATCH_SIZE"]); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::AllMiniLML6V2Q), + 256, + "384-dim models use 256-text mini-batches" + ); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::EmbeddingGemma300MQ4), + 128, + "768-dim models use 128-text mini-batches" + ); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::BGELargeENV15), + 64, + ">768-dim models use 64-text mini-batches" + ); + + let _override = crate::testing::EnvRestore::set(&[("CODESEARCH_BATCH_SIZE", "7")]); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::EmbeddingGemma300MQ4), + 7, + "CODESEARCH_BATCH_SIZE must win over the adaptive default" + ); + } + + /// The interleaved embed path must not even load the model for an empty + /// batch; this is its no-op arm and the only one testable without ONNX. + #[test] + fn embed_chunks_yielding_empty_is_a_noop() { + let pool = EmbeddingServicePool::new(None); + let out = pool + .embed_chunks_yielding(ModelType::default(), Vec::new(), std::time::Duration::ZERO) + .expect("empty embed must succeed"); + assert!(out.is_empty()); + } + /// The index-metadata reader must invert `write_metadata_fields`, and must /// report "no answer" (None) for unknown/missing names rather than silently /// claiming the default — callers decide the fallback. diff --git a/src/index/manager.rs b/src/index/manager.rs index ff190f91..c256211b 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -200,9 +200,32 @@ pub struct SharedStores { /// Counter for number of file changes processed (indexed + removed) since serve start. /// Incremented by FSW batches and incremental refreshes. Read by TUI/dashboard. pub changes_count: std::sync::atomic::AtomicU64, + /// Process-wide heavy-job gate. `Some` in serve mode (attached by + /// `ServeState::try_open_stores`), `None` in standalone CLI/MCP processes. + /// Every whole-repo refresh/reindex/build takes one permit from it; see + /// [`crate::limits`]. + pub job_gate: Option>, } impl SharedStores { + /// Attach a process-wide heavy-job gate to these stores (serve mode). + pub fn with_job_gate(mut self, gate: Arc) -> Self { + self.job_gate = Some(gate); + self + } + + /// Acquire one heavy-job permit, or `None` when no gate is attached + /// (standalone processes keep their previous unbounded behaviour). + /// + /// Hold the returned permit for the whole job: that is what bounds how + /// many repos chunk/embed/build at once. Queries never take this permit. + pub async fn acquire_job_permit(&self) -> Option { + match self.job_gate.as_ref() { + Some(gate) => Some(gate.acquire().await), + None => None, + } + } + /// Create new shared stores from the database path (read-write mode). /// /// This acquires a writer lock. If another process already has the lock, @@ -238,6 +261,7 @@ impl SharedStores { writer_lock: lock, readonly: false, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, }) } @@ -257,6 +281,7 @@ impl SharedStores { writer_lock: None, readonly: true, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, }) } @@ -339,6 +364,18 @@ fn embed_service_choice(pool: Option<&Arc>) -> EmbeddingSo } } +/// True when the current refresh batch should stop accumulating files before +/// reading the next one. +/// +/// `consumed > 0` guarantees at least one file per batch, so a single file +/// larger than `max_chunks` still makes progress; `max_chunks == 0` disables +/// the cap entirely (the file-count batch size remains the only bound). +/// Extracted as a pure predicate so its edges are unit-testable without +/// loading ONNX. +fn chunk_cap_reached(consumed: usize, accumulated_chunks: usize, max_chunks: usize) -> bool { + consumed > 0 && max_chunks > 0 && accumulated_chunks >= max_chunks +} + impl IndexManager { /// Cancellation guard shared by every cancellable indexing function. /// @@ -600,6 +637,36 @@ impl IndexManager { cancel_token: &CancellationToken, embedding_pool: Option<&Arc>, heartbeat: Option<&IndexingHeartbeat>, + ) -> Result<()> { + // Serialise this whole refresh behind the process-wide heavy-job gate + // (when one is attached): the permit is held across the file walk, the + // chunk/embed batches and the final HNSW build, so at most + // `CODESEARCH_INDEX_JOBS` repos do this at once. Queries never take it. + let _job_permit = stores.acquire_job_permit().await; + Self::perform_incremental_refresh_inner( + codebase_path, + db_path, + stores, + cancel_token, + embedding_pool, + heartbeat, + ) + .await + } + + /// Gate-free body of [`Self::perform_incremental_refresh_with_stores`]. + /// + /// Split out for `force_reindex_with_stores`, which holds the gate permit + /// across the destructive clear phase as well and must call this directly: + /// re-entering the public wrapper would re-acquire the gate and deadlock + /// under `CODESEARCH_INDEX_JOBS=1`. + async fn perform_incremental_refresh_inner( + codebase_path: &Path, + db_path: &Path, + stores: &SharedStores, + cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, + heartbeat: Option<&IndexingHeartbeat>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::chunker::SemanticChunker; @@ -799,24 +866,43 @@ impl IndexManager { // sync. Batching bounds peak memory to O(batch), not O(total delta), // so a delta of any size is now safe — it just takes longer, spread // across sequential batches. See `INCREMENTAL_REFRESH_BATCH_SIZE`. + // + // A batch is additionally cut short once its accumulated chunks reach + // `CODESEARCH_MAX_CHUNKS_PER_BATCH` (when set): file count alone is a + // poor memory proxy when a few files can produce tens of thousands of + // chunks. if !changed_files.is_empty() { let batch_size = std::env::var(crate::constants::INCREMENTAL_REFRESH_BATCH_SIZE_ENV) .ok() .and_then(|s| s.parse::().ok()) .filter(|&n| n > 0) .unwrap_or(crate::constants::INCREMENTAL_REFRESH_BATCH_SIZE); - let total_batches = changed_files.len().div_ceil(batch_size); + let max_chunks = crate::limits::max_chunks_per_batch(); + let embed_pause = crate::limits::embed_pause(); + // Upper-bound estimate for the log line; the chunk cap can split + // this into more batches than the file count alone implies. + let estimated_batches = changed_files.len().div_ceil(batch_size); info!( - "🔄 Processing {} changed files in {} batch(es) of up to {} file(s) each...", + "🔄 Processing {} changed files in ~{} batch(es) of up to {} file(s) each \ + (chunk cap per batch: {})...", changed_files.len(), - total_batches, - batch_size + estimated_batches, + batch_size, + if max_chunks == 0 { + "none".to_string() + } else { + max_chunks.to_string() + } ); let cache_dir = crate::constants::get_global_models_cache_dir()?; let mut total_indexed = 0usize; + let mut cursor = 0usize; + let mut batch_idx = 0usize; - for (batch_idx, file_batch) in changed_files.chunks(batch_size).enumerate() { + while cursor < changed_files.len() { + let take = (changed_files.len() - cursor).min(batch_size); + let files_for_embed = changed_files[cursor..cursor + take].to_vec(); // Abort between batches if the repo was removed mid-index. Self::ensure_indexing_active(cancel_token)?; // A batch of a large delta can run for minutes — renew the @@ -832,7 +918,11 @@ impl IndexManager { // so it never runs on a tokio worker thread. The `EmbeddingService` // and `SemanticChunker` are built inside the closure because they // are not needed on the async side and may not be `Send`. - let files_for_embed = file_batch.to_vec(); + // + // The closure consumes as many of the offered files as fit the + // chunk cap (always at least one, so every iteration makes + // progress) and reports how many it took, so the outer loop's + // bookkeeping covers exactly the files that were processed. let cache_dir_for_batch = cache_dir.clone(); let root_for_batch = codebase_path.to_path_buf(); // Clone the token into the blocking closure so a cancel arriving @@ -840,18 +930,30 @@ impl IndexManager { // per-file, not only once the whole batch returns. let batch_cancel = cancel_token.clone(); let embed_source = embed_service_choice(embedding_pool); - let embedded_chunks = tokio::task::spawn_blocking( - move || -> Result> { + let (embedded_chunks, consumed) = tokio::task::spawn_blocking( + move || -> Result<(Vec, usize)> { let mut chunker = SemanticChunker::new(100, 2000, 10); let mut all_chunks = Vec::new(); + let mut consumed = 0usize; for file in &files_for_embed { + // Stop once this batch has reached its chunk budget. + // `consumed > 0` guarantees at least one file per + // iteration even if that one file alone exceeds the + // cap, so the outer loop always advances. + if chunk_cap_reached(consumed, all_chunks.len(), max_chunks) { + break; + } // Mid-embed cancellation point: abort inside the // spawn_blocking task so we stop reading/chunking/ // embedding further files in this batch promptly. if batch_cancel.is_cancelled() { return Err(anyhow::anyhow!("indexing cancelled")); } + // Count the file before the read: a read failure + // still advances the outer cursor, so the loop can + // never stall on an unreadable file. + consumed += 1; let content = match std::fs::read_to_string(&file.path) { Ok(c) => c, Err(_) => continue, @@ -867,54 +969,46 @@ impl IndexManager { } if all_chunks.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), consumed)); } - // NOTE: embed_chunks runs a single ONNX inference over - // the whole batch atomically, so it is not interruptible - // mid-call. Worst-case cancel latency is bounded to one - // batch's embed (INCREMENTAL_REFRESH_BATCH_SIZE=200 - // files); the per-file check above bounds the read/chunk - // phase that precedes it. - match embed_source { - // Serve process: the pool already holds the - // persistent cache's LMDB env open — a fresh - // EmbeddingService here is refused by the - // process-global registry and would re-embed - // everything without cache. + // Serve process: the pool already holds the persistent + // cache's LMDB env open — a fresh `EmbeddingService` + // here is refused by the process-global registry and + // would re-embed everything without cache. + // + // `embed_chunks_yielding` holds the per-model lock for + // one ONNX mini-batch at a time (plus the configured + // pause), so an interactive query embedding is not stuck + // behind this whole file batch. + let embedded = match embed_source { EmbeddingSource::Pool(pool) => { - let service = pool.get(embed_model)?; - // Recover from poisoning: a panicked embed - // service still yields a valid guard (the - // batch's embed_chunks result is what carries - // correctness); a poison here must not - // hard-fail every later batch for the - // process lifetime. - let mut guard = service - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.embed_chunks(all_chunks) + pool.embed_chunks_yielding(embed_model, all_chunks, embed_pause)? } EmbeddingSource::Fresh => { let mut embedding_service = EmbeddingService::with_cache_dir( embed_model, Some(cache_dir_for_batch.as_path()), )?; - embedding_service.embed_chunks(all_chunks) + embedding_service.embed_chunks(all_chunks)? } - } + }; + Ok((embedded, consumed)) }, ) .await .map_err(|e| { - anyhow::anyhow!( - "chunk+embed task panicked (batch {}/{}): {}", - batch_idx + 1, - total_batches, - e - ) + anyhow::anyhow!("chunk+embed task panicked (batch {}): {}", batch_idx + 1, e) })??; + // The closure always consumes at least one file; guard the + // cursor arithmetic against a future regression that would + // otherwise spin forever. + debug_assert!(consumed > 0, "refresh batch made no progress"); + let file_batch = &changed_files[cursor..cursor + consumed]; + cursor += consumed; + batch_idx += 1; + // A cancel arriving after embed completed but before we commit // the batch to the stores must skip the insert + the final // build_index, so a removed repo never receives fresh data. @@ -922,9 +1016,8 @@ impl IndexManager { if !embedded_chunks.is_empty() { info!( - "📦 Batch {}/{}: embedding {} chunks with model {}...", - batch_idx + 1, - total_batches, + "📦 Batch {}: embedding {} chunks with model {}...", + batch_idx, embedded_chunks.len(), embed_model.short_name() ); @@ -1007,7 +1100,7 @@ impl IndexManager { info!( "✅ Indexed {} chunks across {} batch(es)", - total_indexed, total_batches + total_indexed, batch_idx ); } @@ -1079,6 +1172,12 @@ impl IndexManager { // Bail before clearing any store data if the repo was already removed. Self::ensure_indexing_active(cancel_token)?; + // Hold the process-wide heavy-job gate across the destructive clear AND + // the full reindex. The nested call goes to the gate-free inner body so + // the permit is not re-acquired (which would deadlock under + // `CODESEARCH_INDEX_JOBS=1`). + let _job_permit = stores.acquire_job_permit().await; + // ── Step 0: Read and preserve metadata BEFORE clearing anything ── // This is defensive: the DB may be incomplete (no metadata.json at all), // or clear() may indirectly remove it. We preserve model info so we can @@ -1187,7 +1286,7 @@ impl IndexManager { info!("✅ Stores cleared, metadata preserved. Starting full reindex..."); // ── Step 5: Reindex — all files treated as "changed" since metadata is empty ── - Self::perform_incremental_refresh_with_stores( + Self::perform_incremental_refresh_inner( codebase_path, db_path, stores, @@ -1970,6 +2069,11 @@ impl IndexManager { // Bail before touching any store if the repo was removed. Self::ensure_indexing_active(cancel_token)?; + // One permit for the whole watcher batch: a batch can hold hundreds of + // files, each re-embedded and followed by an HNSW rebuild, and every + // repo's watcher would otherwise run one at the same time. + let _job_permit = stores.acquire_job_permit().await; + // Enable quiet mode during FSW batch processing to suppress verbose embedding output set_quiet(true); @@ -2131,6 +2235,9 @@ impl IndexManager { // Abort before the filesystem walk if the repo was already removed. Self::ensure_indexing_active(cancel_token)?; + // One permit for the whole branch-refresh pass (walk, delete, re-index). + let _job_permit = stores.acquire_job_permit().await; + let result: Result<()> = async { // Phase 1: Discover current files on disk. // `walk()` is synchronous + I/O-heavy — offload off the async executor. @@ -2469,11 +2576,11 @@ impl IndexManager { // registry and would degrade this path to cache-less re-embedding. let embedded_chunks = match embed_service_choice(embedding_pool) { EmbeddingSource::Pool(pool) => { - let service = pool.get(embed_model)?; - let mut guard = service - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.embed_chunks(chunks)? + // The FSW paths hold a job permit (`process_batch_with_stores` + // / `refresh_index_with_stores`); interleave the model lock per + // mini-batch so a concurrent query embedding is not stuck + // behind this file. + pool.embed_chunks_yielding(embed_model, chunks, std::time::Duration::ZERO)? } EmbeddingSource::Fresh => { let cache_dir = crate::constants::get_global_models_cache_dir()?; @@ -2712,6 +2819,7 @@ mod tests { writer_lock: None, readonly: false, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, } } @@ -2753,6 +2861,132 @@ mod tests { assert!(db_path.join(WRITER_LOCK_FILE).exists()); } + /// The chunk-cap predicate is the batch-splitting decision; pin its edges + /// here because the loop that uses it runs inside `spawn_blocking` and + /// would otherwise need the ONNX embedding model. + #[test] + fn chunk_cap_predicate_table() { + let cases = [ + // (consumed, accumulated_chunks, cap, expected) + (0, 10_000, 5_000, false), // first file of a batch always proceeds + (1, 4_999, 5_000, false), // below the cap + (1, 5_000, 5_000, true), // exactly at the cap + (1, 5_001, 5_000, true), // one file overshot the cap + (7, 10_000, 0, false), // cap disabled + (1, 0, 0, false), // cap disabled, nothing accumulated + ]; + for (consumed, accumulated, cap, expected) in cases { + assert_eq!( + chunk_cap_reached(consumed, accumulated, cap), + expected, + "chunk_cap_reached(consumed={consumed}, accumulated={accumulated}, cap={cap})" + ); + } + } + + /// Standalone CLI / `codesearch mcp` stores carry no gate and must keep + /// their previous (unbounded) behaviour. + #[tokio::test] + async fn acquire_job_permit_is_none_for_ungated_stores() { + let temp = tempdir().unwrap(); + let stores = create_test_stores(&temp.path().join("db"), 4).await; + assert!(stores.acquire_job_permit().await.is_none()); + } + + /// A gated refresh must (a) wait while the only slot is held and (b) + /// release the slot when it finishes. The repo has NO changed files, so the + /// refresh completes without loading the ONNX model. + #[tokio::test] + async fn gated_refresh_waits_for_a_free_slot_and_releases_it() { + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + + let gate = crate::limits::JobGate::with_config(1, 0); + let held = gate.acquire().await; + let stores = Arc::new( + create_test_stores(&db_path, 4) + .await + .with_job_gate(Arc::clone(&gate)), + ); + + let mut refresh = tokio::spawn({ + let stores = Arc::clone(&stores); + let codebase_path = codebase_path.clone(); + let db_path = db_path.clone(); + async move { + IndexManager::perform_incremental_refresh_with_stores( + &codebase_path, + &db_path, + &stores, + &CancellationToken::new(), + None, + None, + ) + .await + } + }); + + assert!( + tokio::time::timeout(std::time::Duration::from_millis(150), &mut refresh) + .await + .is_err(), + "the refresh must wait while the only job slot is held" + ); + + drop(held); + let result = tokio::time::timeout(std::time::Duration::from_secs(10), refresh) + .await + .expect("refresh must complete once the slot is free") + .expect("refresh task must not panic"); + assert!(result.is_ok(), "refresh failed: {:?}", result.err()); + + // The permit was released: the gate can be acquired again immediately. + let _again = tokio::time::timeout(std::time::Duration::from_secs(1), gate.acquire()) + .await + .expect("the refresh must release its job permit"); + } + + /// Regression for the wrapper/inner split: `force_reindex_with_stores` + /// holds the gate permit across the destructive clear and then calls the + /// gate-free inner body. Calling the public wrapper again would re-acquire + /// the only permit and hang forever under `CODESEARCH_INDEX_JOBS=1` (this + /// test times out instead of hanging only because of the explicit + /// `timeout`). + #[tokio::test] + async fn force_reindex_under_gate_does_not_reacquire_the_permit() { + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + + let gate = crate::limits::JobGate::with_config(1, 0); + let stores = create_test_stores(&db_path, 4) + .await + .with_job_gate(Arc::clone(&gate)); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + IndexManager::force_reindex_with_stores( + &codebase_path, + &db_path, + &stores, + None, + &CancellationToken::new(), + None, + ), + ) + .await + .expect("force reindex must not deadlock on its own job permit"); + + assert!(result.is_ok(), "force reindex failed: {:?}", result.err()); + } + #[tokio::test] async fn test_refresh_no_metadata_early_return() { // When metadata.json doesn't exist, refresh should return Ok early diff --git a/src/lib.rs b/src/lib.rs index f50f2ae3..7539e797 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod federation; pub mod file; pub mod fts; pub mod index; +pub mod limits; pub mod lmdb_registry; pub mod logger; pub mod mcp; diff --git a/src/limits.rs b/src/limits.rs new file mode 100644 index 00000000..1910310f --- /dev/null +++ b/src/limits.rs @@ -0,0 +1,219 @@ +//! Process-wide limits for heavy indexing work. +//! +//! `codesearch serve` hosts many repos, and each repo's refresh/reindex pass +//! is CPU- and memory-heavy: file reads, tree-sitter chunking, ONNX +//! embedding, HNSW graph construction. Without a process-wide budget, N repos +//! with work start N concurrent passes — every core is saturated, and N +//! file/chunk windows are held in memory at once even though ONNX inference +//! itself already serialises on the per-model mutex. +//! +//! This module provides one shared budget ([`JobGate`]) plus the per-inference +//! and per-batch knobs that bound what a single job costs: +//! +//! | Variable | Default | Meaning | +//! |---|---|---| +//! | `CODESEARCH_INDEX_JOBS` | `1` | Max concurrent heavy repo jobs per process | +//! | `CODESEARCH_EMBED_THREADS` | unset (ONNX default = all cores) | ONNX intra-op threads per session | +//! | `CODESEARCH_EMBED_PAUSE_MS` | `0` | Pause between background embedding mini-batches | +//! | `CODESEARCH_MIN_FREE_MB` | `0` (off) | Free-memory floor before a job may start | +//! | `CODESEARCH_MAX_CHUNKS_PER_BATCH` | `0` (off) | Hard cap on chunks held per refresh batch | +//! +//! All values are read from the environment so a systemd unit or shell can set +//! them without a config file. See the README section "Indexing limits" for +//! recommended values on shared machines. +//! +//! The gate is attached to [`crate::index::SharedStores`] at open time in +//! serve mode; standalone processes (CLI index, `codesearch mcp`) leave it +//! `None` and behave exactly as before. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +/// Max concurrent heavy repo jobs in this process (>= 1). +pub const INDEX_JOBS_ENV: &str = "CODESEARCH_INDEX_JOBS"; +/// ONNX intra-op thread count per embedding session (>= 1; unset = all cores). +pub const EMBED_THREADS_ENV: &str = "CODESEARCH_EMBED_THREADS"; +/// Milliseconds to pause between background embedding mini-batches. +pub const EMBED_PAUSE_MS_ENV: &str = "CODESEARCH_EMBED_PAUSE_MS"; +/// Free-memory floor (MB) before a heavy job may start (0 = disabled). +pub const MIN_FREE_MB_ENV: &str = "CODESEARCH_MIN_FREE_MB"; +/// Hard cap on chunks held in memory per refresh batch (0 = disabled). +pub const MAX_CHUNKS_PER_BATCH_ENV: &str = "CODESEARCH_MAX_CHUNKS_PER_BATCH"; + +/// Default for [`index_jobs`]: strictly one heavy repo job at a time. +/// +/// Embedding is already serialised by the per-model mutex, so running another +/// repo's refresh concurrently buys little throughput while multiplying peak +/// memory (each queued job holds its own chunk window) and store-lock +/// contention. Raise it deliberately, with `CODESEARCH_EMBED_THREADS` set, on +/// machines with cores and memory to spare. +pub const DEFAULT_INDEX_JOBS: usize = 1; + +const DEFAULT_EMBED_PAUSE_MS: u64 = 0; +const DEFAULT_MIN_FREE_MB: u64 = 0; +const DEFAULT_MAX_CHUNKS_PER_BATCH: usize = 0; + +/// How often the memory admission check re-reads available memory. +const MEMORY_POLL_INTERVAL: Duration = Duration::from_millis(500); +/// How long a job waits for memory before proceeding with a warning. A +/// permanent wait would be a deadlock (the running job's ONNX arena does not +/// return memory to the OS), so this is deliberately bounded. +const MEMORY_WAIT_MAX: Duration = Duration::from_secs(30); +/// Log when a job waited at least this long for a free slot. +const SLOW_ACQUIRE_LOG_AFTER: Duration = Duration::from_secs(2); + +fn env_parsed(name: &str) -> Option { + std::env::var(name) + .ok() + .and_then(|s| s.trim().parse::().ok()) +} + +/// Max concurrent heavy repo jobs. Values below 1 fall back to the default. +pub fn index_jobs() -> usize { + env_parsed::(INDEX_JOBS_ENV) + .filter(|&n| n >= 1) + .unwrap_or(DEFAULT_INDEX_JOBS) +} + +/// ONNX intra-op threads per embedding session. +/// +/// `None` (unset, or a value below 1) leaves fastembed/ONNX Runtime at its +/// default, which uses every available core. Set a value to cap one inference +/// on a shared machine. +pub fn embed_threads() -> Option { + env_parsed::(EMBED_THREADS_ENV).filter(|&n| n >= 1) +} + +/// Duty-cycle pause between background embedding mini-batches. +pub fn embed_pause() -> Duration { + Duration::from_millis(env_parsed::(EMBED_PAUSE_MS_ENV).unwrap_or(DEFAULT_EMBED_PAUSE_MS)) +} + +/// Free-memory floor in MB for starting a heavy job (0 = disabled). +pub fn min_free_mb() -> u64 { + env_parsed::(MIN_FREE_MB_ENV).unwrap_or(DEFAULT_MIN_FREE_MB) +} + +/// Hard cap on chunks held in memory per refresh batch (0 = disabled). +pub fn max_chunks_per_batch() -> usize { + env_parsed::(MAX_CHUNKS_PER_BATCH_ENV).unwrap_or(DEFAULT_MAX_CHUNKS_PER_BATCH) +} + +/// Available (reclaimable) system memory in MB, read fresh on each call. +pub fn available_memory_mb() -> u64 { + let mut sys = sysinfo::System::new(); + sys.refresh_memory(); + sys.available_memory() / (1024 * 1024) +} + +/// Bounds how many heavy repo jobs run concurrently in one process. +/// +/// A "heavy job" is a whole refresh / reindex / vector-index build pass, not a +/// single inference or a query. Query embeddings never take this gate, so +/// interactive search stays responsive while a background job runs. +#[derive(Debug)] +pub struct JobGate { + permits: Arc, + index_jobs: usize, + min_free_mb: u64, + memory_wait: Duration, +} + +impl JobGate { + /// Build a gate from the environment (the normal serve path). + pub fn from_env() -> Arc { + Self::with_config(index_jobs(), min_free_mb()) + } + + /// Build a gate with an explicit job count and memory floor. + pub fn with_config(index_jobs: usize, min_free_mb: u64) -> Arc { + Self::with_config_and_wait(index_jobs, min_free_mb, MEMORY_WAIT_MAX) + } + + /// Build a gate with a custom memory wait ceiling. Test seam: production + /// callers use [`Self::from_env`]. + pub fn with_config_and_wait( + index_jobs: usize, + min_free_mb: u64, + memory_wait: Duration, + ) -> Arc { + let index_jobs = index_jobs.max(1); + Arc::new(Self { + permits: Arc::new(Semaphore::new(index_jobs)), + index_jobs, + min_free_mb, + memory_wait, + }) + } + + /// Number of concurrent job slots this gate allows. + pub fn index_jobs(&self) -> usize { + self.index_jobs + } + + /// Acquire one job slot, then (when enabled) wait, bounded, for the free + /// memory floor. The caller holds the returned permit for the whole job. + pub async fn acquire(&self) -> JobPermit { + let wait_start = Instant::now(); + let permit = self + .permits + .clone() + .acquire_owned() + .await + .expect("job gate semaphore is never closed"); + let waited = wait_start.elapsed(); + if waited >= SLOW_ACQUIRE_LOG_AFTER { + tracing::info!( + "⏳ Indexing job waited {:.1}s for a free slot ({} slot(s) total; CODESEARCH_INDEX_JOBS)", + waited.as_secs_f64(), + self.index_jobs + ); + } + if self.min_free_mb > 0 { + self.wait_for_memory().await; + } + JobPermit { _permit: permit } + } + + async fn wait_for_memory(&self) { + let free = available_memory_mb(); + if free >= self.min_free_mb { + return; + } + tracing::info!( + "⏳ Free memory is {} MB, below the {} MB floor (CODESEARCH_MIN_FREE_MB); \ + waiting up to {:.0}s before starting the job", + free, + self.min_free_mb, + self.memory_wait.as_secs_f64() + ); + let deadline = Instant::now() + self.memory_wait; + loop { + tokio::time::sleep(MEMORY_POLL_INTERVAL).await; + let free = available_memory_mb(); + if free >= self.min_free_mb { + return; + } + if Instant::now() >= deadline { + tracing::warn!( + "⚠️ Free memory still below CODESEARCH_MIN_FREE_MB \ + ({} MB available, {} MB required); starting the job anyway", + free, + self.min_free_mb + ); + return; + } + } + } +} + +/// Held for the duration of one heavy job; dropping it releases the slot. +#[derive(Debug)] +pub struct JobPermit { + _permit: OwnedSemaphorePermit, +} + +#[cfg(test)] +#[path = "limits_tests.rs"] +mod tests; diff --git a/src/limits_tests.rs b/src/limits_tests.rs new file mode 100644 index 00000000..fac73292 --- /dev/null +++ b/src/limits_tests.rs @@ -0,0 +1,125 @@ +use super::*; +use crate::testing::EnvRestore; +use serial_test::serial; + +/// Remove every limit env var so an inherited shell value cannot change the +/// default assertions. The process environment is shared, so every test that +/// touches it is `#[serial]` and guarded by `EnvRestore`. +fn clear_env() -> EnvRestore { + EnvRestore::remove(&[ + INDEX_JOBS_ENV, + EMBED_THREADS_ENV, + EMBED_PAUSE_MS_ENV, + MIN_FREE_MB_ENV, + MAX_CHUNKS_PER_BATCH_ENV, + ]) +} + +#[test] +#[serial] +fn defaults_are_conservative() { + let _env = clear_env(); + assert_eq!(index_jobs(), DEFAULT_INDEX_JOBS); + assert_eq!( + embed_threads(), + None, + "unset must leave ONNX at its default (all cores)" + ); + assert_eq!(embed_pause(), Duration::ZERO); + assert_eq!(min_free_mb(), 0, "memory floor is opt-in"); + assert_eq!(max_chunks_per_batch(), 0, "chunk cap is opt-in"); +} + +#[test] +#[serial] +fn env_overrides_parse() { + let _env = clear_env(); + let _set = EnvRestore::set(&[ + (INDEX_JOBS_ENV, "3"), + (EMBED_THREADS_ENV, "4"), + (EMBED_PAUSE_MS_ENV, "25"), + (MIN_FREE_MB_ENV, "4096"), + (MAX_CHUNKS_PER_BATCH_ENV, "5000"), + ]); + assert_eq!(index_jobs(), 3); + assert_eq!(embed_threads(), Some(4)); + assert_eq!(embed_pause(), Duration::from_millis(25)); + assert_eq!(min_free_mb(), 4096); + assert_eq!(max_chunks_per_batch(), 5000); +} + +#[test] +#[serial] +fn invalid_values_fall_back() { + let _env = clear_env(); + let _set = EnvRestore::set(&[ + (INDEX_JOBS_ENV, "0"), + (EMBED_THREADS_ENV, "0"), + (EMBED_PAUSE_MS_ENV, "soon"), + (MIN_FREE_MB_ENV, "lots"), + (MAX_CHUNKS_PER_BATCH_ENV, "-1"), + ]); + assert_eq!( + index_jobs(), + DEFAULT_INDEX_JOBS, + "0 job slots is meaningless" + ); + assert_eq!(embed_threads(), None, "0 threads means 'ONNX default'"); + assert_eq!(embed_pause(), Duration::ZERO); + assert_eq!(min_free_mb(), 0); + assert_eq!(max_chunks_per_batch(), 0); +} + +#[tokio::test] +async fn gate_serialises_jobs_when_configured_to_one() { + let gate = JobGate::with_config(1, 0); + let first = gate.acquire().await; + assert!( + tokio::time::timeout(Duration::from_millis(50), gate.acquire()) + .await + .is_err(), + "a second job must wait while the only slot is held" + ); + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), gate.acquire()) + .await + .expect("dropping the permit must release the slot"); + drop(second); +} + +#[tokio::test] +async fn gate_allows_the_configured_number_of_parallel_jobs() { + let gate = JobGate::with_config(2, 0); + let first = gate.acquire().await; + let second = gate.acquire().await; + assert!( + tokio::time::timeout(Duration::from_millis(50), gate.acquire()) + .await + .is_err(), + "a third job must wait once both slots are held" + ); + drop(first); + drop(second); +} + +#[tokio::test] +async fn memory_floor_zero_never_waits() { + let gate = JobGate::with_config_and_wait(1, 0, Duration::from_secs(3600)); + let permit = tokio::time::timeout(Duration::from_millis(200), gate.acquire()) + .await + .expect("no memory wait when the floor is disabled"); + drop(permit); +} + +#[tokio::test] +async fn memory_wait_is_bounded() { + // A floor no machine can meet: acquire must return after the (shortened) + // memory wait instead of blocking forever. + let gate = JobGate::with_config_and_wait(1, u64::MAX, Duration::from_millis(50)); + let start = Instant::now(); + let permit = tokio::time::timeout(Duration::from_secs(5), gate.acquire()) + .await + .expect("the memory wait must be bounded"); + assert!(start.elapsed() >= Duration::from_millis(50)); + drop(permit); +} diff --git a/src/main.rs b/src/main.rs index a08f1cb1..24035e65 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ mod federation; mod file; mod fts; mod index; +mod limits; mod lmdb_registry; mod logger; mod mcp; diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 5ea6d09d..6e1a7773 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -319,6 +319,12 @@ pub(crate) struct ServeState { /// with the model of the repo it targets. Mirrors the `symbol_registry` /// pattern. embedding_pool: Arc, + /// Process-wide heavy-job gate (`CODESEARCH_INDEX_JOBS`): bounds how many + /// repos run a refresh / reindex / HNSW build at once. It is attached to + /// every [`SharedStores`] this state opens, so cold opens, warmup and + /// file-watcher refreshes all draw from the same budget. Queries never + /// take it. See [`crate::limits`]. + job_gate: Arc, /// Serve-wide default embedding model for newly created indexes /// (`codesearch serve --model `), or `None` for the built-in default. /// @@ -420,6 +426,7 @@ impl ServeState { embedding_pool: Arc::new(crate::embed::EmbeddingServicePool::new( crate::constants::get_global_models_cache_dir().ok(), )), + job_gate: crate::limits::JobGate::from_env(), default_model: None, legacy_model_warned: DashMap::new(), tool_call_counts: DashMap::new(), @@ -2510,6 +2517,9 @@ impl ServeState { } }; if let Some(total_chunks) = needs_build { + // HNSW construction is one of the heaviest single operations a repo + // can do; draw the process-wide job permit around it. + let _job_permit = stores.acquire_job_permit().await; info!( "Warmup '{}': building vector index ({} existing chunks)", alias, total_chunks @@ -2704,6 +2714,8 @@ impl ServeState { // "Index not built" until the background refresh completes. // build_index() is CPU-heavy — offload to the blocking pool so the async // runtime is not stalled while building the HNSW index for large repos. + // The open itself is part of a heavy job: hold a process-wide permit. + let _job_permit = stores.acquire_job_permit().await; { let vector_store = Arc::clone(&stores.vector_store); let alias_owned = alias.to_string(); @@ -2983,7 +2995,7 @@ impl ServeState { return match SharedStores::new_readonly(db_path, dims) { Ok(s) => { info!("Opened repo in readonly mode (forced by config): {}", alias); - let stores_arc = Arc::new(s); + let stores_arc = Arc::new(s.with_job_gate(Arc::clone(&self.job_gate))); self.repos.insert( alias.to_string(), RepoState::Readonly { @@ -3003,7 +3015,9 @@ impl ServeState { match SharedStores::new(db_path, dims) { Ok(s) => { info!("Opened repo in write mode: {}", alias); - Ok(OpenedStores::Write(Arc::new(s))) + Ok(OpenedStores::Write(Arc::new( + s.with_job_gate(Arc::clone(&self.job_gate)), + ))) } Err(write_err) => { if allow_create { @@ -3015,7 +3029,7 @@ impl ServeState { match SharedStores::new_readonly(db_path, dims) { Ok(s) => { info!("Opened repo in readonly mode: {}", alias); - let stores_arc = Arc::new(s); + let stores_arc = Arc::new(s.with_job_gate(Arc::clone(&self.job_gate))); self.repos.insert( alias.to_string(), RepoState::Readonly { @@ -4878,7 +4892,9 @@ async fn add_repo_handler( } // Build vector index from freshly indexed data. - // build_index() is CPU-heavy — offload to the blocking pool. + // build_index() is CPU-heavy — offload to the blocking pool, under the + // same process-wide job permit as every other heavy pass. + let _job_permit = stores.acquire_job_permit().await; { let vector_store = Arc::clone(&stores.vector_store); let alias_bi = alias_bg.clone(); @@ -5694,6 +5710,22 @@ pub async fn run_serve( // — it never polls a federated peer on a timer at all. let serve_state = Arc::new(ServeState::new(config, None).with_default_model(default_model)); + // One line that answers "why is it slower / less parallel than before?" + // without grepping the code: the effective heavy-work budget. + info!( + "⚙️ Indexing limits: index_jobs={} (CODESEARCH_INDEX_JOBS), embed_threads={} \ + (CODESEARCH_EMBED_THREADS), embed_pause_ms={} (CODESEARCH_EMBED_PAUSE_MS), \ + min_free_mb={} (CODESEARCH_MIN_FREE_MB), max_chunks_per_batch={} \ + (CODESEARCH_MAX_CHUNKS_PER_BATCH)", + serve_state.job_gate.index_jobs(), + crate::limits::embed_threads() + .map(|n| n.to_string()) + .unwrap_or_else(|| "default".to_string()), + crate::limits::embed_pause().as_millis(), + crate::limits::min_free_mb(), + crate::limits::max_chunks_per_batch(), + ); + // Construct the bind address from resolved host + port. // Using `format!` with `parse::()` handles both IPv4 and IPv6. // For IPv6 literals, users should pass e.g. `[::1]` (with brackets). diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 1e71a301..b63f328d 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -2,6 +2,33 @@ use super::*; use serial_test::serial; use std::io::Write; +#[tokio::test] +async fn try_open_stores_attaches_the_serve_job_gate() { + // Serve-opened stores must carry the same process-wide gate as the + // ServeState, so cold opens, warmup and FSW refreshes all draw from one + // heavy-job budget (CODESEARCH_INDEX_JOBS). + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("repo").join(DB_DIR_NAME); + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + + let stores = match state + .try_open_stores("repo", &db_path, true, false, None) + .expect("a fresh repo must open in write mode") + { + OpenedStores::Write(s) => s, + OpenedStores::Readonly(_) => panic!("allow_create=true must open write mode"), + }; + + let store_gate = stores + .job_gate + .as_ref() + .expect("serve-opened stores must carry the job gate"); + assert!( + std::sync::Arc::ptr_eq(store_gate, &state.job_gate), + "the store gate must be the ServeState gate" + ); +} + #[test] fn test_api_key_matches() { assert!(api_key_matches("secret-key", "secret-key")); From 905a8b097d881bcee6b48b9cf3f308772d41993e Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Fri, 2 Oct 2026 21:01:27 -0400 Subject: [PATCH 36/77] feat(integrations): add OpenCode codesearch plugin Self-contained OpenCode v2 plugin that makes codesearch structural instead of advisory: project scope resolution with guidance injection, zero-hit grep/glob rescue, nudge/prune/block codesearch-first guards (fail-open), serve hub health with optional auto-start, unindexed-index and freshness notices, compaction outlines, /codesearch* commands, a codesearch_scope tool and a codesearch skill. Ships with a full README (install, configuration reference, examples, troubleshooting, security notes) and a mock-host smoke test. --- integrations/opencode/.gitignore | 2 + integrations/opencode/README.md | 466 +++++++ integrations/opencode/codesearch.ts | 1812 +++++++++++++++++++++++++++ integrations/opencode/package.json | 16 + integrations/opencode/test/smoke.ts | 386 ++++++ integrations/opencode/tsconfig.json | 17 + 6 files changed, 2699 insertions(+) create mode 100644 integrations/opencode/.gitignore create mode 100644 integrations/opencode/README.md create mode 100644 integrations/opencode/codesearch.ts create mode 100644 integrations/opencode/package.json create mode 100644 integrations/opencode/test/smoke.ts create mode 100644 integrations/opencode/tsconfig.json diff --git a/integrations/opencode/.gitignore b/integrations/opencode/.gitignore new file mode 100644 index 00000000..504afef8 --- /dev/null +++ b/integrations/opencode/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/integrations/opencode/README.md b/integrations/opencode/README.md new file mode 100644 index 00000000..453a7226 --- /dev/null +++ b/integrations/opencode/README.md @@ -0,0 +1,466 @@ +# OpenCode integration: making codesearch structural + +## The problem + +codesearch publishes usage instructions to every MCP client on connect (the +`initialize` handshake, see main [README § Agent Guidance](../../README.md#agent-guidance-making-agents-use-codesearch-not-grep)). +OpenCode surfaces those instructions and the model follows them reasonably +well. But they are **advisory** and **connection-scoped**: they cannot see the +host state that decides whether the preference actually holds up in practice. + +In OpenCode that shows up as several recurring failure modes: + +1. **Scope friction.** In multi-repo serve mode every tool call needs + `project=` or `group=`. The model has to guess the alias from the directory + name; when it guesses wrong it burns a turn on `scope_required`, and when it + guesses right it still may have picked the wrong project. +2. **Grep wins by default.** `grep`/`glob` are always loaded, zero-friction and + local. The instruction "prefer codesearch" loses to convenience, and an + **empty grep** (the exact case where the index would help) simply dead-ends + the turn. +3. **Worktrees and clones are invisible.** A git worktree is a separate + indexable repo. The model cannot tell that the current directory is + unregistered, so it greps a tree codesearch has never seen. +4. **The hub can be down.** `codesearch serve` is a local daemon. If it is not + running (or an MCP connect failed), the model keeps trying codesearch tools + and gets transport errors instead of falling back gracefully. +5. **Fresh edits lag the index.** The file watcher reindexes shortly after a + write; a query made in that window can miss the change and send the model + back to grep. +6. **Compaction loses structure.** A long session compacts, and the file + outlines that made navigation cheap are summarised away. + +The plugin below fixes these by adding what only the host knows: the working +directory, the session lifecycle, tool calls, and file edits. + +## The fix + +A self-contained [OpenCode v2](https://opencode.ai/docs/plugins/) plugin +(`codesearch.ts`, no imports, no runtime dependencies) that turns the +preference into behaviour: + +| Feature | Default | What it does | +|---|---|---| +| **Scope resolution** | on | Maps the session directory to a registered project alias (and its groups), from `~/.codesearch/repos.json` or a live `status` snapshot. | +| **Scope-aware guidance** | once per session | Injects a compact strategy block plus the resolved scope into system context, so the first tool call already knows `project=`. | +| **Zero-hit rescue** | on | When `grep`/`glob` returns nothing in an indexed repo, runs a literal codesearch query and appends the index hits to the tool result. | +| **Code-search-first guards** | `nudge` | `off`, `nudge` (one tip per session), `prune` (hide `grep`/`glob` from the model), or `block` (deny with the exact codesearch call to use). | +| **Hub health** | on | Probes `/healthz`, asks OpenCode to reload failed MCP servers, and can auto-start a local `codesearch serve` (opt-in). | +| **Unindexed warning** | on | Flags a registered repo with an empty or model-less index, so "no results" is not mistaken for "not there". | +| **Freshness note** | on | Tracks `file.edited` events and reminds the model that a just-edited file may lag the watcher's refresh. | +| **Compaction outlines** | on | Before compaction, captures `explore` outlines of recently edited files and injects them into the summary prompt. | +| **Commands** | on | `/codesearch `, `/codesearch-status`, `/codesearch-index`. | +| **`codesearch_scope` tool** | on | Lets the model ask which project/groups the current directory maps to, and whether the index is healthy. | +| **Skill** | on | Registers a `codesearch` skill with the tool-routing playbook and worked examples. | +| **Auto-recall** | off | Opt-in: runs a semantic search for the user message and injects the top chunks as background context. | + +Everything is **fail-open**: every hook is wrapped, guards only engage while +the hub is confirmed reachable and the repo is registered, and a plugin or +network failure degrades to plain OpenCode behaviour. Nothing here can block a +session or wedge the model. + +## Requirements + +- **OpenCode v2** (tested with 2.0.22). V1 is not supported. +- **A codesearch serve hub** reachable over HTTP (streamable MCP). The default + is `http://127.0.0.1:39725/mcp`; remote serve is supported with an API key. +- Nothing else: the plugin has no npm dependencies and does not need Node or + Bun on PATH (it uses the host's runtime and, when enabled, spawns the same + `codesearch` binary that runs your hub). + +Stdio-only setups (`codesearch mcp` without `serve`) do not expose the HTTP +endpoint this plugin needs. The plugin stays installed and inert there (and +says so in its logs); guards never fire, and the MCP server's own instructions +remain in effect. + +## Install + +The plugin is a single file. Drop it into OpenCode's plugin directory and +restart, or point at it from `opencode.jsonc`. + +### User scope (applies to every project) + +```bash +# macOS / Linux +mkdir -p ~/.config/opencode/plugins +cp integrations/opencode/codesearch.ts ~/.config/opencode/plugins/ +``` + +```powershell +# Windows / PowerShell +New-Item -ItemType Directory -Force "$env:USERPROFILE\.config\opencode\plugins" +Copy-Item integrations\opencode\codesearch.ts "$env:USERPROFILE\.config\opencode\plugins\" +``` + +### Project scope (this repository only) + +```bash +mkdir -p .opencode/plugins +cp integrations/opencode/codesearch.ts .opencode/plugins/ +``` + +### Or reference the path from config + +```jsonc +// ~/.config/opencode/opencode.jsonc +{ + "plugins": ["/absolute/path/to/codesearch.ts"] +} +``` + +OpenCode watches the plugin directories and reloads on change. If the plugin +does not appear, restart the service: + +```sh +opencode service restart +``` + +### Verify + +```sh +opencode plugin list # should list: codesearch local .../codesearch.ts +``` + +Then, inside a session, run `/codesearch-status`. A loaded plugin answers with +the hub URL, health, the directory's resolved project and its index state. + +### Uninstall + +Delete `codesearch.ts` (and optionally `~/.config/opencode/codesearch.json`) +and restart OpenCode. The plugin never installs or edits anything else. + +## Configuration + +Defaults live under `~/.config/opencode/codesearch.json` (override the path +with `CODESEARCH_CONFIG`). Values may use `{env:NAME}` placeholders. +Precedence, lowest to highest: + +1. built-in defaults +2. plugin options in `opencode.jsonc` (object form) +3. the config file +4. environment variables + +```jsonc +{ + // Optional. Usually auto-resolved (see below). + "url": "http://127.0.0.1:39725/mcp", + // Optional bearer token for a remote / authenticated serve. + "token": "{env:CODESEARCH_API_KEY}", + // Name of the MCP server entry in opencode.jsonc to mirror. + "mcpServer": "codesearch", + // Optional override for ~/.codesearch/repos.json. + "reposConfig": "", + "debug": false, + + "scope": { + "enabled": true, + // How often the scope+guidance block is injected: + // "session" (first message), "message" (every message), "off". + "inject": "session", + "includeGroups": true, + "siblings": true, + "maxSiblings": 12, + "warnUnindexed": true + }, + + "guidance": { + "enabled": true, + // Optional Markdown file (or inline text) replacing the built-in guidance. + "file": "", + "text": "" + }, + + "guards": { + // "off" | "nudge" | "prune" | "block" (see the guard section below). + "mode": "nudge", + "tools": ["grep", "glob"], + "rescue": true, + "rescueLimit": 5, + "rescueTimeoutMs": 3000 + }, + + "health": { + "enabled": true, + "intervalMs": 90000, + "initialDelayMs": 60000, + "maxBackoffMs": 900000, + "reconnect": true, + // Opt-in: start a local serve hub when it is not running. + "autoStart": false, + "startCommand": "codesearch serve --quiet" + }, + + "freshness": { "enabled": true, "maxFiles": 8 }, + "compaction": { "enabled": true, "maxFiles": 3, "timeoutMs": 4000, "budgetChars": 2400 }, + + // Opt-in auto-recall. + "recall": { "enabled": false, "limit": 3, "timeoutMs": 2500, "budgetChars": 1800 }, + + "commands": { "enabled": true, "indexCommand": "codesearch index" }, + "tools": { "scope": true }, + "skill": { "enabled": true, "autoinvoke": true }, + + "log": { "scope": false, "guidance": false, "guards": true, "rescue": true, "health": true, "recall": false, "commands": true } +} +``` + +### Environment variables + +| Variable | Default | Meaning | +|---|---|---| +| `CODESEARCH_URL` | – | Full MCP URL (`.../mcp`). Wins over every discovery step. | +| `CODESEARCH_SERVER` | – | Serve **base** URL (e.g. `http://127.0.0.1:39725`), same meaning as the Claude Code guard hooks use. `/mcp` is appended. | +| `CODESEARCH_API_KEY` | – | Bearer token for an authenticated/remote serve. | +| `CODESEARCH_CONFIG` | `~/.config/opencode/codesearch.json` | Config file path. | +| `CODESEARCH_REPOS_CONFIG` | `~/.codesearch/repos.json` | Registry path (shared with the hub and the Claude Code hooks). | +| `CODESEARCH_MCP_SERVER` | `codesearch` | MCP entry name to mirror for URL/token. | +| `CODESEARCH_PLUGIN_GUARDS` | `nudge` | `off` / `nudge` / `prune` / `block`. | +| `CODESEARCH_PLUGIN_SCOPE` | `session` | `session` / `message` / `off`. | +| `CODESEARCH_PLUGIN_RESCUE` | `true` | Zero-hit rescue on/off. | +| `CODESEARCH_PLUGIN_RECALL` | `false` | Auto-recall on/off. | +| `CODESEARCH_PLUGIN_COMPACTION` | `true` | Compaction outlines on/off. | +| `CODESEARCH_PLUGIN_COMMANDS` | `true` | Slash commands on/off. | +| `CODESEARCH_PLUGIN_SKILL` | `true` | Skill registration on/off. | +| `CODESEARCH_PLUGIN_HEALTH` | `true` | Health probing on/off. | +| `CODESEARCH_PLUGIN_DEBUG` | `false` | Verbose plugin logs. | + +### Hub discovery order + +The plugin resolves the MCP endpoint in this order, first non-empty wins: + +1. `url` from config / `CODESEARCH_URL` +2. `CODESEARCH_SERVER` +3. the `url` of the MCP entry named by `mcpServer` in `opencode.jsonc` + (its `Authorization` header is used as the token, minus `Bearer`) +4. `~/.codesearch/serve_url` (written by `codesearch serve`) +5. `http://127.0.0.1:39725/mcp` + +A base URL without `/mcp` is normalised by appending it, so pointing +`CODESEARCH_SERVER` at the hub root is enough. + +## Guards and rescue in detail + +`guards.mode` controls how strongly codesearch replaces `grep`/`glob` in a +**covered** directory (one whose path maps to a registered, indexed repo): + +- **`off`** — hooks are not even registered. Nothing changes. +- **`nudge`** (default) — greps work normally. The first successful + `grep`/`glob` in a session gets a one-line tip appended to its result, and an + **empty** grep/glob is rescued (below). +- **`prune`** — `grep` and `glob` are removed from the tool list for covered + sessions, so the model must use the codesearch tools. The scope block says + so explicitly. Everything else is untouched, and reading/editing files is + unaffected. +- **`block`** — `grep`/`glob` calls are denied with a message naming the exact + `project=` value and the codesearch calls to use instead. Denial happens + **only** while the hub is confirmed reachable (`/healthz` answered) and the + repo is registered; in any other state the call proceeds. This mirrors the + Claude Code `grep-guard` semantics and fails open the same way. + +**Zero-hit rescue** (part of `nudge` and `prune`, and independently +controlled by `guards.rescue`) is the highest-value, lowest-risk piece: it +only ever fires when a tool already returned nothing. For a plain-text pattern +it runs `search` with `mode="literal", phrase=true`; for a regex-shaped +pattern it uses `regex=true`. Hits are appended as metadata-style lines +(path/line/kind/signature), not full code, and the model is pointed at +`get_chunk` for the body: + +```text +[codesearch] grep returned nothing, but the index has 2 match(es) for "retryUpload" in project "my-service": + - src/uploader.ts:42 (Function) function retryUpload(job: Job, attempt: number) + - src/queue.ts:13 (Class) class UploadQueue +Use search/find/get_chunk for the full result. +``` + +Glob patterns that are pure filesystem wildcards (`**/*.ts`) are skipped; +there is nothing semantic to look up. + +## Commands + +| Command | What it does | +|---|---| +| `/codesearch ` | Runs a semantic search in the resolved project and injects the top hits as a prompt, with a pointer to `get_chunk`. Falls back to guidance if the directory is unregistered. | +| `/codesearch-status` | Probes health, refreshes the status snapshot, and reports hub URL, health, directory, resolved project, chunk/file counts, model and lock state, groups and hub repo count. | +| `/codesearch-index` | Runs `commands.indexCommand` (`codesearch index` by default) in the current directory, detached, then reports that indexing started. Use this to register an unregistered clone or worktree. | + +## The `codesearch_scope` tool and the skill + +`codesearch_scope` returns JSON for the current directory: + +```json +{ + "directory": "/work/my-service", + "project": "my-service", + "groups": ["platform"], + "registered": true, + "siblings": [], + "indexed": true, + "chunks": 4123, + "model": "embeddinggemma-q4", + "lockStatus": "warm", + "serveUrl": "http://127.0.0.1:39725/mcp", + "serveHealth": "ok" +} +``` + +Use it when the directory is unusual (a worktree, a multi-repo parent) or when +you want to confirm the index health before trusting a result. It is the one +piece of state the MCP server itself cannot see. + +The plugin also registers a **`codesearch` skill** containing the same routing +playbook as the guidance plus worked examples, so a session can pull the +strategy in deliberately. Disable with `skill.enabled: false`. + +## Health and auto-start + +The health runner probes the serve hub's unauthenticated `/healthz` endpoint +(any HTTP response counts as reachable; only connection-level failures count +as down), with exponential backoff while down. On failure it asks OpenCode to +reload its MCP servers (which re-establishes a failed `codesearch` MCP entry). +With `health.autoStart: true` and a loopback URL it also runs +`codesearch serve --quiet` once, detached, and keeps probing. + +While the hub is down, the injected scope block says so and explicitly allows +`grep` as a fallback; rescue skips its lookup (no timeout cost). + +## Compaction outlines (and optional recall) + +Before a session is compacted, the plugin collects the recently edited files +(tracked from `file.edited` events), fetches `explore kind="outline"` for up +to `compaction.maxFiles`, and appends a compact symbol list to the +compaction prompt, so file structure survives the summary: + +```text +# codesearch outlines (captured before compaction) +- src/uploader.ts: function retryUpload(job: Job, attempt: number):41; class UploadManager:88 +- src/queue.ts: class UploadQueue:13; function enqueue(job: Job):55 +``` + +**Auto-recall** (`recall.enabled`, default `false`) is the opt-in mirror of +the guidance injection: for each new user message that looks like a code +question, it runs a semantic search and injects the top `recall.limit` chunks, +truncated to `budgetChars`. It costs a lookup plus context on every turn, which +is why it is off by default. Label it as background context; it is code from +the index, not instructions. + +## Examples + +**1. Multi-repo scope, no detour.** A workspace registers `my-service`, +`billing-api` and `docs`; the session opens in `my-service`. + +```text +user> where does upload retry backoff live? +# plugin injects: project "my-service", group "platform" (cross-repo fan-out) +model> search(project:"my-service", query:"upload retry backoff") -> get_chunk(...) +``` + +Without the plugin, the first call is a guess: `project:"my_service"` or no +scope at all, and the turn pays for a `scope_required` round-trip. + +**2. Empty grep becomes a hit.** The model greps for an old function name: + +```text +model> grep(pattern:"retryUpload") +tool> [codesearch] grep returned nothing, but the index has 2 match(es) for "retryUpload" in project "my-service": ... +model> get_chunk(chunk_id: 7) +``` + +**3. Guard modes.** `prune` removes `grep`/`glob` from the tool list in +covered sessions; `block` returns: + +```text +codesearch: grep is disabled here — this repository is indexed as project "my-service". +Use the codesearch MCP tools instead: + - search { project: "my-service", query: "" } ... +``` + +Both fail open when the hub is down or the directory is unregistered. + +**4. A dead hub recovers.** `codesearch serve` stops or an MCP connect fails: + +```text +[codesearch] hub unreachable at http://127.0.0.1:39725/mcp: fetch failed +[codesearch] asked OpenCode to reload MCP servers +``` + +With `health.autoStart: true` the plugin starts the hub and logs when it comes +back; the scope block tells the model grep is acceptable until then. + +**5. A new worktree.** The session opens in an unregistered worktree of a +registered repo. The scope block says it is unregistered and points at +`/codesearch-index`; one command registers and indexes it (`codesearch index`, +which produces a separate index per worktree by design). + +**6. Compaction keeps the map.** After a long refactor, the compaction summary +carries the outline block above instead of losing every symbol location. + +## Why this exists alongside the MCP instructions + +The MCP instructions tell the model *what* to do. The plugin makes it +*true* in the host: + +- The server cannot know the session directory, so it cannot resolve + `project=`. The plugin can, because it runs inside the host. +- The server cannot see a `grep` call, let alone an empty one. +- The server cannot recover its own MCP connection or restart itself. +- The server cannot see file edits or pending compaction. + +Together they behave like the Claude Code hooks from +[`integrations/claude-code/`](../claude-code/README.md), but implemented in +the host's own plugin API instead of shell guards: no `jq`, no `curl`, no +per-call process spawn, and coverage decisions come from the same registry the +hub uses. + +## Troubleshooting + +- **Plugin not listed** — confirm the file is `~/.config/opencode/plugins/codesearch.ts` + (or referenced in `opencode.jsonc`), then `opencode service restart`. Look + for `failed to load plugin` and `codesearch` in OpenCode's log + (`~/.local/share/opencode/log/opencode.log`). +- **No scope / `project=null`** — the directory is not registered. `/codesearch-index` + runs `codesearch index`; verify with `codesearch index list` and inspect + `~/.codesearch/repos.json` (or your `CODESEARCH_REPOS_CONFIG`). +- **Rescue never fires** — check `guards.rescue`, `guards.tools`, and health + (`/codesearch-status`). Rescue is skipped while the hub is down. +- **Block mode too strict** — `guards.mode: "nudge"` (or `prune`) is the + intended middle ground. Blocking deliberately requires a live hub. +- **Top-level `scope` tool missing / other plugins conflict** — the tool is + registered as `codesearch_scope` precisely to avoid generic names. +- **Remote serve** — set `url` (or `CODESEARCH_SERVER`) and `token` (or + `CODESEARCH_API_KEY`). `/healthz` stays probeable behind auth. +- **Skill/commands not visible** — they need an OpenCode build with skill and + command transforms (2.x). Registration failures are logged and non-fatal; + everything else keeps working. + +## Security notes + +- The plugin reads `opencode.jsonc`, its own config, and `repos.json`. It + sends network requests only to the resolved serve URL (`/mcp`, `/healthz`). +- **Auto-recall sends the current user message to the serve hub** and injects + retrieved code into the model context. It is off by default; enable it only + where that is acceptable. Treat injected chunks as data, not instructions. +- **Auto-start** is off by default. When enabled it runs + `health.startCommand` with your privileges; keep it pointed at + `codesearch serve`. +- Guards fail open. `block` mode never denies while the hub is unreachable or + the repo is unregistered, so a misconfigured guard cannot trap a session. +- No secrets are stored by the plugin; use `{env:...}` placeholders or the + environment for tokens, and keep the config file readable only by you. + +## Development + +The plugin is dependency-free at runtime. For type checking and the smoke +test (which runs the plugin against a mock OpenCode context and a mock MCP +server): + +```bash +cd integrations/opencode +npm install # dev-only: typescript + @types/node +npm run typecheck # tsc --noEmit +npm run smoke # node test/smoke.ts (Node >= 23.6, or: bun test/smoke.ts) +``` + +`test/smoke.ts` covers scope resolution, guidance injection and deduplication, +zero-hit rescue, the one-time nudge, prune and block modes (including +fail-open for unregistered directories), the commands, the scope tool and the +skill registration. diff --git a/integrations/opencode/codesearch.ts b/integrations/opencode/codesearch.ts new file mode 100644 index 00000000..ae319b89 --- /dev/null +++ b/integrations/opencode/codesearch.ts @@ -0,0 +1,1812 @@ +/** + * codesearch — OpenCode plugin + * ============================ + * + * Makes the codesearch MCP server *structural* inside OpenCode instead of + * advisory: resolves the current directory to an indexed project, injects the + * usage guidance once per session, nudges (or prunes/blocks) grep/glob, rescues + * empty greps with index results, keeps the serve hub healthy, and adds slash + * commands, a scope tool, a skill and compaction assistance. + * + * The MCP server already publishes usage instructions on connect, but those are + * advisory and connection-scoped. This plugin adds what only the host knows: + * the working directory, the session lifecycle, tool calls and file edits. + * + * Design constraints (see README.md in this directory): + * - Self-contained: no imports, no runtime dependencies. The host executes + * this file as-is; fs/child_process come from `process.getBuiltinModule`. + * - Fail open: every hook is wrapped so a plugin or network failure can never + * break a session. Guards only block while the hub is confirmed reachable. + * - Generic: no assumption about repo names, groups or layout. Everything + * comes from `~/.codesearch/repos.json`, the serve hub, or the config file. + * + * Targets OpenCode v2 (`opencode` 2.x). Install: copy this file to + * `~/.config/opencode/plugins/codesearch.ts` and restart OpenCode. + */ + +/* ------------------------------------------------------------------ * + * Types (structural views; no @opencode/plugin import so the file runs + * as-is under the host's module loader) + * ------------------------------------------------------------------ */ + +type Loose = Record + +interface Registration { + dispose(): Promise +} + +interface SystemPart { + type: "text" + text: string + [key: string]: unknown +} + +interface PluginContext { + app?: { version?: string } + location?: { + directory?: string + workspaceID?: string + project?: { id?: string; canonical?: string } + } + options?: Record + session?: { + hook( + name: string, + callback: (event: Loose) => Promise | void, + options?: { providerID?: string }, + ): Promise + prompt?(input: Loose): Promise + context?(input: { sessionID: string }): Promise + } + tool?: { + hook(name: string, callback: (event: Loose) => Promise | void): Promise + transform(callback: (editor: ToolEditor) => void): Promise + } + command?: { + transform(callback: (editor: CommandEditor) => void): Promise + } + skill?: { + transform(callback: (editor: SkillEditor) => void): Promise + } + mcp?: { + list(): Promise + reload?(): Promise + } + event?: { + subscribe(options?: { signal?: AbortSignal }): AsyncIterable + } +} + +interface ToolEditor { + namespace(namespace: { name: string; description: string }): void + add(tool: Loose): void + get?(id: string): Loose | undefined + update?(id: string, update: (tool: Loose) => void): void + remove?(id: string): void +} + +interface CommandEditor { + add(definition: Loose): void +} + +interface SkillEditor { + add(skill: Loose): void + get?(id: string): Loose | undefined + update?(id: string, update: (skill: Loose) => void): void +} + +type GuardMode = "off" | "nudge" | "prune" | "block" + +interface CodeSearchSettings { + url: string + token: string + mcpServer: string + reposConfig: string + debug: boolean + scope: { + enabled: boolean + inject: "session" | "message" | "off" + includeGroups: boolean + siblings: boolean + maxSiblings: number + warnUnindexed: boolean + } + guidance: { enabled: boolean; file: string; text: string } + guards: { + mode: GuardMode + tools: string[] + rescue: boolean + rescueLimit: number + rescueTimeoutMs: number + } + health: { + enabled: boolean + intervalMs: number + initialDelayMs: number + maxBackoffMs: number + reconnect: boolean + autoStart: boolean + startCommand: string + } + freshness: { enabled: boolean; maxFiles: number } + compaction: { enabled: boolean; maxFiles: number; timeoutMs: number; budgetChars: number } + recall: { enabled: boolean; limit: number; timeoutMs: number; budgetChars: number } + commands: { enabled: boolean; indexCommand: string } + tools: { scope: boolean } + skill: { enabled: boolean; autoinvoke: boolean } + log: { + scope: boolean + guidance: boolean + guards: boolean + rescue: boolean + health: boolean + recall: boolean + commands: boolean + } +} + +interface ScopeInfo { + directory: string + project?: string + projectPath?: string + groups: string[] + siblings: string[] + registered: boolean + hasGit: boolean +} + +interface StatusRepo { + alias: string + project_path?: string + total_chunks?: number + total_files?: number + model?: string + lock_status?: string +} + +/* ------------------------------------------------------------------ * + * Constants + * ------------------------------------------------------------------ */ + +const DEFAULTS: CodeSearchSettings = { + url: "", + token: "", + mcpServer: "codesearch", + reposConfig: "", + debug: false, + scope: { + enabled: true, + inject: "session", + includeGroups: true, + siblings: true, + maxSiblings: 12, + warnUnindexed: true, + }, + guidance: { enabled: true, file: "", text: "" }, + guards: { + mode: "nudge", + tools: ["grep", "glob"], + rescue: true, + rescueLimit: 5, + rescueTimeoutMs: 3000, + }, + health: { + enabled: true, + intervalMs: 90_000, + initialDelayMs: 60_000, + maxBackoffMs: 900_000, + reconnect: true, + autoStart: false, + startCommand: "codesearch serve --quiet", + }, + freshness: { enabled: true, maxFiles: 8 }, + compaction: { enabled: true, maxFiles: 3, timeoutMs: 4000, budgetChars: 2400 }, + recall: { enabled: false, limit: 3, timeoutMs: 2500, budgetChars: 1800 }, + commands: { enabled: true, indexCommand: "codesearch index" }, + tools: { scope: true }, + skill: { enabled: true, autoinvoke: true }, + log: { + scope: false, + guidance: false, + guards: true, + rescue: true, + health: true, + recall: false, + commands: true, + }, +} + +const DEFAULT_SERVE_URL = "http://127.0.0.1:39725/mcp" +const PROTOCOL_VERSION = "2025-06-18" +const FALLBACK_PROTOCOL_VERSION = "2024-11-05" +const MAX_SEEN = 20_000 + +/** + * Built-in guidance. Deliberately strategy-only: per-tool parameter detail + * lives in the MCP tool schemas and is injected by the server's `initialize` + * instructions, so it is not duplicated here. + */ +const DEFAULT_GUIDANCE = `codesearch indexes this workspace and serves semantic code search over MCP. Prefer it over grep/glob for anything conceptual, cross-file, or symbol-shaped. + +Tool routing: +- search — concepts, unknown locations, mixed natural language + identifiers. Default mode is semantic (hybrid vector + BM25); use mode="literal" with regex=true or phrase=true for exact syntax. +- find — where a symbol is defined (kind="definition"), its call-sites (kind="usages"), a file's imports (kind="imports"), or a module's dependents (kind="dependents"). +- find_impact — precise transitive call sites before a rename or edit (SCIP: C#, TypeScript; on a "no backend" answer fall back to find kind="usages"). +- explore — outline a file's top-level symbols (kind="outline") or find chunks similar to a known hit (kind="similar"). +- get_chunk — read the code behind a search hit, without loading the whole file. +- status — index health and registered repos. + +Rules: +- Multi-repo serve mode requires a scope on every call except status: pass project="" (returned paths are the server's paths; read content via get_chunk, not the local filesystem), or group="" to fan out. +- Start broad with search, narrow with find/explore, read with get_chunk. Keep compact=true (the default) until you know which chunk you need. +- The serve hub watches files and refreshes indexes automatically; if a just-made change seems missing, retry the query once before falling back to grep.` + +/** Skill playbook: the same strategy plus worked examples. */ +const SKILL_CONTENT = `${DEFAULT_GUIDANCE} + +Examples: +- "Where is the upload retried?" -> search(project=..., query="upload retry backoff") -> get_chunk(top hit). +- "Who calls validateToken?" in a TypeScript repo -> find_impact(project=..., symbol_name="validateToken") -> rename/edit with the full caller list. +- "Add a field to the settings struct" -> find(project=..., symbol="Settings", kind="definition") -> find(project=..., symbol="Settings", kind="usages"). +- Exact syntax like "Vec" or "foo = null" -> search(project=..., mode="literal", regex=true, query="Vec<"). +- "How does this file relate to the rest?" -> explore(project=..., kind="outline", target=) -> explore(kind="similar", target=). +- Freshly edited file returning stale hits -> wait a moment, rerun the query; the watcher is reindexing.` + +/* ------------------------------------------------------------------ * + * Small utilities + * ------------------------------------------------------------------ */ + +function bool(value: string | undefined, fallback: boolean): boolean { + if (value === undefined || value === "") return fallback + return !/^(0|false|no|off)$/i.test(value) +} + +function truncate(input: string, max: number): string { + if (input.length <= max) return input + return input.slice(0, Math.max(0, max - 1)) + "…" +} + +function shortHash(input: string): string { + let h = 5381 + for (let i = 0; i < input.length; i++) h = ((h << 5) + h + input.charCodeAt(i)) | 0 + return (h >>> 0).toString(36) +} + +function isPlainObject(value: unknown): value is Loose { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +/** Deep-merge plain objects; arrays and scalars replace. */ +function mergeDeep(base: T, override: unknown): T { + if (!isPlainObject(override)) return base + const out: Loose = { ...(base as unknown as Loose) } + for (const [key, value] of Object.entries(override)) { + const current = out[key] + if (isPlainObject(current) && isPlainObject(value)) out[key] = mergeDeep(current, value) + else out[key] = value + } + return out as T +} + +function expandEnv(value: string): string { + return value.replace(/\{env:([A-Za-z_][A-Za-z0-9_]*)\}/g, (_m, name: string) => process.env[name] ?? "") +} + +function expandEnvDeep(value: T): T { + if (typeof value === "string") return expandEnv(value) as unknown as T + if (Array.isArray(value)) return value.map((v) => expandEnvDeep(v)) as unknown as T + if (isPlainObject(value)) { + const out: Loose = {} + for (const [k, v] of Object.entries(value)) out[k] = expandEnvDeep(v) + return out as unknown as T + } + return value +} + +/** Inline builtin loader: works under Node (getBuiltinModule) and Bun (require). */ +function builtin(name: string): any { + const p = process as unknown as { getBuiltinModule?: (n: string) => unknown } + if (typeof p.getBuiltinModule === "function") { + try { + const mod = p.getBuiltinModule(name) + if (mod) return mod + } catch { + /* fall through */ + } + } + const g = globalThis as unknown as { require?: (n: string) => unknown } + if (typeof g.require === "function") { + try { + return g.require(name) + } catch { + /* fall through */ + } + } + return undefined +} + +function homeDir(): string { + return process.env.HOME || process.env.USERPROFILE || "" +} + +function configDir(): string { + return process.env.XDG_CONFIG_HOME || `${homeDir()}/.config` +} + +/** + * Strip `//` and block comments plus trailing commas from JSONC, honouring + * string literals. Enough for OpenCode's config files. + */ +function stripJsonc(text: string): string { + let out = "" + let inString = false + let escaped = false + for (let i = 0; i < text.length; i++) { + const ch = text[i] + const next = text[i + 1] + if (inString) { + out += ch + if (escaped) escaped = false + else if (ch === "\\") escaped = true + else if (ch === '"') inString = false + continue + } + if (ch === '"') { + inString = true + out += ch + continue + } + if (ch === "/" && next === "/") { + while (i < text.length && text[i] !== "\n") i++ + out += "\n" + continue + } + if (ch === "/" && next === "*") { + i += 2 + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++ + i++ + continue + } + if (ch === ",") { + let j = i + 1 + while (j < text.length && /\s/.test(text[j])) j++ + if (text[j] === "}" || text[j] === "]") continue // drop trailing comma + } + out += ch + } + return out +} + +function readJsonFile(path: string): Loose | undefined { + const fs = builtin("node:fs") + if (!fs || !path) return undefined + try { + const raw = fs.readFileSync(path, "utf8") + const parsed = JSON.parse(stripJsonc(raw)) + return isPlainObject(parsed) ? parsed : undefined + } catch { + return undefined + } +} + +function fileExists(path: string): boolean { + const fs = builtin("node:fs") + if (!fs || !path) return false + try { + fs.statSync(path) + return true + } catch { + return false + } +} + +function readTextFile(path: string): string | undefined { + const fs = builtin("node:fs") + if (!fs || !path) return undefined + try { + return fs.readFileSync(path, "utf8") + } catch { + return undefined + } +} + +/* ------------------------------------------------------------------ * + * Path helpers + * ------------------------------------------------------------------ */ + +function normalizePath(input: string): string { + let p = (input || "").trim() + if (!p) return "" + // Windows UNC prefix (\\?\C:\...) and backslashes. + p = p.replace(/^\\\\\?\\/, "") + p = p.replace(/\\/g, "/") + p = p.replace(/\/+$/, "") + // Collapse duplicate slashes (keep the leading two for UNC-style roots). + p = p.replace(/([^:])\/{2,}/g, "$1/") + if (process.platform === "win32") p = p.toLowerCase() + return p +} + +/** True when `child` is `parent` or lives under it. */ +function isUnder(child: string, parent: string): boolean { + const c = normalizePath(child) + const p = normalizePath(parent) + if (!c || !p) return false + return c === p || c.startsWith(p + "/") +} + +/** Path relative to `root`, or the original when not under it. */ +function relativeTo(path: string, root: string): string { + const p = normalizePath(path) + const r = normalizePath(root) + if (!p || !r) return path + if (p === r) return "." + if (p.startsWith(r + "/")) return p.slice(r.length + 1) + return path +} + +/* ------------------------------------------------------------------ * + * Configuration resolution + * ------------------------------------------------------------------ */ + +function opencodeConfigPath(): string { + const candidates = [ + `${configDir()}/opencode/opencode.jsonc`, + `${configDir()}/opencode/opencode.json`, + ] + for (const candidate of candidates) if (fileExists(candidate)) return candidate + return candidates[0] +} + +/** + * Read the `mcp` entry named `serverName` from the OpenCode config, supporting + * both the v2 shape (`mcp.servers.`) and the v1 shape (`mcp.`). + */ +function readMcpEntry(serverName: string): { url: string; authorization: string } { + const config = readJsonFile(opencodeConfigPath()) + if (!config) return { url: "", authorization: "" } + const mcp = isPlainObject(config.mcp) ? config.mcp : undefined + if (!mcp) return { url: "", authorization: "" } + const servers = isPlainObject(mcp.servers) ? mcp.servers : mcp + const entry = isPlainObject(servers[serverName]) ? servers[serverName] : undefined + if (!entry) return { url: "", authorization: "" } + const url = typeof entry.url === "string" ? expandEnv(entry.url) : "" + const headers = isPlainObject(entry.headers) ? entry.headers : undefined + const rawAuth = headers && typeof headers.Authorization === "string" ? expandEnv(headers.Authorization) : "" + const authorization = rawAuth.replace(/^Bearer\s+/i, "") + return { url, authorization } +} + +function resolveSettings(ctx: PluginContext): CodeSearchSettings { + let settings = mergeDeep(DEFAULTS, ctx.options ?? {}) + + const configPath = + process.env.CODESEARCH_CONFIG || `${configDir()}/opencode/codesearch.json` + const fileConfig = readJsonFile(configPath) + if (fileConfig) settings = mergeDeep(settings, expandEnvDeep(fileConfig)) + + const env = process.env + settings.url = env.CODESEARCH_URL ?? settings.url + settings.token = env.CODESEARCH_API_KEY ?? settings.token + settings.mcpServer = env.CODESEARCH_MCP_SERVER ?? settings.mcpServer + settings.reposConfig = env.CODESEARCH_REPOS_CONFIG ?? settings.reposConfig + settings.debug = bool(env.CODESEARCH_PLUGIN_DEBUG, settings.debug) + + if (env.CODESEARCH_PLUGIN_GUARDS) settings.guards.mode = env.CODESEARCH_PLUGIN_GUARDS as GuardMode + if (env.CODESEARCH_PLUGIN_SCOPE) settings.scope.inject = env.CODESEARCH_PLUGIN_SCOPE as CodeSearchSettings["scope"]["inject"] + if (env.CODESEARCH_PLUGIN_RESCUE) settings.guards.rescue = bool(env.CODESEARCH_PLUGIN_RESCUE, settings.guards.rescue) + if (env.CODESEARCH_PLUGIN_RECALL) settings.recall.enabled = bool(env.CODESEARCH_PLUGIN_RECALL, settings.recall.enabled) + if (env.CODESEARCH_PLUGIN_COMPACTION) settings.compaction.enabled = bool(env.CODESEARCH_PLUGIN_COMPACTION, settings.compaction.enabled) + if (env.CODESEARCH_PLUGIN_COMMANDS) settings.commands.enabled = bool(env.CODESEARCH_PLUGIN_COMMANDS, settings.commands.enabled) + if (env.CODESEARCH_PLUGIN_SKILL) settings.skill.enabled = bool(env.CODESEARCH_PLUGIN_SKILL, settings.skill.enabled) + if (env.CODESEARCH_PLUGIN_HEALTH) settings.health.enabled = bool(env.CODESEARCH_PLUGIN_HEALTH, settings.health.enabled) + + if (!["off", "nudge", "prune", "block"].includes(settings.guards.mode)) settings.guards.mode = "nudge" + if (!["session", "message", "off"].includes(settings.scope.inject)) settings.scope.inject = "session" + settings.guards.tools = (settings.guards.tools || []).map((t) => String(t).toLowerCase()) + + return settings +} + +function normalizeMcpUrl(value: string): string { + let v = (value || "").trim() + if (!v) return "" + if (!/^https?:\/\//i.test(v)) v = `http://${v}` + v = v.replace(/\/+$/, "") + if (!/\/mcp$/i.test(v)) v += "/mcp" + return v +} + +/** Resolve the serve URL: config/env > CODESEARCH_SERVER > MCP entry > serve_url file > default. */ +function resolveServeUrl(settings: CodeSearchSettings): string { + const mcpEntry = readMcpEntry(settings.mcpServer) + const serveUrlFile = readTextFile(`${homeDir()}/.codesearch/serve_url`)?.trim() ?? "" + const candidates = [ + settings.url, + process.env.CODESEARCH_SERVER ?? "", + mcpEntry.url, + serveUrlFile, + DEFAULT_SERVE_URL, + ] + for (const candidate of candidates) { + const normalized = normalizeMcpUrl(candidate) + if (normalized) return normalized + } + return DEFAULT_SERVE_URL +} + +function resolveToken(settings: CodeSearchSettings): string { + if (settings.token) return settings.token + const mcpEntry = readMcpEntry(settings.mcpServer) + return mcpEntry.authorization || "" +} + +/* ------------------------------------------------------------------ * + * MCP client (streamable HTTP, JSON-RPC) + * ------------------------------------------------------------------ */ + +interface CallOptions { + timeoutMs?: number + retry?: boolean +} + +class McpClient { + private sessionId: string | null = null + private connected = false + private connecting: Promise | null = null + private idSeq = 0 + private chain: Promise = Promise.resolve() + private protocolVersion = PROTOCOL_VERSION + private readonly url: string + private readonly token: string + private readonly debug: boolean + private readonly label: string + + constructor(url: string, token: string, debug: boolean, label: string) { + this.url = url + this.token = token + this.debug = debug + this.label = label + } + + private log(...args: unknown[]): void { + if (this.debug) console.log(`[codesearch:${this.label}]`, ...args) + } + + /** Serialise requests per client so reconnect/session state never races. */ + private serial(fn: () => Promise): Promise { + const run = this.chain.then(fn, fn) + this.chain = run.then( + () => undefined, + () => undefined, + ) + return run + } + + async call(tool: string, args: Record, options: CallOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 30_000 + const retry = options.retry ?? true + return this.serial(async () => { + await this.connect() + const payload = { jsonrpc: "2.0", id: ++this.idSeq, method: "tools/call", params: { name: tool, arguments: args } } + try { + return this.resultOf(tool, await this.post(payload, timeoutMs)) + } catch (err) { + if (!retry) throw err + const message = err instanceof Error ? err.message : String(err) + if (/network|fetch failed|session|ECONN|socket|timed? ?out|abort|4[0-9][0-9]/i.test(message)) { + this.connected = false + this.sessionId = null + await this.connect() + return this.resultOf(tool, await this.post(payload, timeoutMs)) + } + throw err + } + }) + } + + private async connect(): Promise { + if (this.connected) return + if (this.connecting) return this.connecting + this.connecting = this.doConnect().finally(() => { + this.connecting = null + }) + return this.connecting + } + + private async doConnect(): Promise { + const attempt = async (version: string): Promise => { + const init = { + jsonrpc: "2.0", + id: 0, + method: "initialize", + params: { + protocolVersion: version, + capabilities: {}, + clientInfo: { name: "opencode-codesearch-plugin", version: "1.0.0" }, + }, + } + const { session, messages } = await this.postRaw(init, 15_000) + const first = messages.find((m) => (m as Loose).id === 0) as Loose | undefined + if (!first?.result) return false + this.sessionId = session + this.protocolVersion = version + await this.postRaw({ jsonrpc: "2.0", method: "notifications/initialized", params: {} }, 15_000) + return true + } + if (!(await attempt(this.protocolVersion)) && this.protocolVersion !== FALLBACK_PROTOCOL_VERSION) { + if (!(await attempt(FALLBACK_PROTOCOL_VERSION))) { + throw new Error(`codesearch initialize failed at ${this.url}`) + } + } + this.connected = true + this.log("connected", this.sessionId ? `session ${this.sessionId}` : "(stateless)") + } + + private async post(payload: Record, timeoutMs: number): Promise { + const { messages } = await this.postRaw(payload, timeoutMs) + return messages + } + + private async postRaw( + payload: Record, + timeoutMs: number, + ): Promise<{ session: string | null; messages: Loose[] }> { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + } + if (this.token) headers.authorization = `Bearer ${this.token}` + if (this.sessionId) headers["mcp-session-id"] = this.sessionId + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), timeoutMs) + let response: Response + try { + response = await fetch(this.url, { + method: "POST", + headers, + body: JSON.stringify(payload), + signal: controller.signal, + }) + } finally { + clearTimeout(timer) + } + if (!response.ok) throw new Error(`codesearch HTTP ${response.status} ${response.statusText}`) + const session = response.headers.get("mcp-session-id") + const body = await response.text() + const messages = parseMcpBody(body) as Loose[] + if (messages.length === 0 && payload.id !== undefined) { + throw new Error(`codesearch returned no JSON-RPC message for ${String(payload.method)}`) + } + return { session, messages } + } + + private resultOf(tool: string, messages: Loose[]): unknown { + const message = messages.find((m) => m.result || m.error) + if (!message) throw new Error(`No result for ${tool}`) + if (message.error) throw new Error(`codesearch ${tool}: ${message.error.message ?? message.error.code ?? "error"}`) + const result = message.result as Loose | undefined + if (result?.isError) { + const text = result.content?.[0]?.text || "unknown tool error" + throw new Error(`codesearch ${tool} failed: ${String(text).slice(0, 500)}`) + } + const text = (result?.content ?? []) + .filter((c: Loose) => c.type === "text" && typeof c.text === "string") + .map((c: Loose) => c.text as string) + .join("\n") + if (text.trim() === "") return undefined + try { + return JSON.parse(text) + } catch { + return text + } + } +} + +/** Parse a streamable-HTTP body (plain JSON or SSE `data:` lines). */ +function parseMcpBody(body: string): unknown[] { + const out: unknown[] = [] + const trimmed = body.trim() + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + try { + out.push(JSON.parse(trimmed)) + return out + } catch { + /* fall through to SSE parsing */ + } + } + for (const line of body.split(/\r?\n/)) { + const text = line.trim() + if (!text.startsWith("data:")) continue + const raw = text.slice(5).trim() + if (!raw) continue + try { + out.push(JSON.parse(raw)) + } catch { + /* ignore malformed frames */ + } + } + return out +} + +/* ------------------------------------------------------------------ * + * Registry + status cache + * ------------------------------------------------------------------ */ + +class RepoRegistry { + private repos = new Map() + private groups = new Map() + private loadedAt = 0 + private signature = "" + private readonly path: string + + constructor(path: string) { + this.path = path + } + + load(force = false): void { + const now = Date.now() + if (!force && now - this.loadedAt < 30_000) return + this.loadedAt = now + const data = readJsonFile(this.path) + const signature = data ? JSON.stringify(data) : "" + if (signature === this.signature && !force) return + this.signature = signature + const repos = new Map() + const groups = new Map() + if (data) { + const rawRepos = isPlainObject(data.repos) ? data.repos : {} + for (const [alias, path] of Object.entries(rawRepos)) { + if (typeof path === "string" && path) repos.set(alias, normalizePath(path)) + } + const rawGroups = isPlainObject(data.groups) ? data.groups : {} + for (const [name, members] of Object.entries(rawGroups)) { + if (Array.isArray(members)) groups.set(name, members.map(String)) + } + } + this.repos = repos + this.groups = groups + } + + get size(): number { + return this.repos.size + } + + /** Registered project whose path contains (or equals) `directory`. */ + projectFor(directory: string): { alias: string; path: string } | undefined { + this.load() + const dir = normalizePath(directory) + if (!dir) return undefined + let best: { alias: string; path: string } | undefined + for (const [alias, path] of this.repos) { + if (!isUnder(dir, path)) continue + if (!best || path.length > best.path.length) best = { alias, path } + } + return best + } + + /** Registered repos nested under `directory` (multi-repo workspace roots). */ + siblingsFor(directory: string): string[] { + this.load() + const dir = normalizePath(directory) + if (!dir) return [] + const out: string[] = [] + for (const [alias, path] of this.repos) { + if (isUnder(path, dir) && normalizePath(path) !== dir) out.push(alias) + } + return out.sort() + } + + groupsFor(alias: string): string[] { + this.load() + const out: string[] = [] + for (const [name, members] of this.groups) if (members.includes(alias)) out.push(name) + return out.sort() + } + + findAliasByPath(path: string): string | undefined { + this.load() + const target = normalizePath(path) + for (const [alias, repoPath] of this.repos) if (repoPath === target) return alias + return undefined + } +} + +/** Cached `status(kind="projects")` payload, used for scope fallback and health. */ +class StatusCache { + private repos: StatusRepo[] = [] + private groups: Record = {} + private fetchedAt = 0 + + async refresh(client: McpClient, timeoutMs = 10_000): Promise { + const result = (await client.call("status", { kind: "projects" }, { timeoutMs, retry: false })) as Loose | undefined + if (!result) return + if (Array.isArray(result.repos)) this.repos = result.repos as StatusRepo[] + if (isPlainObject(result.groups)) this.groups = result.groups as Record + this.fetchedAt = Date.now() + } + + get ageMs(): number { + return Date.now() - this.fetchedAt + } + + get isFresh(): boolean { + return this.fetchedAt > 0 + } + + get all(): StatusRepo[] { + return this.repos + } + + repo(alias: string): StatusRepo | undefined { + return this.repos.find((r) => r.alias === alias) + } + + projectFor(directory: string): { alias: string; path: string } | undefined { + const dir = normalizePath(directory) + if (!dir) return undefined + let best: { alias: string; path: string } | undefined + for (const repo of this.repos) { + if (!repo.project_path) continue + const path = normalizePath(repo.project_path) + if (!isUnder(dir, path)) continue + if (!best || path.length > best.path.length) best = { alias: repo.alias, path } + } + return best + } + + groupsFor(alias: string): string[] { + const out: string[] = [] + for (const [name, members] of Object.entries(this.groups)) if (members.includes(alias)) out.push(name) + return out.sort() + } + + siblingsFor(directory: string): string[] { + const dir = normalizePath(directory) + if (!dir) return [] + const out: string[] = [] + for (const repo of this.repos) { + if (!repo.project_path) continue + const path = normalizePath(repo.project_path) + if (isUnder(path, dir) && path !== dir) out.push(repo.alias) + } + return out.sort() + } +} + +/* ------------------------------------------------------------------ * + * Message helpers (new-user-message detection) + * ------------------------------------------------------------------ */ + +function messageText(message: Loose): string { + const parts = message?.parts + if (Array.isArray(parts)) { + return parts + .map((p: Loose) => (p && typeof p.text === "string" ? p.text : "")) + .filter(Boolean) + .join("\n") + } + if (typeof message?.content === "string") return message.content + if (Array.isArray(message?.content)) { + return message.content + .map((c: Loose) => (c && typeof c.text === "string" ? c.text : "")) + .filter(Boolean) + .join("\n") + } + return "" +} + +function roleOf(message: Loose): string { + return String(message?.info?.role ?? message?.role ?? "") +} + +function lastUserMessage(messages: Loose[]): { id: string; text: string } | null { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if (roleOf(message) !== "user") continue + const id = String(message?.info?.id ?? message?.id ?? "") + const text = messageText(message) + if (!text.trim()) continue + return { id: id || shortHash(text), text } + } + return null +} + +/** Heuristic for "this user message is a question about code" (recall gate). */ +function isLikelyCodeQuestion(text: string): boolean { + const trimmed = text.trim() + if (trimmed.length < 20 || trimmed.length > 4000) return false + if (/^\//.test(trimmed)) return false + return ( + /\b(where|how|why|what|who|which|find|locate|implement|handle|call|use|flow|work|refactor|fix|add|change|rename|test)\b/i.test(trimmed) || + /[A-Za-z_][A-Za-z0-9_]*\(|::|->|\.\w+\b|_[a-z]/.test(trimmed) + ) +} + +/* ------------------------------------------------------------------ * + * Guidance and scope text + * ------------------------------------------------------------------ */ + +function buildScopePrelude(scope: ScopeInfo, extra: string[]): string { + const lines: string[] = [] + if (scope.project) { + lines.push(`- Current directory is indexed by codesearch as project "${scope.project}"${scope.projectPath ? ` (${scope.projectPath})` : ""}.`) + if (scope.groups.length) { + lines.push(`- Project group${scope.groups.length > 1 ? "s" : ""}: ${scope.groups.map((g) => `"${g}"`).join(", ")} (use group= for cross-repo fan-out).`) + } + } else if (scope.registered && scope.siblings.length) { + lines.push(`- This workspace contains ${scope.siblings.length} registered codesearch project(s): ${scope.siblings.join(", ")}. Pass project= for one, or group= to fan out.`) + } else if (scope.hasGit) { + lines.push(`- This git repository is not registered with codesearch. Register it with \`codesearch index\` (or the /codesearch-index command) to enable semantic search here.`) + } + for (const line of extra) lines.push(line) + return lines.length ? `codesearch scope:\n${lines.join("\n")}` : "" +} + +/* ------------------------------------------------------------------ * + * Health runner + * ------------------------------------------------------------------ */ + +class HealthRunner { + private timer: ReturnType | null = null + private stopped = false + private failures = 0 + state: "unknown" | "ok" | "down" = "unknown" + lastError = "" + private readonly ctx: PluginContext + private readonly url: string + private readonly settings: CodeSearchSettings + private readonly onRecovered: () => void + private readonly log: (...args: unknown[]) => void + private readonly warn: (...args: unknown[]) => void + + constructor( + ctx: PluginContext, + url: string, + settings: CodeSearchSettings, + onRecovered: () => void, + log: (...args: unknown[]) => void, + warn: (...args: unknown[]) => void, + ) { + this.ctx = ctx + this.url = url + this.settings = settings + this.onRecovered = onRecovered + this.log = log + this.warn = warn + } + + start(): void { + if (!this.settings.health.enabled) return + this.schedule(this.settings.health.initialDelayMs) + } + + stop(): void { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + } + + private schedule(ms: number): void { + if (this.stopped) return + const timer = (this.timer = setTimeout(() => void this.tick(), ms)) + ;(timer as unknown as { unref?: () => void }).unref?.() + } + + private async tick(): Promise { + if (this.stopped) return + const before = this.state + await this.probe() + if (before === "down" && this.state === "ok") { + this.log(`hub recovered at ${this.url}`) + this.onRecovered() + } + let delay = this.settings.health.intervalMs + if (this.state === "down") { + this.failures += 1 + delay = Math.min(this.settings.health.intervalMs * 2 ** (this.failures - 1), this.settings.health.maxBackoffMs) + if (this.settings.health.reconnect) await this.tryReconnect() + if (this.settings.health.autoStart && this.failures === 1) await this.tryAutoStart() + } else { + this.failures = 0 + } + this.schedule(delay) + } + + /** Liveness probe: any HTTP response means reachable; only network errors mean down. */ + async probe(): Promise { + const base = this.url.replace(/\/mcp\/?$/i, "") + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 2000) + try { + await fetch(`${base}/healthz`, { signal: controller.signal }) + if (this.state !== "ok") this.log("hub reachable") + this.state = "ok" + this.lastError = "" + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (this.state !== "down") this.warn(`hub unreachable at ${this.url}: ${message}`) + this.state = "down" + this.lastError = message + } finally { + clearTimeout(timer) + } + } + + private async tryReconnect(): Promise { + if (!this.settings.health.reconnect) return + try { + if (this.ctx.mcp && typeof this.ctx.mcp.reload === "function") { + await this.ctx.mcp.reload() + this.log("asked OpenCode to reload MCP servers") + } + } catch (err) { + this.warn(`MCP reload failed: ${err instanceof Error ? err.message : err}`) + } + } + + private async tryAutoStart(): Promise { + if (!/^https?:\/\/(127\.0\.0\.1|localhost|\[::1\])/i.test(this.url)) return + const childProcess = builtin("node:child_process") + if (!childProcess || !this.settings.health.startCommand.trim()) return + try { + const child = childProcess.spawn(this.settings.health.startCommand, { + cwd: this.ctx.location?.directory || process.cwd(), + shell: true, + detached: true, + stdio: "ignore", + }) + child.unref?.() + this.log(`started: ${this.settings.health.startCommand}`) + } catch (err) { + this.warn(`auto-start failed: ${err instanceof Error ? err.message : err}`) + } + } +} + +/* ------------------------------------------------------------------ * + * Guard helpers + * ------------------------------------------------------------------ */ + +function resultOfToolEvent(event: Loose): { output?: string; metadata?: Loose } { + const result = isPlainObject(event?.result) ? event.result : undefined + if (!result) return {} + const output = typeof result.output === "string" ? result.output : undefined + const metadata = isPlainObject(result.metadata) ? result.metadata : undefined + return { output, metadata } +} + +function looksEmptyResult(event: Loose): boolean { + const { output, metadata } = resultOfToolEvent(event) + if (metadata) { + for (const key of ["matches", "count", "numMatches", "total"]) { + if (typeof metadata[key] === "number" && metadata[key] === 0) return true + } + } + if (output === undefined) return false + const trimmed = output.trim() + if (trimmed === "") return true + return /^(no (files|matches|results) found|0 (matches|results)|no matches)\b/i.test(trimmed) +} + +function extractPattern(event: Loose): string { + const input = isPlainObject(event?.input) ? event.input : undefined + if (!input) return "" + for (const key of ["pattern", "query", "search"]) { + const value = input[key] + if (typeof value === "string" && value.trim()) return value.trim() + } + return "" +} + +const REGEX_META = /[\\^$.*+?()[\]{}|]/ + +function formatSearchHits(result: Loose | undefined, limit: number): string[] { + const results = Array.isArray(result?.results) ? (result!.results as Loose[]) : [] + const lines: string[] = [] + for (const hit of results.slice(0, limit)) { + const path = typeof hit.path === "string" ? hit.path : "?" + const line = typeof hit.start_line === "number" ? `:${hit.start_line + 1}` : "" + const kind = typeof hit.kind === "string" ? ` (${hit.kind})` : "" + const signature = typeof hit.signature === "string" && hit.signature ? ` ${truncate(hit.signature, 90)}` : "" + lines.push(` - ${path}${line}${kind}${signature}`) + } + return lines +} + +/* ------------------------------------------------------------------ * + * Plugin + * ------------------------------------------------------------------ */ + +export default { + id: "codesearch", + + async setup(ctx: PluginContext) { + const settings = resolveSettings(ctx) + const log = (...args: unknown[]) => console.log("[codesearch]", ...args) + const warn = (...args: unknown[]) => console.warn("[codesearch]", ...args) + const directory = normalizePath(ctx.location?.directory ?? "") + + const url = resolveServeUrl(settings) + const token = resolveToken(settings) + const registry = new RepoRegistry( + settings.reposConfig || `${homeDir()}/.codesearch/repos.json`, + ) + registry.load() + const status = new StatusCache() + + // Interactive client (rescue/commands/recall) and a background client + // (health/status/compaction assistance) so a slow background call can + // never queue ahead of an interactive one. + const client = new McpClient(url, token, settings.debug, "client") + const background = new McpClient(url, token, settings.debug, "background") + + const disposables: Registration[] = [] + const timers: Array> = [] + const controller = new AbortController() + + const logGuard = (...args: unknown[]) => { + if (settings.log.guards) log(...args) + } + const logRescue = (...args: unknown[]) => { + if (settings.log.rescue) log(...args) + } + const logScope = (...args: unknown[]) => { + if (settings.log.scope) log(...args) + } + + if (settings.debug) log(`loaded in ${directory || "(unknown directory)"}; serve ${url}`) + + /* ---------------- scope resolution ---------------- */ + + const scopeCache = { at: 0, value: undefined as ScopeInfo | undefined } + + function resolveScope(): ScopeInfo { + const now = Date.now() + if (scopeCache.value && now - scopeCache.at < 15_000) return scopeCache.value + + registry.load() + const dir = directory + let project = dir ? registry.projectFor(dir) : undefined + if (!project && status.isFresh && dir) project = status.projectFor(dir) + + let groups: string[] = [] + if (project) { + groups = registry.groupsFor(project.alias) + if (!groups.length && status.isFresh) groups = status.groupsFor(project.alias) + } + + let siblings: string[] = [] + if (!project && dir) { + siblings = registry.siblingsFor(dir) + if (!siblings.length && status.isFresh) siblings = status.siblingsFor(dir) + siblings = siblings.slice(0, settings.scope.maxSiblings) + } + + const hasGit = dir ? fileExists(`${dir}/.git`) : false + const value: ScopeInfo = { + directory: dir, + project: project?.alias, + projectPath: project?.path, + groups: settings.scope.includeGroups ? groups : [], + siblings: settings.scope.siblings ? siblings : [], + registered: !!project, + hasGit, + } + scopeCache.at = now + scopeCache.value = value + return value + } + + /** Repo alias coverage: for guards and rescue. Uses the plugin's location. */ + function coveredAlias(): string | undefined { + return resolveScope().project + } + + /* ---------------- health + status refresh ---------------- */ + + const refreshStatus = async (): Promise => { + try { + await status.refresh(background) + scopeCache.at = 0 // pick up registry changes discovered via status + } catch (err) { + if (settings.debug) warn(`status refresh failed: ${err instanceof Error ? err.message : err}`) + } + } + + const health = new HealthRunner( + ctx, + url, + settings, + () => { + void refreshStatus() + }, + log, + warn, + ) + + // Startup probe + first status refresh, off the critical path. + void health.probe().then(() => void refreshStatus()) + + /* ---------------- freshness (file edits) ---------------- */ + + const editedPaths = new Map() + const recordEdited = (path: string): void => { + const normalized = normalizePath(path) + if (!normalized) return + if (!/\./.test(normalized)) return // skip directories / odd values + editedPaths.set(normalized, Date.now()) + while (editedPaths.size > 64) { + const oldest = editedPaths.keys().next().value as string | undefined + if (oldest === undefined) break + editedPaths.delete(oldest) + } + } + + const collectEditedFromEvent = (event: Loose): void => { + const data = isPlainObject(event?.data) ? event.data : event + for (const key of ["file", "path", "filePath", "filename"]) { + const value = data?.[key] + if (typeof value === "string") recordEdited(value) + } + } + + if (settings.freshness.enabled && ctx.event) { + const loop = (async () => { + try { + for await (const event of ctx.event!.subscribe({ signal: controller.signal })) { + try { + const type = String((event as Loose)?.type ?? "") + if (type.startsWith("file.")) collectEditedFromEvent(event as Loose) + } catch { + /* ignore malformed events */ + } + } + } catch (err) { + if (!(err instanceof Error && err.name === "AbortError")) { + if (settings.debug) warn(`event loop ended: ${err instanceof Error ? err.message : err}`) + } + } + })() + void loop + } + + function recentEdits(max: number): string[] { + const entries = [...editedPaths.entries()].sort((a, b) => b[1] - a[1]) + return entries.map(([path]) => path).slice(0, max) + } + + /* ---------------- guidance text ---------------- */ + + function loadGuidance(): string { + if (!settings.guidance.enabled) return "" + const inline = settings.guidance.text.trim() + if (inline) return inline + const path = expandEnv(settings.guidance.file).trim() + if (path) { + const text = readTextFile(path)?.trim() + if (text) return text + } + return DEFAULT_GUIDANCE + } + + const guidanceText = loadGuidance() + + /* ---------------- context hook (scope, guidance, prune, recall) ---------------- */ + + const seenMessages = new Set() + const seenSessions = new Set() + const nudged = new Set() + + function markSeen(set: Set, key: string): boolean { + if (set.has(key)) return false + set.add(key) + if (set.size > MAX_SEEN) set.clear() + return true + } + + async function onContext(event: Loose): Promise { + try { + const sessionID = String(event?.sessionID ?? "unknown") + const userMessage = lastUserMessage((event?.messages ?? []) as Loose[]) + if (!userMessage || !userMessage.text.trim()) return + const messageKey = `${sessionID}:${userMessage.id}` + if (!markSeen(seenMessages, messageKey)) return + const firstForSession = !seenSessions.has(sessionID) + seenSessions.add(sessionID) + if (seenSessions.size > MAX_SEEN) seenSessions.clear() + + const scope = resolveScope() + const system = Array.isArray(event?.system) ? (event.system as SystemPart[]) : undefined + if (!system) return + + // 1. Opt-in auto-recall: inject index context for the user message. + if (settings.recall.enabled && scope.project && isLikelyCodeQuestion(userMessage.text)) { + await injectRecall(system, sessionID, userMessage.text, scope) + } + + // 2. Prune guard: hide grep/glob from the model when enforcement is + // structural, so the first move is a codesearch call. + const pruneActive = settings.guards.mode === "prune" && !!scope.project + if (pruneActive && isPlainObject(event.tools)) { + for (const tool of settings.guards.tools) delete (event.tools as Loose)[tool] + } + + // 3. Guidance + scope, once per session (or per message). + const injectGuidance = settings.scope.enabled || guidanceText + const wantsThisMessage = + settings.scope.inject === "message" || (settings.scope.inject === "session" && firstForSession) + if (injectGuidance && wantsThisMessage) { + const extra: string[] = [] + if (pruneActive) { + extra.push(`- grep and glob are intentionally unavailable in this project (codesearch guard mode "prune"); use the codesearch tools for discovery.`) + } + if (health.state === "down") { + extra.push(`- The codesearch serve hub is unreachable right now; grep/glob is an acceptable fallback until it recovers.`) + } + if (freshUnindexedWarning(scope)) { + extra.push(`- The registered index for "${scope.project}" is empty or was built with an unknown model; searches will miss until \`codesearch index\` refreshes it.`) + } + if (settings.freshness.enabled) { + const edits = recentEdits(settings.freshness.maxFiles) + if (edits.length) { + const shown = edits.map((p) => relativeTo(p, scope.projectPath || directory) || p) + extra.push(`- Files edited this session: ${shown.join(", ")}. The serve hub reindexes on write; if a query misses a very recent change, retry it once.`) + editedPaths.clear() + } + } + const prelude = buildScopePrelude(scope, extra) + const text = [prelude, guidanceText].filter(Boolean).join("\n\n") + if (text) { + system.push({ type: "text", text }) + logScope(`injected scope/guidance for session ${sessionID} (project=${scope.project ?? "none"})`) + } + } + } catch (err) { + if (settings.debug) warn(`context hook error: ${err instanceof Error ? err.message : err}`) + } + } + + function freshUnindexedWarning(scope: ScopeInfo): boolean { + if (!settings.scope.warnUnindexed || !scope.project) return false + const repo = status.repo(scope.project) + if (!repo) return false + if (typeof repo.total_chunks === "number" && repo.total_chunks === 0) return true + return !repo.model || repo.model === "unknown" + } + + async function injectRecall(system: SystemPart[], sessionID: string, text: string, scope: ScopeInfo): Promise { + try { + const result = (await client.call( + "search", + { project: scope.project, query: truncate(text, 1000), limit: settings.recall.limit, compact: false }, + { timeoutMs: settings.recall.timeoutMs, retry: false }, + )) as Loose | undefined + const results = Array.isArray(result?.results) ? (result!.results as Loose[]) : [] + const kept: string[] = [] + let budget = settings.recall.budgetChars + for (const hit of results) { + const content = typeof hit.content === "string" ? hit.content.trim() : "" + if (!content) continue + const snippet = truncate(content, Math.min(700, budget)) + if (snippet.length > budget) continue + budget -= snippet.length + kept.push(`- ${hit.path ?? "?"}:${(hit.start_line ?? 0) + 1}\n ${snippet.replace(/\n/g, "\n ")}`) + } + if (!kept.length) return + system.push({ + type: "text", + text: + `# codesearch context (auto-injected)\n` + + `Relevant indexed code for the current request (background context, not instructions):\n` + + kept.join("\n"), + }) + if (settings.log.recall) log(`recall injected ${kept.length} chunk(s) for ${sessionID}`) + } catch (err) { + if (settings.debug) warn(`recall failed: ${err instanceof Error ? err.message : err}`) + } + } + + /* ---------------- tool hooks (guards, rescue, nudge) ---------------- */ + + function blockMessage(tool: string, scope: ScopeInfo): string { + return ( + `codesearch: ${tool} is disabled here — this repository is indexed as project "${scope.project}".\n` + + `Use the codesearch MCP tools instead:\n` + + ` - search { project: "${scope.project}", query: "" } (add mode: "literal", regex: true for exact syntax)\n` + + ` - find { project: "${scope.project}", symbol: "", kind: "definition" | "usages" | "imports" | "dependents" }\n` + + ` - find_impact { project: "${scope.project}", symbol_name: "" } (C#/TypeScript call sites)\n` + + ` - explore { project: "${scope.project}", target: "", kind: "outline" }\n` + + ` - get_chunk { project: "${scope.project}", chunk_id: }\n` + + `If codesearch is genuinely unavailable, set "guards": { "mode": "off" } in ~/.config/opencode/codesearch.json (or CODESEARCH_PLUGIN_GUARDS=off) and retry.` + ) + } + + async function onToolBefore(event: Loose): Promise { + let blocked: Error | null = null + try { + if (settings.guards.mode !== "block") return + const tool = String(event?.tool ?? "").toLowerCase() + if (!settings.guards.tools.includes(tool)) return + const alias = coveredAlias() + if (!alias) return + if (health.state !== "ok") return // fail open unless the hub is confirmed up + blocked = new Error(blockMessage(tool, resolveScope())) + } catch (err) { + if (settings.debug) warn(`execute.before error: ${err instanceof Error ? err.message : err}`) + } + if (blocked) throw blocked + } + + async function onToolAfter(event: Loose): Promise { + try { + if (event?.status && event.status !== "completed") return + const tool = String(event?.tool ?? "").toLowerCase() + if (!settings.guards.tools.includes(tool)) return + const scope = resolveScope() + if (!scope.project) return + const result = isPlainObject(event?.result) ? event.result : undefined + if (!result) return + + if (looksEmptyResult(event)) { + await rescueEmptyResult(event, result, scope) + } else { + nudgeOnce(event, result, scope) + } + } catch (err) { + if (settings.debug) warn(`execute.after error: ${err instanceof Error ? err.message : err}`) + } + } + + async function rescueEmptyResult(event: Loose, result: Loose, scope: ScopeInfo): Promise { + if (!settings.guards.rescue) return + if (health.state === "down") return + const pattern = extractPattern(event) + if (!pattern || pattern.length > 500) return + // Glob patterns are not search queries: skip when the pattern is just a + // filesystem wildcard ("**/*.ts") — there is nothing semantic to look up. + if (String(event?.tool ?? "").toLowerCase() === "glob" && /[*?[\]{}]/.test(pattern)) return + const plain = !REGEX_META.test(pattern) + try { + const hits = (await client.call( + "search", + { + project: scope.project, + query: pattern, + mode: "literal", + ...(plain ? { phrase: true } : { regex: true }), + limit: settings.guards.rescueLimit, + compact: true, + }, + { timeoutMs: settings.guards.rescueTimeoutMs, retry: false }, + )) as Loose | undefined + const lines = formatSearchHits(hits, settings.guards.rescueLimit) + if (!lines.length) return + const notice = + `\n[codesearch] ${String(event?.tool ?? "grep")} returned nothing, but the index has ` + + `${lines.length >= settings.guards.rescueLimit ? `${settings.guards.rescueLimit}+` : lines.length} ` + + `match(es) for ${plain ? `"${pattern}"` : `/${pattern}/`} in project "${scope.project}":\n` + + lines.join("\n") + + `\nUse search/find/get_chunk for the full result.` + const existing = typeof result.output === "string" ? result.output : "" + result.output = existing + notice + logRescue(`rescued empty ${String(event?.tool ?? "grep")} (${scope.project})`) + } catch (err) { + if (settings.debug) warn(`rescue failed: ${err instanceof Error ? err.message : err}`) + } + } + + function nudgeOnce(event: Loose, result: Loose, scope: ScopeInfo): void { + if (settings.guards.mode === "off") return + const sessionID = String(event?.sessionID ?? "unknown") + if (!markSeen(nudged, sessionID)) return + const existing = typeof result.output === "string" ? result.output : "" + result.output = + existing + + `\n[codesearch] tip: this repository is indexed as project "${scope.project}" — ` + + `codesearch search is faster and more token-efficient than ${String(event?.tool ?? "grep")} for cross-file or symbol lookups.` + logGuard(`nudged session ${sessionID} after ${String(event?.tool ?? "grep")}`) + } + + /* ---------------- commands ---------------- */ + + async function promptSession(sessionID: string, prompt: Loose, text: string): Promise { + if (!ctx.session?.prompt) { + warn("session.prompt is unavailable; cannot deliver command output") + return + } + await ctx.session.prompt({ ...prompt, sessionID, text, delivery: (prompt as Loose)?.delivery ?? "queue" }) + } + + function formatStatusText(scope: ScopeInfo): string { + const lines: string[] = [] + lines.push(`codesearch serve: ${url}`) + lines.push(`health: ${health.state}${health.lastError ? ` (${health.lastError})` : ""}`) + lines.push(`directory: ${directory || "?"}`) + if (scope.project) { + const repo = status.repo(scope.project) + lines.push( + `project: ${scope.project}` + + (repo + ? ` — ${repo.total_chunks ?? "?"} chunks, ${repo.total_files ?? "?"} files, model=${repo.model ?? "?"}, state=${repo.lock_status ?? "?"}` + : " (not in the latest status snapshot)"), + ) + } else if (scope.siblings.length) { + lines.push(`workspace projects: ${scope.siblings.join(", ")}`) + } else { + lines.push(`project: not registered${scope.hasGit ? " (git repo — run /codesearch-index to register)" : ""}`) + } + if (scope.groups.length) lines.push(`groups: ${scope.groups.join(", ")}`) + if (status.isFresh) lines.push(`hub repos: ${status.all.length}`) + return lines.join("\n") + } + + async function registerCommands(): Promise { + if (!settings.commands.enabled || !ctx.command?.transform) return + const registration = await ctx.command.transform((editor) => { + editor.add({ + name: "codesearch", + description: "Semantic code search (codesearch) — query the index and inject the top hits", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + const query = String(prompt.text ?? invocation?.arguments ?? "").trim() + const scope = resolveScope() + if (!query) { + await promptSession(sessionID, prompt, "Usage: /codesearch ") + return + } + if (!scope.project) { + await promptSession( + sessionID, + prompt, + `codesearch: this directory is not registered with a codesearch project, so there is nothing to search yet. Run /codesearch-index first.`, + ) + return + } + try { + const hits = (await client.call( + "search", + { project: scope.project, query, limit: 8 }, + { timeoutMs: 8_000, retry: false }, + )) as Loose | undefined + const lines = formatSearchHits(hits, 8) + await promptSession( + sessionID, + prompt, + lines.length + ? `codesearch results for "${query}" (project "${scope.project}"):\n${lines.join("\n")}\n\nFetch full code with get_chunk, or refine with find/explore.` + : `codesearch returned no results for "${query}" in project "${scope.project}". Try different wording, or mode literal with regex=true for exact syntax.`, + ) + if (settings.log.commands) log(`/codesearch "${truncate(query, 80)}" -> ${lines.length} hit(s)`) + } catch (err) { + await promptSession( + sessionID, + prompt, + `codesearch search failed: ${err instanceof Error ? err.message : String(err)}`, + ) + } + }, + }) + + editor.add({ + name: "codesearch-status", + description: "Show codesearch serve health, current project scope and index state", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + await health.probe() + await refreshStatus() + scopeCache.at = 0 + await promptSession(sessionID, prompt, formatStatusText(resolveScope())) + }, + }) + + editor.add({ + name: "codesearch-index", + description: "Register and index the current directory with codesearch (runs in the background)", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + const childProcess = builtin("node:child_process") + const command = settings.commands.indexCommand.trim() + if (!command) { + await promptSession(sessionID, prompt, "codesearch: commands.indexCommand is empty; nothing to run.") + return + } + if (!childProcess) { + await promptSession(sessionID, prompt, `codesearch: cannot spawn processes here. Run \`${command}\` in ${directory} manually.`) + return + } + try { + const child = childProcess.spawn(command, { + cwd: directory || process.cwd(), + shell: true, + detached: true, + stdio: "ignore", + }) + child.unref?.() + await promptSession( + sessionID, + prompt, + `codesearch indexing started in ${directory} (\`${command}\`). It runs in the background; check /codesearch-status in a minute (large repos can take longer).`, + ) + if (settings.log.commands) log(`/codesearch-index started: ${command}`) + } catch (err) { + await promptSession(sessionID, prompt, `codesearch index failed to start: ${err instanceof Error ? err.message : String(err)}`) + } + }, + }) + }) + disposables.push(registration) + } + + /* ---------------- scope tool ---------------- */ + + async function registerScopeTool(): Promise { + if (!settings.tools.scope || !ctx.tool?.transform) return + const registration = await ctx.tool.transform((editor) => { + editor.add({ + name: "codesearch_scope", + description: + "Return the codesearch scope for the current directory: project alias, groups, index state, serve health. " + + "Use it when you are unsure which project= value to pass to codesearch MCP tools.", + input: { type: "object", properties: {}, additionalProperties: false }, + execute: async () => { + await refreshStatus().catch(() => undefined) + scopeCache.at = 0 + const scope = resolveScope() + const repo = scope.project ? status.repo(scope.project) : undefined + const info = { + directory: scope.directory, + project: scope.project ?? null, + groups: scope.groups, + registered: scope.registered, + siblings: scope.siblings, + indexed: repo ? (repo.total_chunks ?? 0) > 0 : undefined, + chunks: repo?.total_chunks, + model: repo?.model, + lockStatus: repo?.lock_status, + serveUrl: url, + serveHealth: health.state, + } + return { content: JSON.stringify(info, null, 2) } + }, + }) + }) + disposables.push(registration) + } + + /* ---------------- skill ---------------- */ + + async function registerSkill(): Promise { + if (!settings.skill.enabled || !ctx.skill?.transform) return + const registration = await ctx.skill.transform((editor) => { + editor.add({ + id: "codesearch", + name: "codesearch navigation", + description: + "Use the codesearch MCP tools (search, find, find_impact, explore, get_chunk) to navigate code: " + + "project scope rules, tool routing, and exact-syntax patterns.", + path: "codesearch-plugin/skill.md", + content: SKILL_CONTENT, + }) + if (settings.skill.autoinvoke && typeof editor.get === "function" && typeof editor.update === "function") { + const existing = editor.get("codesearch") + if (existing) editor.update("codesearch", (skill: Loose) => void (skill.autoinvoke = true)) + } + }) + disposables.push(registration) + } + + /* ---------------- compaction assist ---------------- */ + + function formatOutline(target: string, result: unknown): string { + const entries = Array.isArray(result) ? (result as Loose[]) : [] + const symbols = entries + .filter((entry) => entry && typeof entry.signature === "string" && entry.signature) + .filter((entry) => !/^imports\b/i.test(String(entry.signature))) + .slice(0, 12) + .map((entry) => `${truncate(String(entry.signature), 100)}:${(entry.start_line ?? 0) + 1}`) + const label = target.replace(/\\/g, "/") + return symbols.length ? `- ${label}: ${symbols.join("; ")}` : "" + } + + async function onCompaction(event: Loose): Promise { + try { + if (!settings.compaction.enabled) return + const scope = resolveScope() + if (!scope.project || !Array.isArray(event?.system)) return + const candidates: string[] = [] + for (const path of recentEdits(settings.compaction.maxFiles * 2)) { + if (!candidates.includes(path)) candidates.push(path) + } + if (!candidates.length) return + const deadline = Date.now() + settings.compaction.timeoutMs + const outlines: string[] = [] + let budget = settings.compaction.budgetChars + for (const path of candidates) { + if (outlines.length >= settings.compaction.maxFiles || Date.now() >= deadline || budget <= 0) break + try { + const target = path.startsWith(scope.project + "/") ? path.slice(scope.project.length + 1) : path + const result = await client.call( + "explore", + { project: scope.project, target, kind: "outline" }, + { timeoutMs: Math.max(500, deadline - Date.now()), retry: false }, + ) + const line = formatOutline(target, result) + if (line) { + outlines.push(line) + budget -= line.length + } + } catch { + /* best effort */ + } + } + if (!outlines.length) return + ;(event.system as SystemPart[]).push({ + type: "text", + text: + `# codesearch outlines (captured before compaction)\n` + + `Pre-compaction structure of recently edited files (from the code index):\n` + + outlines.join("\n"), + }) + if (settings.log.scope) log(`compaction outlines: ${outlines.length} file(s)`) + } catch (err) { + if (settings.debug) warn(`compaction hook error: ${err instanceof Error ? err.message : err}`) + } + } + + /* ---------------- register everything (fail soft) ---------------- */ + + const safeRegister = async (name: string, fn: () => Promise): Promise => { + try { + await fn() + if (settings.debug) log(`registered: ${name}`) + } catch (err) { + warn(`could not register ${name}: ${err instanceof Error ? err.message : err}`) + } + } + + if (ctx.session?.hook) { + await safeRegister("session context hook", async () => { + disposables.push(await ctx.session!.hook("context", onContext)) + }) + } + if (ctx.tool?.hook && settings.guards.mode !== "off") { + await safeRegister("tool execute.before hook", async () => { + disposables.push(await ctx.tool!.hook("execute.before", onToolBefore)) + }) + await safeRegister("tool execute.after hook", async () => { + disposables.push(await ctx.tool!.hook("execute.after", onToolAfter)) + }) + } + if (settings.compaction.enabled && ctx.session?.hook) { + await safeRegister("compaction hook", async () => { + disposables.push(await ctx.session!.hook("compaction", onCompaction)) + }) + } + await safeRegister("commands", registerCommands) + await safeRegister("scope tool", registerScopeTool) + await safeRegister("skill", registerSkill) + + /* ---------------- timers ---------------- */ + + if (settings.health.enabled) { + health.start() + const statusTimer = setInterval(() => void refreshStatus(), 10 * 60_000) + ;(statusTimer as unknown as { unref?: () => void }).unref?.() + timers.push(statusTimer) + } + + log(`ready (project=${resolveScope().project ?? "none"}, guards=${settings.guards.mode}, serve=${url})`) + + return async () => { + controller.abort() + health.stop() + for (const timer of timers) clearTimeout(timer) + for (const registration of disposables) await registration.dispose().catch(() => undefined) + } + }, +} diff --git a/integrations/opencode/package.json b/integrations/opencode/package.json new file mode 100644 index 00000000..c6c6e60a --- /dev/null +++ b/integrations/opencode/package.json @@ -0,0 +1,16 @@ +{ + "name": "opencode-codesearch-plugin", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "OpenCode v2 plugin that makes codesearch structural: project scope resolution, usage guidance, grep/glob guards with zero-hit rescue, serve health, commands, a scope tool, a skill and compaction assistance.", + "license": "Apache-2.0", + "scripts": { + "typecheck": "tsc --noEmit", + "smoke": "node test/smoke.ts" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.6.0" + } +} diff --git a/integrations/opencode/test/smoke.ts b/integrations/opencode/test/smoke.ts new file mode 100644 index 00000000..4de3c0d0 --- /dev/null +++ b/integrations/opencode/test/smoke.ts @@ -0,0 +1,386 @@ +/** + * Smoke test for the codesearch OpenCode plugin. + * + * Runs the plugin against a mock OpenCode context and a mock codesearch MCP + * server (streamable HTTP) so scope resolution, guidance injection, zero-hit + * rescue, the nudge/prune/block guards, commands, the scope tool and the skill + * registration are exercised end-to-end without a real model session. + * + * Run (Node >= 23.6 strips types natively): + * node test/smoke.ts + * or: + * bun test/smoke.ts + */ + +import assert from "node:assert/strict" +import http from "node:http" +import { mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "../codesearch.ts" + +type Loose = Record +type Hook = (event: Loose) => any + +const HITS = [ + { + chunk_id: 7, + path: "my-service/src/uploader.ts", + start_line: 41, + end_line: 63, + kind: "Function", + signature: "function retryUpload(job: Job, attempt: number)", + }, + { + chunk_id: 9, + path: "my-service/src/queue.ts", + start_line: 12, + end_line: 30, + kind: "Class", + signature: "class UploadQueue", + }, +] + +const STATUS = { + repos: [ + { + alias: "my-service", + project_path: "/work/my-service", + total_chunks: 120, + total_files: 30, + model: "minilm-l6-q", + lock_status: "warm", + }, + { + alias: "sibling-api", + project_path: "/work/sibling-api", + total_chunks: 0, + total_files: 0, + model: "unknown", + lock_status: "available", + }, + ], + groups: { platform: ["my-service", "sibling-api"] }, + hub: { indexed: true, status: "ready" }, +} + +function startMockServer(): Promise<{ url: string; close: () => Promise }> { + const server = http.createServer((req, res) => { + let body = "" + req.on("data", (chunk) => (body += chunk)) + req.on("end", () => { + if (req.url === "/healthz") { + res.writeHead(200, { "content-type": "application/json" }) + res.end('{"status":"ok"}') + return + } + if (req.url !== "/mcp" || req.method !== "POST") { + res.writeHead(404) + res.end() + return + } + let message: Loose + try { + message = JSON.parse(body) as Loose + } catch { + res.writeHead(400) + res.end() + return + } + const reply = (payload: Loose, session = false): void => { + res.writeHead(200, { + "content-type": "application/json", + ...(session ? { "mcp-session-id": "smoke-session" } : {}), + }) + res.end(JSON.stringify(payload)) + } + if (message.method === "initialize") { + reply( + { + jsonrpc: "2.0", + id: message.id, + result: { + protocolVersion: message.params?.protocolVersion ?? "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "mock-codesearch", version: "smoke" }, + }, + }, + true, + ) + return + } + if (message.method === "notifications/initialized") { + res.writeHead(202) + res.end() + return + } + if (message.method === "tools/call") { + const name = String(message.params?.name ?? "") + const args = (message.params?.arguments ?? {}) as Loose + let payload: unknown = {} + if (name === "status") { + payload = STATUS + } else if (name === "search") { + const query = String(args.query ?? "").toLowerCase() + payload = query.includes("none-at-all") ? { results: [] } : { results: HITS } + } else if (name === "explore") { + payload = [ + { chunk_id: 1, kind: "Function", signature: "function retryUpload(job: Job)", start_line: 40, end_line: 63 }, + ] + } + reply({ + jsonrpc: "2.0", + id: message.id, + result: { content: [{ type: "text", text: JSON.stringify(payload) }] }, + }) + return + } + reply({ jsonrpc: "2.0", id: message.id, result: {} }) + }) + }) + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() + const port = typeof address === "object" && address ? address.port : 0 + resolve({ + url: `http://127.0.0.1:${port}/mcp`, + close: () => + new Promise((done) => { + server.close(() => done(undefined)) + }), + }) + }) + }) +} + +function makeCtx(directory: string) { + const hooks: Record = {} + const toolHooks: Record = {} + const tools: Loose[] = [] + const commands: Loose[] = [] + const skills: Loose[] = [] + const prompts: string[] = [] + + const context: Loose = { + app: { version: "smoke" }, + location: { directory }, + session: { + hook: async (name: string, callback: Hook) => { + ;(hooks[name] ||= []).push(callback) + return { dispose: async () => {} } + }, + prompt: async (input: Loose) => { + prompts.push(String(input.text ?? "")) + return {} + }, + }, + tool: { + hook: async (name: string, callback: Hook) => { + ;(toolHooks[name] ||= []).push(callback) + return { dispose: async () => {} } + }, + transform: async (callback: (editor: Loose) => void) => { + callback({ + namespace: () => {}, + add: (tool: Loose) => tools.push(tool), + get: () => undefined, + update: () => {}, + }) + return { dispose: async () => {} } + }, + }, + command: { + transform: async (callback: (editor: Loose) => void) => { + callback({ add: (command: Loose) => commands.push(command) }) + return { dispose: async () => {} } + }, + }, + skill: { + transform: async (callback: (editor: Loose) => void) => { + callback({ + add: (skill: Loose) => skills.push(skill), + get: () => undefined, + update: () => {}, + }) + return { dispose: async () => {} } + }, + }, + mcp: { list: async () => [] }, + event: { + subscribe: async function* (options?: { signal?: AbortSignal }) { + const signal = options?.signal + if (signal?.aborted) return + await new Promise((resolve) => signal?.addEventListener("abort", () => resolve())) + }, + }, + } + + return { context, hooks, toolHooks, tools, commands, skills, prompts } +} + +async function setupPlugin(directory: string) { + const app = makeCtx(directory) + const cleanup = await plugin.setup(app.context as any) + return { app, cleanup } +} + +function userMessage(sessionID: string, messageID: string, text: string): Loose { + return { + sessionID, + system: [], + tools: { grep: {}, glob: {}, read: {} }, + messages: [{ info: { role: "user", id: messageID }, parts: [{ type: "text", text }] }], + } +} + +async function main(): Promise { + const mock = await startMockServer() + const tmp = mkdtempSync(join(tmpdir(), "codesearch-plugin-smoke-")) + + const reposPath = join(tmp, "repos.json") + writeFileSync( + reposPath, + JSON.stringify({ + repos: { "my-service": "/work/my-service", "sibling-api": "/work/sibling-api" }, + groups: { platform: ["my-service", "sibling-api"] }, + }), + ) + const configPath = join(tmp, "codesearch.json") + writeFileSync( + configPath, + JSON.stringify({ + url: "", + log: { guards: false, rescue: false, health: false, commands: false, scope: false, recall: false }, + }), + ) + + process.env.CODESEARCH_CONFIG = configPath + process.env.CODESEARCH_REPOS_CONFIG = reposPath + process.env.CODESEARCH_URL = mock.url + process.env.CODESEARCH_PLUGIN_HEALTH = "0" + delete process.env.CODESEARCH_PLUGIN_GUARDS + + /* ---------------- nudge mode (default) ---------------- */ + + const one = await setupPlugin("/work/my-service") + assert.ok(one.app.hooks.context?.length === 1, "context hook registered") + assert.ok(one.app.toolHooks["execute.before"]?.length === 1, "execute.before hook registered") + assert.ok(one.app.toolHooks["execute.after"]?.length === 1, "execute.after hook registered") + assert.ok(one.app.hooks.compaction?.length === 1, "compaction hook registered") + assert.deepEqual( + one.app.commands.map((c) => c.name).sort(), + ["codesearch", "codesearch-index", "codesearch-status"], + "commands registered", + ) + assert.equal(one.app.tools.length, 1, "scope tool registered") + assert.equal(one.app.tools[0].name, "codesearch_scope") + assert.equal(one.app.skills.length, 1, "skill registered") + assert.equal(one.app.skills[0].id, "codesearch") + + // Guidance + scope injection, once per session. + const first = userMessage("s1", "m1", "Where is the upload retried after a failure?") + await one.app.hooks.context[0](first) + const injected = first.system.map((p: Loose) => p.text).join("\n") + assert.match(injected, /project "my-service"/, "scope line names the resolved project") + assert.match(injected, /group: "platform"/, "scope line lists groups") + assert.match(injected, /codesearch/, "guidance is injected") + assert.ok(first.tools.grep, "nudge mode keeps grep visible") + + const second = userMessage("s1", "m2", "And where is it queued?") + await one.app.hooks.context[0](second) + assert.equal(second.system.length, 0, "session mode injects once per session") + + const duplicate = userMessage("s1", "m1", "Where is the upload retried after a failure?") + await one.app.hooks.context[0](duplicate) + assert.equal(duplicate.system.length, 0, "the same user message never injects twice") + + // Zero-hit rescue turns an empty grep into codesearch hits. + const rescueEvent: Loose = { + tool: "grep", + status: "completed", + sessionID: "s1", + callID: "c1", + input: { pattern: "retryUpload" }, + result: { output: "" }, + } + await one.app.toolHooks["execute.after"][0](rescueEvent) + assert.match(String(rescueEvent.result.output), /codesearch/, "rescue appends a codesearch section") + assert.match(String(rescueEvent.result.output), /uploader\.ts/, "rescue includes index hits") + + // A successful grep gets a one-time nudge per session. + const nudgeEvent: Loose = { + tool: "grep", + status: "completed", + sessionID: "s1", + callID: "c2", + input: { pattern: "queue" }, + result: { output: "src/queue.ts:1:const queue = []", metadata: { count: 1 } }, + } + await one.app.toolHooks["execute.after"][0](nudgeEvent) + assert.match(String(nudgeEvent.result.output), /tip:/, "first successful grep is nudged") + const nudgeAgain: Loose = { ...nudgeEvent, callID: "c3", result: { output: "hit", metadata: { count: 1 } } } + await one.app.toolHooks["execute.after"][0](nudgeAgain) + assert.doesNotMatch(String(nudgeAgain.result.output), /tip:/, "the nudge fires once per session") + + // Scope tool reports the resolved project and index state. + const scopeResult = (await one.app.tools[0].execute({}, {})) as { content: string } + const scopeInfo = JSON.parse(scopeResult.content) + assert.equal(scopeInfo.project, "my-service") + assert.equal(scopeInfo.indexed, true) + assert.equal(scopeInfo.chunks, 120) + + // Commands resolve and deliver results through session.prompt. + const searchCommand = one.app.commands.find((c) => c.name === "codesearch") + assert.ok(searchCommand) + await searchCommand.execute({ sessionID: "s1", prompt: { text: "upload retry" } }) + assert.match(one.app.prompts.at(-1) ?? "", /uploader\.ts/, "/codesearch injects hits") + + const statusCommand = one.app.commands.find((c) => c.name === "codesearch-status") + assert.ok(statusCommand) + await statusCommand.execute({ sessionID: "s1", prompt: { text: "" } }) + assert.match(one.app.prompts.at(-1) ?? "", /project: my-service/, "/codesearch-status reports scope") + + await one.cleanup() + + /* ---------------- prune mode ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "prune" + const pruned = await setupPlugin("/work/my-service") + const prunedEvent = userMessage("s2", "p1", "How does the uploader work?") + await pruned.app.hooks.context[0](prunedEvent) + assert.equal(prunedEvent.tools.grep, undefined, "prune removes grep") + assert.equal(prunedEvent.tools.glob, undefined, "prune removes glob") + assert.ok(prunedEvent.tools.read, "prune keeps unrelated tools") + assert.match(prunedEvent.system.map((p: Loose) => p.text).join("\n"), /intentionally unavailable/) + await pruned.cleanup() + + /* ---------------- block mode (fail-open everywhere else) ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "block" + const blocked = await setupPlugin("/work/my-service") + const before = blocked.app.toolHooks["execute.before"][0] + let threw = "" + for (let i = 0; i < 30 && !threw; i++) { + try { + await before({ tool: "grep", input: { pattern: "x" }, sessionID: "s3" }) + } catch (err) { + threw = String(err) + } + if (!threw) await new Promise((r) => setTimeout(r, 100)) + } + assert.match(threw, /codesearch/, "block mode denies grep once the hub is reachable") + assert.match(threw, /project "my-service"/, "denial names the project scope to use") + + const uncovered = await setupPlugin("/work/unregistered") + await uncovered.app.toolHooks["execute.before"][0]({ tool: "grep", input: {}, sessionID: "s4" }) + await uncovered.cleanup() + await blocked.cleanup() + + delete process.env.CODESEARCH_PLUGIN_GUARDS + // Give any in-flight startup probe a moment before the mock hub disappears. + await new Promise((resolve) => setTimeout(resolve, 200)) + await mock.close() + console.log("smoke: OK") +} + +await main() diff --git a/integrations/opencode/tsconfig.json b/integrations/opencode/tsconfig.json new file mode 100644 index 00000000..393d8e8a --- /dev/null +++ b/integrations/opencode/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "noUnusedLocals": true, + "noUnusedParameters": false + }, + "include": ["codesearch.ts", "test/smoke.ts"] +} From be03bcc26c2c902453b7e035f6890f87e5257b5c Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Fri, 2 Oct 2026 21:01:31 -0400 Subject: [PATCH 37/77] docs: reference the OpenCode integration from README and CHANGELOG --- CHANGELOG.md | 6 ++++++ README.md | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa375c13..d24dcd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.4.12] + +### Added + +- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. + ## [1.4.11] ### Changed diff --git a/README.md b/README.md index a09b30db..099637cc 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ If your agent skips codesearch and falls back to grep/glob too often, paste this > Prefer codesearch for semantic, cross-file, or symbol-oriented lookup ("where is X implemented", "find usages of Y", "how does Z flow"). Use plain grep/glob for a single known file, trivial one-line edits, or exact literal searches. In remote-serve mode, returned paths are from the **server's** filesystem — read content via `get_chunk` rather than opening paths locally, and unindexed dirs (`.venv`, `node_modules`, `build/`) simply return nothing. -OpenCode: put this in the user-level `~/.config/opencode/AGENTS.md` (applies across all projects). Claude Code reads a project-level `AGENTS.md`, so add it per-project (or symlink a shared one). +OpenCode: put this in the user-level `~/.config/opencode/AGENTS.md` (applies across all projects), or install the [OpenCode plugin](integrations/opencode/README.md), which injects this guidance with the resolved project scope automatically. Claude Code reads a project-level `AGENTS.md`, so add it per-project (or symlink a shared one). **Claude Code specifically** tends to ignore this advice more than other clients — its MCP tool schemas are deferred (an extra `ToolSearch` call is needed before codesearch tools are even callable), while Grep/Glob are always fully loaded and zero-friction, and spawned subagents don't inherit `AGENTS.md` or the MCP `initialize` instructions at all. @@ -253,6 +253,8 @@ codesearch hooks claude install --project # project scope (./.claude) The native command embeds the hook scripts in the binary (no source tree needed) and merges the registrations into `settings.json`. The equivalent from-source installers still live in [`integrations/claude-code/`](integrations/claude-code/) (`install.ps1` / `install.sh`) if you'd rather run them directly. +**OpenCode** gets the same structural treatment through the OpenCode v2 plugin in [`integrations/opencode/`](integrations/opencode/README.md): it resolves the session directory to an indexed project, injects scope-aware guidance once per session, rescues empty greps with index hits, optionally prunes or blocks `grep`/`glob`, keeps the serve hub healthy (with optional auto-start), and adds `/codesearch*` commands, a `codesearch_scope` tool, a skill and compaction assistance. Install by copying `codesearch.ts` into `~/.config/opencode/plugins/` — see the [integration README](integrations/opencode/README.md). + Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` override) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. ## MCP Tools Reference From 566b6a9ffeb0de025eb9e509be013db6ac2dab2e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:05:36 +0000 Subject: [PATCH 38/77] chore: bump version to 1.4.13 (auto, PR #293 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dc819f7..958b8a08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.12" +version = "1.4.13" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 4533ace3..57a05499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.12" +version = "1.4.13" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From d0d48132cf628bce60f15da728eaf133b877a93d Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 11:49:30 +0200 Subject: [PATCH 39/77] fix: simple-name extraction strips parameters before segmenting (key-format 2.0 regression) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_simple_name split on '#'/'.' BEFORE stripping parameters. Key-format 2.0 canonical keys carry fully qualified parameter types, full of dots — rsplit('.') landed inside the parameter list, so the stored simple name was a parameter fragment ('Activity, int)', 'APIs)') instead of the method name. Every symbol WITH parameters became unresolvable by name; only parameterless methods survived. Latent since the 2.0 key format landed, armed for every repo at once by the builder-version gate's rebuild wave. New order: after the last '#', strip at '(' first, then at backtick (generic arity), trim './#' suffix, take the last non-empty dot segment — so segmentation can never land inside a parameter list. The table test pins FQN-parameter keys, generics, top-level and nested names; every row fails against the old implementation. Validation: fmt clean, clippy -D warnings clean, cargo test --lib --bins 777+773 passed / 0 failed. --- src/symbols/csharp.rs | 43 ++++++++++++++++++++++++------------- src/symbols/csharp_tests.rs | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 9df5a929..482ac427 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -371,23 +371,36 @@ fn read_ref_warnings(env: &TrackedEnv, rtxn: &heed::RoTxn<'_>, canonical: &str) /// - `"csharp App . FieldDefinition#Validate()."` → `"Validate"` /// - `"csharp SmallSolution.Library . Calculator#Add(int, int)."` → `"Add"` /// - `"csharp . . . Namespace.TopLevel"` → `"TopLevel"` -fn extract_simple_name(scip_symbol: &str) -> String { - // Strip trailing suffix chars: "Validate()." → "Validate", "MyService#" → "MyService" - let cleaned = scip_symbol - .trim_end_matches('.') - .trim_end_matches("()") - .trim_end_matches('#'); - // Take last non-empty segment after '#' or '.' - let last_segment = cleaned - .rsplit(['#', '.']) - .find(|s| !s.trim().is_empty()) - .unwrap_or(cleaned) - .trim(); - // Strip method parameters (e.g. "Add(int, int)" → "Add") - last_segment +/// +/// Parameter lists are stripped BEFORE segmenting: key format 2.0 writes +/// fully-qualified parameter types (`SaveBatchAsync(int, System.Collections +/// .Generic.IReadOnlyList)`), and splitting on `.` first would +/// land inside the parameters — the "Activity, int)" fragments that left +/// every parameterized method unresolvable by fuzzy name. +pub(crate) fn extract_simple_name(scip_symbol: &str) -> String { + // Method/field segment: after the last '#' (types without '#' keep the + // whole string — their name is the last '.'-segment below; a bare + // trailing '#' falls back to the pre-# text). + let after_hash = scip_symbol.rsplit('#').next().unwrap_or(scip_symbol); + let base = if after_hash.trim().is_empty() { + scip_symbol + } else { + after_hash + }; + // Strip parameters FIRST, then the generic-arity marker: keys group all + // overloads and arities under one queryable name. + let name = base .split('(') .next() - .unwrap_or(last_segment) + .unwrap_or(base) + .split('`') + .next() + .unwrap_or(base); + let cleaned = name.trim().trim_end_matches('.').trim_end_matches('#'); + cleaned + .rsplit('.') + .find(|s| !s.trim().is_empty()) + .unwrap_or(cleaned) .trim() .to_string() } diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs index 662d29c3..26259db3 100644 --- a/src/symbols/csharp_tests.rs +++ b/src/symbols/csharp_tests.rs @@ -334,3 +334,44 @@ fn rebuild_meta_write_stamps_version_and_writes_or_clears_warnings() { "a degraded rebuild's warnings must be persisted for the answer path" ); } + +// ── Simple-name extraction (fuzzy-lookup keys) ──────────────────────────── + +/// The live defect this pins: key format 2.0 writes fully-qualified +/// parameter types, and the old extractor split on '.' BEFORE stripping the +/// parameter list — landing inside the parameters. Every parameterized +/// method then keyed `scip_simple_names` under a fragment like +/// "Activity, int)" and fuzzy find_impact resolved NOTHING repository-wide +/// after the version-gate rebuilt all indexes to format 2.0. +#[test] +fn extract_simple_name_strips_fqn_parameters_before_segmenting() { + let cases = [ + // (canonical key, expected simple name) + ("csharp App . FieldDefinition#Validate().", "Validate"), + ( + "csharp SmallSolution.Library . Calculator#Add(int, int).", + "Add", + ), + ( + "csharp Acme.Catalog.Azure.TableStorage.DataStore . ActivityVersionStore#SaveBatchAsync(int, System.Collections.Generic.IReadOnlyList).", + "SaveBatchAsync", + ), + ( + "csharp Acme.Catalog.Import.CmdInfra . Arguments#GetVariableValue`1(string, Acme.Catalog.Import.CmdInfra.ArgumentValidator).", + "GetVariableValue", + ), + ("csharp . . . Namespace.TopLevel", "TopLevel"), + ("csharp App . MyService#", "MyService"), + ("csharp Ns.Sub . Class#_field", "_field"), + // Generic TYPE arity lives on the type segment — stripped too, not + // just the method-side backtick. + ("csharp Ns . List`1#", "List"), + ]; + for (key, expected) in cases { + assert_eq!( + super::csharp::extract_simple_name(key), + expected, + "key: {key}" + ); + } +} From 2771c7f6778dd254e021f3299b031382dba42a96 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:06:47 +0000 Subject: [PATCH 40/77] chore: bump version to 1.4.14 (auto, PR #294 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 958b8a08..d0418caa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.13" +version = "1.4.14" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 57a05499..746ee432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.13" +version = "1.4.14" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From abc77ca62a56f5054d4811e8c7f757d71223308c Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Sat, 3 Oct 2026 10:53:47 -0400 Subject: [PATCH 41/77] fix(integrations): health-gate prune and fix compaction target resolution Addresses the review on the OpenCode plugin: - prune mode now engages only while the hub is confirmed reachable, like block mode; while the hub is down grep/glob stay visible and the injected guidance says grep/glob is the fallback, matching the fail-open contract the header and README already documented. - compaction outlines resolve edited files against the project path with relativeTo() instead of stripping an alias prefix, which produced absolute targets that never matched on Windows (silent no-op). - compaction uses the background MCP client its comment already described, so a slow outline call cannot queue ahead of an interactive one. - postRaw keeps its abort timer armed through response.text(), so a hung body cannot block the serial call chain past the configured timeout. - README documents that settings are resolved once at setup. - smoke test: the mock server records MCP calls and the mock host emits file events; new regression tests cover prune-while-down and the project-relative compaction target (both fail without the fixes). --- integrations/opencode/README.md | 5 ++ integrations/opencode/codesearch.ts | 43 ++++++++----- integrations/opencode/test/smoke.ts | 93 +++++++++++++++++++++++++---- 3 files changed, 116 insertions(+), 25 deletions(-) diff --git a/integrations/opencode/README.md b/integrations/opencode/README.md index 453a7226..71f58ca2 100644 --- a/integrations/opencode/README.md +++ b/integrations/opencode/README.md @@ -224,6 +224,11 @@ Precedence, lowest to highest: | `CODESEARCH_PLUGIN_HEALTH` | `true` | Health probing on/off. | | `CODESEARCH_PLUGIN_DEBUG` | `false` | Verbose plugin logs. | +Settings are resolved **once, at plugin setup** (the host calls `setup()` when +the plugin loads). Editing `codesearch.json` or a `CODESEARCH_*` variable +mid-session therefore does not take effect until OpenCode restarts or the +plugin is reloaded. + ### Hub discovery order The plugin resolves the MCP endpoint in this order, first non-empty wins: diff --git a/integrations/opencode/codesearch.ts b/integrations/opencode/codesearch.ts index ae319b89..f3659b23 100644 --- a/integrations/opencode/codesearch.ts +++ b/integrations/opencode/codesearch.ts @@ -661,25 +661,28 @@ class McpClient { const controller = new AbortController() const timer = setTimeout(() => controller.abort(), timeoutMs) - let response: Response + // The timer (and its abort signal) must stay armed through the body read: + // `response.text()` is as capable of hanging as the response headers are + // (undici's ~300 s bodyTimeout), and clearing the timer early would leave + // the serial call chain blocked far past the requested timeout. try { - response = await fetch(this.url, { + const response = await fetch(this.url, { method: "POST", headers, body: JSON.stringify(payload), signal: controller.signal, }) + if (!response.ok) throw new Error(`codesearch HTTP ${response.status} ${response.statusText}`) + const session = response.headers.get("mcp-session-id") + const body = await response.text() + const messages = parseMcpBody(body) as Loose[] + if (messages.length === 0 && payload.id !== undefined) { + throw new Error(`codesearch returned no JSON-RPC message for ${String(payload.method)}`) + } + return { session, messages } } finally { clearTimeout(timer) } - if (!response.ok) throw new Error(`codesearch HTTP ${response.status} ${response.statusText}`) - const session = response.headers.get("mcp-session-id") - const body = await response.text() - const messages = parseMcpBody(body) as Loose[] - if (messages.length === 0 && payload.id !== undefined) { - throw new Error(`codesearch returned no JSON-RPC message for ${String(payload.method)}`) - } - return { session, messages } } private resultOf(tool: string, messages: Loose[]): unknown { @@ -1327,8 +1330,12 @@ export default { } // 2. Prune guard: hide grep/glob from the model when enforcement is - // structural, so the first move is a codesearch call. - const pruneActive = settings.guards.mode === "prune" && !!scope.project + // structural, so the first move is a codesearch call. Like block + // mode, only engage while the hub is confirmed reachable — + // otherwise the removed tools would contradict the "grep/glob is an + // acceptable fallback" guidance injected below. + const pruneActive = + settings.guards.mode === "prune" && !!scope.project && health.state === "ok" if (pruneActive && isPlainObject(event.tools)) { for (const tool of settings.guards.tools) delete (event.tools as Loose)[tool] } @@ -1342,7 +1349,7 @@ export default { if (pruneActive) { extra.push(`- grep and glob are intentionally unavailable in this project (codesearch guard mode "prune"); use the codesearch tools for discovery.`) } - if (health.state === "down") { + if (health.state === "down" && !pruneActive) { extra.push(`- The codesearch serve hub is unreachable right now; grep/glob is an acceptable fallback until it recovers.`) } if (freshUnindexedWarning(scope)) { @@ -1729,8 +1736,14 @@ export default { for (const path of candidates) { if (outlines.length >= settings.compaction.maxFiles || Date.now() >= deadline || budget <= 0) break try { - const target = path.startsWith(scope.project + "/") ? path.slice(scope.project.length + 1) : path - const result = await client.call( + // Strip the project root the same way `relativeTo` does for the + // freshness notice: it normalizes both sides (including the + // win32 lowercase rule), so absolute paths, alias-prefixed paths + // and already-relative paths all resolve consistently. Stripping + // a literal alias prefix here silently produced absolute targets + // on Windows, where the case-normalized path never matched. + const target = relativeTo(path, scope.projectPath || directory) || path + const result = await background.call( "explore", { project: scope.project, target, kind: "outline" }, { timeoutMs: Math.max(500, deadline - Date.now()), retry: false }, diff --git a/integrations/opencode/test/smoke.ts b/integrations/opencode/test/smoke.ts index 4de3c0d0..7d8163c5 100644 --- a/integrations/opencode/test/smoke.ts +++ b/integrations/opencode/test/smoke.ts @@ -64,7 +64,8 @@ const STATUS = { hub: { indexed: true, status: "ready" }, } -function startMockServer(): Promise<{ url: string; close: () => Promise }> { +function startMockServer(): Promise<{ url: string; close: () => Promise; calls: Loose[] }> { + const calls: Loose[] = [] const server = http.createServer((req, res) => { let body = "" req.on("data", (chunk) => (body += chunk)) @@ -117,6 +118,7 @@ function startMockServer(): Promise<{ url: string; close: () => Promise }> if (message.method === "tools/call") { const name = String(message.params?.name ?? "") const args = (message.params?.arguments ?? {}) as Loose + calls.push({ name, args }) let payload: unknown = {} if (name === "status") { payload = STATUS @@ -148,6 +150,7 @@ function startMockServer(): Promise<{ url: string; close: () => Promise }> new Promise((done) => { server.close(() => done(undefined)) }), + calls, }) }) }) @@ -160,6 +163,8 @@ function makeCtx(directory: string) { const commands: Loose[] = [] const skills: Loose[] = [] const prompts: string[] = [] + const eventQueue: Loose[] = [] + let eventWake: (() => void) | undefined const context: Loose = { app: { version: "smoke" }, @@ -209,13 +214,34 @@ function makeCtx(directory: string) { event: { subscribe: async function* (options?: { signal?: AbortSignal }) { const signal = options?.signal - if (signal?.aborted) return - await new Promise((resolve) => signal?.addEventListener("abort", () => resolve())) + while (!signal?.aborted) { + if (!eventQueue.length) { + await new Promise((resolve) => { + eventWake = resolve + signal?.addEventListener("abort", () => resolve(), { once: true }) + }) + } + while (eventQueue.length) yield eventQueue.shift() as Loose + } }, }, } - return { context, hooks, toolHooks, tools, commands, skills, prompts } + return { + context, + hooks, + toolHooks, + tools, + commands, + skills, + prompts, + emit: (event: Loose) => { + eventQueue.push(event) + const wake = eventWake + eventWake = undefined + wake?.() + }, + } } async function setupPlugin(directory: string) { @@ -322,6 +348,25 @@ async function main(): Promise { await one.app.toolHooks["execute.after"][0](nudgeAgain) assert.doesNotMatch(String(nudgeAgain.result.output), /tip:/, "the nudge fires once per session") + // Compaction assist: recently edited files become explore outlines, and the + // target sent to the server must be project-relative. Regression guard: an + // alias-prefix strip left absolute targets that could never match on + // Windows, where the plugin lowercases paths but the server strips them + // case-sensitively — a silent compaction no-op. + one.app.emit({ type: "file.edited", data: { file: "/work/my-service/src/uploader.ts" } }) + await new Promise((resolve) => setTimeout(resolve, 150)) + const compactionEvent: Loose = { system: [] } + await one.app.hooks.compaction[0](compactionEvent) + const compacted = compactionEvent.system.map((p: Loose) => p.text).join("\n") + assert.match(compacted, /src\/uploader\.ts/, "compaction outline names the relative file") + const exploreCall = mock.calls.filter((c) => c.name === "explore").at(-1) + assert.ok(exploreCall, "compaction asked the server for an outline") + assert.equal( + exploreCall!.args.target, + "src/uploader.ts", + "compaction passes the project-relative target, not an absolute/alias-prefixed path", + ) + // Scope tool reports the resolved project and index state. const scopeResult = (await one.app.tools[0].execute({}, {})) as { content: string } const scopeInfo = JSON.parse(scopeResult.content) @@ -346,12 +391,21 @@ async function main(): Promise { process.env.CODESEARCH_PLUGIN_GUARDS = "prune" const pruned = await setupPlugin("/work/my-service") - const prunedEvent = userMessage("s2", "p1", "How does the uploader work?") - await pruned.app.hooks.context[0](prunedEvent) - assert.equal(prunedEvent.tools.grep, undefined, "prune removes grep") - assert.equal(prunedEvent.tools.glob, undefined, "prune removes glob") - assert.ok(prunedEvent.tools.read, "prune keeps unrelated tools") - assert.match(prunedEvent.system.map((p: Loose) => p.text).join("\n"), /intentionally unavailable/) + // Prune only engages once the startup health probe has confirmed the hub is + // reachable; poll with fresh sessions instead of racing the probe (the + // guidance is injected once per session, so each attempt needs its own). + let prunedEvent: Loose | undefined + for (let i = 0; i < 30 && !prunedEvent; i++) { + const candidate = userMessage(`s2-${i}`, `p${i}`, "How does the uploader work?") + await pruned.app.hooks.context[0](candidate) + if (candidate.tools.grep === undefined) prunedEvent = candidate + else await new Promise((r) => setTimeout(r, 100)) + } + assert.ok(prunedEvent, "prune engages once the hub is reachable") + assert.equal(prunedEvent!.tools.grep, undefined, "prune removes grep") + assert.equal(prunedEvent!.tools.glob, undefined, "prune removes glob") + assert.ok(prunedEvent!.tools.read, "prune keeps unrelated tools") + assert.match(prunedEvent!.system.map((p: Loose) => p.text).join("\n"), /intentionally unavailable/) await pruned.cleanup() /* ---------------- block mode (fail-open everywhere else) ---------------- */ @@ -376,6 +430,25 @@ async function main(): Promise { await uncovered.cleanup() await blocked.cleanup() + /* ---------------- prune fails open while the hub is down ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "prune" + process.env.CODESEARCH_URL = "http://127.0.0.1:1/mcp" + const downPrune = await setupPlugin("/work/my-service") + // Let the startup probe fail (connection refused) so health settles on down. + await new Promise((resolve) => setTimeout(resolve, 300)) + const downEvent = userMessage("s5", "d1", "Where is the uploader?") + await downPrune.app.hooks.context[0](downEvent) + assert.ok(downEvent.tools.grep, "prune keeps grep visible while the hub is down") + assert.ok(downEvent.tools.glob, "prune keeps glob visible while the hub is down") + assert.match( + downEvent.system.map((p: Loose) => p.text).join("\n"), + /unreachable|acceptable fallback/, + "guidance names the grep/glob fallback while the hub is down", + ) + await downPrune.cleanup() + process.env.CODESEARCH_URL = mock.url + delete process.env.CODESEARCH_PLUGIN_GUARDS // Give any in-flight startup probe a moment before the mock hub disappears. await new Promise((resolve) => setTimeout(resolve, 200)) From 57bc3c7a0deac38954799d0644173e3a2815be40 Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Sat, 3 Oct 2026 11:04:58 -0400 Subject: [PATCH 42/77] test(integrations): unit-test stripJsonc and HealthRunner Closes the two test gaps left from the review: - export stripJsonc and HealthRunner for tests (the host consumes the default export only) and add test/unit.ts covering the JSONC scanner (comments, escapes, trailing commas) and the health runner (probe transitions, backoff/reconnect, recovery callback, stop latch, disabled mode). - fix stripJsonc dropping a trailing comma followed by a comment ("a": 1, // last member \n }), a common JSONC shape that previously produced invalid JSON; the lookahead now skips comments and consumes the full */ pair. - add npm run unit and npm test (typecheck + unit + smoke); document the coverage in the README. Both new failure modes were verified by reintroducing the defect and watching the test fail: the comment-separated trailing comma and the HealthRunner stop() latch (an in-flight probe must not reschedule). --- integrations/opencode/README.md | 17 +- integrations/opencode/codesearch.ts | 31 +++- integrations/opencode/package.json | 4 +- integrations/opencode/test/smoke.ts | 3 +- integrations/opencode/test/unit.ts | 230 ++++++++++++++++++++++++++++ 5 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 integrations/opencode/test/unit.ts diff --git a/integrations/opencode/README.md b/integrations/opencode/README.md index 71f58ca2..fac15b84 100644 --- a/integrations/opencode/README.md +++ b/integrations/opencode/README.md @@ -454,18 +454,23 @@ hub uses. ## Development -The plugin is dependency-free at runtime. For type checking and the smoke -test (which runs the plugin against a mock OpenCode context and a mock MCP -server): +The plugin is dependency-free at runtime. For type checking and the tests +(unit tests for the pure helpers and the health runner, plus an end-to-end +smoke test against a mock OpenCode host and MCP server): ```bash cd integrations/opencode npm install # dev-only: typescript + @types/node npm run typecheck # tsc --noEmit -npm run smoke # node test/smoke.ts (Node >= 23.6, or: bun test/smoke.ts) +npm run unit # node test/unit.ts — stripJsonc + HealthRunner (Node >= 23.6) +npm run smoke # node test/smoke.ts — end-to-end vs mock host/MCP (or: bun) +npm test # typecheck + unit + smoke ``` +`test/unit.ts` covers the JSONC scanner (comments, string escapes, trailing +commas including comment-separated ones) and the health runner (probe +transitions, backoff/reconnect, recovery callback, stop, disabled mode). `test/smoke.ts` covers scope resolution, guidance injection and deduplication, zero-hit rescue, the one-time nudge, prune and block modes (including -fail-open for unregistered directories), the commands, the scope tool and the -skill registration. +fail-open for unregistered directories and an unreachable hub), compaction +outlines, the commands, the scope tool and the skill registration. diff --git a/integrations/opencode/codesearch.ts b/integrations/opencode/codesearch.ts index f3659b23..84819c47 100644 --- a/integrations/opencode/codesearch.ts +++ b/integrations/opencode/codesearch.ts @@ -337,7 +337,11 @@ function configDir(): string { * Strip `//` and block comments plus trailing commas from JSONC, honouring * string literals. Enough for OpenCode's config files. */ -function stripJsonc(text: string): string { +/** + * Strip JSONC comments and trailing commas. Exported for `test/unit.ts`; the + * host consumes the default export only. + */ +export function stripJsonc(text: string): string { let out = "" let inString = false let escaped = false @@ -368,8 +372,25 @@ function stripJsonc(text: string): string { continue } if (ch === ",") { + // A trailing comma may sit before whitespace AND comments, e.g. + // "a": 1, // keep this member last + // } + // Skip both while looking ahead for the closing bracket. let j = i + 1 - while (j < text.length && /\s/.test(text[j])) j++ + for (;;) { + while (j < text.length && /\s/.test(text[j])) j++ + if (text[j] === "/" && text[j + 1] === "/") { + while (j < text.length && text[j] !== "\n") j++ + continue + } + if (text[j] === "/" && text[j + 1] === "*") { + j += 2 + while (j < text.length && !(text[j] === "*" && text[j + 1] === "/")) j++ + j += 2 // skip the closing */ + continue + } + break + } if (text[j] === "}" || text[j] === "]") continue // drop trailing comma } out += ch @@ -951,7 +972,11 @@ function buildScopePrelude(scope: ScopeInfo, extra: string[]): string { * Health runner * ------------------------------------------------------------------ */ -class HealthRunner { +/** + * Periodic `/healthz` probe with backoff and optional MCP reload. Exported for + * `test/unit.ts`; the host consumes the default export only. + */ +export class HealthRunner { private timer: ReturnType | null = null private stopped = false private failures = 0 diff --git a/integrations/opencode/package.json b/integrations/opencode/package.json index c6c6e60a..0ff721e0 100644 --- a/integrations/opencode/package.json +++ b/integrations/opencode/package.json @@ -7,7 +7,9 @@ "license": "Apache-2.0", "scripts": { "typecheck": "tsc --noEmit", - "smoke": "node test/smoke.ts" + "unit": "node test/unit.ts", + "smoke": "node test/smoke.ts", + "test": "npm run typecheck && npm run unit && npm run smoke" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/integrations/opencode/test/smoke.ts b/integrations/opencode/test/smoke.ts index 7d8163c5..1e555b0a 100644 --- a/integrations/opencode/test/smoke.ts +++ b/integrations/opencode/test/smoke.ts @@ -3,7 +3,8 @@ * * Runs the plugin against a mock OpenCode context and a mock codesearch MCP * server (streamable HTTP) so scope resolution, guidance injection, zero-hit - * rescue, the nudge/prune/block guards, commands, the scope tool and the skill + * rescue, the nudge/prune/block guards (including prune fail-open while the + * hub is down), compaction outlines, commands, the scope tool and the skill * registration are exercised end-to-end without a real model session. * * Run (Node >= 23.6 strips types natively): diff --git a/integrations/opencode/test/unit.ts b/integrations/opencode/test/unit.ts new file mode 100644 index 00000000..814335c0 --- /dev/null +++ b/integrations/opencode/test/unit.ts @@ -0,0 +1,230 @@ +/** + * Unit tests for the codesearch OpenCode plugin's pure helpers and the health + * runner. Complements test/smoke.ts, which exercises the plugin end-to-end + * against a mock OpenCode host and a mock codesearch MCP server. + * + * Run (Node >= 23.6 strips types natively): + * node test/unit.ts + * or: + * npm run unit + */ + +import assert from "node:assert/strict" +import http from "node:http" +import type { AddressInfo } from "node:net" +import { HealthRunner, stripJsonc } from "../codesearch.ts" + +type Loose = Record + +/* ------------------------------------------------------------------ * + * stripJsonc + * ------------------------------------------------------------------ */ + +function testStripJsonc(): void { + // Plain JSON passes through unchanged and still parses. + const plain = '{"a":1,"b":[1,2],"c":"x"}' + assert.equal(stripJsonc(plain), plain, "plain JSON is unchanged") + assert.deepEqual(JSON.parse(stripJsonc(plain)), JSON.parse(plain)) + + // Line comments (leading, trailing, at EOF) are removed. + assert.deepEqual(JSON.parse(stripJsonc('{\n // note\n "a": 1 // tail\n}\n')), { a: 1 }) + assert.deepEqual(JSON.parse(stripJsonc('{"a":1} // done')), { a: 1 }, "comment at EOF without newline") + + // Block comments, including inline and multi-line. + assert.deepEqual(JSON.parse(stripJsonc('{/* one */"a":/* two\nlines */1}')), { a: 1 }) + + // Comment markers inside strings are data, not comments. + assert.deepEqual( + JSON.parse(stripJsonc('{"url":"http://example.com/a//b","glob":"/* literal */"}')), + { url: "http://example.com/a//b", glob: "/* literal */" }, + ) + + // Escaped quotes do not terminate the string early. + assert.deepEqual(JSON.parse(stripJsonc('{"s":"a\\"//not-a-comment"}')), { s: 'a"//not-a-comment' }) + + // Trailing commas in objects and arrays are dropped. + assert.deepEqual(JSON.parse(stripJsonc('{\n "a": [1, 2,],\n "b": {"c": 3,},\n}')), { + a: [1, 2], + b: { c: 3 }, + }) + + // Regression: a trailing comma may be followed by a comment before the + // closing bracket ("a": 1, // last member \n }). Comments must be skipped + // while looking ahead, not mistaken for the next member. + assert.deepEqual(JSON.parse(stripJsonc('{"a": 1, // last member\n}')), { a: 1 }) + assert.deepEqual(JSON.parse(stripJsonc('{"a": [1, /* c */ ]}')), { a: [1] }) + + // A comma followed by a real member is preserved. + assert.deepEqual(JSON.parse(stripJsonc('{"a":1, "b":2}')), { a: 1, b: 2 }) + + // CRLF line comments terminate at the newline, not at a literal \r. + assert.deepEqual(JSON.parse(stripJsonc('{\r\n // note\r\n "a": 1\r\n}\r\n')), { a: 1 }) +} + +/* ------------------------------------------------------------------ * + * HealthRunner + * ------------------------------------------------------------------ */ + +function healthSettings(overrides: Loose = {}): Loose { + return { + health: { + enabled: true, + intervalMs: 40, + initialDelayMs: 10, + maxBackoffMs: 80, + reconnect: true, + autoStart: false, + startCommand: "", + ...overrides, + }, + } +} + +async function waitFor(predicate: () => boolean, what: string, timeoutMs = 4000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`timed out waiting for ${what}`) +} + +/** Minimal /healthz server that counts probes, can stop/restart in place, and + * can hold responses so a probe can be kept in flight deterministically. */ +async function startHealthServer(port = 0) { + let count = 0 + let hold = false + let pending: Array<() => void> = [] + const make = () => + http.createServer((_req, res) => { + count += 1 + const respond = () => { + res.writeHead(200, { "content-type": "application/json" }) + res.end('{"status":"ok"}') + } + if (hold) pending.push(respond) + else respond() + }) + let server = make() + await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)) + const actual = (server.address() as AddressInfo).port + return { + url: `http://127.0.0.1:${actual}/mcp`, + hits: () => count, + pendingProbes: () => pending.length, + hold: (value: boolean) => { + hold = value + }, + release: () => { + const waiting = pending + pending = [] + for (const respond of waiting) respond() + }, + close: () => new Promise((resolve) => server.close(() => resolve())), + restart: async () => { + server = make() + await new Promise((resolve) => server.listen(actual, "127.0.0.1", resolve)) + }, + } +} + +async function testHealthRunner(): Promise { + const hub = await startHealthServer() + const logs: string[] = [] + const warnings: string[] = [] + const log = (...args: unknown[]) => logs.push(args.join(" ")) + const warn = (...args: unknown[]) => warnings.push(args.join(" ")) + + // probe() reflects reachability in state and lastError. + { + const runner = new HealthRunner({} as any, hub.url, healthSettings() as any, () => {}, log, warn) + await runner.probe() + assert.equal(runner.state, "ok", "probe marks a reachable hub ok") + assert.equal(runner.lastError, "", "ok state clears lastError") + await hub.close() + await runner.probe() + assert.equal(runner.state, "down", "probe marks an unreachable hub down") + assert.ok(runner.lastError.length > 0, "down state records the error") + assert.ok( + warnings.some((line) => /hub unreachable/.test(line)), + "the first down transition is warned about", + ) + runner.stop() + } + + // start() probes after the initial delay, backs off while down, asks the + // host to reload failed MCP servers, and reports recovery exactly once. + { + await hub.restart() + let reloads = 0 + let recovered = 0 + const ctx = { + mcp: { + reload: async () => { + reloads += 1 + }, + }, + } + const runner = new HealthRunner( + ctx as any, + hub.url, + healthSettings() as any, + () => { + recovered += 1 + }, + log, + warn, + ) + runner.start() + await waitFor(() => runner.state === "ok", "the initial probe to mark the hub ok") + await hub.close() + await waitFor(() => runner.state === "down", "the hub to be detected down") + await waitFor(() => reloads >= 1, "the reconnect reload call") + await hub.restart() + await waitFor(() => runner.state === "ok", "the hub to be detected recovered") + await waitFor(() => recovered === 1, "the recovery callback") + assert.ok( + logs.some((line) => /hub recovered/.test(line)), + "recovery is logged", + ) + // stop() must latch: even a probe that is already in flight must not + // reschedule another one afterwards. + hub.hold(true) + await waitFor(() => hub.pendingProbes() >= 1, "an in-flight probe to hold") + const hitsAtStop = hub.hits() + runner.stop() + hub.release() + await new Promise((resolve) => setTimeout(resolve, 250)) + assert.equal(hub.hits(), hitsAtStop, "stop() halts probing, including from an in-flight tick") + hub.hold(false) + hub.release() + } + + // start() is a no-op when health probing is disabled. + { + const hits = hub.hits() + const runner = new HealthRunner( + {} as any, + hub.url, + healthSettings({ enabled: false }) as any, + () => {}, + log, + warn, + ) + runner.start() + await new Promise((resolve) => setTimeout(resolve, 120)) + assert.equal(hub.hits(), hits, "disabled health never probes") + assert.equal(runner.state, "unknown", "state stays unknown while disabled") + runner.stop() + } + + await hub.close() +} + +async function main(): Promise { + testStripJsonc() + await testHealthRunner() + console.log("unit: OK") +} + +await main() From 80e5c4375f18b79afab57b839f34e1eef81759f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:37:48 +0000 Subject: [PATCH 43/77] chore: bump version to 1.4.15 (auto, PR #295 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0418caa..c182f4bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.14" +version = "1.4.15" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 746ee432..4b4ca011 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.14" +version = "1.4.15" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From a01e437e85b453ba0e4a6430aaebf05c20961f7a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:53:50 +0000 Subject: [PATCH 44/77] chore: bump version to 1.4.16 (auto, PR #292 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7e86623..e6113c4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.15" +version = "1.4.16" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 4b4ca011..0c82abd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.15" +version = "1.4.16" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From c327a8550bd43fe988b079c0be71d5576313306c Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 18:57:36 +0200 Subject: [PATCH 45/77] =?UTF-8?q?docs(cloud):=20build=20the=20image=20loca?= =?UTF-8?q?lly=20=E2=80=94=20ACR=20Basic/Standard=20agents=20fail=20the=20?= =?UTF-8?q?model-warmup=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integrations/cloud/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integrations/cloud/README.md b/integrations/cloud/README.md index 24999902..0352d6dd 100644 --- a/integrations/cloud/README.md +++ b/integrations/cloud/README.md @@ -42,6 +42,14 @@ consumed by the serve replica). There is no shared filesystem or database betwee environment, a storage account, and container app/job resources. - The `codesearch` container image published to a registry the platform can pull from. +> ⚠️ **Build locally, not via `az acr build`, on a Basic/Standard-SKU registry.** The +> builder stage compiles the release binary, then downloads and warms up the embedding +> model before tarring it — memory/disk-heavy on top of the compile. ACR's default build +> agent (Basic/Standard SKU; Agent Pools with bigger VMs need Premium) reliably fails at +> the `COPY --from=builder /models.tar.gz` step with `failed to create image: failed to +> get layer sha256:...: layer does not exist`, after a ~25 min compile — confirmed twice, +> not a one-off. Build with local Docker Desktop and `docker push` to the registry instead. + ## Provisioning (generic) Replace every `<...>` placeholder with your own values. From a378b3259b4cb810ec112457ef4589178634f00e Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 19:16:41 +0200 Subject: [PATCH 46/77] docs: changelog for always-on symbol indexes + cloud build note --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fea1bed1..f3e37bcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,17 @@ finalized in place with a date — no renaming/migration step needed. - **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. - **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. +- **Always-on symbol indexes (C# + TypeScript).** Symbol (SCIP) indexes are no longer a manual or incidental artifact: serve warmup, cold opens, file-watcher batches and reindexes each drive a per-language symbol pass wherever chunk indexes are built, so `find_impact` evidence exists without a separate rebuild step. Startup inspects the on-disk symbol indexes with a builder-version stamp and rebuilds those an older binary produced (degraded retry instead of a hard failure), so a upgraded serve never answers from stale symbol data. ### Changed - **`codesearch serve` runs heavy repo jobs one at a time by default (`CODESEARCH_INDEX_JOBS=1`).** Warmup, cold opens, file-watcher batches and reindexes now queue behind a running refresh instead of all running at once; raise the value deliberately on machines with cores and memory to spare. Standalone `codesearch index` / `codesearch mcp` runs are unchanged (no gate is attached). `CODESEARCH_EMBED_THREADS` unset keeps the previous ONNX default (all cores), so single-repo index throughput is unchanged unless the variable is set. +- **`find_impact` is honest about partial evidence; the TUI says how healthy the symbol indexes are.** Index-level rebuild warnings (non-compiling C# projects, swallowed resolution failures, non-zero `scip-typescript` exits) ride the `find_impact` answer as an additive `warnings` array instead of vanishing into the log. The scip-csharp helper's workspace-load diagnostics are aggregated into a single warnings line and never escalate to errors during workspace load. The serve TUI's symbol-index indicators use severity language: a degraded C# index renders red (`C#⚠`) and the info line names the failures. +- **Cloud image: build locally, not via `az acr build`.** The Dockerfile's builder stage compiles the release binary and warms up the embedding model before tarring — memory/disk-heavy, and ACR Basic/Standard build agents reliably fail the model-tar layer after a ~25 min compile (confirmed twice). `integrations/cloud/README.md` now documents building with local Docker Desktop and `docker push` instead. + +### Fixed + +- **SCIP key-format 2.0 regression: simple-name extraction now strips parameters before segmenting.** Fully qualified symbol keys that carry parameter types (real C# methods do) segmented on the parameter list instead of the path segments, producing wrong simple names for the affected symbols. ## [1.4.11] From 502f18139e022a306d0989f21a93ebf8170ebb9e Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 19:26:40 +0200 Subject: [PATCH 47/77] chore: drop branch-local AGENTS.md before finalization (convention: never reaches develop) --- AGENTS.md | 264 ------------------------------------------------------ 1 file changed, 264 deletions(-) delete mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index c915512c..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,264 +0,0 @@ -# AGENTS.develop.md - -This file is the **develop-branch reference** for all coding agents. -It is committed on `develop` and merged from every feature branch. - -`AGENTS.md` and `CLAUDE.md` are **branch-local working files**: committed on the -feature branch while building, dropped before the PR merges — they never appear -on `develop` or `master`. See *For agents starting a new feature branch* below. - -## Setup (once per machine) - -```bash -git config core.hooksPath .githooks -``` - -This enables the hooks: `post-checkout` auto-creates `AGENTS.md` (from this -file) and the `CLAUDE.md` pointer on feature branches, `pre-commit` runs -`cargo fmt` + the root-md allowlist guard (branch-aware, see below), `pre-push` -blocks direct pushes to `master`. See `.githooks/README.md`. - -## For agents starting a new feature branch - -If `AGENTS.md` was not auto-created by the hook, create it manually: - -```bash -cp AGENTS.develop.md AGENTS.md -``` - -Then append your work plan as a `## Plan` section at the end of `AGENTS.md`. -Leave the architecture sections intact — they provide context. - -**Commit `AGENTS.md`/`CLAUDE.md` on the feature branch as you work** — the plan -travels through the PR and survives branch switches. - -**At the end of the branch, before the PR (the changelog moment):** -1. Update `## Active feature branches` below (remove this branch) -2. Add one line to `## Changelog highlights` -3. Add the CHANGELOG.md entry under the pending-version heading (functionality changes only — pure tooling/docs churn gets the `no-changelog` PR label instead; workflow changes like this one never go in the changelog) -4. Drop the branch-local files and commit: - `git rm AGENTS.md CLAUDE.md` → `chore: drop branch-local agent files` - -Enforced mechanically (forgetting step 4 cannot leak the files): -- `pre-commit` rejects introducing root `AGENTS.md`/`CLAUDE.md` on - `develop`/`master` (direct commits and conflict resolutions). -- `pre-merge-commit` runs the same guard on the merge result — a clean - merge never invokes pre-commit. -- The `agent-files-check` CI workflow flags any PR into `develop`/`master` - whose head still carries them. - -**No active work plan lives here.** Feature branches carry their own `AGENTS.md`. -This file contains only architecture, conventions, and changelog. - ---- - -## ⚠️ Branching & PR workflow (READ FIRST) - -This repo uses a **`develop`-based** gitflow. The GitHub default branch is `master` (`origin/HEAD → origin/master`), but `master` is **NOT** the integration branch. - -- **Integration branch = `develop`.** All feature/fix/release branches merge into `develop`. -- **ALL PRs target `develop`** — pass `--base develop` to `gh pr create`. NEVER target `master` (releases only, cut at release time). -- **Merge style:** feature/fix → `develop` = merge commits (`--merge`); `develop` → `master` release PR = **squash**, with `--body "$(scripts/release-coauthors.sh)"` so contributors stay credited on the default branch. -- **Review requirement** is enforced by a repo ruleset. As repo owner, override with `gh pr merge --merge --admin --delete-branch`. -- Before creating a PR, **verify the base**: `gh pr view --json baseRefName`. If it says `master`, retarget: `gh pr edit --base develop`. -- **Release squashes regress the merge-base** → a release PR may fail with "cannot be cleanly created" even when content is fine. Fix: cut `release/vX.Y.Z` off develop, run `git merge -s ours origin/master` in it, verify the diff vs master is the intended release delta, PR that into master. Never merge master into develop directly. - -Common mistake: a subagent creates a PR with no explicit `--base`, the tooling picks `master` (GitHub default), and the PR lands against the wrong branch. Always specify `--base develop`. - ---- - -## What codesearch is - -A fast, local, offline MCP server for semantic code search. Single Rust binary. -No Docker, no cloud, no external services. Designed for coding agents (OpenCode, -Claude Code, Claude Desktop) that need to search and navigate large codebases efficiently. - -Core stack: Tantivy (BM25 FTS) + arroy (HNSW vectors) + fastembed/ONNX (embeddings) + -tree-sitter (AST chunking) + LMDB (persistent storage) + rmcp 1.5.0 (MCP protocol). -Hybrid BM25 + vector search fused via RRF. - -## MCP tools - -Five tools exposed to agents: - -| Tool | Description | -|---|---| -| `search` | Hybrid semantic + BM25 search. Requires `project` or `group` in multi-repo mode. | -| `find` | Symbol navigation: definition, usages, imports, dependents. Requires scope. | -| `explore` | File outline or similar-chunk lookup. Requires scope. | -| `get_chunk` | Retrieve a chunk by ID with optional context lines. Requires `project` in multi-repo mode. | -| `status` | Index and project status. Lightweight (no DB open) when called without scope. | - -All tools return `scope_required` structured errors in multi-repo mode when no `project` -or `group` is specified, with `available_projects`, `available_groups`, and `hint_for_agent`. - -## Multi-repo serve mode - -`codesearch serve` starts an MCP HTTP server on `127.0.0.1:{port}` (default 39725). -Multiple repos register via `repos.json` with aliases. Agents route queries per-alias -or per-group. - -**Repo lifecycle:** -- `Warm` — DB open, vector index ready, no FSW. State after background warmup or fan-out open. -- `Write` — Warm + file system watcher running. Transitions from Warm on first explicit project query. -- `Readonly` — Another process holds the write lock. -- `Idle` (status string was `Closed`) — Evicted by idle reaper after `REPO_IDLE_TIMEOUT_SECS` (30 min default) of inactivity; index stays on disk, stores auto-reopen on the next query. - -**Idle reaper** runs every `REAPER_INTERVAL_SECS` (5 min). Evicts repos not queried within timeout. -All opens update `last_access` so the reaper can track every open. - -**Fan-out rule:** `get_chunk` and group queries use `touch=false` → repos open as Warm only, -no FSW spawned. Only explicit `project=` queries use `touch=true` → Warm→Write transition. - -## TUI - -`codesearch serve` with a TTY starts an embedded ratatui TUI (repo table, status, CPU). -Without TTY: headless, logs only. - -- `codesearch serve --no-tui` — suppress TUI even with TTY -- `codesearch serve tui [--url http://...]` — standalone TUI via HTTP polling of `GET /status` - -## HTTP endpoints (serve mode) - -| Endpoint | Method | Description | -|---|---|---| -| `/health` | GET | Health check JSON | -| `/status` | GET | Lightweight repo state snapshot for TUI polling | -| `/repos` | POST | Register + index + warmup a new repo | -| `/repos/:alias` | DELETE | Stop FSW, evict, unregister, delete DB | -| `/repos/:alias/reindex` | POST | Incremental or force reindex (background) | -| `/mcp` | GET/POST | MCP streamable HTTP endpoint | - -## Supported languages - -17 tree-sitter grammars (table in README.md). `find_impact` has SCIP symbol -precision for C# (bundled `scip-csharp`, resident helper via its `serve` -subcommand) and TypeScript (`npx scip-typescript`). Protobuf is text-aware -chunking only. - -## Release artifacts - -GitHub Actions produces release binaries per tag (plain + `-with-csharp` -variants): `codesearch-{windows-x86_64.zip,linux-x86_64.tar.gz,macos-arm64.tar.gz}`. -Single-file native binaries, no runtime dependencies. macOS build is manual-trigger -only (expensive runners). - ---- - -## Current state - -- **Version:** `Major.Minor.Patch` (semver). Patch auto-bumps +1 on every PR merged to `develop` (`.github/workflows/bump-develop.yml`); minor bumps manually at release (`scripts/bump-version.sh --type minor`, resets patch→0). Per-commit uniqueness: `build.rs` `+`. See `RELEASING.md`. -- **Validation:** `cargo check` for iteration → `cargo clippy --all-targets -- -D warnings` → `cargo test --lib --bins` before a branch is done. No `--release` builds during the fix loop. -- **Deploy:** `..\copy-to-common.ps1` builds + copies both binaries to `~/.local/bin/`. Stop serve first — a running `codesearch.exe` is file-locked on Windows. - -## Implemented features (orientation only — narratives live in CHANGELOG.md) - -- **Federation** — `codesearch remote add/rm/list/mounts`; `@peer` group fan-out; mounted projects addressable as `project=/`; TUI mount management with live status. Remote write verbs (`add`, `reindex --force`) need a read-write peer; `rm` is not durable across a cold start. -- **Cloud indexer-job split** — heavy build job uploads a snapshot, light serve restores it. DOCS repos are `repo_read_only` in `repos.json`; only `custom-kb` stays writable with a bounded incremental reindex per KB `git pull`. -- **`find_impact`** — SCIP-precise transitive call-sites (C#, TypeScript), ambiguity envelope, partial-results warnings, freshness fields. On `busy: true` sleep `retry_after_seconds` and retry the SAME call (busy is progress, never a reason to fall back to text search); `index_head_sha` vs `current_head_sha` drift is surfaced, never auto-reindexed. -- **REST mirrors** — read-only HTTP mirrors of the MCP tools incl. `/find-impact`. - -## ⚠️ Design constraint — never poll a federated peer - -**A federated peer is NEVER contacted on a timer.** Each poll wakes the peer's -scale-to-zero replica, which then self-warms its full idle window (measured ~50% -duty cycle from zero searches). Shipped twice (PR #181/#184), reverted twice — -do not re-attempt. Local repos may be polled in the background; peers are -contacted only by a real federated tool call (activity poke) or the explicit -TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). - -## Open TODOs - -- **Low severity:** literal-mode snippets for markdown chunks sometimes show the chunk's opening line instead of the matched line (the `match_info.unwrap_or_else` fallback), making true hits look like false negatives. - ---- - -## Key conventions for agents - -- **Branch from develop**, never from master. Feature branches: `features/` (or `fix/`, `chore/`). -- **Cargo.toml version** on develop may be one version ahead of the deployed binary — that is expected due to `copy-to-common.ps1` deploy hook. Never flag as inconsistency. -- **Always fix bugs you encounter — including pre-existing ones** the current branch did not introduce. Clean code and tests are the bar: a fix lands with a test that fails without it (reintroduce the defect, watch it fail — the proof rule under *Search errors*). -- **Never write separate `AGENTS_xxx.md` sibling files** unless explicitly requested. OpenCode reads `AGENTS.md` only. Out-of-repo planning goes to `C:\WorkArea\AI\codesearch\instructions\`. -- **Root file hygiene (markdown)**: root markdown is allowlisted — `AGENTS.develop.md`, `README.md`, `README_CSharp.md`, `CHANGELOG.md`, `RELEASING.md` always; `AGENTS.md` and `CLAUDE.md` on **feature branches only** (branch-local, dropped at finalization — the pre-commit guard rejects them on `develop`/`master`). Anything else (diagnoses, plans, test scenarios, worklogs) goes into `.docs/` (gitignored, local-only). Enforced by the `pre-commit` root-md allowlist guard and the `agent-files-check` CI workflow. -- **Path normalization**: all path comparisons must go through a single normalize utility. Windows UNC prefixes (`\\?\C:\`), backslash/forward-slash mismatches, and worktree `.git` file resolution have each caused subtle bugs in the past. -- **ONNX arena allocator**: uses `kNextPowerOfTwo` growth, never returns memory to OS. ~2GB memory during indexing is a known limitation. No local fix available until upstream fastembed exposes `OrtArenaCfg`. -- **Git worktrees**: `find_git_root` returns the worktree directory itself. Each worktree is a separate indexable repo. Groups can be used to search across worktrees of the same base repo. - -## Runtime locations - -- **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. -- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). -- **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. -- **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. - -## Deploying to runtime - -- `..\copy-to-common.ps1` — builds and copies **both** `codesearch.exe` and `scip-csharp.exe` to `~/.local/bin/` (the common execution dir). Use this to update the runtime binaries. **No `--release` builds — always dev/debug.** -- The C# helper is built via: `dotnet publish helpers/csharp/scip-csharp.csproj -r win-x64 --self-contained -c Release` -- Helper output must be **single-file only**: `scip-csharp.exe` (+ optional `.pdb`). The `.csproj` has `PublishSingleFile=true`. -- Do NOT copy framework DLLs, `BuildHost-*` dirs, or `.dll.config` files to the runtime location. - -## Notes for agents - -- **Never use the bundled `codesearch` binary to investigate this repo** (it is the project under development). Use codesearch MCP tools first for discovery (this repo is indexed as `codesearch-git`); `grep`/`Read` for exact refs, other git refs, or when MCP returns nothing. -- **Tests live in sibling `_tests.rs` files**, table-driven preferred over near-duplicate per-case fns. -- **Tests that set env vars must be `#[serial]`** and set them via `crate::testing::EnvRestore` — cargo runs tests as parallel threads of one process, so an unserialised `set_var` races every reader. -- **Never call `.canonicalize()`** — use `safe_canonicalize()`. -- **Windows transient rename errors** (os error 5/32/33 from AV/Search-Indexer races): classify with `is_transient_rename_error()` / `ServeState::is_db_locked_error` and wrap in a bounded retry. Never retry non-transient errors. -- **Counter-then-teardown races:** a background task tearing down state guarded by an in-flight counter must take the write lock BEFORE checking the counter and hold it across check + clear. Consumers increment the counter before acquiring the resource, so `counter == 0` under the write lock proves no consumer exists. - -### LMDB rules - -- **One `EnvOpenOptions::open()` per directory per process.** All access via `get_or_open_stores()` → `Arc`; SCIP opens share a per-directory env (`get_or_open_shared_env`). -- **Open every env with `BASE_ENV_FLAGS`** (`src/lmdb_registry.rs`) — heed refuses to reopen one path with different options. -- **Commit, never drop, a txn whose DB handle you keep** — an aborted txn's DBI is closed by LMDB; using it later yields a bare `EINVAL`. -- **A dropped `TrackedEnv` must close via `prepare_for_closing()`** — heed's `OPENED_ENV` cache keeps a clone, so a plain drop never runs `mdb_env_close` and Windows keeps the files locked for the process lifetime (the `index rm` os-error-32 bug). - -### Search errors must not become empty results - -Never `unwrap_or_default()` a store error on a search path — "no results" and "store down" must stay distinguishable. This defect was re-introduced across sibling handlers in seven review rounds; it is a class, not a site: - -- Render error chains with `{:#}`, never `{}`. Pass "not found" claims through `qualify_empty_result()`; never state a diagnosis you did not verify. -- The rule covers EVERY MCP handler: `find`, `get_chunk`, `explore`, `find_imports`, `find_dependents`, and the single-store `project=` paths. -- **Warnings channels must terminate** on every path that writes to them, and the read must be reachable from the last write. -- **Every `for store in stores` fan-out opens its `*_warnings` channel before the loop.** `Err(_)` over a store result is banned: bind it, render `{e:#}`, carry it. `MultiReadOutcome` is `#[must_use]`. -- **Take the channel as a parameter** — use `respond_with_items()` / `respond_with_object()`, which cannot be called without the channel. -- **New response shapes use the shared exits, not hand-rolled ones.** `serde_json::json!` renders `None` as an explicit `null`, so conditional keys must be *inserted*, not set. -- **Suppress `suggested_tool` when warnings are present.** -- **Verify a batch edit by re-running its detector over the whole file**, including the lines the edit added. -- **Before claiming a test pins a fix, reintroduce the defect and watch it fail.** -- **A caller-facing literal wrapped across lines needs a `\` continuation** — enforced by `tests/caller_facing_literals.rs`. - ---- - -## Active feature branches (not yet merged) - -| Branch | Description | -|---|---| -| *(none)* | | - ---- - -## Changelog highlights (recent) - -- **v1.4.4** — resident C# workspace pool no longer serves stale `find_impact` results after a rebuild: `WorkspacePool::evict` bumps a per-solution generation counter closing a spawn-in-flight race, and `scip_ref_cache` is now cleared unconditionally on both full and incremental rebuilds -- **v1.3.37** — per-index embedding models end-to-end: serve queries, `POST /repos` and CLI index/stats/status honour the model each index records in its `metadata.json`; `serve --model` sets the default for newly created indexes; unrecorded indexes are queried with the built-in model plus a caller-facing warning; mid-rebuild indexes no longer report ready (PR #248) -- **v1.3.23–v1.3.36** — dependency + platform wave: rmcp 3.3, fastembed 6.1 + ort rc.13, tantivy 0.26, axum 0.8, ratatui 0.30 + crossterm 0.29, thiserror 2, notify 8, tree-sitter 0.27, dirs/sha2/scip/sysinfo refresh + dependabot (weekly); clears the open Aikido/RUSTSEC advisories -- **v1.3.19** — `find_impact` ambiguity envelope + `resolved_symbol`; partial-results `warnings`; C# symbol-key uniqueness (index v2.0) -- **v1.3.16** — REST `/find-impact` endpoint -- **v1.3.3** — federation hardening release - -Older entries: see `CHANGELOG.md`. - -## Plan — always-on symbol indexes (features/always-on-symbol-indexes) - -**Doel:** elke geregistreerde repo heeft altijd een actuele SCIP-symbolenindex (C# en TypeScript), ongeacht het registratiepad. - -**Gap-analyse (geverifieerd tegen de bron):** runtime-add (`add_repo_handler`) bouwt tekst+vector+FSW maar geen symbolen; startup phase-2 is C#-only; TS krijgt nooit een initiele build; reindex alleen symbolen bij `?symbols=true`; find_impact antwoordt stilletjes leeg bij ontbrekende index. - -**Stappen:** -1. serve-kant: `trigger_symbol_rebuild` per taal + in-flight guard per (alias,lang) in `ServeState`; `evaluate_symbol_rebuild` (generic gate, C# delegeert naar `evaluate_csharp_rebuild`); phase-2 hernoemd naar `run_phase_2_symbols` itereert alle geinstalleerde talen; `maybe_rebuild_symbols` helper; reindex gebruikt die altijd (gate-gestuurd, `?symbols=true` forceert); add_repo_handler roept hem na `restart_fsw`. -2. find_impact self-heal: bij `!has_index` een fire-and-forget `heal_missing_symbol_index` (serve-mode, alias uit serve-config zodat remote mounts nooit lokaal healen) + warning in het antwoord; drift blijft onaangetast (no-thrash beleid). -3. Tests: in-flight claim/stale-takeover; heal/evaluate op niet-toepasselijke repo; existing suites groen. Validatie: fmt + clippy -D warnings + cargo test --lib --bins. - -**Buiten scope:** deploy naar ~/.local/bin en live verificatie op de draaiende hub (operator-stap: stop serve, copy-to-common.ps1, start serve). From f5f774ae2e37d3ab818cbb26f5d04ec3a6171042 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 19:37:10 +0200 Subject: [PATCH 48/77] docs: credit @donbowman in the changelog for #294 and #295 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e37bcb..4b914a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,8 @@ finalized in place with a date — no renaming/migration step needed. ### Added -- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. -- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. +- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. *(Contributed by @donbowman in #294.)* +- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. *(Contributed by @donbowman in #295, including the follow-up fixes.)* - **Always-on symbol indexes (C# + TypeScript).** Symbol (SCIP) indexes are no longer a manual or incidental artifact: serve warmup, cold opens, file-watcher batches and reindexes each drive a per-language symbol pass wherever chunk indexes are built, so `find_impact` evidence exists without a separate rebuild step. Startup inspects the on-disk symbol indexes with a builder-version stamp and rebuilds those an older binary produced (degraded retry instead of a hard failure), so a upgraded serve never answers from stale symbol data. ### Changed From 90817dba090a4ba04bfe4c4b78be4bf51c0b8d70 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 19:48:27 +0200 Subject: [PATCH 49/77] docs: thank @donbowman properly in the changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b914a93..ef753947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,10 @@ finalized in place with a date — no renaming/migration step needed. ### Added -- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. *(Contributed by @donbowman in #294.)* -- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. *(Contributed by @donbowman in #295, including the follow-up fixes.)* +- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. +- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. + +*A big thank-you to Don Bowman ([@donbowman](https://github.com/donbowman)) for his outstanding, highly value-adding contributions to this release — the indexing-limits rework (#294) and the OpenCode plugin integration (#295), follow-up fixes included.* - **Always-on symbol indexes (C# + TypeScript).** Symbol (SCIP) indexes are no longer a manual or incidental artifact: serve warmup, cold opens, file-watcher batches and reindexes each drive a per-language symbol pass wherever chunk indexes are built, so `find_impact` evidence exists without a separate rebuild step. Startup inspects the on-disk symbol indexes with a builder-version stamp and rebuilds those an older binary produced (degraded retry instead of a hard failure), so a upgraded serve never answers from stale symbol data. ### Changed From 87eb92e3012831b3b1d97bccdd5f6654821465db Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 20:00:18 +0200 Subject: [PATCH 50/77] chore: drop branch-local CLAUDE.md before finalization (convention: never reaches develop) --- CLAUDE.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 5d302681..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -Read AGENTS.md. From d0fb255c5f1adc26f20462f3f227df941c87cf9f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:24:11 +0000 Subject: [PATCH 51/77] chore: bump version to 1.4.17 (auto, PR #296 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6113c4b..c03ec815 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.16" +version = "1.4.17" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 0c82abd8..e2beb4e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.16" +version = "1.4.17" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 4b6f1eb077a323eead3f17c1c93829d3b30c0844 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sat, 3 Oct 2026 20:28:55 +0200 Subject: [PATCH 52/77] =?UTF-8?q?release:=201.5.0=20=E2=80=94=20always-on?= =?UTF-8?q?=20symbol=20indexes,=20indexing=20limits,=20OpenCode=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 +++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef753947..c3970a4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> -## [1.4.12] +## [1.5.0] - 2026-10-03 ### Added @@ -354,6 +354,8 @@ finalized in place with a date — no renaming/migration step needed. ## [1.0.72] - 2026-05-01 - Initial multi-repo release: multi-repo `serve` (HTTP/SSE, per-project/group routing, RRF cross-repo search), stdio MCP proxy with client-side auto-reconnect, tree-sitter chunking (9 langs), persistent SHA-256 embedding cache, repository groups, re-tuned RRF, and LMDB resize crash fix (#30, `MDB_MAP_FULL`). +[1.5.0]: https://github.com/flupkede/codesearch/compare/v1.4.9...v1.5.0 + [1.4.9]: https://github.com/flupkede/codesearch/compare/v1.4.4...v1.4.9 [1.0.171]: https://github.com/flupkede/codesearch/compare/v1.0.162...v1.0.171 [1.0.162]: https://github.com/flupkede/codesearch/compare/v1.0.160...v1.0.162 diff --git a/Cargo.lock b/Cargo.lock index c03ec815..9d902561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.17" +version = "1.5.0" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index e2beb4e5..ae97f570 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.17" +version = "1.5.0" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From c38ffa741624a3f7699805153c726452e60d109b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 3 Oct 2026 23:19:30 +0300 Subject: [PATCH 53/77] =?UTF-8?q?feat(setup):=20implement=20`codesearch=20?= =?UTF-8?q?setup`=20=E2=80=94=20download=20embedding=20models=20via=20fast?= =?UTF-8?q?embed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fills the stubbed `setup` command (TODO: Download model from HuggingFace Hub). Resolves --model through the shared ModelType parser (default stays ModelType::default(), no behaviour change elsewhere), initializes fastembed against the global models cache dir with the download progress bar visible (runtime paths keep it off), and probe-embeds one query so a broken download fails inside setup instead of mid-indexing. Unknown model names now error with the full valid-model list. Also fixes the CLI help text, which advertised a non-existent default model name. Refs the README promise "codesearch setup — Download embedding models". --- src/cli/mod.rs | 2 +- src/cli/setup.rs | 97 ++++++++++++++++++++++++++++++++++++++++--- src/embed/embedder.rs | 23 +++++++++- 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 5b7a007b..8e083385 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -528,7 +528,7 @@ pub enum Commands { /// Download embedding models Setup { - /// Model to download (defaults to mxbai-embed-xsmall-v1) + /// Model to download (defaults to minilm-l6-q) #[arg(long)] model: Option, }, diff --git a/src/cli/setup.rs b/src/cli/setup.rs index 2fcd8c67..bfd29107 100644 --- a/src/cli/setup.rs +++ b/src/cli/setup.rs @@ -1,12 +1,99 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; + +use crate::embed::ModelType; + +/// Resolve the CLI `--model` argument to a [`ModelType`]. +/// +/// `None` falls back to the default model; unknown names are rejected with the +/// full valid-model list so the error is self-explanatory. Pure and +/// network-free, so it is unit-testable. +fn resolve_model(model: Option<&str>) -> Result { + match model { + Some(name) => ModelType::parse(name).ok_or_else(|| { + anyhow!( + "Unknown model '{name}' (valid models: {})", + ModelType::valid_short_names() + ) + }), + None => Ok(ModelType::default()), + } +} pub async fn run(model: Option) -> Result<()> { - let model_name = model.unwrap_or_else(|| "mxbai-embed-xsmall-v1".to_string()); + let model_type = resolve_model(model.as_deref())?; + + let cache_dir = crate::constants::get_global_models_cache_dir()?; + + println!( + "📦 Downloading embedding model: {} ({})", + model_type.name(), + model_type.short_name() + ); + println!(" Cache dir: {}", cache_dir.display()); - println!("📦 Downloading embedding model: {}", model_name); + // Unlike the runtime paths, setup shows the download progress bar: it + // pulls hundreds of MB on purpose and a silent run looks like a hang. + let mut embedder = crate::embed::FastEmbedder::with_cache_dir_progress( + model_type, + Some(&cache_dir), + true, + ) + .map_err(|e| { + anyhow!( + "Failed to initialize embedding model: {e}\n\ + Hint: if huggingface.co is unreachable, set HF_ENDPOINT to a HuggingFace mirror and retry." + ) + })?; - // TODO: Download model from HuggingFace Hub + // Probe inference so a broken download fails here, not mid-indexing. + let dimensions = embedder.embed_query("setup probe")?.len(); + if dimensions != model_type.dimensions() { + return Err(anyhow!( + "Model returned {dimensions} dims, expected {}", + model_type.dimensions() + )); + } - println!("✅ Setup complete!"); + println!( + "✅ Setup complete! Model '{}' ({dimensions} dims) ready at {}", + model_type.short_name(), + cache_dir.display() + ); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_model_is_rejected_with_valid_list() { + let err = resolve_model(Some("not-a-model")).expect_err("unknown model must be rejected"); + let msg = format!("{err:#}"); + assert!( + msg.contains("not-a-model"), + "error must echo the bad name: {msg}" + ); + assert!( + msg.contains(ModelType::valid_short_names().as_str()), + "error must list valid models: {msg}" + ); + } + + #[test] + fn none_resolves_to_default_model() { + assert_eq!(resolve_model(None).unwrap(), ModelType::default()); + } + + #[test] + fn known_short_names_resolve() { + assert_eq!( + resolve_model(Some("embeddinggemma-q4")).unwrap(), + ModelType::EmbeddingGemma300MQ4 + ); + assert_eq!( + resolve_model(Some("minilm-l6-q")).unwrap(), + ModelType::default() + ); + } +} diff --git a/src/embed/embedder.rs b/src/embed/embedder.rs index 5434902c..96065ebf 100644 --- a/src/embed/embedder.rs +++ b/src/embed/embedder.rs @@ -318,6 +318,27 @@ impl FastEmbedder { pub fn with_cache_dir( model_type: ModelType, cache_dir: Option<&std::path::Path>, + ) -> Result { + Self::init(model_type, cache_dir, false) + } + + /// Create a new embedder with the fastembed download progress bar visible. + /// + /// Only `codesearch setup` wants this: it downloads hundreds of MB on + /// purpose, and a silent run looks like a hang. Runtime paths keep the + /// bar off so it does not corrupt TUI/serve output. + pub fn with_cache_dir_progress( + model_type: ModelType, + cache_dir: Option<&std::path::Path>, + show_download_progress: bool, + ) -> Result { + Self::init(model_type, cache_dir, show_download_progress) + } + + fn init( + model_type: ModelType, + cache_dir: Option<&std::path::Path>, + show_download_progress: bool, ) -> Result { // Set cache directory via environment variable if provided // Note: fastembed library uses FASTEMBED_CACHE_DIR (not FASTEMBED_CACHE_PATH) @@ -333,7 +354,7 @@ impl FastEmbedder { let cpu_ep = CPU::default().with_arena_allocator(true).build(); let options = TextInitOptions::new(model_type.to_fastembed_model()) - .with_show_download_progress(false) + .with_show_download_progress(show_download_progress) .with_execution_providers(vec![cpu_ep]); // Cap ONNX intra-op threads when CODESEARCH_EMBED_THREADS is set. From e480f826557b4c7469b6e0ba6877b725588b5c36 Mon Sep 17 00:00:00 2001 From: Don Bowman Date: Sat, 3 Oct 2026 17:35:52 -0400 Subject: [PATCH 54/77] fix(serve): keep unjoinable index tasks tracked and stop reaping queued ones With CODESEARCH_INDEX_JOBS=1 a slow backlog could lose every queued reindex and then serve "Database is locked by another process" until a restart: - await_index_task removed the task entry before waiting and dropped the handle on timeout, so a detached task held its Arc and the single job permit invisibly. - is_indexing evicted any marker older than MAX_INDEXING_SECS and cancelled the task behind it; tasks queued on the job gate never renew their markers, so they were mass-cancelled as "leaked". - The job-gate wait ignored cancellation, parking cancelled tasks on the gate with their stores open. - remove_repo and format recovery deleted the DB dir under a live task. Fix: cancellation-aware gate acquisition (limits::JobAcquire); timeouts keep the task tracked and a periodic sweeper reaps finished tasks, cleans orphaned DB dirs, and re-asserts cancellation for removed repos; DB deletion is deferred while the task runs; markers backed by a live Reindex task are renewed rather than cancelled. --- CHANGELOG.md | 6 + src/constants.rs | 11 +- src/index/manager.rs | 104 ++++++++- src/limits.rs | 59 ++++- src/limits_tests.rs | 53 +++++ src/serve/mod.rs | 545 ++++++++++++++++++++++++++++++++----------- src/serve/tests.rs | 403 +++++++++++++++++++++++++++++--- src/serve/tui.rs | 17 +- 8 files changed, 1005 insertions(+), 193 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3970a4c..d0f90fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.5.1] + +### Fixed + +- **Queued reindexes are no longer cancelled as "leaked", and an unjoinable index task can no longer wedge a hub.** With the process-wide job gate (`CODESEARCH_INDEX_JOBS=1`, the default), tasks queued behind a long-running job held their `active_reindexes` markers past `MAX_INDEXING_SECS` without renewing them, so the lazy stale-marker check evicted the markers and cancelled the parked tasks — leaving empty stores and in-process LMDB handles that made later writes fail with "Database is locked by another process" until a serve restart. Separately, `await_index_task` removed the task entry *before* waiting and dropped the `JoinHandle` on timeout, so a task too deep in the uninterruptible `build_index` to cancel kept its `Arc` and the single job permit invisibly; the job-gate wait ignored cancellation, parking cancelled tasks on the gate; and `remove_repo` / format recovery could delete the DB directory out from under a still-running task. Now: a marker backed by a live tracked `Reindex` task is renewed (with a throttled wedge warning) instead of reaped; gate acquisition is cancellation-aware (`JobAcquire`) so cancelled or removed queued jobs abort and release their stores; a task that cannot be joined stays tracked and is reaped by a periodic sweeper that also deletes orphaned DB directories and re-asserts cancellation for removed repos; DB deletion is deferred while the task runs; format recovery refuses to wipe in that state; and stale markers from other owners are still evicted. + ## [1.5.0] - 2026-10-03 ### Added diff --git a/src/constants.rs b/src/constants.rs index f22fcd04..79616c7a 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -504,14 +504,17 @@ pub const REMOTE_ROW_REFRESH_SECS: u64 = 5; /// cancellation between insert and remove silently leaks the entry, leaving /// the TUI stuck on "Indexing" forever. /// -/// Rather than chase every leak path, entries older than this threshold are +/// Rather than chase every leak path, a marker older than this threshold is /// treated as absent by all read sites (`repo_statuses_lightweight`, -/// `evict_idle_repos`, and the reindex 409 guard) and lazily evicted. This -/// makes the system self-healing regardless of the leak cause. +/// `evict_idle_repos`, and the reindex 409 guard) and lazily evicted — but +/// only when no tracked indexing task for the alias is still alive. A live +/// task (queued on the job gate, or in a long uninterruptible build) renews +/// its marker instead, so a backlog drains rather than being cancelled as +/// "leaked"; the system stays self-healing for genuinely dead tasks without +/// killing slow ones. /// /// 30 minutes is deliberately generous: a force reindex of a very large repo /// or a full scip-csharp solution rebuild can legitimately take many minutes. -/// Any session stuck beyond that is almost certainly leaked. /// Override with `CODESEARCH_MAX_INDEXING_SECS`. pub const MAX_INDEXING_SECS: u64 = 30 * 60; // 30 minutes diff --git a/src/index/manager.rs b/src/index/manager.rs index f1743494..57442b60 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -230,6 +230,31 @@ impl SharedStores { } } + /// Cancellation-aware variant of [`Self::acquire_job_permit`]. + /// + /// Returns [`crate::limits::JobAcquire::Cancelled`] when `cancel_token` + /// fires (or already fired) before a slot is granted, so the caller can + /// abort the job promptly and release its stores instead of parking on + /// the gate for the permit holder's whole run. + pub async fn acquire_job_permit_or_cancel( + &self, + cancel_token: &CancellationToken, + ) -> crate::limits::JobAcquire { + match self.job_gate.as_ref() { + Some(gate) => match gate.acquire_or_cancel(cancel_token).await { + Some(permit) => crate::limits::JobAcquire::Ready(Some(permit)), + None => crate::limits::JobAcquire::Cancelled, + }, + None => { + if cancel_token.is_cancelled() { + crate::limits::JobAcquire::Cancelled + } else { + crate::limits::JobAcquire::Ready(None) + } + } + } + } + /// Create new shared stores from the database path (read-write mode). /// /// This acquires a writer lock. If another process already has the lock, @@ -646,7 +671,14 @@ impl IndexManager { // (when one is attached): the permit is held across the file walk, the // chunk/embed batches and the final HNSW build, so at most // `CODESEARCH_INDEX_JOBS` repos do this at once. Queries never take it. - let _job_permit = stores.acquire_job_permit().await; + // Cancellation-aware: a queued job that was cancelled aborts here and + // drops its stores instead of parking for the holder's whole run. + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; Self::perform_incremental_refresh_inner( codebase_path, db_path, @@ -1179,8 +1211,18 @@ impl IndexManager { // Hold the process-wide heavy-job gate across the destructive clear AND // the full reindex. The nested call goes to the gate-free inner body so // the permit is not re-acquired (which would deadlock under - // `CODESEARCH_INDEX_JOBS=1`). - let _job_permit = stores.acquire_job_permit().await; + // `CODESEARCH_INDEX_JOBS=1`). The wait is cancellation-aware so a repo + // removed while queued aborts before the destructive clear. + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; + // Re-check after the (possibly long) wait: a cancellation that landed + // while queued must not start the clear, and the stores are still + // intact at this point. + Self::ensure_indexing_active(cancel_token)?; // ── Step 0: Read and preserve metadata BEFORE clearing anything ── // This is defensive: the DB may be incomplete (no metadata.json at all), @@ -2090,7 +2132,12 @@ impl IndexManager { // One permit for the whole watcher batch: a batch can hold hundreds of // files, each re-embedded and followed by an HNSW rebuild, and every // repo's watcher would otherwise run one at the same time. - let _job_permit = stores.acquire_job_permit().await; + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; // Enable quiet mode during FSW batch processing to suppress verbose embedding output set_quiet(true); @@ -2254,7 +2301,12 @@ impl IndexManager { Self::ensure_indexing_active(cancel_token)?; // One permit for the whole branch-refresh pass (walk, delete, re-index). - let _job_permit = stores.acquire_job_permit().await; + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; let result: Result<()> = async { // Phase 1: Discover current files on disk. @@ -2911,6 +2963,48 @@ mod tests { assert!(stores.acquire_job_permit().await.is_none()); } + /// Cancellation-aware acquisition: ungated stores must distinguish + /// "proceed without a gate" from "cancelled", and a waiter cancelled while + /// the only slot is held must abort promptly instead of parking. + #[tokio::test] + async fn acquire_job_permit_or_cancel_distinguishes_cancel_from_ungated() { + let temp = tempdir().unwrap(); + let stores = create_test_stores(&temp.path().join("db"), 4).await; + + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert!(matches!( + stores.acquire_job_permit_or_cancel(&cancelled).await, + crate::limits::JobAcquire::Cancelled + )); + assert!(matches!( + stores + .acquire_job_permit_or_cancel(&CancellationToken::new()) + .await, + crate::limits::JobAcquire::Ready(None) + )); + + // Gated: a waiter cancelled while the only slot is held aborts. + let gate = crate::limits::JobGate::with_config(1, 0); + let gated = Arc::new( + create_test_stores(&temp.path().join("db2"), 4) + .await + .with_job_gate(Arc::clone(&gate)), + ); + let held = gate.acquire().await; + let waiter = CancellationToken::new(); + let canceller = waiter.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + canceller.cancel(); + }); + assert!(matches!( + gated.acquire_job_permit_or_cancel(&waiter).await, + crate::limits::JobAcquire::Cancelled + )); + drop(held); + } + /// A gated refresh must (a) wait while the only slot is held and (b) /// release the slot when it finishes. The repo has NO changed files, so the /// refresh completes without loading the ONNX model. diff --git a/src/limits.rs b/src/limits.rs index 1910310f..68ebee2e 100644 --- a/src/limits.rs +++ b/src/limits.rs @@ -29,6 +29,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::sync::CancellationToken; /// Max concurrent heavy repo jobs in this process (>= 1). pub const INDEX_JOBS_ENV: &str = "CODESEARCH_INDEX_JOBS"; @@ -154,14 +155,37 @@ impl JobGate { /// Acquire one job slot, then (when enabled) wait, bounded, for the free /// memory floor. The caller holds the returned permit for the whole job. + /// + /// Prefer [`Self::acquire_or_cancel`] for jobs that carry a cancellation + /// token; this variant waits unbounded because there is nothing to cancel + /// it with. pub async fn acquire(&self) -> JobPermit { - let wait_start = Instant::now(); - let permit = self - .permits - .clone() - .acquire_owned() + self.acquire_or_cancel(&CancellationToken::new()) .await - .expect("job gate semaphore is never closed"); + .expect("a never-cancelled token can never abort the acquire") + } + + /// Acquire one job slot, aborting promptly when `cancel` fires while + /// queued. + /// + /// Returns `None` without consuming a slot when the token was already + /// cancelled or fires before a slot is granted. A job cancelled while + /// queued must abort rather than park: parking kept a removed or cancelled + /// job's `Arc` (and its LMDB env) alive for the permit + /// holder's whole run, leaving "locked by another process" / double-open + /// failures behind with nothing left to release them. + pub async fn acquire_or_cancel(&self, cancel: &CancellationToken) -> Option { + let wait_start = Instant::now(); + if cancel.is_cancelled() { + return None; + } + let permit = tokio::select! { + biased; + _ = cancel.cancelled() => return None, + permit = self.permits.clone().acquire_owned() => { + permit.expect("job gate semaphore is never closed") + } + }; let waited = wait_start.elapsed(); if waited >= SLOW_ACQUIRE_LOG_AFTER { tracing::info!( @@ -173,7 +197,12 @@ impl JobGate { if self.min_free_mb > 0 { self.wait_for_memory().await; } - JobPermit { _permit: permit } + // The memory wait above is bounded; a cancellation that lands during + // it still wins — release the slot rather than start cancelled work. + if cancel.is_cancelled() { + return None; + } + Some(JobPermit { _permit: permit }) } async fn wait_for_memory(&self) { @@ -214,6 +243,22 @@ pub struct JobPermit { _permit: OwnedSemaphorePermit, } +/// Cancellation-aware acquisition outcome for a heavy job. +/// +/// `#[must_use]`: dropping a [`Self::Ready`] value releases the job slot +/// immediately (the job would run ungated), and ignoring [`Self::Cancelled`] +/// means starting work that was already cancelled. Always match on it. +#[derive(Debug)] +#[must_use] +pub enum JobAcquire { + /// The job may proceed. The permit is `None` when no gate is attached + /// (standalone processes keep their previous unbounded behaviour). + Ready(Option), + /// The cancellation token fired before a slot was granted; no slot was + /// consumed and the job must abort promptly, dropping its stores. + Cancelled, +} + #[cfg(test)] #[path = "limits_tests.rs"] mod tests; diff --git a/src/limits_tests.rs b/src/limits_tests.rs index fac73292..1c0fd3f2 100644 --- a/src/limits_tests.rs +++ b/src/limits_tests.rs @@ -123,3 +123,56 @@ async fn memory_wait_is_bounded() { assert!(start.elapsed() >= Duration::from_millis(50)); drop(permit); } + +#[tokio::test] +async fn cancelled_gate_waiter_aborts_without_consuming_a_slot() { + // Regression: a reindex queued behind a long-running job parked on the + // gate holding its store handles. A cancellation while queued must abort + // the wait promptly and leave the slot untouched. + let gate = JobGate::with_config(1, 0); + let held = gate.acquire().await; + let token = CancellationToken::new(); + let canceller = token.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + canceller.cancel(); + }); + + let start = Instant::now(); + let outcome = gate.acquire_or_cancel(&token).await; + assert!( + outcome.is_none(), + "a cancelled waiter must not receive a permit" + ); + assert!( + start.elapsed() < Duration::from_secs(2), + "cancellation must abort the wait promptly" + ); + + drop(held); + // The slot is still free: the cancelled wait never consumed it. + assert!(gate + .acquire_or_cancel(&CancellationToken::new()) + .await + .is_some()); +} + +#[tokio::test] +async fn pre_cancelled_token_never_takes_a_slot() { + let gate = JobGate::with_config(1, 0); + let token = CancellationToken::new(); + token.cancel(); + assert!(gate.acquire_or_cancel(&token).await.is_none()); + assert!(gate + .acquire_or_cancel(&CancellationToken::new()) + .await + .is_some()); +} + +#[tokio::test] +async fn uncancelled_token_proceeds_through_the_gate() { + let gate = JobGate::with_config(2, 0); + let token = CancellationToken::new(); + assert!(gate.acquire_or_cancel(&token).await.is_some()); + assert!(gate.acquire_or_cancel(&token).await.is_some()); +} diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 36b968df..eb3c86cb 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -48,6 +48,7 @@ use crate::index::{ CSharpRebuildNotifier, IndexManager, IndexingHeartbeat, IndexingStatusCallback, SharedStores, SymbolRebuildSignal, }; +use crate::limits::JobAcquire; use crate::mcp::types::HealthResponse; use crate::symbols::{csharp, RebuildScope, SymbolIndexerRegistry}; @@ -221,6 +222,19 @@ pub(crate) enum IndexingOwner { Symbol, } +/// A tracked background indexing task. +/// +/// `db_path` travels with the entry so the DB directory of a repo that was +/// removed mid-index can be cleaned up after the task finally exits, and +/// `started_at` lets the periodic sweeper report tasks that have been alive +/// past the stale threshold (a wedge signal for operators). +struct IndexTask { + handle: tokio::task::JoinHandle<()>, + token: CancellationToken, + db_path: PathBuf, + started_at: Instant, +} + /// Shared state for the serve mode. pub(crate) struct ServeState { /// Repo alias → opened stores (or conflicted marker). @@ -255,22 +269,28 @@ pub(crate) struct ServeState { /// `stop_fsw`) so the LMDB `Environment` drops and releases the file /// handles BEFORE the DB directory is deleted. See `await_fsw_shutdown`. fsw_tasks: DashMap>, - /// Repo alias → `(JoinHandle, CancellationToken)` of its background - /// *indexing* task (the heavy `add_repo`/`reindex` embed pass), separate - /// from `fsw_tasks` because `restart_fsw` reuses the `fsw_tasks` slot for - /// the continuous watcher loop. + /// Repo alias → its tracked background *indexing* task (the heavy + /// `add_repo`/`reindex` embed pass), separate from `fsw_tasks` because + /// `restart_fsw` reuses the `fsw_tasks` slot for the continuous watcher + /// loop. /// /// This exists to fix the index-cancellation no-op (BUG1): `add_repo` and /// `reindex` used to spawn detached, untracked `tokio::spawn` tasks that /// neither observed the cancel token nor could be awaited, so `remove_repo` /// reported success while a full-corpus embed pass kept running (and /// writing) on the removed alias — 6 GB / 52% CPU runaway. Registering the - /// handle here lets `await_index_task` (called from `remove_repo`) cancel - /// the token AND await the task before the DB directory is deleted, so the - /// task's `Arc` (and the LMDB mmap handles it keeps alive) - /// drop first. The token is stored alongside the handle so `remove_repo` - /// can cancel regardless of the repo's `RepoState` variant. - index_tasks: DashMap, CancellationToken)>, + /// task here lets `await_index_task` (called from `remove_repo`) cancel the + /// token AND wait for the task, so the task's `Arc` (and the + /// LMDB mmap handles it keeps alive) drop first. The entry is *never* + /// dropped while the task may still run: when the cooperative wait cannot + /// join it (uninterruptible `build_index`), the handle goes back into this + /// map so [`Self::sweep_index_tasks`] can reap it — and clean up a removed + /// repo's DB directory — once it finally exits. + index_tasks: DashMap, + /// Repo alias → last time [`Self::is_indexing`] / [`Self::sweep_index_tasks`] + /// warned that a *live* task's marker crossed `MAX_INDEXING_SECS`. + /// Throttles the wedge warning to once per alias per few minutes. + stale_live_warned: DashMap, /// Aliases detected with LMDB storage-format corruption (e.g. /// `MDB_BAD_VALSIZE` after a storage-layer major upgrade such as /// arroy 0.5→0.8 / heed 0.20→0.22), queued for a wipe + full force @@ -446,6 +466,7 @@ impl ServeState { open_locks: DashMap::new(), fsw_tasks: DashMap::new(), index_tasks: DashMap::new(), + stale_live_warned: DashMap::new(), format_recovery_queue: std::sync::Mutex::new(std::collections::VecDeque::new()), format_recovery_done: DashMap::new(), format_recovery_worker_started: std::sync::atomic::AtomicBool::new(false), @@ -706,18 +727,37 @@ impl ServeState { } /// Mark `alias` as actively indexing for `owner`, returning `true` if the - /// caller may proceed. Returns `false` when **any** owner holds a - /// non-stale marker (i.e. another indexing run is genuinely in progress) — - /// in that case the caller should return HTTP 409. Stale markers from - /// leaked/crashed tasks are silently dropped first. + /// caller may proceed. Returns `false` when **any** owner holds a live + /// marker (i.e. another indexing run is genuinely in progress) — in that + /// case the caller should return HTTP 409. Stale markers from leaked + /// tasks are dropped first, but a marker whose tracked task is still + /// alive is never treated as stale: queued-behind-the-gate jobs can wait + /// far past `MAX_INDEXING_SECS` without being leaks. /// /// This is the guard used by `reindex_handler`, `add_repo_handler`, and /// `spawn_force_reindex` to reject concurrent reindexes. fn begin_indexing(&self, alias: &str, owner: IndexingOwner) -> bool { let now = Instant::now(); let max = self.indexing_timeout(); + // Liveness is read before locking the marker map so the two DashMaps + // are never locked in a nested order. + let task_live = self.index_task_is_live(alias); let mut owners = self.active_reindexes.entry(alias.to_string()).or_default(); - owners.retain(|_, ts| now.duration_since(*ts) < max); + owners.retain(|owner, ts| { + if now.duration_since(*ts) < max { + return true; + } + if task_live && *owner == IndexingOwner::Reindex { + // Not a leak: renew rather than drop, or a concurrent begin + // could double-run the repo. Only the tracked index task + // (Reindex) supplies liveness evidence; other owners' stale + // markers are dropped as before. + *ts = now; + true + } else { + false + } + }); if owners.is_empty() { owners.insert(owner, now); true @@ -857,7 +897,18 @@ impl ServeState { self.repos.remove(alias); self.last_access.remove(alias); self.await_fsw_shutdown(alias).await; - self.await_index_task(alias).await; + if !self.await_index_task(alias).await { + // The task is still alive (typically parked in the uninterruptible + // build_index). Wiping the DB dir from under it is unsafe — it can + // wedge the task while it holds the store handles and the + // process-wide job permit. Refuse and let the next corruption + // detection retry once the task exits. + return Err(format!( + "in-flight indexing task for '{}' did not stop within the cooperative \ + budget; refusing to wipe its DB dir", + alias + )); + } let deadline = Instant::now() + Duration::from_secs(crate::constants::DB_DELETE_RETRY_BUDGET_SECS); @@ -958,7 +1009,7 @@ impl ServeState { let cancelled = self .index_tasks .get(alias) - .is_some_and(|entry| entry.value().1.is_cancelled()); + .is_some_and(|entry| entry.value().token.is_cancelled()); if cancelled { return Err(format!( "rebuild for '{}' was cancelled before it completed (stale indexing marker); \ @@ -969,14 +1020,50 @@ impl ServeState { Ok(()) } - /// Returns `true` if `alias` is currently (non-stale) indexing for any - /// owner. + /// True when a tracked indexing task for `alias` has not finished. /// - /// Stale markers — those older than [`MAX_INDEXING_SECS`] — are lazily - /// evicted here. This is the self-healing mechanism: even if a - /// fire-and-forget background task panics or is cancelled between - /// `begin_indexing` and `end_indexing`, its marker eventually expires and - /// the TUI returns to the correct state without a server restart. + /// Live-task evidence overrides marker age: a task queued behind the + /// process-wide job gate can wait far longer than `MAX_INDEXING_SECS` + /// without being a leak, and treating it as one (evict marker + cancel) + /// cancels healthy queued work whenever the backlog drains slower than the + /// threshold. + fn index_task_is_live(&self, alias: &str) -> bool { + self.index_tasks + .get(alias) + .is_some_and(|entry| !entry.value().handle.is_finished()) + } + + /// Throttled gate for the "live task past the stale threshold" warning: + /// true at most once per alias per five minutes. + fn stale_warn_due(&self, alias: &str) -> bool { + const WARN_EVERY: Duration = Duration::from_secs(300); + let now = Instant::now(); + let due = match self.stale_live_warned.get_mut(alias) { + Some(mut last) => { + if now.duration_since(*last) < WARN_EVERY { + false + } else { + *last = now; + true + } + } + None => true, + }; + if due { + self.stale_live_warned.insert(alias.to_string(), now); + } + due + } + + /// Returns `true` if `alias` is currently indexing for any owner. + /// + /// Markers older than [`MAX_INDEXING_SECS`] are aged out unless their + /// owner carries liveness evidence: a `Reindex` marker backed by a live + /// tracked index task is renewed instead. That keeps the self-healing + /// behavior for panicked or crashed tasks while never cancelling + /// live-but-slow work — a task queued on the process-wide job gate or a + /// long `build_index` routinely exceeds the threshold, and a backlog of + /// queued reindexes must drain instead of being cancelled as "leaked". /// /// Eviction runs under the shard's write lock (`get_mut` + `retain`), so a /// concurrent `begin_indexing`/`renew_indexing` that refreshed a marker in @@ -986,16 +1073,26 @@ impl ServeState { /// predicate + removal). pub(crate) fn is_indexing(&self, alias: &str) -> bool { let max = self.indexing_timeout(); + // Liveness is read before locking the marker map so the two DashMaps + // are never locked in a nested order. + let task_live = self.index_task_is_live(alias); let mut evicted: Vec = Vec::new(); + let mut renewed: Vec = Vec::new(); let mut any_left = false; if let Some(mut owners) = self.active_reindexes.get_mut(alias) { let now = Instant::now(); owners.retain(|owner, ts| { - let fresh = now.duration_since(*ts) < max; - if !fresh { + if now.duration_since(*ts) < max { + return true; + } + if task_live && *owner == IndexingOwner::Reindex { + *ts = now; + renewed.push(*owner); + true + } else { evicted.push(*owner); + false } - fresh }); any_left = !owners.is_empty(); } @@ -1004,56 +1101,123 @@ impl ServeState { // raced this eviction keeps its fresh marker. self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); } + if !renewed.is_empty() && self.stale_warn_due(alias) { + tracing::warn!( + "⏳ Indexing marker(s) for '{}' exceeded {}s but the tracked task is still \ + live (queued on the job gate, or a long-running build) — renewing, not \ + cancelling. Owners: {:?}", + alias, + max.as_secs(), + renewed + ); + } if !evicted.is_empty() { tracing::warn!( "🧹 Evicted stale indexing marker(s) for '{}' (older than {}s, owners \ - {:?}) — likely a leaked/crashed background task", + {:?}) — no tracked task is alive; reaping", alias, max.as_secs(), evicted ); - self.cancel_stale_index_task(alias); + self.reap_finished_index_task(alias); } any_left } - /// Cancel the background index task still registered for `alias` after its - /// indexing marker was evicted as stale. + /// Reap the tracked index task for `alias` once its handle has finished. /// - /// Dropping the marker only fixes what the TUI and the reindex guard - /// *believe*; the task itself keeps running, and with it the - /// `Arc` it captured — so the LMDB env and the - /// `.writer.lock` stay held for the process lifetime. Every later write - /// (reindex, format recovery, `POST /repos`) then fails with "Database is - /// locked by another process" even though the repo looks idle and closed. - /// Cancelling the task's token releases those handles at its next - /// cancellation point. + /// Called after stale markers are evicted (there is no live task by + /// definition) and from the periodic [`Self::sweep_index_tasks`]. A + /// running task is never cancelled here, however old its marker: only + /// [`Self::remove_repo`] cancels, and only for its own alias. + fn reap_finished_index_task(&self, alias: &str) { + let Some(entry) = self.index_tasks.get(alias) else { + return; + }; + if !entry.value().handle.is_finished() { + return; + } + let db_path = entry.value().db_path.clone(); + drop(entry); + if self + .index_tasks + .remove_if(alias, |_, v| v.handle.is_finished()) + .is_some() + { + self.stale_live_warned.remove(alias); + // A finished task that left markers behind panicked or errored + // without `end_indexing`; with no task left they are stale state. + // Only the Reindex owner belongs to the index task — the other + // owners (Warmup, Watcher, Symbol) manage their own lifecycle. + if !self.index_tasks.contains_key(alias) { + if let Some(mut owners) = self.active_reindexes.get_mut(alias) { + owners.remove(&IndexingOwner::Reindex); + } + self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); + } + // Covers the removed-mid-build case where the task exited via a + // path that never reached its post-build self-clean. + self.self_clean_if_unregistered(alias, &db_path); + } + } + + /// Periodic sweep of tracked index tasks (runs on the idle-reaper tick). /// - /// Cooperative only: the handle is never aborted (see - /// [`Self::await_index_task`] — an abort would detach the blocking - /// `build_index` that owns its own store clone and drop the post-build - /// self-cleanup), and the entry stays registered so `remove_repo` can - /// still join it. A task that already finished is reaped here instead. - fn cancel_stale_index_task(&self, alias: &str) { - let finished = match self.index_tasks.get(alias) { - Some(entry) => { - let (handle, token) = entry.value(); - if handle.is_finished() { - true - } else { - token.cancel(); - tracing::warn!( - "🧹 Cancelled the leaked index task for '{}' — releasing its store \ - handles and writer lock", - alias - ); - false + /// Finishes what cooperative removal could not: reaps entries whose tasks + /// finally exited (clearing leftover markers and deleting a removed + /// repo's DB directory), re-asserts cancellation for tasks whose repo is + /// gone, and loudly reports tasks alive far past the stale threshold — + /// the wedge signal an operator needs. Live tasks are never cancelled + /// here. + fn sweep_index_tasks(&self) { + let entries: Vec<(String, bool, Duration)> = self + .index_tasks + .iter() + .map(|e| { + let v = e.value(); + ( + e.key().clone(), + v.handle.is_finished(), + v.started_at.elapsed(), + ) + }) + .collect(); + let max = self.indexing_timeout(); + for (alias, finished, age) in entries { + if finished { + self.reap_finished_index_task(&alias); + continue; + } + let registered = self + .config + .read() + .map(|c| c.resolve(&alias).is_some()) + .unwrap_or(true); + if !registered { + // A removed alias's task has no reason to keep going. The + // token was cancelled by remove_repo; re-assert it + // (idempotent, cooperative) so the entry can be reaped at its + // next cancellation point. + if let Some(entry) = self.index_tasks.get(&alias) { + if !entry.value().token.is_cancelled() { + entry.value().token.cancel(); + tracing::warn!( + "🧹 Index task for removed repo '{}' is still live — \ + cancellation re-asserted", + alias + ); + } } + continue; + } + if age > max && self.stale_warn_due(&alias) { + tracing::warn!( + "⏳ Index task for '{}' has been live for {}s with no completion — \ + possible wedge; it stays tracked and will be reaped when it exits", + alias, + age.as_secs() + ); } - None => return, - }; - if finished { - self.index_tasks.remove(alias); } } @@ -1948,6 +2112,21 @@ impl ServeState { /// This is the shared logic used by both the HTTP `DELETE /repos/{alias}` handler /// and the TUI confirmation flow. pub(crate) async fn remove_repo(&self, alias: &str) -> Result { + self.remove_repo_with_budget( + alias, + std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), + ) + .await + } + + /// [`Self::remove_repo`] with an injectable cooperative budget for the + /// index-task wait (tests use a short one; production passes + /// `BG_TASK_COOPERATIVE_TIMEOUT_SECS`). + pub(crate) async fn remove_repo_with_budget( + &self, + alias: &str, + index_task_budget: Duration, + ) -> Result { // 1. Resolve project path from config let project_path = { let config = self @@ -1981,9 +2160,15 @@ impl ServeState { // above cancelled a token nobody listened to, this await found nothing // to wait on, and the embed pass kept running (writing chunks, holding // the LMDB mmap open) long after remove_repo reported success. - // await_index_task cancels the task's OWN token and awaits its exit, so - // its Arc drops before the DB delete below. - self.await_index_task(alias).await; + // await_index_task cancels the task's OWN token and waits for its + // exit, so its Arc drops before the DB delete below. It + // returns false when the task could not be joined within the + // cooperative budget (e.g. parked in an uninterruptible build): the DB + // delete is then DEFERRED instead of pulling the directory out from + // under the live build. + let index_task_exited = self + .await_index_task_with_budget(alias, index_task_budget) + .await; // 3. Unregister from repos.json { @@ -2022,7 +2207,24 @@ impl ServeState { // report honestly. let mut db_deleted = !db_path.exists(); let mut db_delete_error: Option = None; - if db_path.exists() { + if db_path.exists() && !index_task_exited { + // The index task is still alive (typically parked in the + // uninterruptible `build_index`). Deleting the directory out from + // under it can wedge a serve process: the task's + // mmapped LMDB/arroy/Tantivy files vanished mid-build, it never + // reached a cancellation point, and it then held its + // `Arc` plus the process-wide job permit for the + // process lifetime. Defer instead: the task's post-build guard + // (`self_clean_if_unregistered`) deletes the dir once it observes + // the removal, and [`Self::sweep_index_tasks`] retries if a panic + // skips that path. + tracing::warn!( + "Database dir for '{}' left in place: its index task is still running \ + (cleanup deferred to the task's post-build guard / sweeper)", + alias + ); + db_delete_error = Some("index task still running; DB cleanup deferred".to_string()); + } else if db_path.exists() { // Deadline-bounded exponential-backoff retry. We ONLY retry on // lock-class errors (sharing/lock violation or access-denied on // Windows, or a message hinting the dir is in use) — a genuine @@ -2252,58 +2454,85 @@ impl ServeState { } } - /// Cancel and await the background *indexing* task for `alias` - /// (`add_repo`/`reindex` embed pass), if one is registered in + /// Cancel and wait (bounded) for the background *indexing* task for + /// `alias` (`add_repo`/`reindex` embed pass), if one is registered in /// [`Self::index_tasks`]. /// - /// Cancels the task's token first (so an in-flight embed pass aborts at the - /// next batch/phase boundary), then awaits its `JoinHandle` with a 5s - /// timeout. The await is what guarantees the task's `Arc` — - /// and the LMDB mmap handles it keeps alive on Windows — have actually - /// dropped before `remove_repo` deletes the DB directory. Without this, - /// `remove_repo` would delete `repos.json` while the detached task kept - /// writing chunks into a soon-to-be-orphaned `.codesearch.db`. - async fn await_index_task(&self, alias: &str) { - if let Some((_, (handle, token))) = self.index_tasks.remove(alias) { - token.cancel(); - // Bounded cooperative join. We deliberately do NOT abort the task - // on timeout. An indexing task can be parked inside `build_index`'s - // synchronous arroy HNSW build, which runs on a `spawn_blocking` - // thread and has no cancellation point Tokio can interrupt. - // Aborting the OUTER `JoinHandle` would only detach that blocking - // task (it keeps its own `Arc>` clone, so the - // LMDB mmap stays open regardless) AND drop the post-build - // continuation — including the self-cleanup that deletes the - // orphaned `.codesearch.db` dir once the build finishes. So on - // timeout we detach the outer task ON PURPOSE: its post-build guard - // (`remove_orphaned_db_dir`) releases the handles and self-cleans - // the directory. The deadline-bounded delete retry in `remove_repo` - // covers builds that finish within its budget; a serve restart reaps - // anything left over. - match tokio::time::timeout( - std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), - handle, - ) - .await - { - Ok(Ok(())) => { - tracing::debug!("Index task for '{}' exited cleanly", alias); - } - Ok(Err(join_err)) => { - tracing::warn!( - "Index task for '{}' panicked during shutdown: {}", - alias, - join_err - ); - } - Err(_) => { - tracing::warn!( - "Index task for '{}' still in an uninterruptible build_index after {}s; \ - detaching — its post-build guard will self-clean the DB dir", - alias, - crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS, - ); - } + /// Cancels the task's token first (so an in-flight embed pass aborts at + /// the next batch/phase boundary), then waits up to the cooperative + /// budget for its exit. Returns `true` when the task exited within the + /// budget, `false` when it is still running (typically parked inside the + /// uninterruptible `build_index`). + /// + /// The entry is never lost: on timeout the still-pending handle goes back + /// into [`Self::index_tasks`]. The old code removed the entry before + /// waiting and dropped the handle on timeout, so a detached task held its + /// `Arc` (and the process-wide job permit) invisibly for the + /// process lifetime — the leak class this tracking exists to prevent. Keeping + /// the handle tracked lets [`Self::sweep_index_tasks`] reap it and clean + /// up a removed repo's DB directory once it finally exits. + async fn await_index_task(&self, alias: &str) -> bool { + self.await_index_task_with_budget( + alias, + std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), + ) + .await + } + + /// [`Self::await_index_task`] with an injectable budget (tests use a short + /// one; production passes `BG_TASK_COOPERATIVE_TIMEOUT_SECS`). + async fn await_index_task_with_budget(&self, alias: &str, budget: Duration) -> bool { + let Some((_, task)) = self.index_tasks.remove(alias) else { + return true; + }; + task.token.cancel(); + let IndexTask { + mut handle, + token, + db_path, + started_at, + } = task; + // Bounded cooperative wait; never abort. An indexing task can be + // parked inside `build_index`'s synchronous arroy HNSW build, which + // runs on a `spawn_blocking` thread and has no cancellation point + // Tokio can interrupt. Aborting the OUTER `JoinHandle` would only + // detach that blocking task (it keeps its own + // `Arc>` clone, so the LMDB mmap stays open + // regardless) AND drop the post-build continuation — including the + // self-cleanup that deletes the orphaned `.codesearch.db` dir once the + // build finishes. + match tokio::time::timeout(budget, &mut handle).await { + Ok(Ok(())) => { + tracing::debug!("Index task for '{}' exited cleanly", alias); + true + } + Ok(Err(join_err)) => { + tracing::warn!( + "Index task for '{}' panicked during shutdown: {}", + alias, + join_err + ); + true + } + Err(_) => { + tracing::warn!( + "Index task for '{}' did not exit within {}s; keeping it tracked — the \ + stale-task sweeper will reap it and clean up its DB dir once it exits", + alias, + budget.as_secs() + ); + // `timeout(&mut handle)` leaves the handle owned; put it back + // so the task can never become invisible. + self.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token, + db_path, + started_at, + }, + ); + false } } } @@ -2412,8 +2641,8 @@ impl ServeState { .unwrap_or(true); if registered { tracing::info!( - "Task for '{}' was cancelled but the repo is still registered — keeping its DB \ - dir (handles released)", + "Index task for '{}' ended but the repo is still registered — keeping its DB \ + dir", alias ); return; @@ -3689,16 +3918,17 @@ impl ServeState { /// closed. Naming the holders turns a two-day silent failure into one /// warning line. /// - /// Gated on a still-running index task: the reaper only evicts repos that - /// are not indexing, so a live task here means its marker was already - /// evicted as stale — the leak signature. Without that gate the warning - /// would fire on every eviction, because a just-cancelled FSW drains - /// asynchronously and still holds the env for a moment. + /// Gated on a still-running index task: the idle reaper evicts repos only + /// when they are not indexing, so a live task at eviction time means its + /// marker is missing or was dropped by another flow (e.g. a stale + /// non-Reindex marker) — the leak/teardown signature. Without that gate + /// the warning would fire on every eviction, because a just-cancelled FSW + /// drains asynchronously and still holds the env for a moment. fn warn_if_still_held(&self, alias: &str) { let leaked_task = self .index_tasks .get(alias) - .is_some_and(|entry| !entry.value().0.is_finished()); + .is_some_and(|entry| !entry.value().handle.is_finished()); if !leaked_task { return; } @@ -4731,6 +4961,9 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry + // so a removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "Force reindex for '{}': clearing stores and reindexing", @@ -4796,9 +5029,15 @@ async fn reindex_handler( g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); - state - .index_tasks - .insert(alias.to_string(), (handle, reindex_token)); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); } else { // Incremental refresh: ensure the repo is opened, then refresh let stores = match state.get_or_open_stores(&alias, true).await { @@ -4822,6 +5061,9 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry + // so a removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "🔄 Incremental reindex triggered for '{}' via HTTP API", @@ -4869,9 +5111,15 @@ async fn reindex_handler( g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); - state - .index_tasks - .insert(alias.to_string(), (handle, reindex_token)); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); } ( @@ -5139,6 +5387,9 @@ async fn add_repo_handler( // indexing task, so cancelling it (stop_fsw) did nothing and the task ran // the full embed pass to completion on a removed alias. let token_for_task = cancel_token.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry so a + // removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let index_handle = tokio::spawn(async move { tracing::info!( @@ -5208,8 +5459,22 @@ async fn add_repo_handler( // Build vector index from freshly indexed data. // build_index() is CPU-heavy — offload to the blocking pool, under the - // same process-wide job permit as every other heavy pass. - let _job_permit = stores.acquire_job_permit().await; + // same process-wide job permit as every other heavy pass. The wait is + // cancellation-aware: a repo removed while queued behind another build + // must not keep its stores (and a queue position) parked. + let _job_permit = match stores.acquire_job_permit_or_cancel(&token_for_task).await { + JobAcquire::Ready(permit) => permit, + JobAcquire::Cancelled => { + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); + return; + } + }; + // Re-check liveness after the wait: a removal that landed while the + // task was parked must not resurrect the alias's vector index. + if !state_bg.is_alias_live(&alias_bg, &token_for_task) { + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); + return; + } { let vector_store = Arc::clone(&stores.vector_store); let alias_bi = alias_bg.clone(); @@ -5263,10 +5528,17 @@ async fn add_repo_handler( // Register the indexing task so remove_repo can cancel + await it (BUG1). // Storing the token alongside the handle means remove_repo can cancel - // regardless of the repo's RepoState variant. - state - .index_tasks - .insert(alias.clone(), (index_handle, cancel_token)); + // regardless of the repo's RepoState variant. The DB path travels with + // the entry for deferred cleanup if the repo is removed mid-index. + state.index_tasks.insert( + alias.clone(), + IndexTask { + handle: index_handle, + token: cancel_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); ( StatusCode::ACCEPTED, @@ -6270,6 +6542,7 @@ pub async fn run_serve( tokio::select! { _ = tokio::time::sleep(interval) => { reaper_state.evict_idle_repos(); + reaper_state.sweep_index_tasks(); // Dashboard refresh handled by TUI auto-refresh (TTY) or not needed (non-TTY) } _ = reaper_cancel.cancelled() => { diff --git a/src/serve/tests.rs b/src/serve/tests.rs index f15bd119..235329ba 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -190,9 +190,15 @@ async fn await_index_task_cancels_and_joins_indexing_task() { } done_clone.store(true, std::sync::atomic::Ordering::SeqCst); }); - state - .index_tasks - .insert("repo-x".to_string(), (handle, token)); + state.index_tasks.insert( + "repo-x".to_string(), + IndexTask { + handle, + token, + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); state.await_index_task("repo-x").await; assert!( !state.index_tasks.contains_key("repo-x"), @@ -329,9 +335,15 @@ async fn remove_repo_during_active_build_self_cleans_db_dir() { // orphaned — self-clean it now that the build's handles are released. ServeState::remove_orphaned_db_dir("buildrepo", &db_path_for_cleanup); }); - state - .index_tasks - .insert("buildrepo".to_string(), (handle, token)); + state.index_tasks.insert( + "buildrepo".to_string(), + IndexTask { + handle, + token, + db_path: db_path.clone(), + started_at: Instant::now(), + }, + ); // remove_repo lands WHILE the spawn_blocking build is still sleeping. let outcome = state @@ -3025,13 +3037,15 @@ async fn a_second_format_recovery_for_the_same_alias_is_refused() { #[tokio::test] #[serial] -async fn stale_indexing_marker_cancels_the_leaked_index_task() { - // Regression: evicting the stale marker used to fix only what the TUI - // believed. The task itself kept running and kept its `Arc` - // — so the LMDB env and `.writer.lock` stayed held and every later write - // failed with "Database is locked by another process" on a repo that - // logged as idle and closed. - let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "1")]); +async fn await_index_task_cancels_and_releases_the_task_handles() { + // Regression: a leaked index task used to keep its `Arc` + // after every cooperative path forgot it — the LMDB env and + // `.writer.lock` stayed held, and later writes failed with "Database is + // locked by another process" on a repo that logged as idle. Cancellation + // must be delivered AND the handle joined (not detached), so the stores + // are released. The staleness check no longer cancels live tasks (see + // `stale_marker_with_live_task_is_renewed_not_cancelled`); + // `await_index_task` is the cancellation trigger. let state = Arc::new(ServeState::new(ReposConfig::default(), None)); // Stand-in for the store handles the real task captures. @@ -3043,39 +3057,32 @@ async fn stale_indexing_marker_cancels_the_leaked_index_task() { task_token.cancelled().await; drop(stores_task); }); - state - .index_tasks - .insert("leaky".to_string(), (handle, token.clone())); - state.active_reindexes.insert( + state.index_tasks.insert( "leaky".to_string(), - std::iter::once(( - IndexingOwner::Reindex, - Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("monotonic clock at least 2s old"), - )) - .collect(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, ); assert!( - !state.is_indexing("leaky"), - "a marker older than the timeout must not read as indexing" + state.await_index_task("leaky").await, + "a task observing cancellation must join within the cooperative budget" ); assert!( token.is_cancelled(), - "the leaked task must be cancelled, not merely forgotten" + "the cancelled task must observe cancellation" + ); + assert!( + !state.index_tasks.contains_key("leaky"), + "a joined task must be reaped from index_tasks" ); - - let (handle, _) = state - .index_tasks - .remove("leaky") - .expect("task entry kept") - .1; - handle.await.expect("cancelled task joins cleanly"); assert_eq!( Arc::strong_count(&stores), 1, - "the leaked task must have released its store handle" + "the cancelled task must have released its store handle" ); } @@ -3088,9 +3095,15 @@ async fn fresh_indexing_marker_leaves_its_index_task_running() { let token = CancellationToken::new(); let task_token = token.clone(); let handle = tokio::spawn(async move { task_token.cancelled().await }); - state - .index_tasks - .insert("busy".to_string(), (handle, token.clone())); + state.index_tasks.insert( + "busy".to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); state.begin_indexing("busy", IndexingOwner::Reindex); assert!( @@ -3851,3 +3864,317 @@ fn evaluate_stays_fresh_when_index_is_current_and_clean() { "a clean index must not consume the degraded retry" ); } + +// --------------------------------------------------------------------------- +// Queue-backlog regressions: tracked index tasks, stale-marker liveness, +// deferred DB cleanup. +// --------------------------------------------------------------------------- + +/// Age `alias`'s Reindex marker past the stale threshold. +fn aged_marker(state: &ServeState, alias: &str) { + state + .active_reindexes + .entry(alias.to_string()) + .or_default() + .insert( + IndexingOwner::Reindex, + Instant::now() - Duration::from_secs(MAX_INDEXING_SECS * 2), + ); +} + +/// A handle whose task has already completed (the handle is kept, not awaited). +async fn finished_handle() -> tokio::task::JoinHandle<()> { + let handle = tokio::spawn(async {}); + while !handle.is_finished() { + tokio::task::yield_now().await; + } + handle +} + +fn tracked_task(handle: tokio::task::JoinHandle<()>, db_path: std::path::PathBuf) -> IndexTask { + IndexTask { + handle, + token: CancellationToken::new(), + db_path, + started_at: Instant::now(), + } +} + +#[tokio::test] +#[serial] +async fn stale_marker_with_live_task_is_renewed_not_cancelled() { + // Regression: a reindex task queued behind the process-wide job gate can + // outlive MAX_INDEXING_SECS without being a leak. Evicting its marker and + // cancelling the task cancels healthy queued work whenever a backlog + // outlives the threshold. + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "live-task-repo"; + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + let token = CancellationToken::new(); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + aged_marker(&state, alias); + assert!( + !state.begin_indexing(alias, IndexingOwner::Reindex), + "a live task must block a second run even with a stale marker" + ); + + // A stale marker owned by another flow (Watcher) carries no liveness + // evidence of its own and must still be evicted, even while the index + // task lives. + state + .active_reindexes + .entry(alias.to_string()) + .or_default() + .insert( + IndexingOwner::Watcher, + Instant::now() - Duration::from_secs(MAX_INDEXING_SECS * 2), + ); + aged_marker(&state, alias); + assert!( + state.is_indexing(alias), + "a live task must keep reporting indexing" + ); + assert!( + state + .active_reindexes + .get(alias) + .is_some_and(|owners| !owners.contains_key(&IndexingOwner::Watcher)), + "a stale non-Reindex marker must still be evicted" + ); + assert!( + !token.is_cancelled(), + "the marker check must never cancel a live task" + ); + let renewed = state + .active_reindexes + .get(alias) + .and_then(|owners| owners.get(&IndexingOwner::Reindex).copied()) + .expect("the marker must survive the check"); + assert!( + Instant::now().duration_since(renewed) < Duration::from_secs(5), + "a live marker must be renewed, not evicted" + ); + + { + let entry = state.index_tasks.get(alias); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} + +#[tokio::test] +#[serial] +async fn stale_marker_without_live_task_is_evicted_and_entry_reaped() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "dead-task-repo"; + let handle = finished_handle().await; + state.index_tasks.insert( + alias.to_string(), + tracked_task(handle, std::path::PathBuf::from("/nonexistent-db")), + ); + aged_marker(&state, alias); + + assert!( + !state.is_indexing(alias), + "with no live task the stale marker must be evicted" + ); + assert!( + !state.index_tasks.contains_key(alias), + "the finished entry must be reaped" + ); + assert!( + !state.active_reindexes.contains_key(alias), + "the evicted marker must be cleaned up" + ); +} + +#[tokio::test] +#[serial] +async fn sweeper_reaps_finished_unregistered_task_and_cleans_db() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("gone-repo").join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("metadata.json"), b"{}").unwrap(); + + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let handle = finished_handle().await; + state.index_tasks.insert( + "gone-repo".to_string(), + tracked_task(handle, db_path.clone()), + ); + state + .active_reindexes + .entry("gone-repo".to_string()) + .or_default() + .insert(IndexingOwner::Reindex, Instant::now()); + + state.sweep_index_tasks(); + + assert!(!state.index_tasks.contains_key("gone-repo")); + assert!(!db_path.exists(), "the orphaned DB dir must be deleted"); + assert!(!state.active_reindexes.contains_key("gone-repo")); +} + +#[tokio::test] +#[serial] +async fn await_index_task_timeout_keeps_task_tracked() { + // Regression: the old code removed the entry before waiting and dropped + // the handle on timeout, making the detached task invisible while it held + // its stores and the job permit. + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "parked-task"; + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + let exited = state + .await_index_task_with_budget(alias, Duration::from_millis(50)) + .await; + assert!(!exited, "a parked task cannot exit within the budget"); + assert!( + state.index_tasks.contains_key(alias), + "the task must stay tracked — never invisible" + ); + assert!(token.is_cancelled(), "cancellation must still be signalled"); + + { + let entry = state.index_tasks.get(alias); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} + +#[tokio::test] +#[serial] +async fn await_index_task_returns_true_when_task_exits() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let handle = tokio::spawn(async {}); + state.index_tasks.insert( + "quick-task".to_string(), + tracked_task(handle, std::path::PathBuf::from("/nonexistent-db")), + ); + assert!( + state + .await_index_task_with_budget("quick-task", Duration::from_secs(5)) + .await + ); + assert!(!state.index_tasks.contains_key("quick-task")); +} + +#[tokio::test] +#[serial] +async fn remove_repo_defers_db_delete_while_index_task_is_still_running() { + // Regression: deleting the DB dir under a live (uninterruptible) index + // task is what can wedge a hub — the task never reaches a cancellation + // point and keeps its stores and the job permit for the process lifetime. + let (_tmp, repo_path, state) = state_with_repo("slowrepo"); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("data.mdb"), "fake").unwrap(); + + // A task that ignores cancellation entirely, like a parked build. + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + "slowrepo".to_string(), + IndexTask { + handle, + token, + db_path: db_path.clone(), + started_at: Instant::now(), + }, + ); + + let outcome = state + .remove_repo_with_budget("slowrepo", Duration::from_millis(100)) + .await + .expect("remove_repo should succeed"); + + assert!( + !outcome.db_deleted, + "the DB dir must NOT be deleted out from under a live task" + ); + assert!(db_path.exists(), "the dir stays until the task exits"); + assert!( + state.index_tasks.contains_key("slowrepo"), + "the running task must stay tracked for the sweeper" + ); + + // Simulate the task finally exiting (aborted here; a real one would run + // its post-build guard): the sweeper is the backstop that reaps the entry + // and deletes the orphaned dir. + if let Some(entry) = state.index_tasks.get("slowrepo") { + entry.value().handle.abort(); + } + loop { + let finished = state + .index_tasks + .get("slowrepo") + .is_some_and(|entry| entry.value().handle.is_finished()); + if finished { + break; + } + tokio::task::yield_now().await; + } + state.sweep_index_tasks(); + + assert!(!state.index_tasks.contains_key("slowrepo")); + assert!( + !db_path.exists(), + "the sweeper must clean the orphaned DB dir" + ); +} + +#[tokio::test] +#[serial] +async fn sweeper_reasserts_cancellation_for_a_removed_live_task() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + "removed-repo".to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + state.sweep_index_tasks(); + + assert!( + token.is_cancelled(), + "a removed repo's live task must have cancellation re-asserted" + ); + assert!( + state.index_tasks.contains_key("removed-repo"), + "a live task must stay tracked until it exits" + ); + + { + let entry = state.index_tasks.get("removed-repo"); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} diff --git a/src/serve/tui.rs b/src/serve/tui.rs index 3b7af57e..3678d47d 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -7,7 +7,7 @@ use std::io; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use ratatui::backend::CrosstermBackend; use ratatui::layout::{Constraint, Layout}; @@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken; use super::tui_common::{ self, KeyAction, OverlayKeyAction, OverlayState, RemoteIndexStats, RemoteStatsState, RepoRow, }; -use super::{IndexingOwner, ServeState}; +use super::{IndexTask, IndexingOwner, ServeState}; use crate::cli::doctor; use crate::constants::{DB_DIR_NAME, LANG_CSHARP, LANG_TYPESCRIPT}; use crate::index::IndexManager; @@ -1165,6 +1165,9 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei // `index_tasks` so `remove_repo` can cancel + await it (BUG1). let reindex_token = CancellationToken::new(); let reindex_token_task = reindex_token.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry so a + // removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "TUI: Force reindex for '{}': clearing stores and reindexing", @@ -1224,7 +1227,15 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei // Remove guard state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); }); - state.index_tasks.insert(alias, (handle, reindex_token)); + state.index_tasks.insert( + alias, + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); ReindexLaunch::Started } From 09819320a0728f8d746849519d12ca6c1454e5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 00:51:01 +0300 Subject: [PATCH 55/77] feat(constants): add CODESEARCH_HOME relocatable global root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ~/.codesearch holds repos.json, the embedding-models cache, serve logs, serve_url and the global ignore/extension files, with no way to relocate the root — awkward on machines that keep large artifacts off the home dir (e.g. under ~/Development). All global-path accessors now derive from a single resolution point, codesearch_home(): $CODESEARCH_HOME when set and non-empty (must be absolute — a relative value would silently resolve against the cwd and fragment state between runs), ~/.codesearch otherwise. config_dir() in db_discovery delegates to it, so repos.json and serve_url follow automatically; the models cache, serve logs, global .codesearchignore and extensions.json attach to the same root. Pre-existing per-file overrides (CODESEARCH_REPOS_CONFIG, CODESEARCH_EXTENSION_MAP) still win when set. Empty or unset values keep the existing default — no behaviour change otherwise. Documents the knob in the README env-var table next to its REPOS_CONFIG sibling. --- README.md | 1 + src/constants.rs | 175 +++++++++++++++++++++++++++++++++----- src/db_discovery/repos.rs | 5 +- 3 files changed, 156 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index defa3148..e828e74a 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,7 @@ In the `codesearch serve` TUI, mounts appear in **italic/cyan**, distinguishing | `CODESEARCH_ALLOWED_HOSTS` | Comma-separated hostname allowlist for the MCP streamable-HTTP transport (unset = loopback only: `localhost`, `127.0.0.1`, `::1`). Set this to your container/service hostname when serve runs behind a container network or reverse proxy — see [Security](#security). | | `CODESEARCH_DISABLE_HOST_VALIDATION` | `1`/`true` disables the MCP transport's Host-header allowlist entirely (DNS-rebinding protection off). Only safe behind a reverse proxy/firewall that already restricts inbound Host headers — see [Security](#security). | | `CODESEARCH_MCP_MODE` | MCP mode: auto, client, local | +| `CODESEARCH_HOME` | Relocate the entire global root (`repos.json`, models cache, logs, `serve_url`, global ignore/extension files) away from `~/.codesearch`. Must be an absolute path; unset = `~/.codesearch`. Per-file overrides like `CODESEARCH_REPOS_CONFIG` still win | | `CODESEARCH_REPOS_CONFIG` | Path to repos.json | | `CODESEARCH_REPO_IDLE_TIMEOUT_SECS` | Idle eviction timeout (default: 1800) | | `CODESEARCH_CACHE_MAX_MEMORY` | Embedding cache MB (default: 500) | diff --git a/src/constants.rs b/src/constants.rs index f22fcd04..72cc2ce1 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -59,18 +59,50 @@ pub const DEFAULT_LOG_MAX_FILES: usize = 5; /// Default log retention period in days pub const DEFAULT_LOG_RETENTION_DAYS: u64 = 5; -/// Get the global models cache directory (~/.codesearch/models/). +/// Environment variable relocating the entire global codesearch root. /// -/// This centralizes embedding model downloads so they are shared across all -/// databases instead of being duplicated per-project. The directory is created -/// if it does not exist. +/// When set to a non-empty absolute path, every global artifact — repos.json, +/// the embedding-models cache, serve logs, serve_url, the global +/// .codesearchignore and extensions.json — resolves under it instead of +/// `~/.codesearch`. This is the single knob for machines that keep large +/// artifacts off the home directory. Per-file overrides such as +/// [`REPOS_CONFIG_ENV`] and [`EXTENSION_MAP_ENV`] still win when set. +pub const CODESEARCH_HOME_ENV: &str = "CODESEARCH_HOME"; + +/// Resolve the global codesearch root directory. /// -/// Falls back to a temp directory if the home directory cannot be determined. -pub fn get_global_models_cache_dir() -> anyhow::Result { - let base = - dirs::home_dir().ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?; +/// Returns `$CODESEARCH_HOME` when set and non-empty (it must be absolute — a +/// relative value would silently resolve against the current working +/// directory and fragment state between runs), otherwise `~/.codesearch`. +/// All global-path accessors derive from this single point, so one hook +/// relocates everything. +pub fn codesearch_home() -> anyhow::Result { + if let Ok(dir) = std::env::var(CODESEARCH_HOME_ENV) { + let dir = dir.trim(); + if !dir.is_empty() { + let path = PathBuf::from(dir); + if !path.is_absolute() { + anyhow::bail!( + "{} must be an absolute path, got: {}", + CODESEARCH_HOME_ENV, + dir + ); + } + return Ok(path); + } + } + dirs::home_dir() + .map(|home| home.join(CONFIG_DIR_NAME)) + .ok_or_else(|| anyhow::anyhow!("Could not determine home directory")) +} - let models_dir = base.join(CONFIG_DIR_NAME).join(MODELS_SUBDIR); +/// Get the global models cache directory (`/models/`). +/// +/// This centralizes embedding model downloads so they are shared across all +/// databases instead of being duplicated per-project. Relocated wholesale via +/// [`CODESEARCH_HOME_ENV`]. The directory is created if it does not exist. +pub fn get_global_models_cache_dir() -> anyhow::Result { + let models_dir = codesearch_home()?.join(MODELS_SUBDIR); if !models_dir.exists() { std::fs::create_dir_all(&models_dir).map_err(|e| { @@ -85,13 +117,17 @@ pub fn get_global_models_cache_dir() -> anyhow::Result { Ok(models_dir) } -/// Get the global cache directory (~/.codesearch/). +/// Get the global cache directory (`/`). /// /// Used for client/auto mode logging when no local DB is available. -/// The directory is created if it does not exist. +/// Relocated wholesale via [`CODESEARCH_HOME_ENV`]. The directory is created +/// if it does not exist. pub fn get_global_cache_dir() -> PathBuf { - let base = dirs::home_dir().unwrap_or_else(|| std::env::current_dir().unwrap_or_default()); - let cache_dir = base.join(CONFIG_DIR_NAME); + let cache_dir = codesearch_home().unwrap_or_else(|_| { + std::env::current_dir() + .unwrap_or_default() + .join(CONFIG_DIR_NAME) + }); if !cache_dir.exists() { let _ = std::fs::create_dir_all(&cache_dir); } @@ -101,18 +137,19 @@ pub fn get_global_cache_dir() -> PathBuf { /// Name of the global codesearchignore file in ~/.codesearch/ pub const GLOBAL_CODESEARCHIGNORE_FILE: &str = ".codesearchignore"; -/// Get the path to the global .codesearchignore file (~/.codesearch/.codesearchignore). +/// Get the path to the global .codesearchignore file +/// (`/.codesearchignore`). /// /// This file uses the same gitignore syntax and is applied to all indexed repos, /// providing a way to set ignore rules without modifying repo-local files. /// Repo-local `.codesearchignore` and `.gitignore` take precedence. +/// Relocated wholesale via [`CODESEARCH_HOME_ENV`]. /// -/// Returns `None` only if the home directory cannot be determined. +/// Returns `None` only if the root cannot be resolved. pub fn global_codesearchignore_path() -> Option { - dirs::home_dir().map(|home| { - home.join(CONFIG_DIR_NAME) - .join(GLOBAL_CODESEARCHIGNORE_FILE) - }) + codesearch_home() + .ok() + .map(|root| root.join(GLOBAL_CODESEARCHIGNORE_FILE)) } /// Name of the global extension→language map file in ~/.codesearch/ @@ -132,15 +169,17 @@ pub const EXTENSION_MAP_ENV: &str = "CODESEARCH_EXTENSION_MAP"; /// the binary. User overrides take precedence over the built-in extension table. /// /// The path resolves to `$CODESEARCH_EXTENSION_MAP` when set and non-empty, -/// otherwise `~/.codesearch/extensions.json`. Returns `None` only when neither -/// the env var nor the home directory is available. +/// otherwise `/extensions.json`. Returns `None` only when +/// neither the env var nor a resolvable root is available. pub fn global_extension_map_path() -> Option { if let Ok(p) = std::env::var(EXTENSION_MAP_ENV) { if !p.is_empty() { return Some(PathBuf::from(p)); } } - dirs::home_dir().map(|home| home.join(CONFIG_DIR_NAME).join(GLOBAL_EXTENSION_MAP_FILE)) + codesearch_home() + .ok() + .map(|root| root.join(GLOBAL_EXTENSION_MAP_FILE)) } /// Name of the repos configuration file @@ -981,4 +1020,96 @@ mod tests { "Filename should match GLOBAL_CODESEARCHIGNORE_FILE constant" ); } + + #[test] + #[serial_test::serial] + fn codesearch_home_relocates_all_global_paths() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + // A developer machine with either per-file override exported would + // otherwise win over CODESEARCH_HOME and fail this test spuriously. + let _clear_specific = + crate::testing::EnvRestore::remove(&[REPOS_CONFIG_ENV, EXTENSION_MAP_ENV]); + let _set = + crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, root.to_str().unwrap())]); + + assert_eq!(codesearch_home().unwrap(), root); + assert_eq!( + get_global_models_cache_dir().unwrap(), + root.join(MODELS_SUBDIR), + "models cache must live under CODESEARCH_HOME" + ); + assert!(root.join(MODELS_SUBDIR).exists()); + assert_eq!(get_global_cache_dir(), root); + assert_eq!( + global_codesearchignore_path(), + Some(root.join(GLOBAL_CODESEARCHIGNORE_FILE)) + ); + assert_eq!( + global_extension_map_path(), + Some(root.join(GLOBAL_EXTENSION_MAP_FILE)) + ); + assert_eq!( + crate::db_discovery::repos::config_path().unwrap(), + root.join(REPOS_CONFIG_FILE), + "repos.json default must follow CODESEARCH_HOME" + ); + } + + #[test] + #[serial_test::serial] + fn relative_codesearch_home_is_rejected() { + let _set = crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, "relative/dir")]); + + let err = codesearch_home().unwrap_err(); + assert!( + err.to_string().contains("absolute"), + "error must mention the absolute-path requirement, got: {err}" + ); + assert!( + get_global_models_cache_dir().is_err(), + "a bad root must not silently fall back to the default models dir" + ); + assert_eq!( + global_codesearchignore_path(), + None, + "a bad root must not silently resolve the global ignore file" + ); + } + + #[test] + #[serial_test::serial] + fn empty_or_unset_codesearch_home_falls_back_to_default() { + let expected = dirs::home_dir().unwrap().join(CONFIG_DIR_NAME); + + let _empty = crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, "")]); + assert_eq!( + codesearch_home().unwrap(), + expected, + "an empty override must fall back to ~/.codesearch" + ); + + let _unset = crate::testing::EnvRestore::remove(&[CODESEARCH_HOME_ENV]); + assert_eq!(codesearch_home().unwrap(), expected); + } + + #[test] + #[serial_test::serial] + fn specific_overrides_win_over_codesearch_home() { + let tmp = tempfile::tempdir().unwrap(); + let map_path = tmp.path().join("my-extensions.json"); + let repos_path = tmp.path().join("my-repos.json"); + let _set = crate::testing::EnvRestore::set(&[ + (CODESEARCH_HOME_ENV, tmp.path().to_str().unwrap()), + (EXTENSION_MAP_ENV, map_path.to_str().unwrap()), + (REPOS_CONFIG_ENV, repos_path.to_str().unwrap()), + ]); + + assert_eq!(global_extension_map_path(), Some(map_path)); + assert_eq!( + crate::db_discovery::repos::config_path().unwrap(), + repos_path, + "CODESEARCH_REPOS_CONFIG must win over CODESEARCH_HOME" + ); + } } diff --git a/src/db_discovery/repos.rs b/src/db_discovery/repos.rs index 36ce1bd2..318c8208 100644 --- a/src/db_discovery/repos.rs +++ b/src/db_discovery/repos.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::cache::{normalize_user_path, safe_canonicalize, strip_unc_prefix}; -use crate::constants::{CONFIG_DIR_NAME, REPOS_CONFIG_FILE}; +use crate::constants::REPOS_CONFIG_FILE; /// A remote `codesearch serve` peer that can be queried for federation. /// @@ -1147,8 +1147,7 @@ impl ReposConfig { } pub fn config_dir() -> Result { - let home_dir = dirs::home_dir().ok_or_else(|| anyhow::anyhow!("No home directory found"))?; - Ok(home_dir.join(CONFIG_DIR_NAME)) + crate::constants::codesearch_home() } pub fn config_path() -> Result { From 413973096445eedbdc9426c968314323c5f23bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 01:18:14 +0300 Subject: [PATCH 56/77] feat(hooks): honour CODESEARCH_HOME in guard hooks and OpenCode plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Claude Code guard hooks and the OpenCode plugin resolve repos.json and serve_url from bash/PowerShell/TypeScript, independently of the Rust binary, with fallbacks hardcoded to $HOME/.codesearch — once the global root moves via CODESEARCH_HOME they would see no registration and silently stop enforcing (the guards fail open). They now mirror constants.rs codesearch_home(): a per-file CODESEARCH_REPOS_CONFIG override wins, then $CODESEARCH_HOME/, then ~/.codesearch/ — across codesearch-common.{sh,ps1}, web-guard.{sh,ps1}, the grep-guard coverage docs and the OpenCode plugin's codesearchHome(). The embedded post-checkout worktree hook gets the same fallback chain (its -f test would otherwise silently skip auto-registration after relocation). run-tests.sh pins the relocated registry end-to-end (covered repo denied, unregistered fails open) and a cli test pins the hook block; READMEs document the chain wherever it was already described. --- README.md | 6 ++-- integrations/claude-code/README.md | 8 +++-- .../claude-code/hooks/codesearch-common.ps1 | 4 ++- .../claude-code/hooks/codesearch-common.sh | 10 +++++-- integrations/claude-code/hooks/edit-guard.ps1 | 4 +-- integrations/claude-code/hooks/edit-guard.sh | 3 +- integrations/claude-code/hooks/grep-guard.ps1 | 3 +- integrations/claude-code/hooks/grep-guard.sh | 5 ++-- integrations/claude-code/hooks/run-tests.sh | 29 +++++++++++++++---- integrations/claude-code/hooks/web-guard.ps1 | 1 + integrations/claude-code/hooks/web-guard.sh | 2 +- integrations/opencode/README.md | 11 +++---- integrations/opencode/codesearch.ts | 9 ++++-- src/cli/mod.rs | 20 +++++++++++-- 14 files changed, 84 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e828e74a..2398b4a1 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ The native command embeds the hook scripts in the binary (no source tree needed) **OpenCode** gets the same structural treatment through the OpenCode v2 plugin in [`integrations/opencode/`](integrations/opencode/README.md): it resolves the session directory to an indexed project, injects scope-aware guidance once per session, rescues empty greps with index hits, optionally prunes or blocks `grep`/`glob`, keeps the serve hub healthy (with optional auto-start), and adds `/codesearch*` commands, a `codesearch_scope` tool, a skill and compaction assistance. Install by copying `codesearch.ts` into `~/.config/opencode/plugins/` — see the [integration README](integrations/opencode/README.md). -Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` override) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. +Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` / `CODESEARCH_HOME` overrides) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. ## MCP Tools Reference @@ -446,12 +446,12 @@ When using `git worktree add` to create parallel working directories, codesearch codesearch hooks git install ``` -This installs a `post-checkout` hook that POSTs the worktree path to the running serve instance whenever a new worktree is checked out. The hook reads the serve URL from `~/.codesearch/serve_url` (automatically managed by `codesearch serve`). +This installs a `post-checkout` hook that POSTs the worktree path to the running serve instance whenever a new worktree is checked out. The hook reads the serve URL from `~/.codesearch/serve_url` (`$CODESEARCH_HOME/serve_url` when the global root is relocated; automatically managed by `codesearch serve`). The install target is resolved with `git rev-parse --git-path hooks`, so it honours `core.hooksPath` (and, inside a linked worktree, the shared common-dir hooks) rather than assuming `.git/hooks/`. An existing `post-checkout` is not overwritten — codesearch's logic is chained in as a marker-delimited block. **How it works:** -1. `codesearch serve` writes its URL to `~/.codesearch/serve_url` on startup (deletes on shutdown) +1. `codesearch serve` writes its URL to `~/.codesearch/serve_url` (`$CODESEARCH_HOME/serve_url` when relocated) on startup (deletes on shutdown) 2. The `post-checkout` hook reads that file and POSTs the working directory to `POST /repos` 3. Serve registers the worktree path and begins indexing (deduped — won't re-register existing paths) diff --git a/integrations/claude-code/README.md b/integrations/claude-code/README.md index 1831f031..4e01bb3d 100644 --- a/integrations/claude-code/README.md +++ b/integrations/claude-code/README.md @@ -40,7 +40,8 @@ that make the preference *structural* instead of advisory: Windows-style roots, so POSIX absolute paths were still resolved against the cwd). Coverage is decided by **registration**: the target's git root must be one of the repos in the hub's `~/.codesearch/repos.json` - (honoring `CODESEARCH_REPOS_CONFIG`), which also carves out nested + (honoring `CODESEARCH_REPOS_CONFIG`; `CODESEARCH_HOME` relocates the + root), which also carves out nested repos for free — an unregistered clone inside a registered repo resolves to its own git root and is treated as uncovered. Grep is auto-allowed **only** when codesearch is genuinely @@ -73,7 +74,8 @@ that make the preference *structural* instead of advisory: everything else — making the caller-aware-editing protocol structural. Coverage uses the same registration model as grep-guard (shared `codesearch-common` helpers): the file's git root must be listed in - `~/.codesearch/repos.json` (or `CODESEARCH_SERVER` is set); unregistered + `~/.codesearch/repos.json` (relocatable via `CODESEARCH_HOME`; or + `CODESEARCH_SERVER` is set); unregistered repos and non-git paths are never blocked. - **`edit-guard-post`** — a `PostToolUse` hook on @@ -182,7 +184,7 @@ whose commands point at `hooks/codesearch/`) from `settings.json`, and delete **registration with the serve hub**: the target's git root must be one of the repos listed in `~/.codesearch/repos.json` (the same registration list the hub itself resolves queries by, honoring the - `CODESEARCH_REPOS_CONFIG` override), or an explicit `CODESEARCH_SERVER` + `CODESEARCH_REPOS_CONFIG` / `CODESEARCH_HOME` overrides), or an explicit `CODESEARCH_SERVER` env var for pure remote-serve setups with no local registration. A local `.codesearch.db` directory is deliberately **not** a coverage signal anymore (#199): a stale db from a since-unregistered repo used diff --git a/integrations/claude-code/hooks/codesearch-common.ps1 b/integrations/claude-code/hooks/codesearch-common.ps1 index 00c01d89..35b25501 100644 --- a/integrations/claude-code/hooks/codesearch-common.ps1 +++ b/integrations/claude-code/hooks/codesearch-common.ps1 @@ -5,12 +5,14 @@ # PowerShell twin of codesearch-common.sh — keep the two behaviorally # equivalent. Coverage model: a repo is covered when its git root is # REGISTERED in ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides -# the location), or when CODESEARCH_SERVER opts a pure remote-serve setup +# the location, CODESEARCH_HOME relocates the whole global root), or when +# CODESEARCH_SERVER opts a pure remote-serve setup # in. Everything here fails open: an unresolvable coverage question must # allow, never deny. function Get-CodesearchReposConfigFile { if ($env:CODESEARCH_REPOS_CONFIG) { return $env:CODESEARCH_REPOS_CONFIG } + if ($env:CODESEARCH_HOME) { return (Join-Path $env:CODESEARCH_HOME 'repos.json') } return (Join-Path $HOME '.codesearch/repos.json') } diff --git a/integrations/claude-code/hooks/codesearch-common.sh b/integrations/claude-code/hooks/codesearch-common.sh index a77e51ac..3ea79549 100644 --- a/integrations/claude-code/hooks/codesearch-common.sh +++ b/integrations/claude-code/hooks/codesearch-common.sh @@ -5,7 +5,8 @@ # # Coverage model (shared with the Rust serve hub, src/db_discovery/repos.rs): # a repo is covered when its git root is REGISTERED in -# ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides the location), +# ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides the location, +# CODESEARCH_HOME relocates the whole global root), # or when CODESEARCH_SERVER opts a pure remote-serve setup in. Everything # here fails open: an unresolvable coverage question must allow, never deny. # Requires: jq @@ -17,11 +18,14 @@ jq_str() { tr -d '\r' } -# repos.json location — mirrors src/db_discovery/repos.rs `config_path()`: -# CODESEARCH_REPOS_CONFIG override > ~/.codesearch/repos.json. +# repos.json location — mirrors src/db_discovery/repos.rs `config_path()` +# and constants.rs `codesearch_home()`: +# CODESEARCH_REPOS_CONFIG > $CODESEARCH_HOME/repos.json > ~/.codesearch/repos.json. repos_config_file() { if [ -n "${CODESEARCH_REPOS_CONFIG:-}" ]; then printf '%s' "$CODESEARCH_REPOS_CONFIG" + elif [ -n "${CODESEARCH_HOME:-}" ]; then + printf '%s' "${CODESEARCH_HOME%/}/repos.json" else printf '%s' "${HOME:-}/.codesearch/repos.json" fi diff --git a/integrations/claude-code/hooks/edit-guard.ps1 b/integrations/claude-code/hooks/edit-guard.ps1 index a45b0af6..066d5b9d 100644 --- a/integrations/claude-code/hooks/edit-guard.ps1 +++ b/integrations/claude-code/hooks/edit-guard.ps1 @@ -3,8 +3,8 @@ # two behaviorally equivalent. # # When the edited file's repo is codesearch-registered (git root listed in -# ~/.codesearch/repos.json, honoring CODESEARCH_REPOS_CONFIG, or a -# CODESEARCH_SERVER opt-in), every touched file needs a marker proving the +# ~/.codesearch/repos.json, honoring CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME, +# or a CODESEARCH_SERVER opt-in), every touched file needs a marker proving the # agent consulted codesearch for exactly that path within the last 5 # minutes: mcp__codesearch__find_impact for SCIP-backed languages # (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for diff --git a/integrations/claude-code/hooks/edit-guard.sh b/integrations/claude-code/hooks/edit-guard.sh index c37da6f5..45b8a027 100644 --- a/integrations/claude-code/hooks/edit-guard.sh +++ b/integrations/claude-code/hooks/edit-guard.sh @@ -5,7 +5,8 @@ # # When the edited file's repo is codesearch-registered (shared coverage model # with grep-guard: git root listed in ~/.codesearch/repos.json, honoring -# CODESEARCH_REPOS_CONFIG, or a CODESEARCH_SERVER opt-in), every touched file +# CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME, or a CODESEARCH_SERVER opt-in), +# every touched file # needs a marker proving the agent consulted codesearch for exactly that path # within the last 5 minutes: mcp__codesearch__find_impact for SCIP-backed # languages (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for diff --git a/integrations/claude-code/hooks/grep-guard.ps1 b/integrations/claude-code/hooks/grep-guard.ps1 index ee919d8b..cdc732de 100644 --- a/integrations/claude-code/hooks/grep-guard.ps1 +++ b/integrations/claude-code/hooks/grep-guard.ps1 @@ -82,7 +82,8 @@ $targetRoot = Resolve-TargetGitRoot $path if (-not $targetRoot) { exit 0 } # Coverage: the TARGET repo's git root is REGISTERED with the serve hub -# (~/.codesearch/repos.json — the same list the hub itself resolves by), +# (~/.codesearch/repos.json, honoring the CODESEARCH_REPOS_CONFIG / +# CODESEARCH_HOME overrides — the same list the hub itself resolves by), # matching the bash twin and edit-guard; a running serve hub alone is NOT a # signal (it covers many repos and being alive says nothing about this one). # CODESEARCH_SERVER stays the explicit opt-in for pure remote-serve setups. diff --git a/integrations/claude-code/hooks/grep-guard.sh b/integrations/claude-code/hooks/grep-guard.sh index cd24b3b1..ce310553 100644 --- a/integrations/claude-code/hooks/grep-guard.sh +++ b/integrations/claude-code/hooks/grep-guard.sh @@ -59,8 +59,9 @@ path=$(echo "$raw" | jq -r '.tool_input.path // empty' | jq_str) # every absolute path. # # Coverage signal (#199): the target's git root is REGISTERED with the -# local serve hub (~/.codesearch/repos.json — the same registration list -# the hub itself resolves queries by). A `.codesearch.db` directory at +# local serve hub (~/.codesearch/repos.json, honoring the +# CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME overrides — the same +# registration list the hub itself resolves queries by). A `.codesearch.db` directory at # the git root was only ever a proxy for that and is wrong in both # directions: a stale db from a since-unregistered repo denied Grep while # the hub could not actually answer for it (unknown alias), and a diff --git a/integrations/claude-code/hooks/run-tests.sh b/integrations/claude-code/hooks/run-tests.sh index e536bdc3..31b6a7d0 100644 --- a/integrations/claude-code/hooks/run-tests.sh +++ b/integrations/claude-code/hooks/run-tests.sh @@ -8,10 +8,11 @@ # # Coverage is simulated without a real index: repoA is "registered" via a # temp repos.json passed through CODESEARCH_REPOS_CONFIG (the same override -# the guards honour at runtime), repoB is not listed. The serve hub is only -# contacted by the grep-guard smoke case, which forces a dead port so the -# answer is deterministic. State lives in a temp TMPDIR; the env overrides -# are process-scoped and die with this script. +# the guards honour at runtime), repoB is not listed. One case pins the +# CODESEARCH_HOME relocation fallback with REPOS_CONFIG unset. The serve hub +# is only contacted by the grep-guard smoke case, which forces a dead port +# so the answer is deterministic. State lives in a temp TMPDIR; the env +# overrides are process-scoped and die with this script. set -u @@ -25,7 +26,7 @@ STATE_DIR="$TMP_ROOT/state" mkdir -p "$STATE_DIR" cleanup() { - unset CODESEARCH_REPOS_CONFIG CODESEARCH_SERVER TMPDIR + unset CODESEARCH_REPOS_CONFIG CODESEARCH_SERVER CODESEARCH_HOME TMPDIR rm -rf "$TMP_ROOT" } trap cleanup EXIT @@ -152,6 +153,23 @@ case_grep_guard_smoke() { fi } +case_covered_via_codesearch_home() { + # Relocation pin: with REPOS_CONFIG unset, guards must resolve repos.json + # from $CODESEARCH_HOME (mirrors constants.rs codesearch_home()) — and + # only from there, so a dev's real global registry cannot leak in. + local home="$TMP_ROOT/cs-home" saved_cfg="${CODESEARCH_REPOS_CONFIG:-}" + mkdir -p "$home" + jq -n --arg a "$ROOT_A" '{repos: {repoA: $a}}' > "$home/repos.json" + unset CODESEARCH_REPOS_CONFIG + export CODESEARCH_HOME="$home" + run_edit "$(edit_event "$REPO_A/x.cs")" + expect_deny "CODESEARCH_HOME registry: covered .cs denied" "find_impact" + run_edit "$(edit_event "$REPO_B/b.cs")" + expect_allow "CODESEARCH_HOME registry: unregistered repo fails open" + unset CODESEARCH_HOME + export CODESEARCH_REPOS_CONFIG="$saved_cfg" +} + case_no_coverage_allows case_covered_cs_denied case_covered_ts_denied @@ -163,6 +181,7 @@ case_multiedit_partial_marks_denied case_find_usages_marker_allows case_window_expiry_denies_again case_grep_guard_smoke +case_covered_via_codesearch_home echo echo "edit-guard self-tests: $PASS passed, $FAIL failed" diff --git a/integrations/claude-code/hooks/web-guard.ps1 b/integrations/claude-code/hooks/web-guard.ps1 index 6dddaac1..d519068c 100644 --- a/integrations/claude-code/hooks/web-guard.ps1 +++ b/integrations/claude-code/hooks/web-guard.ps1 @@ -76,6 +76,7 @@ if ([string]::IsNullOrWhiteSpace($q)) { exit 0 } # names — the opt-in allowlist). No mounts -> nothing to prefer -> allow. # ------------------------------------------------------------------ $config = if ($env:CODESEARCH_REPOS_CONFIG) { $env:CODESEARCH_REPOS_CONFIG } + elseif ($env:CODESEARCH_HOME) { Join-Path $env:CODESEARCH_HOME 'repos.json' } else { Join-Path $HOME '.codesearch/repos.json' } if (-not (Test-Path $config)) { exit 0 } diff --git a/integrations/claude-code/hooks/web-guard.sh b/integrations/claude-code/hooks/web-guard.sh index 6c72a633..31eff40c 100644 --- a/integrations/claude-code/hooks/web-guard.sh +++ b/integrations/claude-code/hooks/web-guard.sh @@ -54,7 +54,7 @@ q=$(echo "$raw" | jq -r '.tool_input.query // .tool_input.url // empty') # names — the opt-in allowlist). No mounts -> nothing to prefer -> allow the # web call unimpeded. # ------------------------------------------------------------------ -config="${CODESEARCH_REPOS_CONFIG:-$HOME/.codesearch/repos.json}" +config="${CODESEARCH_REPOS_CONFIG:-${CODESEARCH_HOME:-$HOME/.codesearch}/repos.json}" [ -f "$config" ] || exit 0 # NOTE: strip CR. On Windows/Git Bash jq emits CRLF, and a trailing \r silently diff --git a/integrations/opencode/README.md b/integrations/opencode/README.md index fac15b84..b120ad28 100644 --- a/integrations/opencode/README.md +++ b/integrations/opencode/README.md @@ -41,7 +41,7 @@ preference into behaviour: | Feature | Default | What it does | |---|---|---| -| **Scope resolution** | on | Maps the session directory to a registered project alias (and its groups), from `~/.codesearch/repos.json` or a live `status` snapshot. | +| **Scope resolution** | on | Maps the session directory to a registered project alias (and its groups), from `~/.codesearch/repos.json` (relocatable via `CODESEARCH_HOME`) or a live `status` snapshot. | | **Scope-aware guidance** | once per session | Injects a compact strategy block plus the resolved scope into system context, so the first tool call already knows `project=`. | | **Zero-hit rescue** | on | When `grep`/`glob` returns nothing in an indexed repo, runs a literal codesearch query and appends the index hits to the tool result. | | **Code-search-first guards** | `nudge` | `off`, `nudge` (one tip per session), `prune` (hide `grep`/`glob` from the model), or `block` (deny with the exact codesearch call to use). | @@ -148,7 +148,8 @@ Precedence, lowest to highest: "token": "{env:CODESEARCH_API_KEY}", // Name of the MCP server entry in opencode.jsonc to mirror. "mcpServer": "codesearch", - // Optional override for ~/.codesearch/repos.json. + // Optional override for the registry (default: ~/.codesearch/repos.json, + // relocated wholesale by CODESEARCH_HOME). "reposConfig": "", "debug": false, @@ -212,7 +213,7 @@ Precedence, lowest to highest: | `CODESEARCH_SERVER` | – | Serve **base** URL (e.g. `http://127.0.0.1:39725`), same meaning as the Claude Code guard hooks use. `/mcp` is appended. | | `CODESEARCH_API_KEY` | – | Bearer token for an authenticated/remote serve. | | `CODESEARCH_CONFIG` | `~/.config/opencode/codesearch.json` | Config file path. | -| `CODESEARCH_REPOS_CONFIG` | `~/.codesearch/repos.json` | Registry path (shared with the hub and the Claude Code hooks). | +| `CODESEARCH_REPOS_CONFIG` | `$CODESEARCH_HOME/repos.json` (`~/.codesearch` when unset) | Registry path (shared with the hub and the Claude Code hooks). | | `CODESEARCH_MCP_SERVER` | `codesearch` | MCP entry name to mirror for URL/token. | | `CODESEARCH_PLUGIN_GUARDS` | `nudge` | `off` / `nudge` / `prune` / `block`. | | `CODESEARCH_PLUGIN_SCOPE` | `session` | `session` / `message` / `off`. | @@ -237,7 +238,7 @@ The plugin resolves the MCP endpoint in this order, first non-empty wins: 2. `CODESEARCH_SERVER` 3. the `url` of the MCP entry named by `mcpServer` in `opencode.jsonc` (its `Authorization` header is used as the token, minus `Bearer`) -4. `~/.codesearch/serve_url` (written by `codesearch serve`) +4. `~/.codesearch/serve_url`, or `$CODESEARCH_HOME/serve_url` when the global root is relocated (written by `codesearch serve`) 5. `http://127.0.0.1:39725/mcp` A base URL without `/mcp` is normalised by appending it, so pointing @@ -424,7 +425,7 @@ hub uses. (`~/.local/share/opencode/log/opencode.log`). - **No scope / `project=null`** — the directory is not registered. `/codesearch-index` runs `codesearch index`; verify with `codesearch index list` and inspect - `~/.codesearch/repos.json` (or your `CODESEARCH_REPOS_CONFIG`). + `~/.codesearch/repos.json` (or your `CODESEARCH_REPOS_CONFIG`; `CODESEARCH_HOME` relocates the default). - **Rescue never fires** — check `guards.rescue`, `guards.tools`, and health (`/codesearch-status`). Rescue is skipped while the hub is down. - **Block mode too strict** — `guards.mode: "nudge"` (or `prune`) is the diff --git a/integrations/opencode/codesearch.ts b/integrations/opencode/codesearch.ts index 84819c47..36e6395c 100644 --- a/integrations/opencode/codesearch.ts +++ b/integrations/opencode/codesearch.ts @@ -333,6 +333,11 @@ function configDir(): string { return process.env.XDG_CONFIG_HOME || `${homeDir()}/.config` } +/** Global codesearch root: $CODESEARCH_HOME, else ~/.codesearch (mirrors constants.rs `codesearch_home()`). */ +function codesearchHome(): string { + return process.env.CODESEARCH_HOME || `${homeDir()}/.codesearch` +} + /** * Strip `//` and block comments plus trailing commas from JSONC, honouring * string literals. Enough for OpenCode's config files. @@ -541,7 +546,7 @@ function normalizeMcpUrl(value: string): string { /** Resolve the serve URL: config/env > CODESEARCH_SERVER > MCP entry > serve_url file > default. */ function resolveServeUrl(settings: CodeSearchSettings): string { const mcpEntry = readMcpEntry(settings.mcpServer) - const serveUrlFile = readTextFile(`${homeDir()}/.codesearch/serve_url`)?.trim() ?? "" + const serveUrlFile = readTextFile(`${codesearchHome()}/serve_url`)?.trim() ?? "" const candidates = [ settings.url, process.env.CODESEARCH_SERVER ?? "", @@ -1159,7 +1164,7 @@ export default { const url = resolveServeUrl(settings) const token = resolveToken(settings) const registry = new RepoRegistry( - settings.reposConfig || `${homeDir()}/.codesearch/repos.json`, + settings.reposConfig || `${codesearchHome()}/repos.json`, ) registry.load() const status = new StatusCache() diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 8e083385..3baf386b 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1821,8 +1821,9 @@ fn codesearch_hook_block() -> String { # Only react to branch/worktree checkouts ($3 = 1). `git worktree add` fires # post-checkout with flag 1; a file checkout (`git checkout -- path`) fires with # flag 0 and must not re-register (it changes no repo location, just wastes a POST). -if [ "$3" = "1" ] && [ -f "$HOME/.codesearch/serve_url" ]; then - __cs_url=$(cat "$HOME/.codesearch/serve_url") +__cs_home="${CODESEARCH_HOME:-$HOME/.codesearch}" +if [ "$3" = "1" ] && [ -f "$__cs_home/serve_url" ]; then + __cs_url=$(cat "$__cs_home/serve_url") if [ -n "$__cs_url" ]; then # Git Bash `pwd` yields an msys path (/c/...) that codesearch serve # cannot canonicalize (HTTP 400); `pwd -W` yields a native C:/ path. @@ -2059,6 +2060,21 @@ mod tests { ); } + #[test] + fn test_hook_block_resolves_serve_url_via_codesearch_home() { + let block = codesearch_hook_block(); + // serve_url lives under the relocatable global root: the hook must + // fall back through CODESEARCH_HOME, mirroring codesearch_home(). + assert!( + block.contains("${CODESEARCH_HOME:-$HOME/.codesearch}"), + "hook must honour CODESEARCH_HOME before ~/.codesearch" + ); + assert!( + !block.contains("$HOME/.codesearch/serve_url"), + "no direct ~/.codesearch/serve_url path may remain" + ); + } + #[test] fn test_full_hook_has_posix_shebang() { let hook = codesearch_full_hook(); From e9f1c3f05b0a71d76612ebaf56047643f5f5d492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 01:46:08 +0300 Subject: [PATCH 57/77] fix(index): exclude dependency lock files from indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live search evaluation on real repos surfaced lock files in the results: lock names whose extension alone is indexable (pnpm-lock.yaml, package-lock.json, npm-shrinkwrap.json, bun.lockb) slipped past the extension-based skips, and generated churn added noise to semantic hits. Add the whole lock-file class (JS/TS, Rust, Go, Python, PHP, Ruby, Nix) to ALWAYS_EXCLUDED as exact filenames so the walker and the watcher exclude them uniformly — extension-based .lock/.sum skips still apply, but the name match closes the .yaml/.json/.lockb gap and makes the exclusion explicit and extension-independent. Also correct the ALWAYS_EXCLUDED doc comment: it claimed the list is written to .gitignore/.codesearchignore, which it never was — the check is purely in memory. Existing indexes keep previously indexed lock files until the next reindex. --- src/constants.rs | 28 ++++++++++++++++++++++++++-- src/file/mod.rs | 19 +++++++++++++++++++ src/watch/mod.rs | 7 +++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 72cc2ce1..1675635e 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -947,8 +947,12 @@ pub const ALWAYS_SKIP_FILENAME_SUFFIXES: &[&str] = &[ ".orig", ]; -/// Directories and files that should always be excluded from indexing -/// These are added to both .gitignore and .codesearchignore automatically +/// Directories and files that should always be excluded from indexing. +/// +/// This list is applied purely in memory: the file walker filters these names +/// during traversal and the watcher skips paths containing them. Nothing is +/// written to `.gitignore` or `.codesearchignore`, and previously indexed +/// content keeps appearing in search results until the next reindex. pub const ALWAYS_EXCLUDED: &[&str] = &[ // Codesearch databases ".codesearch", @@ -986,6 +990,26 @@ pub const ALWAYS_EXCLUDED: &[&str] = &[ "coverage", ".nyc_output", ".cache", + // Dependency lock files — generated by package managers, never source. + // Matched by exact filename so the walker and watcher exclude them even + // when the extension alone is not skipped (pnpm-lock.yaml, + // package-lock.json, npm-shrinkwrap.json, bun.lockb). + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "bun.lock", + "bun.lockb", + "deno.lock", + "Cargo.lock", + "go.sum", + "poetry.lock", + "Pipfile.lock", + "uv.lock", + "pdm.lock", + "composer.lock", + "Gemfile.lock", + "flake.lock", ]; #[cfg(test)] diff --git a/src/file/mod.rs b/src/file/mod.rs index b98c3d1d..6cfa21e3 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -322,6 +322,25 @@ mod tests { assert_eq!(files[0].path.file_name().unwrap(), "index.js"); } + /// Dependency lock files are excluded by exact filename: their extensions + /// (.yaml/.json) are indexable on their own, so without the name match the + /// walker would index every churn commit of a generated lock file. + #[test] + fn test_excluded_lock_files() { + let dir = TempDir::new().unwrap(); + + fs::write(dir.path().join("pnpm-lock.yaml"), "lockVersion: '9.0'").unwrap(); + fs::write(dir.path().join("package-lock.json"), "{}").unwrap(); + fs::write(dir.path().join("yarn.lock"), "# yarn lockfile").unwrap(); + fs::write(dir.path().join("main.rs"), "fn main() {}").unwrap(); + + let walker = FileWalker::new(dir.path()); + let (files, _) = walker.walk().unwrap(); + + assert_eq!(files.len(), 1); + assert_eq!(files[0].path.file_name().unwrap(), "main.rs"); + } + /// A root whose own name matches an `ALWAYS_EXCLUDED` entry (e.g. `.git`) /// must be rejected at `walk()` time — otherwise the depth==0 short-circuit /// in `filter_entry` would let every internal file be indexed. diff --git a/src/watch/mod.rs b/src/watch/mod.rs index b10d4229..3b39e4de 100644 --- a/src/watch/mod.rs +++ b/src/watch/mod.rs @@ -702,6 +702,13 @@ mod tests { assert!(!watcher.is_watchable(Path::new("/tmp/image.png"))); assert!(!watcher.is_watchable(Path::new("/tmp/data.bin"))); + // Should NOT watch (dependency lock files — name-based, covers names + // whose extension alone is indexable like .yaml/.json/.lockb) + assert!(!watcher.is_watchable(Path::new("/tmp/pnpm-lock.yaml"))); + assert!(!watcher.is_watchable(Path::new("/tmp/package-lock.json"))); + assert!(!watcher.is_watchable(Path::new("/tmp/npm-shrinkwrap.json"))); + assert!(!watcher.is_watchable(Path::new("/tmp/bun.lockb"))); + // SHOULD watch (code files) assert!(watcher.is_watchable(Path::new("/tmp/src/main.rs"))); assert!(watcher.is_watchable(Path::new("/tmp/src/lib.ts"))); From aa8555ca53500546ee015737304b2726cd3e0961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 01:47:59 +0300 Subject: [PATCH 58/77] fix(find-impact): serialize unknown index age as null, not u64::MAX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `index_age` returns u64::MAX whenever the symbol index cannot be opened or read, and FindImpactResult carried that sentinel straight into the JSON response — a consumer saw "index_age_seconds": 18446744073709551615 and had to know the internal convention to read it as "unknown". Serialize the field with a custom mapper that turns the sentinel into JSON null and passes real ages through unchanged. The in-process consumers are untouched: the field stays u64 and `SymbolLookupFailure::classify` keeps matching on u64::MAX — only the wire representation changes. --- src/mcp/find_impact_tests.rs | 30 ++++++++++++++++++++++++++++++ src/symbols/mod.rs | 19 ++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index 72ef5139..66ec3513 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -286,6 +286,36 @@ fn fingerprint_fields_present_when_set_and_omitted_when_none() { assert!(!keys.contains(&"resolved_symbol"), "keys: {keys:?}"); } +#[test] +fn index_age_sentinel_serializes_as_null_and_real_ages_as_numbers() { + use crate::symbols::{FindImpactResult, SymbolReference}; + + let result = |age: u64| FindImpactResult { + symbol: "FieldDefinition.Validate".to_string(), + resolved_symbol: None, + references: vec![SymbolReference { + file: PathBuf::from("a.cs"), + start_line: 1, + end_line: 1, + kind: "definition".to_string(), + }], + warnings: Vec::new(), + index_age_seconds: age, + language: "csharp".to_string(), + scope: "project:p".to_string(), + index_head_sha: None, + current_head_sha: None, + }; + + // u64::MAX is `index_age`'s unreadable-index sentinel; a consumer must + // see "unknown" (null), not 18446744073709551615 seconds. + let unknown: serde_json::Value = serde_json::to_value(result(u64::MAX)).unwrap(); + assert_eq!(unknown["index_age_seconds"], serde_json::Value::Null); + + let known: serde_json::Value = serde_json::to_value(result(12)).unwrap(); + assert_eq!(known["index_age_seconds"], serde_json::json!(12)); +} + #[test] fn ambiguity_envelope_serializes_the_four_documented_fields() { let ambiguity = crate::symbols::SymbolAmbiguity { diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index d662033d..51fa28e8 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -76,7 +76,10 @@ pub struct FindImpactResult { /// answer is as complete as the index knows. #[serde(skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, - /// Seconds since the symbol index was last rebuilt. + /// Seconds since the symbol index was last rebuilt. `u64::MAX` is the + /// internal "age unknown" sentinel `index_age` returns when the index + /// cannot be opened or read; over the wire it serializes as null. + #[serde(serialize_with = "serialize_index_age")] pub index_age_seconds: u64, /// Language that produced this result. pub language: String, @@ -95,6 +98,20 @@ pub struct FindImpactResult { pub current_head_sha: Option, } +/// Serialize `index_age_seconds`, mapping the internal unknown-age sentinel +/// (`u64::MAX`) to JSON null — a consumer must not see +/// `18446744073709551615` and mistake it for an age. Real ages stay numbers. +fn serialize_index_age(age: &u64, serializer: S) -> Result +where + S: serde::Serializer, +{ + if *age == u64::MAX { + serializer.serialize_none() + } else { + serializer.serialize_u64(*age) + } +} + /// A `find_impact` query in the form the adapters resolve it. /// /// The three input variants of `FindImpactRequest` map onto this one-to-one; From a0c00b19714455c3b8d1e3d65e56966388031099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 01:51:13 +0300 Subject: [PATCH 59/77] fix(search): attribute group-fan-out hits to their repo in result paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit search(project=X) and find prefix every hit as "alias/repo-relative", but search(group=all) returned bare paths: build_semantic_response's group branch called prefix_path_multi with an empty alias list, so it could never prefix anything. The live evaluation hit this directly — with five repos fanned out together, a hit could not be attributed to its project. The fan-out accessors already deduplicate by (alias, chunk_id), but the alias is dropped where the per-store results flatten into one vec. Rather than reworking MultiReadOutcome — which would ripple into every fan-out caller — record each surviving chunk's owning alias at the points that already know it: the vector-search closure receives the alias, and the two chunk resolvers know the store they resolved from. semantic_search_multi then prefixes paths with the recorded alias before building the response, mirroring project= routing; the dead group branch in build_semantic_response becomes a plain normalization, and the unused prefix_path_multi helper is removed. Known limitation, unchanged by this commit: chunks with colliding ids across repos are attributed to whichever store the resolve probe finds first, because id-based fusion and lookup are bare-chunk-id keyed. The recorded alias always matches the content actually served. Covered end-to-end: a lexical group query over two seeded stores returns "alpha/src/lib.rs" and "beta/src/util.rs". --- src/mcp/helpers.rs | 18 ------ src/mcp/search.rs | 73 +++++++++++++++++++++--- src/mcp/tests.rs | 134 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 27 deletions(-) diff --git a/src/mcp/helpers.rs b/src/mcp/helpers.rs index 02fbdcd6..77b5efbf 100644 --- a/src/mcp/helpers.rs +++ b/src/mcp/helpers.rs @@ -119,24 +119,6 @@ pub(crate) fn prefix_path_with_alias( } } -/// Prefix a result path with the matching repo alias from a set of aliases and their roots. -/// Used by handlers that have alias/root info but not a full `MultiStoreContext`. -pub(crate) fn prefix_path_multi( - path: &str, - aliases: &[String], - alias_roots: &std::collections::HashMap, -) -> String { - let normalized = crate::cache::normalize_path_str(path); - for alias in aliases { - if let Some(root) = alias_roots.get(alias) { - if normalized.starts_with(root.as_str()) { - return prefix_path_with_alias(path, Some(alias), root); - } - } - } - normalized -} - /// Pick the project root to relativise a result path against for a `filter_path` /// prefix match, so `filter_path` is interpreted **relative to the repo root** /// in every routing mode: diff --git a/src/mcp/search.rs b/src/mcp/search.rs index 5ac0200a..78c73272 100644 --- a/src/mcp/search.rs +++ b/src/mcp/search.rs @@ -499,6 +499,14 @@ impl CodesearchService { let mode = request.mode.as_deref().unwrap_or("auto"); let structural_intent = detect_structural_intent(&request.query); + // Which repo (alias) owns each surviving chunk id, recorded at + // fan-out and chunk-resolution time. The flat fan-out merge drops the + // store of origin, so without this map a group=all response cannot + // attribute a hit to its repo — `project=` results are prefixed + // "alias/path", group results must be too. + let mut alias_by_chunk: std::collections::HashMap = + std::collections::HashMap::new(); + // === Lexical mode: FTS only across all stores === if mode == "lexical" { // Lexical has no second backend, so a failed store here is invisible @@ -545,22 +553,22 @@ impl CodesearchService { .unwrap_or(std::cmp::Ordering::Equal) }); - let results = self + let mut results = self .resolve_fts_to_search_results_multi( &all_fts, limit, &stores, aliases, &mut lexical_warnings, + &mut alias_by_chunk, ) .await; + prefix_paths_with_alias(&mut results, &alias_by_chunk, alias_roots); if let Some(target_kind) = structural_intent { - // We need mutable results but we have them as vectordb::SearchResult - let mut mutable_results = results; - boost_kind(&mut mutable_results, target_kind); + boost_kind(&mut results, target_kind); return self.build_semantic_response( - mutable_results, + results, request, compact, !identifiers.is_empty(), @@ -643,9 +651,13 @@ impl CodesearchService { "internal error: no query embedding resolved for repo '{alias}'" ) })?; - store + let found = store .search(embedding, limit * 5) - .context("Error searching vector store") + .context("Error searching vector store")?; + for r in &found { + alias_by_chunk.insert(r.id, alias.to_string()); + } + Ok(found) }, stores.clone(), aliases, @@ -714,6 +726,7 @@ impl CodesearchService { results.push(r); } } + prefix_paths_with_alias(&mut results, &alias_by_chunk, alias_roots); return self.build_semantic_response( results, request, @@ -807,6 +820,7 @@ impl CodesearchService { &stores, aliases, &mut search_warnings, + &mut alias_by_chunk, ) .await { @@ -820,6 +834,8 @@ impl CodesearchService { boost_kind(&mut mapped, target_kind); } + prefix_paths_with_alias(&mut mapped, &alias_by_chunk, alias_roots); + self.build_semantic_response( mapped, request, @@ -832,6 +848,7 @@ impl CodesearchService { } /// Resolve a single chunk from multiple stores (used for FTS-only hits in multi-store fusion). + #[allow(clippy::too_many_arguments)] async fn resolve_chunk_from_stores( &self, chunk_id: u32, @@ -839,6 +856,7 @@ impl CodesearchService { stores: &[Arc], aliases: &[String], warnings: &mut Vec, + alias_by_chunk: &mut std::collections::HashMap, ) -> Option { for (idx, store_arc) in stores.iter().enumerate() { let store = match bounded_vector_read(&store_arc.vector_store).await { @@ -853,6 +871,12 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { + // Record the owning alias with the content actually chosen: + // path attribution below must match this very chunk, and the + // probe order here is what decided it. + if let Some(alias) = aliases.get(idx) { + alias_by_chunk.insert(chunk_id, alias.clone()); + } return Some(crate::vectordb::SearchResult { id: chunk_id, content: chunk.content, @@ -875,6 +899,7 @@ impl CodesearchService { } /// Resolve FTS results to SearchResult using multiple stores. + #[allow(clippy::too_many_arguments)] async fn resolve_fts_to_search_results_multi( &self, fts_results: &[crate::fts::FtsResult], @@ -882,6 +907,7 @@ impl CodesearchService { stores: &[Arc], aliases: &[String], warnings: &mut Vec, + alias_by_chunk: &mut std::collections::HashMap, ) -> Vec { let mut results = Vec::new(); for fts in fts_results.iter().take(limit) { @@ -903,6 +929,11 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { + // Same as resolve_chunk_from_stores: attribute the alias of + // the store this content was actually read from. + if let Some(alias) = aliases.get(idx) { + alias_by_chunk.insert(fts.chunk_id, alias.clone()); + } results.push(crate::vectordb::SearchResult { id: fts.chunk_id, content: chunk.content, @@ -1090,7 +1121,10 @@ impl CodesearchService { }) .collect(); - // Prefix paths with alias for multi-repo / single-project identification + // `project=` routing prefixes paths with the routed alias here. Group + // fan-out results arrive already alias-prefixed (semantic_search_multi + // recorded each chunk's repo), so this pass only normalizes them; + // stdio without alias roots passes paths through untouched. for item in &mut items { if let Some(alias) = project_alias { if let Some(root) = alias_roots.get(alias) { @@ -1099,7 +1133,7 @@ impl CodesearchService { item.path = crate::cache::normalize_path_str(&item.path); } } else if !alias_roots.is_empty() { - item.path = prefix_path_multi(&item.path, &[], alias_roots); + item.path = crate::cache::normalize_path_str(&item.path); } } @@ -1175,3 +1209,24 @@ impl CodesearchService { } } } + +/// Prefix group-fan-out result paths with the repo alias each chunk was +/// recorded from, mirroring the `project=` routing ("alias/repo-relative"). +/// +/// Results whose alias was not recorded (every survivor is recorded at +/// fan-out or resolution time, so this indicates an internal slip) keep their +/// raw path — `build_semantic_response` still normalizes it. +fn prefix_paths_with_alias( + results: &mut [crate::vectordb::SearchResult], + alias_by_chunk: &std::collections::HashMap, + alias_roots: &std::collections::HashMap, +) { + for r in results.iter_mut() { + let Some(alias) = alias_by_chunk.get(&r.id) else { + continue; + }; + if let Some(root) = alias_roots.get(alias) { + r.path = prefix_path_with_alias(&r.path, Some(alias), root); + } + } +} diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index 79408b7d..8628c2d3 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -565,6 +565,140 @@ fn test_relative_path_empty_alias() { assert_eq!(result, "docs/guide.md"); } +// === group fan-out alias attribution (semantic_search_multi) === + +#[tokio::test] +async fn group_lexical_search_attributes_each_hit_to_its_repo() { + use rmcp::model::ContentBlock; + + // Two repos with disjoint chunk ids. A group query must answer with + // alias-prefixed paths ("alpha/…", "beta/…") exactly like a project= + // query — before the fix the fan-out lost the store of origin and both + // hits came back bare, making a cross-repo response unattributable. + let root_a = tempfile::tempdir().expect("tempdir a"); + let stores_a = std::sync::Arc::new( + crate::index::SharedStores::new(&root_a.path().join(".codesearch.db"), 2) + .expect("stores a"), + ); + { + let mut vs = stores_a.vector_store.write().await; + let chunk = crate::chunker::Chunk::new( + "fn gadget() {}".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/lib.rs".to_string(), + ); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + chunk, + vec![0.0, 1.0], + )]) + .expect("insert chunk a"); + vs.build_index().expect("build index a"); + } + { + let mut fts = stores_a.fts_store.write().await; + fts.add_chunk(0, "fn gadget() {}", "src/lib.rs", None, "Function") + .expect("fts doc a"); + fts.commit().expect("commit a"); + } + + let root_b = tempfile::tempdir().expect("tempdir b"); + let stores_b = std::sync::Arc::new( + crate::index::SharedStores::new(&root_b.path().join(".codesearch.db"), 2) + .expect("stores b"), + ); + { + // insert_chunks assigns ids from 0 per store, so a filler chunk keeps + // the real one at id 1 and the two test repos never share a chunk id. + let mut vs = stores_b.vector_store.write().await; + let filler = crate::chunker::Chunk::new( + "filler".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/filler.rs".to_string(), + ); + let chunk = crate::chunker::Chunk::new( + "gadget helper".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/util.rs".to_string(), + ); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + filler, + vec![0.5, 0.5], + )]) + .expect("insert filler b"); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + chunk, + vec![1.0, 0.0], + )]) + .expect("insert chunk b"); + vs.build_index().expect("build index b"); + } + { + let mut fts = stores_b.fts_store.write().await; + fts.add_chunk(1, "gadget helper", "src/util.rs", None, "Function") + .expect("fts doc b"); + fts.commit().expect("commit b"); + } + + let service = super::CodesearchService::new_with_stores( + Some(root_a.path().to_path_buf()), + Some(stores_a.clone()), + ) + .expect("service"); + + let request = super::SemanticSearchRequest { + query: "gadget".to_string(), + limit: Some(10), + compact: Some(false), + filter_path: None, + mode: Some("lexical".to_string()), + project: None, + group: Some("all".to_string()), + }; + let mut alias_roots = std::collections::HashMap::new(); + alias_roots.insert( + "alpha".to_string(), + crate::cache::normalize_path_str(&root_a.path().to_string_lossy()), + ); + alias_roots.insert( + "beta".to_string(), + crate::cache::normalize_path_str(&root_b.path().to_string_lossy()), + ); + + let result = service + .semantic_search_multi( + &request, + &[], + 10, + false, + vec![stores_a.clone(), stores_b.clone()], + &["alpha".to_string(), "beta".to_string()], + &alias_roots, + ) + .await + .expect("group search must succeed"); + + let json = match &result.content[0] { + ContentBlock::Text(t) => t.text.clone(), + other => panic!("expected text content, got {other:?}"), + }; + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid json response"); + let paths: Vec<&str> = parsed["results"] + .as_array() + .expect("results array") + .iter() + .map(|r| r["path"].as_str().expect("path string")) + .collect(); + assert_eq!(paths.len(), 2, "both repos must answer, paths: {paths:?}"); + assert!(paths.contains(&"alpha/src/lib.rs"), "paths: {paths:?}"); + assert!(paths.contains(&"beta/src/util.rs"), "paths: {paths:?}"); +} + #[test] fn test_normalize_tool_path_relativizes_absolute() { let root = std::path::Path::new("/tmp/proj"); From c352f2c50c4ff501f60eb08f4814a9e53edc6ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=91=D0=B5=D0=BB=D0=BE=D0=B2=20=D0=90=D0=BB=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D0=B0=D0=BD=D0=B4=D1=80=20=D0=A4=D0=B5=D0=B4=D0=BE=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sun, 4 Oct 2026 02:20:27 +0300 Subject: [PATCH 60/77] fix(find-impact): make the missing-index warning state the real heal status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit find_impact answered every query against a never-built symbol index with "A background rebuild was started; retry this call once it comple." — regardless of whether anything could actually run. In live use on a Kotlin/Gradle set of repos that promise was false three ways: the TypeScript adapter is not applicable (it requires a top-level tsconfig.json, these repos have none), and even where applicable, no rebuild starts outside serve mode. The caller kept retrying an answer that would never change. The warning now reflects what find_impact really did. A new SymbolIndexHeal status picks the wording: BackgroundStarted (serve mode with a routed alias — the old text, now only when the heal task is actually spawned), Manual (no serve: says so and names `codesearch index symbol `), NotApplicable (the indexer's own reason via the new SymbolIndexer::applicability_hint — e.g. the scip-typescript top-level tsconfig.json rule). The self-heal spawn is likewise skipped for non-applicable repos, where it could only ever be a no-op. Every variant still names the language and marks the empty reference list as UNKNOWN — that part of the contract is pinned by tests, along with new pins that "a rebuild was started" may only appear when one was. --- src/mcp/find_impact.rs | 50 +++++++++++++++++++++++++++++++----- src/mcp/find_impact_tests.rs | 40 +++++++++++++++++++++++++---- src/symbols/csharp.rs | 4 +++ src/symbols/mod.rs | 8 ++++++ src/symbols/typescript.rs | 4 +++ 5 files changed, 94 insertions(+), 12 deletions(-) diff --git a/src/mcp/find_impact.rs b/src/mcp/find_impact.rs index 904d0d87..5faf63bf 100644 --- a/src/mcp/find_impact.rs +++ b/src/mcp/find_impact.rs @@ -93,14 +93,36 @@ fn dedupe_references( out } +/// What find_impact actually did (or could not do) about a missing symbol +/// index — drives the wording of [`missing_index_warning`] so the answer +/// never claims a rebuild that is not running. +pub(crate) enum SymbolIndexHeal { + /// Serve mode with a routed alias: the self-heal task was spawned. + BackgroundStarted, + /// No serve / no routed alias: nothing runs automatically; the caller + /// must build the index explicitly. + Manual, + /// The indexer does not apply to this repository (e.g. no top-level + /// tsconfig.json for TypeScript) — no rebuild can ever start here. + NotApplicable(&'static str), +} + /// Caller-facing warning for a find_impact answer against a project whose /// symbol index was never built. The empty reference list such an answer /// carries means UNKNOWN — stamping this on the payload is what keeps it from -/// passing for "no callers" while the self-heal rebuild runs. -pub(crate) fn missing_index_warning(lang: &str) -> String { - format!( - "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. A background rebuild was started; retry this call once it completes." - ) +/// passing for "no callers" while the status says what will actually happen. +pub(crate) fn missing_index_warning(lang: &str, heal: SymbolIndexHeal) -> String { + match heal { + SymbolIndexHeal::BackgroundStarted => format!( + "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. A background rebuild was started; retry this call once it completes." + ), + SymbolIndexHeal::Manual => format!( + "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. Background rebuilds only run in serve mode; build it with `codesearch index symbol ` and retry." + ), + SymbolIndexHeal::NotApplicable(reason) => format!( + "No {lang} symbol index exists for this project and none can be built automatically ({reason}) — an empty reference list means UNKNOWN, not 'no references'." + ), + } } #[tool_router(router = find_impact_router, vis = "pub(crate)")] @@ -270,13 +292,27 @@ impl CodesearchService { // warnings channel. Drift (index_head_sha vs current) is // deliberately NOT healed — reindexing on every branch switch would // thrash (see `SymbolIndexer::index_head_sha`). + // + // The warning wording follows the real status: a repo the indexer + // does not apply to (no top-level tsconfig.json / no .sln) can never + // get a rebuilt index, so saying "a rebuild was started" there was a + // promise nothing would fulfil. let index_missing = !indexer.has_index(&db_path); + let mut heal_spawned = false; let heal_warning: Vec = if index_missing { - vec![missing_index_warning(indexer.language())] + let heal = if !indexer.applies_to(&project_root) { + SymbolIndexHeal::NotApplicable(indexer.applicability_hint()) + } else if self.serve_state.is_some() && ctx.project_alias.is_some() { + heal_spawned = true; + SymbolIndexHeal::BackgroundStarted + } else { + SymbolIndexHeal::Manual + }; + vec![missing_index_warning(indexer.language(), heal)] } else { Vec::new() }; - if index_missing { + if heal_spawned { if let (Some(ref serve_state), Some(ref alias)) = (&self.serve_state, &ctx.project_alias) { diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index 66ec3513..60726ce2 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -9,7 +9,7 @@ use super::{ find_impact_with_budget, missing_index_warning, resolve_find_impact_budget_secs, - ImpactLookupOutcome, + ImpactLookupOutcome, SymbolIndexHeal, }; use crate::constants::{DEFAULT_FIND_IMPACT_BUDGET_SECS, FIND_IMPACT_BUDGET_SECS_ENV}; use crate::symbols::{SymbolLookupBusy, SymbolReference}; @@ -864,12 +864,13 @@ async fn clean_resolved_answer_omits_the_warnings_field() { } /// The missing-index warning is what keeps a never-built symbol index's -/// empty reference list from passing for "no callers": it must name the -/// language, state UNKNOWN explicitly, and tell the caller to retry after -/// the background rebuild. +/// empty reference list from passing for "no callers": whatever the heal +/// status, it must name the language and state UNKNOWN explicitly. The +/// wording per status must also stay honest — "a background rebuild was +/// started" is only allowed when one actually was. #[test] fn missing_index_warning_names_language_and_marks_answer_unknown() { - let warning = missing_index_warning("csharp"); + let warning = missing_index_warning("csharp", SymbolIndexHeal::BackgroundStarted); assert!( warning.contains("csharp"), "must name the language, got: {warning}" @@ -884,6 +885,35 @@ fn missing_index_warning_names_language_and_marks_answer_unknown() { ); } +#[test] +fn missing_index_warning_manual_names_the_build_command() { + let warning = missing_index_warning("typescript", SymbolIndexHeal::Manual); + assert!(warning.contains("UNKNOWN"), "got: {warning}"); + assert!( + warning.contains("codesearch index symbol"), + "must tell the caller how to build the index, got: {warning}" + ); + assert!( + !warning.contains("rebuild was started"), + "nothing was started — must not claim it, got: {warning}" + ); +} + +#[test] +fn missing_index_warning_not_applicable_explains_why_and_promises_nothing() { + let reason = "the scip-typescript adapter requires a top-level tsconfig.json (monorepo layouts are not resolved yet)"; + let warning = missing_index_warning("typescript", SymbolIndexHeal::NotApplicable(reason)); + assert!(warning.contains("UNKNOWN"), "got: {warning}"); + assert!( + warning.contains("tsconfig.json"), + "must surface the indexer's own reason, got: {warning}" + ); + assert!( + !warning.contains("rebuild was started"), + "a non-applicable repo can never be rebuilt here — must not claim a rebuild, got: {warning}" + ); +} + /// A never-built index must not answer "no references" silently: the empty /// NotFound answer carries the self-heal warning. Fails without the fix — /// the pre-fix answer is byte-identical to a genuine "no callers", the exact diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 482ac427..1c58ff70 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -2083,6 +2083,10 @@ impl SymbolIndexer for CSharpSymbolIndexer { Self::find_solution(repo_path).is_some() } + fn applicability_hint(&self) -> &'static str { + "the scip-csharp adapter requires a .sln file in the repository root" + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 51fa28e8..e19b8982 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -523,6 +523,14 @@ pub trait SymbolIndexer: Send + Sync { true } + /// Why `applies_to` would refuse a repository, in one clause. Surfaced + /// by find_impact's missing-index warning so a repo that can never get + /// a symbol index says so — instead of promising a rebuild that will + /// not happen. Adapters override next to their `applies_to`. + fn applicability_hint(&self) -> &'static str { + "the repository has no entrypoint this symbol indexer recognizes" + } + /// Downcast to `Any` for concrete-type method access (e.g. `prewarm_ref_cache`). /// /// This is needed because some adapter-specific methods (like Phase 3 pre-warm) diff --git a/src/symbols/typescript.rs b/src/symbols/typescript.rs index 2707b6fe..2925dc35 100644 --- a/src/symbols/typescript.rs +++ b/src/symbols/typescript.rs @@ -780,6 +780,10 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { Self::find_tsconfig(repo_path).is_some() } + fn applicability_hint(&self) -> &'static str { + "the scip-typescript adapter requires a top-level tsconfig.json (monorepo layouts are not resolved yet)" + } + fn as_any(&self) -> &dyn std::any::Any { self } From 3c56d8aa0948518a699acda0e375cda37e473ca4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 07:08:57 +0000 Subject: [PATCH 61/77] chore: bump version to 1.5.1 (auto, PR #299 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d902561..3894e312 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.0" +version = "1.5.1" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index ae97f570..f2b8e1e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.0" +version = "1.5.1" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 45213918645c45d84aababca53dfc336dcab5ead Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 07:20:42 +0000 Subject: [PATCH 62/77] chore: bump version to 1.5.2 (auto, PR #298 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3894e312..94886223 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.1" +version = "1.5.2" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index f2b8e1e4..620ed1fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.1" +version = "1.5.2" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 07f2c1fe2e78c0cd5b64ffd13ca13a19359cb5f2 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 4 Oct 2026 09:26:02 +0200 Subject: [PATCH 63/77] refactor(cli) + docs: centralize --model parsing; changelog for #298 Centralize the parse + valid-models-error pattern (inline in run() and the remote-add arm, plus belovaf's tested setup copy) into parse_model_arg(Option<&str>) -> Result>: None keeps per-query resolution, unknown names fail with the valid list. setup wraps it for its default-model semantics; its tests are unchanged. CHANGELOG: [1.5.1] Added entry crediting belovaf for codesearch setup + CODESEARCH_HOME (#298), with a thank-you line. --- CHANGELOG.md | 6 ++++++ src/cli/mod.rs | 40 +++++++++++++++++++--------------------- src/cli/setup.rs | 16 ++++------------ 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0f90fc8..9ed8daf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ finalized in place with a date — no renaming/migration step needed. ## [1.5.1] +### Added + +- **`codesearch setup` — download embedding models, and a relocatable global root via `CODESEARCH_HOME`.** The stubbed setup command now resolves `--model` through the shared model parser (unknown names error with the valid list), downloads into the global models cache with a visible progress bar (runtime paths stay silent), and probe-embeds one query so a broken download fails in setup instead of mid-indexing. `CODESEARCH_HOME` relocates the entire global root (repos.json, models cache, logs, serve_url, global ignore/extensions) and is honoured by the hub, db-discovery, every Claude Code guard hook and the OpenCode plugin, with tests pinning each layer. *(Contributed by @belovaf in #298 — in the same PR: find-impact warnings that no longer promise rebuilds nothing will run, group-fan-out hits attributed to their repo, and dependency-lockfile indexing exclusions.)* + +*A big thank-you to belovaf ([@belovaf](https://github.com/belovaf)) for his first contribution to this release — a careful, well-tested feature plus four quality fixes landed in one go.* + ### Fixed - **Queued reindexes are no longer cancelled as "leaked", and an unjoinable index task can no longer wedge a hub.** With the process-wide job gate (`CODESEARCH_INDEX_JOBS=1`, the default), tasks queued behind a long-running job held their `active_reindexes` markers past `MAX_INDEXING_SECS` without renewing them, so the lazy stale-marker check evicted the markers and cancelled the parked tasks — leaving empty stores and in-process LMDB handles that made later writes fail with "Database is locked by another process" until a serve restart. Separately, `await_index_task` removed the task entry *before* waiting and dropped the `JoinHandle` on timeout, so a task too deep in the uninterruptible `build_index` to cancel kept its `Arc` and the single job permit invisibly; the job-gate wait ignored cancellation, parking cancelled tasks on the gate; and `remove_repo` / format recovery could delete the DB directory out from under a still-running task. Now: a marker backed by a live tracked `Reindex` task is renewed (with a throttled wedge warning) instead of reaped; gate acquisition is cancellation-aware (`JobAcquire`) so cancelled or removed queued jobs abort and release their stores; a task that cannot be joined stays tracked and is reaped by a periodic sweeper that also deletes orphaned DB directories and re-asserts cancellation for removed repos; DB deletion is deferred while the task runs; format recovery refuses to wipe in that state; and stale markers from other owners are still evicted. diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 3baf386b..6c065e81 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1005,19 +1005,28 @@ fn warn_if_heavier_model(model_type: ModelType) { } } +/// Resolve a CLI `--model` flag to an optional [`ModelType`]. +/// +/// `None` (flag absent) stays `None` so runtime paths keep their per-query +/// model resolution; an unknown name is rejected with the full valid-model +/// list. Shared by every command that takes `--model`. +pub(crate) fn parse_model_arg(model: Option<&str>) -> Result> { + match model { + Some(name) => ModelType::parse(name).map(Some).ok_or_else(|| { + anyhow::anyhow!( + "Unknown model '{name}' (valid models: {})", + ModelType::valid_short_names() + ) + }), + None => Ok(None), + } +} + pub async fn run(cancel_token: CancellationToken) -> Result<()> { let cli = Cli::parse(); // Parse model from CLI flag - let model_type = cli.model.as_ref().and_then(|m| ModelType::parse(m)); - if cli.model.is_some() && model_type.is_none() { - eprintln!( - "Unknown model: '{}'. Available models:", - cli.model.as_deref().unwrap_or_default() - ); - eprintln!(" {}", ModelType::valid_short_names()); - std::process::exit(1); - } + let model_type = parse_model_arg(cli.model.as_deref())?; // Set quiet mode if requested if cli.quiet { @@ -1107,18 +1116,7 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Some(peer_name) = &remote { run_remote_add(peer_name, add_path).await } else { - let mt = model - .as_deref() - .and_then(|m| { - let parsed = ModelType::parse(m); - if parsed.is_none() { - eprintln!("Unknown model: '{}'. Available models:", m); - eprintln!(" {}", ModelType::valid_short_names()); - std::process::exit(1); - } - parsed - }) - .or(model_type); + let mt = parse_model_arg(model.as_deref())?.or(model_type); if let Some(mt) = mt { warn_if_heavier_model(mt); } diff --git a/src/cli/setup.rs b/src/cli/setup.rs index bfd29107..5bc7315f 100644 --- a/src/cli/setup.rs +++ b/src/cli/setup.rs @@ -4,19 +4,11 @@ use crate::embed::ModelType; /// Resolve the CLI `--model` argument to a [`ModelType`]. /// -/// `None` falls back to the default model; unknown names are rejected with the -/// full valid-model list so the error is self-explanatory. Pure and -/// network-free, so it is unit-testable. +/// Thin setup-specific wrapper over [`super::parse_model_arg`]: setup wants +/// the default model when the flag is absent, where other commands keep +/// `None` for per-query resolution. fn resolve_model(model: Option<&str>) -> Result { - match model { - Some(name) => ModelType::parse(name).ok_or_else(|| { - anyhow!( - "Unknown model '{name}' (valid models: {})", - ModelType::valid_short_names() - ) - }), - None => Ok(ModelType::default()), - } + Ok(super::parse_model_arg(model)?.unwrap_or_default()) } pub async fn run(model: Option) -> Result<()> { From 3b13e7fe69e477a7acf9a262a4ceea24dad24709 Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 4 Oct 2026 17:35:37 +0200 Subject: [PATCH 64/77] =?UTF-8?q?release:=201.5.1=20=E2=80=94=20serve=20ta?= =?UTF-8?q?sk=20lifecycle,=20setup=20command,=20CODESEARCH=5FHOME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 +++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ed8daf8..530479c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> -## [1.5.1] +## [1.5.1] - 2026-10-04 ### Added @@ -366,6 +366,8 @@ finalized in place with a date — no renaming/migration step needed. ## [1.0.72] - 2026-05-01 - Initial multi-repo release: multi-repo `serve` (HTTP/SSE, per-project/group routing, RRF cross-repo search), stdio MCP proxy with client-side auto-reconnect, tree-sitter chunking (9 langs), persistent SHA-256 embedding cache, repository groups, re-tuned RRF, and LMDB resize crash fix (#30, `MDB_MAP_FULL`). +[1.5.1]: https://github.com/flupkede/codesearch/compare/v1.5.0...v1.5.1 + [1.5.0]: https://github.com/flupkede/codesearch/compare/v1.4.9...v1.5.0 [1.4.9]: https://github.com/flupkede/codesearch/compare/v1.4.4...v1.4.9 diff --git a/Cargo.lock b/Cargo.lock index 94886223..3894e312 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.2" +version = "1.5.1" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 620ed1fc..f2b8e1e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.2" +version = "1.5.1" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 826f3308f13a6df9d8abb5ba17d34b2bc79c9c2f Mon Sep 17 00:00:00 2001 From: flupkede Date: Sun, 4 Oct 2026 19:33:12 +0200 Subject: [PATCH 65/77] fix(csharp): restore packages before the workspace load; NuGet audit advisories are not index warnings --- CHANGELOG.md | 6 ++ helpers/csharp/Program.cs | 64 ++++++++++++++++++++ helpers/csharp/WorkspaceNoise.cs | 21 +++++++ helpers/csharp/tests/ScipCsharp.Tests.csproj | 1 + helpers/csharp/tests/WorkspaceNoiseTests.cs | 47 ++++++++++++++ 5 files changed, 139 insertions(+) create mode 100644 helpers/csharp/WorkspaceNoise.cs create mode 100644 helpers/csharp/tests/WorkspaceNoiseTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 530479c2..e4002721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.5.2] + +### Fixed + +- **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. + ## [1.5.1] - 2026-10-04 ### Added diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index 334013a9..61a60bb1 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Locator; using Microsoft.CodeAnalysis.MSBuild; @@ -323,10 +324,69 @@ private static MSBuildWorkspace CreateTolerantWorkspace() var workspace = MSBuildWorkspace.Create(properties); workspace.WorkspaceFailed += (_, e) => + { + // NuGet's vulnerability audit surfaces as an Msbuild "failure" but + // is a security advisory, not a load failure — the index is + // unaffected and emitting it would pin the repo's index warning + // until the package is bumped. + if (WorkspaceNoise.IsNuGetAuditNoise(e.Diagnostic?.Message)) + { + return; + } Console.Error.WriteLine($"[WARN] Workspace error: {e.Diagnostic}"); + }; return workspace; } + /// + /// Restores the solution before the design-time load. MSBuildWorkspace + /// does not restore: with stale or missing obj/project.assets.json, + /// package-typed references fail to resolve and the load drowns in CS0246 + /// cascades even though `dotnet build` (which restores first) succeeds. + /// Best effort: a failed or missing `dotnet` is logged and the load + /// proceeds — degraded indexing beats no indexing. + /// + private static void RestoreSolution(string solutionPath) + { + try + { + var psi = new ProcessStartInfo + { + FileName = "dotnet", + Arguments = $"restore \"{solutionPath}\" --nologo -v q", + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + }; + using var process = Process.Start(psi); + if (process is null) + { + return; + } + // Bounded: a hung restore (unreachable feed) must not stall the + // index indefinitely. 5 minutes is generous even for a cold sln. + if (!process.WaitForExit(300_000)) + { + process.Kill(entireProcessTree: true); + Console.Error.WriteLine( + "[WARN] dotnet restore did not finish within 300s — killed; continuing, " + + "package references may be unresolved (degraded symbols)."); + return; + } + if (process.ExitCode != 0) + { + Console.Error.WriteLine( + $"[WARN] dotnet restore exited {process.ExitCode} for '{solutionPath}' — continuing; " + + "package references may be unresolved (degraded symbols)."); + } + } + catch (Exception ex) + { + Console.Error.WriteLine( + $"[WARN] dotnet restore could not start ({ex.GetType().Name}: {ex.Message}) — continuing without it."); + } + } + // ── Solution filtering ─────────────────────────────────────────── /// @@ -338,6 +398,10 @@ private static MSBuildWorkspace CreateTolerantWorkspace() /// private static async Task OpenSolutionFilteredAsync(MSBuildWorkspace workspace, string solutionPath) { + // Restore first: the design-time build below does not restore, and + // stale/missing package assets cascade into CS0246 noise that would + // otherwise pin the repo's index warning. + RestoreSolution(solutionPath); var solutionDir = Path.GetDirectoryName(solutionPath) ?? "."; var lines = await File.ReadAllLinesAsync(solutionPath).ConfigureAwait(false); diff --git a/helpers/csharp/WorkspaceNoise.cs b/helpers/csharp/WorkspaceNoise.cs new file mode 100644 index 00000000..54160f68 --- /dev/null +++ b/helpers/csharp/WorkspaceNoise.cs @@ -0,0 +1,21 @@ +namespace ScipCsharp; + +/// +/// Classifies workspace-load noise that must not surface as index warnings. +/// Compiled into both scip-csharp and the test project via a linked Compile +/// item (same pattern as WorkspaceLoad.cs). +/// +internal static class WorkspaceNoise +{ + /// + /// True when a workspace-failure diagnostic is a NuGet audit advisory + /// ("Package 'X' has a known high severity vulnerability, ...") rather + /// than a load failure — emitting it would pin the repo's index warning + /// until the package is bumped, while the index itself is unaffected. + /// Pure and null-tolerant so it is unit-testable. + /// + public static bool IsNuGetAuditNoise(string? message) => + message is not null + && message.Contains("has a known", StringComparison.OrdinalIgnoreCase) + && message.Contains("vulnerability", StringComparison.OrdinalIgnoreCase); +} diff --git a/helpers/csharp/tests/ScipCsharp.Tests.csproj b/helpers/csharp/tests/ScipCsharp.Tests.csproj index 868ea409..bb6e9a9a 100644 --- a/helpers/csharp/tests/ScipCsharp.Tests.csproj +++ b/helpers/csharp/tests/ScipCsharp.Tests.csproj @@ -29,6 +29,7 @@ + diff --git a/helpers/csharp/tests/WorkspaceNoiseTests.cs b/helpers/csharp/tests/WorkspaceNoiseTests.cs new file mode 100644 index 00000000..792fdb8e --- /dev/null +++ b/helpers/csharp/tests/WorkspaceNoiseTests.cs @@ -0,0 +1,47 @@ +using ScipCsharp; +using Xunit; + +namespace ScipCsharp.Tests; + +/// +/// Unit tests for the workspace-noise policy: NuGet audit advisories must not +/// surface as index warnings (they are security notes, not load failures), +/// while genuine Msbuild load failures still do. +/// +public class WorkspaceNoiseTests +{ + [Fact] + public void NuGetAuditAdvisoryIsFiltered() + { + var message = + "Msbuild failed when processing the file 'Acme.Core.csproj' with message: " + + "Package 'Microsoft.Extensions.Caching.Memory' 8.0.0 has a known high severity " + + "vulnerability, https://github.com/advisories/GHSA-qj66-m88j-hmgj"; + Assert.True(WorkspaceNoise.IsNuGetAuditNoise(message)); + } + + [Fact] + public void GenuineLoadFailureIsKept() + { + var message = + "Msbuild failed when processing the file 'Acme.Core.csproj' with message: " + + "The type or namespace name 'Catalog' does not exist in the namespace 'Acme'"; + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(message)); + } + + [Fact] + public void NullAndUnrelatedMessagesAreKept() + { + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(null)); + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(string.Empty)); + Assert.False(WorkspaceNoise.IsNuGetAuditNoise("Invalid project file, unsupported project type")); + } + + [Theory] + [InlineData("has a KNOWN vulnerability")] + [InlineData("HAS A KNOWN VULNERABILITY")] + public void MatchingIsCaseInsensitive(string message) + { + Assert.True(WorkspaceNoise.IsNuGetAuditNoise(message)); + } +} From 2165c0b7c2254e32fb5ef86432eda4eacd2285d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:36:00 +0000 Subject: [PATCH 66/77] chore: bump version to 1.5.2 (auto, PR #301 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3894e312..94886223 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.1" +version = "1.5.2" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index f2b8e1e4..620ed1fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.1" +version = "1.5.2" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From cf8d824d47439fde02c3825e117920b99d011d2f Mon Sep 17 00:00:00 2001 From: flupkede Date: Mon, 5 Oct 2026 09:04:29 +0200 Subject: [PATCH 67/77] fix(csharp): restore only when assets are stale or missing; demote Warning-severity workspace diagnostics to INFO --- CHANGELOG.md | 1 + helpers/csharp/Program.cs | 45 ++++++-- helpers/csharp/RestorePolicy.cs | 59 +++++++++++ helpers/csharp/tests/RestorePolicyTests.cs | 106 +++++++++++++++++++ helpers/csharp/tests/ScipCsharp.Tests.csproj | 1 + 5 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 helpers/csharp/RestorePolicy.cs create mode 100644 helpers/csharp/tests/RestorePolicyTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e4002721..50317d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ finalized in place with a date — no renaming/migration step needed. ### Fixed +- **Restore runs only when package assets are missing or stale, and warning-severity workspace diagnostics no longer count as index warnings.** Restoring on every reindex was far too heavy for multi-repo hubs and big solutions legitimately exceed any fixed timeout (`dotnet restore did not finish within 300s` was observed live). Now the helper checks each project's `obj/project.assets.json` against its .csproj and restores only when evidence demands it (the steady state of an actively developed repo is a no-op), the kill timeout is tunable via `CODESEARCH_RESTORE_TIMEOUT_SECS`, and restore uses `--ignore-failed-sources` so an unreachable private feed cannot block indexing. Separately, warning-severity workspace diagnostics (e.g. "Found project reference without a matching metadata reference" for excluded or external projects) are benign resolution notes — they log at INFO and no longer pin the repo's index warning; only true load failures (minus NuGet audit advisories) still do. - **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. ## [1.5.1] - 2026-10-04 diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index 61a60bb1..0acc8b90 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Locator; +using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.MSBuild; namespace ScipCsharp; @@ -325,6 +326,16 @@ private static MSBuildWorkspace CreateTolerantWorkspace() var workspace = MSBuildWorkspace.Create(properties); workspace.WorkspaceFailed += (_, e) => { + // Warning-severity diagnostics are benign resolution notes (e.g. + // "Found project reference without a matching metadata reference" + // for excluded or external projects): they do not block indexing + // and would pin the repo's index warning forever. Only true + // failures belong in the warning channel. + if (e.Diagnostic.Kind == WorkspaceDiagnosticKind.Warning) + { + Console.Error.WriteLine($"[INFO] Workspace note: {e.Diagnostic}"); + return; + } // NuGet's vulnerability audit surfaces as an Msbuild "failure" but // is a security advisory, not a load failure — the index is // unaffected and emitting it would pin the repo's index warning @@ -343,17 +354,21 @@ private static MSBuildWorkspace CreateTolerantWorkspace() /// does not restore: with stale or missing obj/project.assets.json, /// package-typed references fail to resolve and the load drowns in CS0246 /// cascades even though `dotnet build` (which restores first) succeeds. - /// Best effort: a failed or missing `dotnet` is logged and the load - /// proceeds — degraded indexing beats no indexing. + /// Gated by ; best effort: a + /// failed or missing `dotnet` is logged and the load proceeds — degraded + /// indexing beats no indexing. /// private static void RestoreSolution(string solutionPath) { + var timeoutSeconds = RestorePolicy.RestoreTimeoutSeconds(); try { var psi = new ProcessStartInfo { FileName = "dotnet", - Arguments = $"restore \"{solutionPath}\" --nologo -v q", + // --ignore-failed-sources: an unreachable private feed must not + // block indexing when the packages are already in the cache. + Arguments = $"restore \"{solutionPath}\" --nologo -v q --ignore-failed-sources", RedirectStandardError = true, RedirectStandardOutput = true, UseShellExecute = false, @@ -364,13 +379,15 @@ private static void RestoreSolution(string solutionPath) return; } // Bounded: a hung restore (unreachable feed) must not stall the - // index indefinitely. 5 minutes is generous even for a cold sln. - if (!process.WaitForExit(300_000)) + // index indefinitely. Timeout is env-tunable for large solutions + // on slow feeds. + if (!process.WaitForExit(timeoutSeconds * 1000)) { process.Kill(entireProcessTree: true); Console.Error.WriteLine( - "[WARN] dotnet restore did not finish within 300s — killed; continuing, " + - "package references may be unresolved (degraded symbols)."); + $"[WARN] dotnet restore did not finish within {timeoutSeconds}s — killed; continuing, " + + "package references may be unresolved (degraded symbols). " + + $"Raise with {RestorePolicy.TimeoutEnvName} if this solution legitimately needs longer."); return; } if (process.ExitCode != 0) @@ -398,10 +415,6 @@ private static void RestoreSolution(string solutionPath) /// private static async Task OpenSolutionFilteredAsync(MSBuildWorkspace workspace, string solutionPath) { - // Restore first: the design-time build below does not restore, and - // stale/missing package assets cascade into CS0246 noise that would - // otherwise pin the repo's index warning. - RestoreSolution(solutionPath); var solutionDir = Path.GetDirectoryName(solutionPath) ?? "."; var lines = await File.ReadAllLinesAsync(solutionPath).ConfigureAwait(false); @@ -417,6 +430,16 @@ private static async Task OpenSolutionFilteredAsync(MSBuildWorkspace workspace, .Where(p => UnsupportedProjectExtensions.Contains(Path.GetExtension(p.RelativePath))) .ToList(); + // Restore only when the evidence says assets are missing or stale: the + // design-time build below does not restore, and missing package assets + // cascade into CS0246 noise that would otherwise pin the repo's index + // warning. But restoring on every reindex is far too heavy for + // multi-repo hubs — fresh assets mean no restore at all. + if (RestorePolicy.NeedsRestore(supported.Select(p => p.RelativePath), solutionDir)) + { + RestoreSolution(solutionPath); + } + // Log skipped projects foreach (var skip in unsupported) { diff --git a/helpers/csharp/RestorePolicy.cs b/helpers/csharp/RestorePolicy.cs new file mode 100644 index 00000000..0d327210 --- /dev/null +++ b/helpers/csharp/RestorePolicy.cs @@ -0,0 +1,59 @@ +using System.Diagnostics; + +namespace ScipCsharp; + +/// +/// Restore policy for the design-time workspace load: MSBuildWorkspace does +/// not restore, but running dotnet restore on every reindex is far too +/// heavy for multi-repo hubs. Restore only when the evidence says assets are +/// missing or stale. Compiled into both scip-csharp and the test project via +/// a linked Compile item (same pattern as WorkspaceLoad.cs). +/// +internal static class RestorePolicy +{ + /// Env override for the restore kill timeout, in seconds. + public const string TimeoutEnvName = "CODESEARCH_RESTORE_TIMEOUT_SECS"; + private const string RestoreTimeoutEnv = TimeoutEnvName; + private const int DefaultRestoreTimeoutSeconds = 300; + + /// + /// True when at least one project lacks obj/project.assets.json or + /// has one older than its .csproj (a package reference or target framework + /// was edited since the last restore). False means every project has fresh + /// assets and restore can be skipped entirely — the steady state of an + /// actively developed repo. + /// + public static bool NeedsRestore(IEnumerable projectRelativePaths, string solutionDir) + { + foreach (var relative in projectRelativePaths) + { + var projectPath = Path.GetFullPath(Path.Combine(solutionDir, relative)); + var projectDir = Path.GetDirectoryName(projectPath) ?? solutionDir; + var assetsPath = Path.Combine(projectDir, "obj", "project.assets.json"); + if (!File.Exists(assetsPath)) + { + return true; + } + if (File.GetLastWriteTimeUtc(projectPath) > File.GetLastWriteTimeUtc(assetsPath)) + { + return true; + } + } + return false; + } + + /// + /// Restore kill timeout in seconds: CODESEARCH_RESTORE_TIMEOUT_SECS + /// when set to a positive integer, otherwise 300. Unparseable values fall + /// back to the default rather than failing the load. + /// + public static int RestoreTimeoutSeconds() + { + var raw = Environment.GetEnvironmentVariable(RestoreTimeoutEnv); + if (int.TryParse(raw, out var seconds) && seconds > 0) + { + return seconds; + } + return DefaultRestoreTimeoutSeconds; + } +} diff --git a/helpers/csharp/tests/RestorePolicyTests.cs b/helpers/csharp/tests/RestorePolicyTests.cs new file mode 100644 index 00000000..c64bba32 --- /dev/null +++ b/helpers/csharp/tests/RestorePolicyTests.cs @@ -0,0 +1,106 @@ +using ScipCsharp; +using Xunit; + +namespace ScipCsharp.Tests; + +/// +/// Unit tests for the restore gating policy: a repo whose projects all carry +/// fresh obj/project.assets.json must skip restore entirely (the steady state +/// of an actively developed repo), while a missing or stale assets file +/// triggers it. +/// +public class RestorePolicyTests : IDisposable +{ + private readonly string _solutionDir; + + public RestorePolicyTests() + { + _solutionDir = Path.Combine(Path.GetTempPath(), "cs-restore-policy-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_solutionDir); + } + + public void Dispose() + { + Directory.Delete(_solutionDir, recursive: true); + } + + private string AddProject(string relativePath, bool withAssets, bool staleAssets = false) + { + var projectPath = Path.Combine(_solutionDir, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(projectPath)!); + File.WriteAllText(projectPath, ""); + if (withAssets) + { + var objDir = Path.Combine(Path.GetDirectoryName(projectPath)!, "obj"); + Directory.CreateDirectory(objDir); + var assetsPath = Path.Combine(objDir, "project.assets.json"); + File.WriteAllText(assetsPath, "{}"); + if (staleAssets) + { + // Make the assets look OLDER than the project file. + File.SetLastWriteTimeUtc(assetsPath, File.GetLastWriteTimeUtc(projectPath).AddMinutes(-5)); + } + } + return projectPath; + } + + [Fact] + public void FreshAssetsOnEveryProjectSkipsRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + AddProject(@"src\Acme.Web\Acme.Web.csproj", withAssets: true); + + Assert.False(RestorePolicy.NeedsRestore( + new[] { @"src\Acme.Core\Acme.Core.csproj", @"src\Acme.Web\Acme.Web.csproj" }, _solutionDir)); + } + + [Fact] + public void MissingAssetsTriggersRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + AddProject(@"src\Acme.Web\Acme.Web.csproj", withAssets: false); + + Assert.True(RestorePolicy.NeedsRestore( + new[] { @"src\Acme.Core\Acme.Core.csproj", @"src\Acme.Web\Acme.Web.csproj" }, _solutionDir)); + } + + [Fact] + public void StaleAssetsOlderThanProjectTriggerRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true, staleAssets: true); + + Assert.True(RestorePolicy.NeedsRestore(new[] { @"src\Acme.Core\Acme.Core.csproj" }, _solutionDir)); + } + + [Fact] + public void EmptyProjectListSkipsRestore() + { + Assert.False(RestorePolicy.NeedsRestore(Array.Empty(), _solutionDir)); + } + + [Fact] + public void RestoreTimeoutDefaultsTo300AndParsesEnvOverride() + { + Assert.Equal(300, RestorePolicy.RestoreTimeoutSeconds()); + + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, "600"); + try + { + Assert.Equal(600, RestorePolicy.RestoreTimeoutSeconds()); + } + finally + { + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, null); + } + + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, "not-a-number"); + try + { + Assert.Equal(300, RestorePolicy.RestoreTimeoutSeconds()); + } + finally + { + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, null); + } + } +} diff --git a/helpers/csharp/tests/ScipCsharp.Tests.csproj b/helpers/csharp/tests/ScipCsharp.Tests.csproj index bb6e9a9a..7a57d219 100644 --- a/helpers/csharp/tests/ScipCsharp.Tests.csproj +++ b/helpers/csharp/tests/ScipCsharp.Tests.csproj @@ -30,6 +30,7 @@ + From 5f9210654940a09e95ec7e6f12f0610b27e58375 Mon Sep 17 00:00:00 2001 From: flupkede Date: Mon, 5 Oct 2026 09:33:53 +0200 Subject: [PATCH 68/77] fix(test): serialize env-dependent constants test against CODESEARCH_HOME tests global_codesearchignore_path_returns_home_codesearch_dir reads a global-path accessor that resolves via CODESEARCH_HOME but was not marked serial, so it could run concurrently with the serial tests that set/clear that env var and observe a relocated or rejected root. Marked serial like its siblings. --- src/constants.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/constants.rs b/src/constants.rs index d69552ce..d07e36d2 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1032,6 +1032,9 @@ mod tests { } #[test] + // Reads global-path accessors that resolve via CODESEARCH_HOME: must not + // overlap the serial tests that set/clear that env var in this binary. + #[serial_test::serial] fn global_codesearchignore_path_returns_home_codesearch_dir() { let path = global_codesearchignore_path(); assert!(path.is_some(), "Should return Some when home dir exists"); From d94d95e9e8ece9c3d8e47f6474742608def7f39a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 07:55:15 +0000 Subject: [PATCH 69/77] chore: bump version to 1.5.3 (auto, PR #303 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94886223..885e68de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.2" +version = "1.5.3" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 620ed1fc..1f0cbd61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.2" +version = "1.5.3" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 37f7400493176693fd10fdeb2c3aad640dca556d Mon Sep 17 00:00:00 2001 From: flupkede Date: Mon, 5 Oct 2026 12:12:56 +0200 Subject: [PATCH 70/77] =?UTF-8?q?fix(csharp):=20a=20timed-out=20restore=20?= =?UTF-8?q?is=20never=20killed=20=E2=80=94=20it=20continues=20in=20the=20b?= =?UTF-8?q?ackground?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Killing the restore at the wait ceiling wasted all of its work and left assets missing, so every reindex re-paid the same wait and stayed degraded (a poison loop for exactly the biggest solutions). On timeout the restore now keeps running detached; this load proceeds degraded and the next reindex picks up the fresh assets. Redirected pipes are drained in the background so the child can neither block on a full pipe nor hold the serve process's stderr past EOF. Timeout warning moved to RestorePolicy.StillRunningWarning with a test pinning the detach contract. --- CHANGELOG.md | 2 +- helpers/csharp/Program.cs | 25 ++++++++++++++-------- helpers/csharp/RestorePolicy.cs | 17 ++++++++++++++- helpers/csharp/tests/RestorePolicyTests.cs | 20 +++++++++++++++++ 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50317d1a..fe32d8a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ finalized in place with a date — no renaming/migration step needed. ### Fixed - **Restore runs only when package assets are missing or stale, and warning-severity workspace diagnostics no longer count as index warnings.** Restoring on every reindex was far too heavy for multi-repo hubs and big solutions legitimately exceed any fixed timeout (`dotnet restore did not finish within 300s` was observed live). Now the helper checks each project's `obj/project.assets.json` against its .csproj and restores only when evidence demands it (the steady state of an actively developed repo is a no-op), the kill timeout is tunable via `CODESEARCH_RESTORE_TIMEOUT_SECS`, and restore uses `--ignore-failed-sources` so an unreachable private feed cannot block indexing. Separately, warning-severity workspace diagnostics (e.g. "Found project reference without a matching metadata reference" for excluded or external projects) are benign resolution notes — they log at INFO and no longer pin the repo's index warning; only true load failures (minus NuGet audit advisories) still do. -- **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. +- **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and when the bounded wait expires the restore is **not killed**: it keeps running in the background, this load proceeds degraded, and the next reindex picks up the fresh assets — a killed restore would waste all of its work and re-pay the same wait on every subsequent reindex. NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. ## [1.5.1] - 2026-10-04 diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index 0acc8b90..bc0e3deb 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -356,7 +356,9 @@ private static MSBuildWorkspace CreateTolerantWorkspace() /// cascades even though `dotnet build` (which restores first) succeeds. /// Gated by ; best effort: a /// failed or missing `dotnet` is logged and the load proceeds — degraded - /// indexing beats no indexing. + /// indexing beats no indexing. The wait is bounded, but the restore is + /// never killed: on timeout it continues in the background and the next + /// reindex picks up the fresh assets. /// private static void RestoreSolution(string solutionPath) { @@ -378,16 +380,21 @@ private static void RestoreSolution(string solutionPath) { return; } - // Bounded: a hung restore (unreachable feed) must not stall the - // index indefinitely. Timeout is env-tunable for large solutions - // on slow feeds. + // Drain both pipes in the background: the child inherits these + // handles, so without readers a chatty restore could block on a + // full pipe, and the serve process reads stderr until EOF. + _ = process.StandardOutput.ReadToEndAsync(); + _ = process.StandardError.ReadToEndAsync(); + // Bounded WAIT, never a kill: a killed restore wastes all of its + // work and leaves assets missing, so every reindex would pay the + // same wait and stay degraded. On timeout the restore keeps + // running in the background (the child outlives this process); + // this load proceeds degraded and the next reindex finds the + // fresh assets and loads clean. Timeout is env-tunable for large + // solutions on slow feeds. if (!process.WaitForExit(timeoutSeconds * 1000)) { - process.Kill(entireProcessTree: true); - Console.Error.WriteLine( - $"[WARN] dotnet restore did not finish within {timeoutSeconds}s — killed; continuing, " + - "package references may be unresolved (degraded symbols). " + - $"Raise with {RestorePolicy.TimeoutEnvName} if this solution legitimately needs longer."); + Console.Error.WriteLine(RestorePolicy.StillRunningWarning(timeoutSeconds)); return; } if (process.ExitCode != 0) diff --git a/helpers/csharp/RestorePolicy.cs b/helpers/csharp/RestorePolicy.cs index 0d327210..16655ddb 100644 --- a/helpers/csharp/RestorePolicy.cs +++ b/helpers/csharp/RestorePolicy.cs @@ -43,7 +43,7 @@ public static bool NeedsRestore(IEnumerable projectRelativePaths, string } /// - /// Restore kill timeout in seconds: CODESEARCH_RESTORE_TIMEOUT_SECS + /// Restore wait timeout in seconds: CODESEARCH_RESTORE_TIMEOUT_SECS /// when set to a positive integer, otherwise 300. Unparseable values fall /// back to the default rather than failing the load. /// @@ -56,4 +56,19 @@ public static int RestoreTimeoutSeconds() } return DefaultRestoreTimeoutSeconds; } + + /// + /// The [WARN] line emitted when the bounded wait expires while + /// dotnet restore is still running. The restore is NOT killed: it + /// keeps running in the background (the child outlives the helper), this + /// load proceeds degraded, and the next reindex finds the fresh assets — + /// a killed restore would waste all of its work and re-pay the same wait + /// on every subsequent reindex. Pinned by test so a "killed" wording + /// cannot sneak back in. + /// + public static string StillRunningWarning(int seconds) => + $"[WARN] dotnet restore still running after {seconds}s — proceeding without it " + + "(degraded symbols this run); it keeps running in the background and the next " + + "reindex picks up the fresh assets. " + + $"Raise with {TimeoutEnvName} if this solution legitimately needs longer."; } diff --git a/helpers/csharp/tests/RestorePolicyTests.cs b/helpers/csharp/tests/RestorePolicyTests.cs index c64bba32..3e38d40c 100644 --- a/helpers/csharp/tests/RestorePolicyTests.cs +++ b/helpers/csharp/tests/RestorePolicyTests.cs @@ -103,4 +103,24 @@ public void RestoreTimeoutDefaultsTo300AndParsesEnvOverride() Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, null); } } + + /// + /// Pins the timeout contract: the wait expiring must announce a restore + /// that is STILL RUNNING in the background, never a killed one — a kill + /// would waste all of the restore's work and re-pay the same wait on + /// every subsequent reindex (the poison loop for the biggest solutions). + /// + [Fact] + public void TimeoutWarningAnnouncesBackgroundContinuationNeverAKill() + { + var warning = RestorePolicy.StillRunningWarning(300); + + Assert.StartsWith("[WARN]", warning); + Assert.Contains("still running after 300s", warning); + Assert.Contains("background", warning); + Assert.Contains("next", warning); + Assert.Contains("fresh assets", warning); + Assert.DoesNotContain("killed", warning, StringComparison.OrdinalIgnoreCase); + Assert.Contains(RestorePolicy.TimeoutEnvName, warning); + } } From 28ea37de45dbbd8272909176b8855a2850141349 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:33:04 +0000 Subject: [PATCH 71/77] chore: bump version to 1.5.4 (auto, PR #304 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 885e68de..436b7304 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.3" +version = "1.5.4" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 1f0cbd61..7caade0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.3" +version = "1.5.4" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 707fa108b7ea324b09b76d84634638a09e3da814 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 06:37:35 +0000 Subject: [PATCH 72/77] chore(deps): bump the cargo-minor-and-patch group with 3 updates Bumps the cargo-minor-and-patch group with 3 updates: [uuid](https://github.com/uuid-rs/uuid), [rmcp](https://github.com/modelcontextprotocol/rust-sdk) and [libc](https://github.com/rust-lang/libc). Updates `uuid` from 1.26.1 to 1.27.0 - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.26.1...v1.27.0) Updates `rmcp` from 3.4.1 to 3.5.0 - [Release notes](https://github.com/modelcontextprotocol/rust-sdk/releases) - [Changelog](https://github.com/modelcontextprotocol/rust-sdk/blob/main/release-plz.toml) - [Commits](https://github.com/modelcontextprotocol/rust-sdk/compare/rmcp-v3.4.1...rmcp-v3.5.0) Updates `libc` from 0.2.189 to 0.2.190 - [Release notes](https://github.com/rust-lang/libc/releases) - [Changelog](https://github.com/rust-lang/libc/blob/0.2.190/CHANGELOG.md) - [Commits](https://github.com/rust-lang/libc/compare/0.2.189...0.2.190) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.27.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo-minor-and-patch - dependency-name: rmcp dependency-version: 3.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo-minor-and-patch - dependency-name: libc dependency-version: 0.2.190 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo-minor-and-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 436b7304..84e8886c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2770,9 +2770,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libfuzzer-sys" @@ -4603,9 +4603,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "3.4.1" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6317cd8c13e3ec9033cf2aa5aa92cd743f0f4f8a93cddc42ea0ae6ce3b8898" +checksum = "fae7019994ae0fe4ada40b732f798f3ff26f0f04facb1477f1bf37eb4f18a2d3" dependencies = [ "async-trait", "base64 0.23.1", @@ -6398,9 +6398,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "atomic", "getrandom 0.4.3", From c2aba670d039bc3e12ffa6445ae08980e60b8475 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:23:04 +0000 Subject: [PATCH 73/77] chore: bump version to 1.5.5 (auto, PR #305 merged to develop) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84e8886c..cd74880b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.4" +version = "1.5.5" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 7caade0b..18f80a7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.4" +version = "1.5.5" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From fdc85a5a6019b086f44522ac60d87238e0161aeb Mon Sep 17 00:00:00 2001 From: flupkede Date: Tue, 6 Oct 2026 10:32:36 +0200 Subject: [PATCH 74/77] =?UTF-8?q?release:=201.5.2=20=E2=80=94=20C#=20resto?= =?UTF-8?q?re=20policy,=20workspace-noise=20filtering,=20never-kill=20deta?= =?UTF-8?q?ch,=20dep=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 +++++++- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe32d8a9..9fcdcd18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,17 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> -## [1.5.2] +## [1.5.2] - 2026-10-05 ### Fixed - **Restore runs only when package assets are missing or stale, and warning-severity workspace diagnostics no longer count as index warnings.** Restoring on every reindex was far too heavy for multi-repo hubs and big solutions legitimately exceed any fixed timeout (`dotnet restore did not finish within 300s` was observed live). Now the helper checks each project's `obj/project.assets.json` against its .csproj and restores only when evidence demands it (the steady state of an actively developed repo is a no-op), the kill timeout is tunable via `CODESEARCH_RESTORE_TIMEOUT_SECS`, and restore uses `--ignore-failed-sources` so an unreachable private feed cannot block indexing. Separately, warning-severity workspace diagnostics (e.g. "Found project reference without a matching metadata reference" for excluded or external projects) are benign resolution notes — they log at INFO and no longer pin the repo's index warning; only true load failures (minus NuGet audit advisories) still do. - **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and when the bounded wait expires the restore is **not killed**: it keeps running in the background, this load proceeds degraded, and the next reindex picks up the fresh assets — a killed restore would waste all of its work and re-pay the same wait on every subsequent reindex. NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. +### Changed + +- **Dependency refresh:** libc 0.2.190, rmcp 3.5.0, uuid 1.27.0 (#305). + ## [1.5.1] - 2026-10-04 ### Added @@ -373,6 +377,8 @@ finalized in place with a date — no renaming/migration step needed. ## [1.0.72] - 2026-05-01 - Initial multi-repo release: multi-repo `serve` (HTTP/SSE, per-project/group routing, RRF cross-repo search), stdio MCP proxy with client-side auto-reconnect, tree-sitter chunking (9 langs), persistent SHA-256 embedding cache, repository groups, re-tuned RRF, and LMDB resize crash fix (#30, `MDB_MAP_FULL`). +[1.5.2]: https://github.com/flupkede/codesearch/compare/v1.5.1...v1.5.2 + [1.5.1]: https://github.com/flupkede/codesearch/compare/v1.5.0...v1.5.1 [1.5.0]: https://github.com/flupkede/codesearch/compare/v1.4.9...v1.5.0 diff --git a/Cargo.lock b/Cargo.lock index cd74880b..45d373fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.5.5" +version = "1.5.2" dependencies = [ "anyhow", "arroy", diff --git a/Cargo.toml b/Cargo.toml index 18f80a7f..620ed1fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.5.5" +version = "1.5.2" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" From 20ca1b7d66ea62becb00b461d12b9fe2c78cec95 Mon Sep 17 00:00:00 2001 From: flupkede Date: Wed, 14 Oct 2026 23:52:34 +0200 Subject: [PATCH 75/77] fix: the per-merge bump never synced the helper version stamp (on master) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pull_request_target executes this workflow as it exists on the DEFAULT branch, and the default branch is master — the fix merged to develop (#332) only lands here at the next release sync, so every develop-PR bump meanwhile still skips the csproj and drifts the helper stamp from the crate version (deploy warning 'codesearch 1.5.2x vs scip-csharp 1.5.9/1.5.2x'). Same fix as #332: rewrite whatever value is there and hard-verify before the commit. Review: @reviewer unavailable (usage limit); worker self-review — identical hunk as the reviewed #332 change, YAML parse OK. --- .github/workflows/bump-develop.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bump-develop.yml b/.github/workflows/bump-develop.yml index bb6fa0ab..71485a12 100644 --- a/.github/workflows/bump-develop.yml +++ b/.github/workflows/bump-develop.yml @@ -75,7 +75,7 @@ jobs: echo "new=$NEW" >> "$GITHUB_OUTPUT" echo "::notice::Bump $CURRENT -> $NEW" - - name: Bump version in Cargo.toml + Cargo.lock + - name: Bump version in Cargo.toml, Cargo.lock + scip-csharp.csproj env: CURRENT: ${{ steps.ver.outputs.current }} NEW: ${{ steps.ver.outputs.new }} @@ -88,9 +88,15 @@ jobs: # dependency versions (unlike `cargo update --workspace`, which would # opportunistically bump transitive deps as a side effect). sed -i "/^name = \"codesearch\"/,/^version = /{ s/^version = \"$CURRENT\"/version = \"$NEW\"/ }" Cargo.lock - # Hard-verify both files now carry the new version before committing. + # scip-csharp.csproj — the helper's feeds its --version line, + # the same-build identity the host logs and copy-to-common compares. + # Rewrite WHATEVER value is there, never anchor on $CURRENT: once the + # files drift apart the substitution silently no-ops forever. + sed -i "0,/[^<]*<\/Version>/s//$NEW<\/Version>/" helpers/csharp/scip-csharp.csproj + # Hard-verify all three files now carry the new version before committing. grep -m1 "^version = \"$NEW\"" Cargo.toml >/dev/null grep -A1 '^name = "codesearch"$' Cargo.lock | grep -m1 "^version = \"$NEW\"" >/dev/null + grep -m1 "$NEW" helpers/csharp/scip-csharp.csproj >/dev/null - name: Commit & push bump env: @@ -100,7 +106,7 @@ jobs: set -euo pipefail git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add Cargo.toml Cargo.lock + git add Cargo.toml Cargo.lock helpers/csharp/scip-csharp.csproj # No-op guard (e.g. a re-merge that already left develop at NEW). if git diff --cached --quiet; then echo "No version change to commit (already at $NEW)." From 237a73c3cbc75e241d8a39a66f1071cc1db47a83 Mon Sep 17 00:00:00 2001 From: Tony Rose <174962249+tony-nexartis@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:07:53 -0700 Subject: [PATCH 76/77] =?UTF-8?q?fix(codesearch):=20attribute=20group=3D?= =?UTF-8?q?=20hits=20to=20their=20owning=20repo=20at=20creation=20(chunk?= =?UTF-8?q?=20ids=20are=20store-local,=20never=20globally=20unique)=20+=20?= =?UTF-8?q?csharp=20restore-policy=20P1/P2=20=E2=80=94=20proven=20cargo=20?= =?UTF-8?q?check=20+=20cargo=20test=20157=20+=20dotnet=20test=207?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tony Rose <174962249+tony-nexartis@users.noreply.github.com> --- helpers/csharp/RestorePolicy.cs | 77 +++++++++++++++-- helpers/csharp/tests/RestorePolicyTests.cs | 20 +++++ src/mcp/search.rs | 96 ++++++++++------------ 3 files changed, 136 insertions(+), 57 deletions(-) diff --git a/helpers/csharp/RestorePolicy.cs b/helpers/csharp/RestorePolicy.cs index 16655ddb..88bf83f4 100644 --- a/helpers/csharp/RestorePolicy.cs +++ b/helpers/csharp/RestorePolicy.cs @@ -17,11 +17,12 @@ internal static class RestorePolicy private const int DefaultRestoreTimeoutSeconds = 300; /// - /// True when at least one project lacks obj/project.assets.json or - /// has one older than its .csproj (a package reference or target framework - /// was edited since the last restore). False means every project has fresh - /// assets and restore can be skipped entirely — the steady state of an - /// actively developed repo. + /// True when at least one project lacks obj/project.assets.json, has + /// one older than its .csproj (a package reference or target framework was + /// edited since the last restore), or has one older than an applicable + /// shared restore input (see ). False + /// means every project has fresh assets and restore can be skipped entirely + /// — the steady state of an actively developed repo. /// public static bool NeedsRestore(IEnumerable projectRelativePaths, string solutionDir) { @@ -34,14 +35,78 @@ public static bool NeedsRestore(IEnumerable projectRelativePaths, string { return true; } - if (File.GetLastWriteTimeUtc(projectPath) > File.GetLastWriteTimeUtc(assetsPath)) + var assetsWritten = File.GetLastWriteTimeUtc(assetsPath); + if (File.GetLastWriteTimeUtc(projectPath) > assetsWritten) { return true; } + if (AnySharedInputNewerThan(projectDir, assetsWritten)) + { + return true; + } + } + return false; + } + + /// + /// True when a shared restore input that applies to + /// was modified after the assets were written. + /// MSBuild/NuGet discover these by walking up from the project directory, + /// and editing one changes the resolved dependency graph without touching + /// the .csproj — under central package management or imported build config + /// a .csproj-only comparison would treat the assets as fresh, skip restore, + /// and load a stale asset graph. Absent files never trigger a restore. + /// + private static bool AnySharedInputNewerThan(string projectDir, DateTime assetsWritten) + { + // Per-project lock file sits beside the .csproj. + if (IsNewerThan(Path.Combine(projectDir, PackagesLockName), assetsWritten)) + { + return true; + } + + // Directory.Build.props / Directory.Packages.props: MSBuild and NuGet + // import the NEAREST one found walking up, so once a name is matched a + // higher directory can no longer supply it. + var nearestPending = new HashSet(NearestInputNames, StringComparer.OrdinalIgnoreCase); + + for (var dir = new DirectoryInfo(projectDir); dir is not null; dir = dir.Parent) + { + // NuGet.config files are merged from every ancestor directory. + if (IsNewerThan(Path.Combine(dir.FullName, NuGetConfigName), assetsWritten)) + { + return true; + } + + foreach (var name in NearestInputNames) + { + if (!nearestPending.Contains(name) || !File.Exists(Path.Combine(dir.FullName, name))) + { + continue; + } + nearestPending.Remove(name); + if (IsNewerThan(Path.Combine(dir.FullName, name), assetsWritten)) + { + return true; + } + } } + return false; } + private static bool IsNewerThan(string path, DateTime reference) => + File.Exists(path) && File.GetLastWriteTimeUtc(path) > reference; + + private const string NuGetConfigName = "NuGet.config"; + private const string PackagesLockName = "packages.lock.json"; + + private static readonly string[] NearestInputNames = + { + "Directory.Packages.props", + "Directory.Build.props", + }; + /// /// Restore wait timeout in seconds: CODESEARCH_RESTORE_TIMEOUT_SECS /// when set to a positive integer, otherwise 300. Unparseable values fall diff --git a/helpers/csharp/tests/RestorePolicyTests.cs b/helpers/csharp/tests/RestorePolicyTests.cs index 3e38d40c..0f988ba4 100644 --- a/helpers/csharp/tests/RestorePolicyTests.cs +++ b/helpers/csharp/tests/RestorePolicyTests.cs @@ -72,6 +72,26 @@ public void StaleAssetsOlderThanProjectTriggerRestore() Assert.True(RestorePolicy.NeedsRestore(new[] { @"src\Acme.Core\Acme.Core.csproj" }, _solutionDir)); } + /// + /// Central package management / imported build config can change the + /// resolved dependency graph without touching any .csproj, so a shared + /// input newer than the assets must force a restore even when every .csproj + /// is older than its obj/project.assets.json. + /// + [Fact] + public void StaleSharedRestoreInputTriggersRestore() + { + var projectPath = AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + var assetsPath = Path.Combine(Path.GetDirectoryName(projectPath)!, "obj", "project.assets.json"); + + // A Directory.Packages.props at the solution root, newer than the assets. + var packagesProps = Path.Combine(_solutionDir, "Directory.Packages.props"); + File.WriteAllText(packagesProps, ""); + File.SetLastWriteTimeUtc(packagesProps, File.GetLastWriteTimeUtc(assetsPath).AddMinutes(5)); + + Assert.True(RestorePolicy.NeedsRestore(new[] { @"src\Acme.Core\Acme.Core.csproj" }, _solutionDir)); + } + [Fact] public void EmptyProjectListSkipsRestore() { diff --git a/src/mcp/search.rs b/src/mcp/search.rs index 78c73272..c47d324e 100644 --- a/src/mcp/search.rs +++ b/src/mcp/search.rs @@ -499,13 +499,15 @@ impl CodesearchService { let mode = request.mode.as_deref().unwrap_or("auto"); let structural_intent = detect_structural_intent(&request.query); - // Which repo (alias) owns each surviving chunk id, recorded at - // fan-out and chunk-resolution time. The flat fan-out merge drops the - // store of origin, so without this map a group=all response cannot - // attribute a hit to its repo — `project=` results are prefixed - // "alias/path", group results must be too. - let mut alias_by_chunk: std::collections::HashMap = - std::collections::HashMap::new(); + // Repo ownership is carried WITH each result, not reconstructed from a + // chunk-id-only map. Every fan-out hit and every chunk resolved from a + // store has its alias baked into its path the moment the owning store + // (alias) is known. Chunk ids are allocated per store and collide + // across repos, so keying ownership by `u32` alone silently mislabels + // (or drops) the result of one repo when two share a local id — the + // reason a `group=` response could point at the wrong repo. Attributing + // at creation keeps group results ("alias/path") consistent with the + // `project=` routing without ever trusting a non-unique id. // === Lexical mode: FTS only across all stores === if mode == "lexical" { @@ -559,11 +561,10 @@ impl CodesearchService { limit, &stores, aliases, + alias_roots, &mut lexical_warnings, - &mut alias_by_chunk, ) .await; - prefix_paths_with_alias(&mut results, &alias_by_chunk, alias_roots); if let Some(target_kind) = structural_intent { boost_kind(&mut results, target_kind); @@ -651,11 +652,17 @@ impl CodesearchService { "internal error: no query embedding resolved for repo '{alias}'" ) })?; - let found = store + let mut found = store .search(embedding, limit * 5) .context("Error searching vector store")?; - for r in &found { - alias_by_chunk.insert(r.id, alias.to_string()); + // Attribute each hit to its own repo here, while the owning + // store (alias) is known: chunk ids are only unique within a + // store, so deferring to a chunk-id-keyed map would collapse + // hits from different repos that share a local id. + if let Some(root) = alias_roots.get(alias) { + for r in &mut found { + r.path = prefix_path_with_alias(&r.path, Some(alias), root); + } } Ok(found) }, @@ -726,7 +733,6 @@ impl CodesearchService { results.push(r); } } - prefix_paths_with_alias(&mut results, &alias_by_chunk, alias_roots); return self.build_semantic_response( results, request, @@ -819,8 +825,8 @@ impl CodesearchService { f.rrf_score, &stores, aliases, + alias_roots, &mut search_warnings, - &mut alias_by_chunk, ) .await { @@ -834,8 +840,6 @@ impl CodesearchService { boost_kind(&mut mapped, target_kind); } - prefix_paths_with_alias(&mut mapped, &alias_by_chunk, alias_roots); - self.build_semantic_response( mapped, request, @@ -855,8 +859,8 @@ impl CodesearchService { score: f32, stores: &[Arc], aliases: &[String], + alias_roots: &std::collections::HashMap, warnings: &mut Vec, - alias_by_chunk: &mut std::collections::HashMap, ) -> Option { for (idx, store_arc) in stores.iter().enumerate() { let store = match bounded_vector_read(&store_arc.vector_store).await { @@ -871,16 +875,21 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { - // Record the owning alias with the content actually chosen: - // path attribution below must match this very chunk, and the - // probe order here is what decided it. - if let Some(alias) = aliases.get(idx) { - alias_by_chunk.insert(chunk_id, alias.clone()); - } + // Attribute the path to the store this content was actually + // read from — the probe order here is what decided the owner. + let path = match ( + aliases.get(idx), + aliases.get(idx).and_then(|a| alias_roots.get(a)), + ) { + (Some(alias), Some(root)) => { + prefix_path_with_alias(&chunk.path, Some(alias.as_str()), root) + } + _ => chunk.path, + }; return Some(crate::vectordb::SearchResult { id: chunk_id, content: chunk.content, - path: chunk.path, + path, start_line: chunk.start_line, end_line: chunk.end_line, kind: chunk.kind, @@ -906,8 +915,8 @@ impl CodesearchService { limit: usize, stores: &[Arc], aliases: &[String], + alias_roots: &std::collections::HashMap, warnings: &mut Vec, - alias_by_chunk: &mut std::collections::HashMap, ) -> Vec { let mut results = Vec::new(); for fts in fts_results.iter().take(limit) { @@ -929,15 +938,21 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { - // Same as resolve_chunk_from_stores: attribute the alias of + // Same as resolve_chunk_from_stores: attribute the path to // the store this content was actually read from. - if let Some(alias) = aliases.get(idx) { - alias_by_chunk.insert(fts.chunk_id, alias.clone()); - } + let path = match ( + aliases.get(idx), + aliases.get(idx).and_then(|a| alias_roots.get(a)), + ) { + (Some(alias), Some(root)) => { + prefix_path_with_alias(&chunk.path, Some(alias.as_str()), root) + } + _ => chunk.path, + }; results.push(crate::vectordb::SearchResult { id: fts.chunk_id, content: chunk.content, - path: chunk.path, + path, start_line: chunk.start_line, end_line: chunk.end_line, kind: chunk.kind, @@ -1209,24 +1224,3 @@ impl CodesearchService { } } } - -/// Prefix group-fan-out result paths with the repo alias each chunk was -/// recorded from, mirroring the `project=` routing ("alias/repo-relative"). -/// -/// Results whose alias was not recorded (every survivor is recorded at -/// fan-out or resolution time, so this indicates an internal slip) keep their -/// raw path — `build_semantic_response` still normalizes it. -fn prefix_paths_with_alias( - results: &mut [crate::vectordb::SearchResult], - alias_by_chunk: &std::collections::HashMap, - alias_roots: &std::collections::HashMap, -) { - for r in results.iter_mut() { - let Some(alias) = alias_by_chunk.get(&r.id) else { - continue; - }; - if let Some(root) = alias_roots.get(alias) { - r.path = prefix_path_with_alias(&r.path, Some(alias), root); - } - } -} From 7b47a9b04c191a982f9e1bad20e53581fd43f9c2 Mon Sep 17 00:00:00 2001 From: Tony Rose <174962249+tony-nexartis@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:46:36 -0700 Subject: [PATCH 77/77] =?UTF-8?q?fix(mcp):=20classify=20the=20unknown-proj?= =?UTF-8?q?ect=20refusal=20(NEE-P-CONSISTENT)=20=E2=80=94=20a=20bad=20proj?= =?UTF-8?q?ect=3D=20alias=20now=20returns=20a=20structured=20project=5Funr?= =?UTF-8?q?esolved=20envelope=20with=20available=5Fprojects/groups=20inste?= =?UTF-8?q?ad=20of=20the=20bare=20"Unknown=20alias=20'x'"=20string;=20+=20?= =?UTF-8?q?unit=20test=20(cargo=20test=20812=20pass,=201=20pre-existing=20?= =?UTF-8?q?macOS=20/private=20symlink=20failure=20in=20db=5Fdiscovery,=20u?= =?UTF-8?q?nrelated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tony Rose <174962249+tony-nexartis@users.noreply.github.com> --- src/mcp/mod.rs | 94 +++++++++++++++++++++++++++++++++++++----------- src/mcp/tests.rs | 25 +++++++++++++ 2 files changed, 98 insertions(+), 21 deletions(-) diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index ac0a38a7..a57ae7d7 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -37,6 +37,29 @@ fn store_lock_wait() -> std::time::Duration { .unwrap_or_else(|| std::time::Duration::from_secs(crate::constants::STORE_LOCK_WAIT_SECS)) } +/// Build the structured `project_unresolved` refusal JSON for an unknown +/// `project=` alias (NEE-P-CONSISTENT: a refusal names its class and remedy). +/// Pure + unit-testable; the caller supplies the serve hub's enumeration. +fn unknown_project_error_json( + alias: &str, + projects: Vec, + groups: Vec, + project_groups: std::collections::HashMap>, +) -> String { + let payload = serde_json::json!({ + "error_code": "project_unresolved", + "message": format!("Unknown project alias '{}'.", alias), + "available_projects": projects, + "available_groups": groups, + "project_groups": project_groups, + "hint_for_agent": format!( + "'{}' is not a registered project. Re-call with one of available_projects, or ask the user which repository to search. project_groups maps each project to its group(s) — prefer group= when the target is listed there.", + alias + ) + }); + payload.to_string() +} + /// Acquire the vector-store read lock with a bounded wait. /// /// A handler queued on this lock keeps its `Arc` — and with it @@ -1145,8 +1168,16 @@ impl CodesearchService { types::validate_project_group(project, group, true)?; if let Some(ref alias) = project { - let stores = serve_state.get_or_open_stores(alias, true).await?; - return Ok(Some((vec![stores], vec![alias.clone()]))); + // A bad alias is a NAMED refusal, not a bare string: surface the same + // available-projects envelope the unscoped case uses (P1, codesearch + // tuning round 2026-10-16) so an agent corrects without guessing. + return match serve_state.get_or_open_stores(alias, true).await { + Ok(stores) => Ok(Some((vec![stores], vec![alias.clone()]))), + Err(e) if e.contains("Unknown alias") => { + Err(self.format_unknown_project_error(alias)) + } + Err(e) => Err(e), + }; } if let Some(ref group_name) = group { @@ -1258,25 +1289,7 @@ impl CodesearchService { /// that picking a single project would miss sibling repos in the same group /// (e.g. a separate config / import-data repo). fn format_scope_error(&self) -> String { - let (projects, mut groups, project_groups) = if let Some(ref serve_state) = self.serve_state - { - let cfg = serve_state.config_snapshot(); - let mut projects: Vec = cfg.repos.keys().cloned().collect(); - // Include opt-in mounted remote projects so an agent can discover and - // route to them by name (they are first-class `project=` targets). - projects.extend( - cfg.mounted_remote_projects() - .into_iter() - .map(|(name, _)| name), - ); - projects.sort(); - projects.dedup(); - let mut groups: Vec = cfg.groups.keys().cloned().collect(); - groups.sort(); - (projects, groups, cfg.project_groups()) - } else { - (vec![], vec![], std::collections::HashMap::new()) - }; + let (projects, mut groups, project_groups) = self.project_scope_payload(); // The "all" virtual group is always available when there are projects to // search — advertise it so agents discover the cross-repo shortcut. // (scope_required only fires when >1 repo is registered, so this is safe.) @@ -1297,6 +1310,45 @@ impl CodesearchService { payload.to_string() } + /// The `(available_projects, available_groups, project_groups)` triple for the + /// active serve hub (empty in local mode). One home for the enumeration shared + /// by the scope and unknown-project refusals. + fn project_scope_payload( + &self, + ) -> ( + Vec, + Vec, + std::collections::HashMap>, + ) { + if let Some(ref serve_state) = self.serve_state { + let cfg = serve_state.config_snapshot(); + let mut projects: Vec = cfg.repos.keys().cloned().collect(); + // Include opt-in mounted remote projects so an agent can discover and + // route to them by name (they are first-class `project=` targets). + projects.extend( + cfg.mounted_remote_projects() + .into_iter() + .map(|(name, _)| name), + ); + projects.sort(); + projects.dedup(); + let mut groups: Vec = cfg.groups.keys().cloned().collect(); + groups.sort(); + (projects, groups, cfg.project_groups()) + } else { + (vec![], vec![], std::collections::HashMap::new()) + } + } + + /// Build a structured `project_unresolved` error JSON for an unknown + /// `project=` alias (NEE-P-CONSISTENT: a refusal names its class and remedy). + /// Previously a bare `Unknown alias 'x'` string reached the agent with no + /// valid-project list — the exact shape that forces a guessing round-trip. + fn format_unknown_project_error(&self, alias: &str) -> String { + let (projects, groups, project_groups) = self.project_scope_payload(); + unknown_project_error_json(alias, projects, groups, project_groups) + } + /// Execute a read-only action against the vector store with an explicit store override. /// /// If `store_override` is provided (from project/group routing), it takes precedence. diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index 8628c2d3..586be728 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -3011,3 +3011,28 @@ fn single_index_status_requires_a_built_graph_to_report_ready() { let (status, _) = super::single_index_status(0, false); assert_eq!(status, "building"); } + +#[test] +fn test_unknown_project_error_is_classified() { + // P1 (codesearch tuning round 2026-10-16): a bad `project=` alias must be a + // NAMED refusal carrying the available-project list, not a bare + // `Unknown alias 'x'` string (NEE-P-CONSISTENT: a refusal names its class + // and remedy so the agent corrects without a guessing round-trip). + let mut project_groups = std::collections::HashMap::new(); + project_groups.insert( + "nexartis".to_string(), + vec!["alpha".to_string(), "beta".to_string()], + ); + let json = super::unknown_project_error_json( + "nope", + vec!["alpha".to_string(), "beta".to_string()], + vec!["all".to_string(), "nexartis".to_string()], + project_groups, + ); + let v: serde_json::Value = serde_json::from_str(&json).expect("valid json"); + assert_eq!(v["error_code"], "project_unresolved"); + assert_eq!(v["available_projects"][0], "alpha"); + assert_eq!(v["available_projects"][1], "beta"); + assert_eq!(v["available_groups"][1], "nexartis"); + assert!(v["hint_for_agent"].as_str().unwrap().contains("nope")); +}