feat: /status + MCP status/projects always answer via a background snapshot

AGENTS.develop.md

@@ -241,6 +241,7 @@ Never `unwrap_or_default()` a store error on a search path — "no results" and
 
 ## Changelog highlights (recent)
 
+- **v1.5.33 (pending)** — /status + MCP status answer from a background snapshot (never measure inline): always responsive under rebuild waves, with `snapshot_age_secs` + a `load` summary for calling agents (`CODESEARCH_STATUS_REFRESH_SECS`, default 2s)
 - **v1.5.31 (pending)** — scip-csharp VS-MSBuild restore fallback: NETSDK1045-class CLI SDK gaps (hollow SDK dir, net10-only-in-VS) retry via vswhere + `MSBuild -t:restore`, and MSBuild registration picks a VS instance when no CLI SDK can target the repo
 - **v1.5.31 (pending)** — serve TUI status: `Indexing` only for real text indexing (pure SCIP rebuilds drive the C#/TS columns only); resting repos read `Idle` again
 - **v1.5.16 (pending)** — C# restore-pending retries 20×60s → 3 attempts with exponential backoff; restore exhaustion renders as yellow C#⚠ partial on every serve path instead of a red error

CHANGELOG.md

@@ -14,6 +14,12 @@ more PRs land; when the release is actually tagged, the same section is
 finalized in place with a date — no renaming/migration step needed.
 -->
 
+## [1.5.33]
+
+### Fixed
+
+- **`GET /status` and the MCP `status` tool always answer, even under a heavy rebuild wave.** They used to measure inline in the request path: per registered repo several filesystem probes plus a SCIP LMDB env open and a sysinfo refresh, synchronous on async workers that a rebuild wave (dotnet restore + scip-csharp children) starves — with 28 repos one call took tens of seconds and polling clients (remote TUI, calling agents) piled up and timed out; the project list died the same death. Now a single background task measures (on the blocking pool, one pass at a time — under load the cadence stretches instead of piling measuring tasks onto a starved machine) and stores a snapshot whose readers answer in microseconds. Every answer carries `snapshot_age_secs` (staleness is visible, never a hang) and a `load` summary (`text_indexing` / `symbol_indexing` / `errored` / `refresh_interval_secs`) so calling agents can throttle themselves; before the first pass completes they answer `"starting"` immediately. Cadence: `CODESEARCH_STATUS_REFRESH_SECS`, default 2s.
+
 ## [1.5.31]
 
 ### Changed

src/constants.rs

@@ -616,6 +616,12 @@ pub const REMOTE_PEER_RETRY_BACKOFF_MS: &[u64] = &[3000, 8000];
 /// single millisecond value (test hook so retry tests don't sleep for real).
 pub const REMOTE_PEER_RETRY_BACKOFF_ENV: &str = "CODESEARCH_REMOTE_RETRY_BACKOFF_MS";
 
+/// Cadence of the background status-snapshot refresher (seconds). Read
+/// endpoints answer from the stored snapshot, so this only trades staleness
+/// (`snapshot_age_secs`) for probing cost — under load the cadence
+/// stretches on its own; this knob only sets the idle pace.
+pub const STATUS_REFRESH_SECS_ENV: &str = "CODESEARCH_STATUS_REFRESH_SECS";
+
 /// Cooperative join window (seconds) for `await_index_task` and
 /// `await_fsw_shutdown`: how long a background indexing / file-watcher task
 /// is given to observe its `CancellationToken` and exit on its own before it

src/mcp/mod.rs

@@ -121,6 +121,8 @@ pub(crate) use helpers::*;
 pub(crate) use instructions::*;
 pub(crate) use responses::*;
 pub use runtime::*;
+/// The projects-measure pass for the serve status-snapshot background task.
+pub(crate) use status::compute_projects_snapshot;
 
 /// Re-order lexical `usages` hits so source-code paths come before everything
 /// else (docs, configs, markdown). Stable: score order is preserved within

src/mcp/status.rs

@@ -16,6 +16,102 @@ const LOCK_STATUS_HELD_IN_PROCESS: &str = "held-in-process";
 const LOCK_STATUS_LOCKED_EXTERNALLY: &str = "locked-externally";
 const LOCK_STATUS_AVAILABLE: &str = "available";
 
+/// One full projects-measure pass for the background status-snapshot task
+/// — the ONLY place these per-repo probes (metadata reads, try_read stats,
+/// lock detection) run. `list_projects` answers from the stored result and
+/// must never call this on the request path.
+pub(crate) fn compute_projects_snapshot(
+    serve_state: &crate::serve::ServeState,
+) -> (Vec<RepoInfo>, Vec<String>) {
+    let config = serve_state.config_snapshot();
+    let project_groups = config.project_groups();
+    let mut repos_info = Vec::new();
+    let mut list_warnings: Vec<String> = Vec::new();
+
+    for (alias, path) in &config.repos {
+        let db_path = path.join(crate::constants::DB_DIR_NAME);
+
+        let (total_chunks, total_files, model, lock_status, error) = if db_path.exists() {
+            let (model_name, _dims) = read_model_metadata(&db_path);
+
+            // For repos already opened in DashMap, use the live SharedStores for stats
+            // WITHOUT opening a new VectorStore connection.
+            // For unopened repos, just report metadata — do NOT open the DB.
+            let opened = serve_state.get_opened_stores(alias);
+            // Never wait on the store lock: an indexing run holds it for
+            // whole batches, and status must answer immediately.
+            let stats_result = opened
+                .as_ref()
+                .and_then(|stores| stores.vector_store.try_read().ok().map(|vs| vs.stats()));
+            if let Some(stats_result) = stats_result {
+                // `0 chunks` alone reads exactly like "not indexed yet" — the
+                // repo may in fact be full and simply failing to answer (the
+                // read-only-incident shape this branch exists for). Attribute
+                // the failure to THIS repo rather than a top-level channel:
+                // list_projects returns one entry per repo, so per-item is the
+                // shape that actually matches the fan-out.
+                // `repo_stats_from_result` carries only the part of this
+                // decision that varies by Ok/Err — see its doc comment.
+                // `record_stats_or_warn` wraps it so this call site cannot
+                // silently drop the warning half without also breaking the
+                // counts it returns — see its own doc comment.
+                let (total_chunks, total_files, error) =
+                    record_stats_or_warn(stats_result, alias, &mut list_warnings);
+                (
+                    total_chunks,
+                    total_files,
+                    model_name,
+                    serve_state
+                        .repo_lock_status(alias)
+                        .unwrap_or("unknown")
+                        .to_string(),
+                    error,
+                )
+            } else {
+                // Not opened, or its store is busy — persisted stats from
+                // metadata.json; do NOT open the DB. Our own holders are
+                // checked before the file-lock probe, which cannot tell
+                // this process apart from another one.
+                let (md_chunks, md_files) = read_metadata_stats(&db_path);
+                let lock_status = if serve_state.is_indexing(alias) {
+                    LOCK_STATUS_INDEXING
+                } else if opened.is_some() {
+                    LOCK_STATUS_BUSY
+                } else if serve_state.is_held_in_process(alias, &db_path) {
+                    LOCK_STATUS_HELD_IN_PROCESS
+                } else if crate::index::is_database_locked(&db_path) {
+                    LOCK_STATUS_LOCKED_EXTERNALLY
+                } else {
+                    LOCK_STATUS_AVAILABLE
+                };
+                (
+                    md_chunks,
+                    md_files,
+                    model_name,
+                    lock_status.to_string(),
+                    None,
+                )
+            }
+        } else {
+            (0, 0, "not indexed".to_string(), "unknown".to_string(), None)
+        };
+
+        repos_info.push(RepoInfo {
+            alias: alias.clone(),
+            project_path: path.display().to_string(),
+            database_path: db_path.display().to_string(),
+            total_chunks,
+            total_files,
+            model,
+            lock_status,
+            groups: project_groups.get(alias).cloned().unwrap_or_default(),
+            error,
+        });
+    }
+
+    (repos_info, list_warnings)
+}
+
 #[tool_router(router = status_router, vis = "pub(crate)")]
 impl CodesearchService {
     /// Unified status tool — dispatches based on `kind`.
@@ -78,7 +174,32 @@ impl CodesearchService {
                 // Count the virtual "all" group when repos are registered, so the
                 // summary doesn't read "0 group(s)" while `all` is actually available.
                 let group_count = config.groups.len() + if config.repos.is_empty() { 0 } else { 1 };
-                let statuses = serve_state.repo_statuses_lightweight();
+                // From the background snapshot, never an inline measure: under a
+                // rebuild wave the snapshot task is the ONLY thing allowed to
+                // touch the filesystem (the GET /status timeout fix) — and this
+                // tool must always answer too. Pre-first-pass answers "starting".
+                let Some(snap) = serve_state.latest_status_snapshot() else {
+                    let response = IndexStatusResponse {
+                        indexed: repo_count > 0,
+                        status: "starting".to_string(),
+                        status_message:
+                            "serve is computing its first status snapshot; retry in a few seconds"
+                                .to_string(),
+                        total_chunks: 0,
+                        total_files: 0,
+                        model: self.model_type.short_name().to_string(),
+                        dimensions: 0,
+                        max_chunk_id: 0,
+                        db_path: format!("({} repos)", repo_count),
+                        project_path: format!("serve mode — {} repo(s)", repo_count),
+                        error_message: None,
+                        mode: self.mcp_mode(),
+                    };
+                    let json =
+                        serde_json::to_string(&response).unwrap_or_else(|_| "{}".to_string());
+                    return Ok(CallToolResult::success(vec![ContentBlock::text(json)]));
+                };
+                let statuses = &snap.repos;
                 let open_count = statuses
                     .iter()
                     .filter(|(_, r)| matches!(r.status, crate::serve::RepoStateLabel::Open))
@@ -351,90 +472,23 @@ impl CodesearchService {
         // When serve is active, use ServeState as source of truth for lock status
         if let Some(ref serve_state) = self.serve_state {
             let config = serve_state.config_snapshot();
-            let project_groups = config.project_groups();
-            let mut repos_info = Vec::new();
-            let mut list_warnings: Vec<String> = Vec::new();
-
-            for (alias, path) in &config.repos {
-                let db_path = path.join(crate::constants::DB_DIR_NAME);
-
-                let (total_chunks, total_files, model, lock_status, error) = if db_path.exists() {
-                    let (model_name, _dims) = read_model_metadata(&db_path);
-
-                    // For repos already opened in DashMap, use the live SharedStores for stats
-                    // WITHOUT opening a new VectorStore connection.
-                    // For unopened repos, just report metadata — do NOT open the DB.
-                    let opened = serve_state.get_opened_stores(alias);
-                    // Never wait on the store lock: an indexing run holds it for
-                    // whole batches, and status must answer immediately.
-                    let stats_result = opened.as_ref().and_then(|stores| {
-                        stores.vector_store.try_read().ok().map(|vs| vs.stats())
-                    });
-                    if let Some(stats_result) = stats_result {
-                        // `0 chunks` alone reads exactly like "not indexed yet" — the
-                        // repo may in fact be full and simply failing to answer (the
-                        // read-only-incident shape this branch exists for). Attribute
-                        // the failure to THIS repo rather than a top-level channel:
-                        // list_projects returns one entry per repo, so per-item is the
-                        // shape that actually matches the fan-out.
-                        // `repo_stats_from_result` carries only the part of this
-                        // decision that varies by Ok/Err — see its doc comment.
-                        // `record_stats_or_warn` wraps it so this call site cannot
-                        // silently drop the warning half without also breaking the
-                        // counts it returns — see its own doc comment.
-                        let (total_chunks, total_files, error) =
-                            record_stats_or_warn(stats_result, alias, &mut list_warnings);
-                        (
-                            total_chunks,
-                            total_files,
-                            model_name,
-                            serve_state
-                                .repo_lock_status(alias)
-                                .unwrap_or("unknown")
-                                .to_string(),
-                            error,
-                        )
-                    } else {
-                        // Not opened, or its store is busy — persisted stats from
-                        // metadata.json; do NOT open the DB. Our own holders are
-                        // checked before the file-lock probe, which cannot tell
-                        // this process apart from another one.
-                        let (md_chunks, md_files) = read_metadata_stats(&db_path);
-                        let lock_status = if serve_state.is_indexing(alias) {
-                            LOCK_STATUS_INDEXING
-                        } else if opened.is_some() {
-                            LOCK_STATUS_BUSY
-                        } else if serve_state.is_held_in_process(alias, &db_path) {
-                            LOCK_STATUS_HELD_IN_PROCESS
-                        } else if crate::index::is_database_locked(&db_path) {
-                            LOCK_STATUS_LOCKED_EXTERNALLY
-                        } else {
-                            LOCK_STATUS_AVAILABLE
-                        };
-                        (
-                            md_chunks,
-                            md_files,
-                            model_name,
-                            lock_status.to_string(),
-                            None,
-                        )
-                    }
-                } else {
-                    (0, 0, "not indexed".to_string(), "unknown".to_string(), None)
-                };
-
-                repos_info.push(RepoInfo {
-                    alias: alias.clone(),
-                    project_path: path.display().to_string(),
-                    database_path: db_path.display().to_string(),
-                    total_chunks,
-                    total_files,
-                    model,
-                    lock_status,
-                    groups: project_groups.get(alias).cloned().unwrap_or_default(),
-                    error,
-                });
-            }
+            // From the background snapshot, never an inline measure — the same
+            // contract as /status and status(kind=index): under a rebuild wave
+            // the measuring task is the only thing that touches the filesystem,
+            // and the project list must still answer (it was the second timeout
+            // in the 2026-10-16 field case). Pre-first-pass answers an empty
+            // list with an explicit warming warning instead of hanging.
+            let (repos_info, list_warnings) = match serve_state.latest_status_snapshot() {
+                Some(snap) => (snap.projects.clone(), snap.projects_warnings.clone()),
+                None => (
+                    Vec::new(),
+                    vec![
+                        "status snapshot still warming up — the project list is empty for a few \
+                     seconds after serve startup; retry shortly"
+                            .to_string(),
+                    ],
+                ),
+            };
 
             let response = ListProjectsResponse {
                 repos: repos_info,

src/mcp/types.rs

@@ -526,7 +526,9 @@ pub struct RemoteProjectInfo {
 }
 
 /// Information about a single registered project/repo.
-#[derive(Debug, Serialize)]
+/// `Clone`: the snapshot stores the computed list and every projects answer
+/// clones it out (the slot lock is never held across response building).
+#[derive(Debug, Clone, Serialize)]
 pub struct RepoInfo {
     pub alias: String,
     pub project_path: String,