diff --git a/.cargo/config.toml b/.cargo/config.toml index 57fad0d6..3942b26c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,9 @@ [build] # Place target directory outside the repository to keep source tree clean target-dir = "../target" + +# Fast debug builds: rust-lld links several times faster than link.exe. +# Debug-only policy (issue #192): release builds ship via release.yml and are +# built there explicitly, never locally. +[target.x86_64-pc-windows-msvc] +linker = "rust-lld" diff --git a/.github/workflows/bump-develop.yml b/.github/workflows/bump-develop.yml index bb6fa0ab..71485a12 100644 --- a/.github/workflows/bump-develop.yml +++ b/.github/workflows/bump-develop.yml @@ -75,7 +75,7 @@ jobs: echo "new=$NEW" >> "$GITHUB_OUTPUT" echo "::notice::Bump $CURRENT -> $NEW" - - name: Bump version in Cargo.toml + Cargo.lock + - name: Bump version in Cargo.toml, Cargo.lock + scip-csharp.csproj env: CURRENT: ${{ steps.ver.outputs.current }} NEW: ${{ steps.ver.outputs.new }} @@ -88,9 +88,15 @@ jobs: # dependency versions (unlike `cargo update --workspace`, which would # opportunistically bump transitive deps as a side effect). sed -i "/^name = \"codesearch\"/,/^version = /{ s/^version = \"$CURRENT\"/version = \"$NEW\"/ }" Cargo.lock - # Hard-verify both files now carry the new version before committing. + # scip-csharp.csproj — the helper's feeds its --version line, + # the same-build identity the host logs and copy-to-common compares. + # Rewrite WHATEVER value is there, never anchor on $CURRENT: once the + # files drift apart the substitution silently no-ops forever. + sed -i "0,/[^<]*<\/Version>/s//$NEW<\/Version>/" helpers/csharp/scip-csharp.csproj + # Hard-verify all three files now carry the new version before committing. grep -m1 "^version = \"$NEW\"" Cargo.toml >/dev/null grep -A1 '^name = "codesearch"$' Cargo.lock | grep -m1 "^version = \"$NEW\"" >/dev/null + grep -m1 "$NEW" helpers/csharp/scip-csharp.csproj >/dev/null - name: Commit & push bump env: @@ -100,7 +106,7 @@ jobs: set -euo pipefail git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add Cargo.toml Cargo.lock + git add Cargo.toml Cargo.lock helpers/csharp/scip-csharp.csproj # No-op guard (e.g. a re-merge that already left develop at NEW). if git diff --cached --quiet; then echo "No version change to commit (already at $NEW)." diff --git a/AGENTS.develop.md b/AGENTS.develop.md index 9088733b..b378a1f2 100644 --- a/AGENTS.develop.md +++ b/AGENTS.develop.md @@ -187,7 +187,7 @@ TUI `i` overlay. The TUI discovery tick is config-only (zero HTTP). ## Runtime locations - **Runtime dir**: `C:\Users\develterf\.local\bin\` — contains `codesearch.exe` and `helpers/csharp/scip-csharp.exe`. This is where `codesearch serve` runs from. -- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. +- **Build:** via `build.ps1` (repo root) — it self-heals the bare-flag quirk and sets `CARGO_TARGET_DIR` so `target/` stays outside the repo. Debug-only (issue #192): `-Release` is refused, release artifacts come from CI; rust-lld is pinned in `.cargo/config.toml` (bin rebuild ~2x faster). - **Build dir**: `target/release/` — lives **outside the repo** (set via `CARGO_TARGET_DIR`). For compilation only. Never run codesearch from this location. - **Logs**: `~\.codesearch\logs\` — codesearch writes structured logs here during serve. Check these for startup errors, rebuild failures, and helper detection messages. diff --git a/CHANGELOG.md b/CHANGELOG.md index fa375c13..9fcdcd18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,49 @@ more PRs land; when the release is actually tagged, the same section is finalized in place with a date — no renaming/migration step needed. --> +## [1.5.2] - 2026-10-05 + +### Fixed + +- **Restore runs only when package assets are missing or stale, and warning-severity workspace diagnostics no longer count as index warnings.** Restoring on every reindex was far too heavy for multi-repo hubs and big solutions legitimately exceed any fixed timeout (`dotnet restore did not finish within 300s` was observed live). Now the helper checks each project's `obj/project.assets.json` against its .csproj and restores only when evidence demands it (the steady state of an actively developed repo is a no-op), the kill timeout is tunable via `CODESEARCH_RESTORE_TIMEOUT_SECS`, and restore uses `--ignore-failed-sources` so an unreachable private feed cannot block indexing. Separately, warning-severity workspace diagnostics (e.g. "Found project reference without a matching metadata reference" for excluded or external projects) are benign resolution notes — they log at INFO and no longer pin the repo's index warning; only true load failures (minus NuGet audit advisories) still do. +- **C# symbol indexing restores packages before the workspace load, and NuGet audit advisories no longer count as index warnings.** The design-time load does not restore: with stale or missing `project.assets.json`, package-typed references collapsed into CS0246 cascades that flagged whole solution families as degraded (⚠ in the serve TUI) even though `dotnet build` succeeds. The helper now runs `dotnet restore` before opening the solution (bounded, best effort — a failed restore logs and continues, degraded indexing beats no indexing), and when the bounded wait expires the restore is **not killed**: it keeps running in the background, this load proceeds degraded, and the next reindex picks up the fresh assets — a killed restore would waste all of its work and re-pay the same wait on every subsequent reindex. NuGet vulnerability advisories surfacing as Msbuild "failures" are filtered from the warning channel: they are security notes for the product, not index-health signals. Compiler *warnings* were already never counted; genuine project compilation errors still are, and indexing stays as complete as the broken projects allow. + +### Changed + +- **Dependency refresh:** libc 0.2.190, rmcp 3.5.0, uuid 1.27.0 (#305). + +## [1.5.1] - 2026-10-04 + +### Added + +- **`codesearch setup` — download embedding models, and a relocatable global root via `CODESEARCH_HOME`.** The stubbed setup command now resolves `--model` through the shared model parser (unknown names error with the valid list), downloads into the global models cache with a visible progress bar (runtime paths stay silent), and probe-embeds one query so a broken download fails in setup instead of mid-indexing. `CODESEARCH_HOME` relocates the entire global root (repos.json, models cache, logs, serve_url, global ignore/extensions) and is honoured by the hub, db-discovery, every Claude Code guard hook and the OpenCode plugin, with tests pinning each layer. *(Contributed by @belovaf in #298 — in the same PR: find-impact warnings that no longer promise rebuilds nothing will run, group-fan-out hits attributed to their repo, and dependency-lockfile indexing exclusions.)* + +*A big thank-you to belovaf ([@belovaf](https://github.com/belovaf)) for his first contribution to this release — a careful, well-tested feature plus four quality fixes landed in one go.* + +### Fixed + +- **Queued reindexes are no longer cancelled as "leaked", and an unjoinable index task can no longer wedge a hub.** With the process-wide job gate (`CODESEARCH_INDEX_JOBS=1`, the default), tasks queued behind a long-running job held their `active_reindexes` markers past `MAX_INDEXING_SECS` without renewing them, so the lazy stale-marker check evicted the markers and cancelled the parked tasks — leaving empty stores and in-process LMDB handles that made later writes fail with "Database is locked by another process" until a serve restart. Separately, `await_index_task` removed the task entry *before* waiting and dropped the `JoinHandle` on timeout, so a task too deep in the uninterruptible `build_index` to cancel kept its `Arc` and the single job permit invisibly; the job-gate wait ignored cancellation, parking cancelled tasks on the gate; and `remove_repo` / format recovery could delete the DB directory out from under a still-running task. Now: a marker backed by a live tracked `Reindex` task is renewed (with a throttled wedge warning) instead of reaped; gate acquisition is cancellation-aware (`JobAcquire`) so cancelled or removed queued jobs abort and release their stores; a task that cannot be joined stays tracked and is reaped by a periodic sweeper that also deletes orphaned DB directories and re-asserts cancellation for removed repos; DB deletion is deferred while the task runs; format recovery refuses to wipe in that state; and stale markers from other owners are still evicted. + +## [1.5.0] - 2026-10-03 + +### Added + +- **Settable indexing/refresh limits: `CODESEARCH_INDEX_JOBS`, `CODESEARCH_EMBED_THREADS`, `CODESEARCH_EMBED_PAUSE_MS`, `CODESEARCH_MIN_FREE_MB`, `CODESEARCH_MAX_CHUNKS_PER_BATCH` (serve flags `--index-jobs`, `--embed-threads`, `--embed-pause-ms`, `--min-free-mb`).** A multi-repo hub used to start one full refresh/reindex/HNSW-build pass per repo the moment that repo had work, so a mass `git pull`, a burst of first queries or several HTTP reindexes ran many CPU-heavy passes at once, held many file/chunk windows in memory, and queued the (already serialised) ONNX inference behind all of them. A process-wide job gate now bounds concurrent heavy repo jobs (default **1**; queries never take it), the ONNX session's intra-op thread count can be capped (`CODESEARCH_EMBED_THREADS`, previously always all cores), background embedding releases the model lock once per mini-batch with an optional duty-cycle pause so interactive query embeddings are not stuck behind a whole batch, the read/chunk window can be capped by chunks as well as files, and a job can wait for a free-memory floor before starting. See the README section "Indexing limits" for recommended values. +- **OpenCode plugin integration (`integrations/opencode/`).** codesearch's MCP `initialize` instructions are advisory and connection-scoped: they cannot resolve the session directory to a registered project, see a `grep` return nothing, recover a dead serve hub, notice an unregistered worktree, or keep file outlines alive across compaction. The new self-contained OpenCode v2 plugin (no runtime dependencies, fail-open throughout) adds host-side behaviour around the same MCP server: it resolves the working directory to a project alias and injects scope-aware guidance once per session, turns a zero-hit `grep`/`glob` into index hits, offers `nudge`/`prune`/`block` codesearch-first guards that deny only while the hub is confirmed reachable and the repo is registered, probes `/healthz` and asks OpenCode to reload failed MCP servers (with optional local `codesearch serve` auto-start), warns when a registered index is empty or model-less, notes just-edited files while the watcher catches up, captures `explore` outlines into the compaction prompt, and registers `/codesearch`, `/codesearch-status` and `/codesearch-index` commands, a `codesearch_scope` tool and a `codesearch` skill. Includes a config reference, install/usage docs and a mock-host smoke test. + +*A big thank-you to Don Bowman ([@donbowman](https://github.com/donbowman)) for his outstanding, highly value-adding contributions to this release — the indexing-limits rework (#294) and the OpenCode plugin integration (#295), follow-up fixes included.* +- **Always-on symbol indexes (C# + TypeScript).** Symbol (SCIP) indexes are no longer a manual or incidental artifact: serve warmup, cold opens, file-watcher batches and reindexes each drive a per-language symbol pass wherever chunk indexes are built, so `find_impact` evidence exists without a separate rebuild step. Startup inspects the on-disk symbol indexes with a builder-version stamp and rebuilds those an older binary produced (degraded retry instead of a hard failure), so a upgraded serve never answers from stale symbol data. + +### Changed + +- **`codesearch serve` runs heavy repo jobs one at a time by default (`CODESEARCH_INDEX_JOBS=1`).** Warmup, cold opens, file-watcher batches and reindexes now queue behind a running refresh instead of all running at once; raise the value deliberately on machines with cores and memory to spare. Standalone `codesearch index` / `codesearch mcp` runs are unchanged (no gate is attached). `CODESEARCH_EMBED_THREADS` unset keeps the previous ONNX default (all cores), so single-repo index throughput is unchanged unless the variable is set. +- **`find_impact` is honest about partial evidence; the TUI says how healthy the symbol indexes are.** Index-level rebuild warnings (non-compiling C# projects, swallowed resolution failures, non-zero `scip-typescript` exits) ride the `find_impact` answer as an additive `warnings` array instead of vanishing into the log. The scip-csharp helper's workspace-load diagnostics are aggregated into a single warnings line and never escalate to errors during workspace load. The serve TUI's symbol-index indicators use severity language: a degraded C# index renders red (`C#⚠`) and the info line names the failures. +- **Cloud image: build locally, not via `az acr build`.** The Dockerfile's builder stage compiles the release binary and warms up the embedding model before tarring — memory/disk-heavy, and ACR Basic/Standard build agents reliably fail the model-tar layer after a ~25 min compile (confirmed twice). `integrations/cloud/README.md` now documents building with local Docker Desktop and `docker push` instead. + +### Fixed + +- **SCIP key-format 2.0 regression: simple-name extraction now strips parameters before segmenting.** Fully qualified symbol keys that carry parameter types (real C# methods do) segmented on the parameter list instead of the path segments, producing wrong simple names for the affected symbols. + ## [1.4.11] ### Changed @@ -334,6 +377,12 @@ finalized in place with a date — no renaming/migration step needed. ## [1.0.72] - 2026-05-01 - Initial multi-repo release: multi-repo `serve` (HTTP/SSE, per-project/group routing, RRF cross-repo search), stdio MCP proxy with client-side auto-reconnect, tree-sitter chunking (9 langs), persistent SHA-256 embedding cache, repository groups, re-tuned RRF, and LMDB resize crash fix (#30, `MDB_MAP_FULL`). +[1.5.2]: https://github.com/flupkede/codesearch/compare/v1.5.1...v1.5.2 + +[1.5.1]: https://github.com/flupkede/codesearch/compare/v1.5.0...v1.5.1 + +[1.5.0]: https://github.com/flupkede/codesearch/compare/v1.4.9...v1.5.0 + [1.4.9]: https://github.com/flupkede/codesearch/compare/v1.4.4...v1.4.9 [1.0.171]: https://github.com/flupkede/codesearch/compare/v1.0.162...v1.0.171 [1.0.162]: https://github.com/flupkede/codesearch/compare/v1.0.160...v1.0.162 diff --git a/Cargo.lock b/Cargo.lock index 9c85609b..45d373fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,11 +181,11 @@ dependencies = [ "nohash", "ordered-float 5.5.0", "page_size", - "rand 0.10.2", + "rand 0.10.3", "rayon", "roaring", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tracing", ] @@ -245,7 +245,7 @@ dependencies = [ "num-traits", "pastey 0.1.1", "rayon", - "thiserror 2.0.20", + "thiserror 2.0.21", "v_frame", "y4m", ] @@ -713,7 +713,7 @@ dependencies = [ [[package]] name = "codesearch" -version = "1.4.11" +version = "1.5.2" dependencies = [ "anyhow", "arroy", @@ -743,7 +743,7 @@ dependencies = [ "ort", "pretty_assertions", "protobuf", - "rand 0.10.2", + "rand 0.10.3", "ratatui", "rayon", "regex", @@ -758,7 +758,7 @@ dependencies = [ "sysinfo 0.39.6", "tantivy", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -1600,9 +1600,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastembed" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c88b42d8347afbf09e3d2fac61fcc22b698b7b2f35a27a6e988bcdad5784b07" +checksum = "3850d724d559da51a4b88f68b0ba111a2627c32279bec6b3c256f073c71a3195" dependencies = [ "hf-hub 0.5.0", "image", @@ -1611,7 +1611,7 @@ dependencies = [ "safetensors", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokenizers", ] @@ -2080,7 +2080,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "ureq", "windows-sys 0.61.2", ] @@ -2105,7 +2105,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio-util", @@ -2127,7 +2127,7 @@ dependencies = [ "more-asserts", "serde", "serde_json", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -2638,7 +2638,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.21", "walkdir", "windows-link", ] @@ -2704,7 +2704,7 @@ checksum = "bde5057d6143cc94e861d90f591b9303d6716c6b9602309150bd068853c10899" dependencies = [ "hashbrown 0.16.1", "portable-atomic", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -2770,9 +2770,9 @@ checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libfuzzer-sys" @@ -4064,7 +4064,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -4080,14 +4080,14 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -4149,9 +4149,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -4230,7 +4230,7 @@ dependencies = [ "palette", "serde", "strum", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-segmentation", "unicode-truncate", "unicode-width", @@ -4329,7 +4329,7 @@ dependencies = [ "rand 0.9.5", "rand_chacha", "simd_helpers", - "thiserror 2.0.20", + "thiserror 2.0.21", "v_frame", "wasm-bindgen", ] @@ -4427,7 +4427,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -4577,7 +4577,7 @@ dependencies = [ "async-trait", "http", "reqwest 0.13.5", - "thiserror 2.0.20", + "thiserror 2.0.21", "tower-service", ] @@ -4603,9 +4603,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23c62fe489ac1d401ab32688cfacac3737a8978dc3343e5361464c7724fd3cb" +checksum = "fae7019994ae0fe4ada40b732f798f3ff26f0f04facb1477f1bf37eb4f18a2d3" dependencies = [ "async-trait", "base64 0.23.1", @@ -4618,14 +4618,14 @@ dependencies = [ "indexmap", "pastey 0.2.3", "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "rmcp-macros", "schemars", "serde", "serde_json", "sse-stream", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -4636,9 +4636,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd740c45d66ceb87e5579082abc27bd771665e464e9660a17a048c721b2a6025" +checksum = "c7e66877136e969d5a2d561ee5841b17717a0619aebd64abb47a2a2f46e34683" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -5419,7 +5419,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "typetag", "uuid", @@ -5621,11 +5621,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -5641,9 +5641,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", @@ -5758,7 +5758,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 2.0.20", + "thiserror 2.0.21", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -5809,7 +5809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "tokio", ] @@ -5939,7 +5939,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.20", + "thiserror 2.0.21", "time", "tracing-subscriber", ] @@ -6398,9 +6398,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "atomic", "getrandom 0.4.3", @@ -7053,7 +7053,7 @@ dependencies = [ "http", "hyper", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "redb", "reqwest 0.13.5", "reqwest-middleware", @@ -7062,7 +7062,7 @@ dependencies = [ "serde_repr", "statrs", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-retry", "tokio_with_wasm", @@ -7093,12 +7093,12 @@ dependencies = [ "itertools 0.14.0", "lz4_flex", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "regex", "safe-transmute", "serde", "static_assertions", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -7121,12 +7121,12 @@ dependencies = [ "http", "itertools 0.14.0", "more-asserts", - "rand 0.10.2", + "rand 0.10.3", "serde", "serde_json", "sha2 0.11.0", "tempfile", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", @@ -7161,13 +7161,13 @@ dependencies = [ "more-asserts", "oneshot", "pin-project", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "serde", "serde_json", "shellexpand", "sysinfo 0.38.4", - "thiserror 2.0.20", + "thiserror 2.0.21", "tokio", "tokio-util", "tokio_with_wasm", diff --git a/Cargo.toml b/Cargo.toml index a7a732b9..620ed1fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codesearch" -version = "1.4.11" +version = "1.5.2" edition = "2021" authors = ["codesearch contributors"] license = "Apache-2.0" diff --git a/README.md b/README.md index a09b30db..2398b4a1 100644 --- a/README.md +++ b/README.md @@ -233,7 +233,7 @@ If your agent skips codesearch and falls back to grep/glob too often, paste this > Prefer codesearch for semantic, cross-file, or symbol-oriented lookup ("where is X implemented", "find usages of Y", "how does Z flow"). Use plain grep/glob for a single known file, trivial one-line edits, or exact literal searches. In remote-serve mode, returned paths are from the **server's** filesystem — read content via `get_chunk` rather than opening paths locally, and unindexed dirs (`.venv`, `node_modules`, `build/`) simply return nothing. -OpenCode: put this in the user-level `~/.config/opencode/AGENTS.md` (applies across all projects). Claude Code reads a project-level `AGENTS.md`, so add it per-project (or symlink a shared one). +OpenCode: put this in the user-level `~/.config/opencode/AGENTS.md` (applies across all projects), or install the [OpenCode plugin](integrations/opencode/README.md), which injects this guidance with the resolved project scope automatically. Claude Code reads a project-level `AGENTS.md`, so add it per-project (or symlink a shared one). **Claude Code specifically** tends to ignore this advice more than other clients — its MCP tool schemas are deferred (an extra `ToolSearch` call is needed before codesearch tools are even callable), while Grep/Glob are always fully loaded and zero-friction, and spawned subagents don't inherit `AGENTS.md` or the MCP `initialize` instructions at all. @@ -253,7 +253,9 @@ codesearch hooks claude install --project # project scope (./.claude) The native command embeds the hook scripts in the binary (no source tree needed) and merges the registrations into `settings.json`. The equivalent from-source installers still live in [`integrations/claude-code/`](integrations/claude-code/) (`install.ps1` / `install.sh`) if you'd rather run them directly. -Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` override) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. +**OpenCode** gets the same structural treatment through the OpenCode v2 plugin in [`integrations/opencode/`](integrations/opencode/README.md): it resolves the session directory to an indexed project, injects scope-aware guidance once per session, rescues empty greps with index hits, optionally prunes or blocks `grep`/`glob`, keeps the serve hub healthy (with optional auto-start), and adds `/codesearch*` commands, a `codesearch_scope` tool, a skill and compaction assistance. Install by copying `codesearch.ts` into `~/.config/opencode/plugins/` — see the [integration README](integrations/opencode/README.md). + +Note: the grep-guard detects "codesearch is available **for this repo**" via that repo's registration with the serve hub (`~/.codesearch/repos.json`, honoring the `CODESEARCH_REPOS_CONFIG` / `CODESEARCH_HOME` overrides) or `CODESEARCH_SERVER` — **not** by checking whether a `codesearch` process is running (that runs almost constantly as a multi-repo hub and would false-fire in every directory), and **not** via a local `.codesearch.db` directory (a stale db from a since-unregistered repo used to deny Grep even though the hub could not answer for it). For a remote-serve setup with no local registration, set `CODESEARCH_SERVER` to opt back into enforcement. ## MCP Tools Reference @@ -444,12 +446,12 @@ When using `git worktree add` to create parallel working directories, codesearch codesearch hooks git install ``` -This installs a `post-checkout` hook that POSTs the worktree path to the running serve instance whenever a new worktree is checked out. The hook reads the serve URL from `~/.codesearch/serve_url` (automatically managed by `codesearch serve`). +This installs a `post-checkout` hook that POSTs the worktree path to the running serve instance whenever a new worktree is checked out. The hook reads the serve URL from `~/.codesearch/serve_url` (`$CODESEARCH_HOME/serve_url` when the global root is relocated; automatically managed by `codesearch serve`). The install target is resolved with `git rev-parse --git-path hooks`, so it honours `core.hooksPath` (and, inside a linked worktree, the shared common-dir hooks) rather than assuming `.git/hooks/`. An existing `post-checkout` is not overwritten — codesearch's logic is chained in as a marker-delimited block. **How it works:** -1. `codesearch serve` writes its URL to `~/.codesearch/serve_url` on startup (deletes on shutdown) +1. `codesearch serve` writes its URL to `~/.codesearch/serve_url` (`$CODESEARCH_HOME/serve_url` when relocated) on startup (deletes on shutdown) 2. The `post-checkout` hook reads that file and POSTs the working directory to `POST /repos` 3. Serve registers the worktree path and begins indexing (deduped — won't re-register existing paths) @@ -573,15 +575,63 @@ In the `codesearch serve` TUI, mounts appear in **italic/cyan**, distinguishing | `CODESEARCH_ALLOWED_HOSTS` | Comma-separated hostname allowlist for the MCP streamable-HTTP transport (unset = loopback only: `localhost`, `127.0.0.1`, `::1`). Set this to your container/service hostname when serve runs behind a container network or reverse proxy — see [Security](#security). | | `CODESEARCH_DISABLE_HOST_VALIDATION` | `1`/`true` disables the MCP transport's Host-header allowlist entirely (DNS-rebinding protection off). Only safe behind a reverse proxy/firewall that already restricts inbound Host headers — see [Security](#security). | | `CODESEARCH_MCP_MODE` | MCP mode: auto, client, local | +| `CODESEARCH_HOME` | Relocate the entire global root (`repos.json`, models cache, logs, `serve_url`, global ignore/extension files) away from `~/.codesearch`. Must be an absolute path; unset = `~/.codesearch`. Per-file overrides like `CODESEARCH_REPOS_CONFIG` still win | | `CODESEARCH_REPOS_CONFIG` | Path to repos.json | | `CODESEARCH_REPO_IDLE_TIMEOUT_SECS` | Idle eviction timeout (default: 1800) | | `CODESEARCH_CACHE_MAX_MEMORY` | Embedding cache MB (default: 500) | | `CODESEARCH_MAX_LMDB_MAP_SIZE_MB` | Hard cap (MB) for LMDB auto-resize on `MDB_MAP_FULL`, applied to both the vector store and the persistent embedding cache (default: 16384 = 16GB; clamped to at least 1024). Raise this for very large corpora (millions of chunks) that legitimately exceed the default cap — see #189. | | `CODESEARCH_BATCH_SIZE` | Embedding batch size | +| `CODESEARCH_INCREMENTAL_BATCH_SIZE` | Files read/chunked per incremental-refresh batch (default: 200) | +| `CODESEARCH_INDEX_JOBS` | Max concurrent heavy repo jobs — refresh / reindex / HNSW build — in one serve process (default: 1; queries are never gated). See [Indexing limits](#indexing-limits) | +| `CODESEARCH_EMBED_THREADS` | ONNX intra-op threads per embedding session (unset = ONNX default, all cores) | +| `CODESEARCH_EMBED_PAUSE_MS` | Pause (ms) between background embedding mini-batches, a duty cycle for shared machines (default: 0) | +| `CODESEARCH_MIN_FREE_MB` | Free-memory floor (MB) a heavy job waits for (max 30 s, then proceeds with a warning) before starting (default: 0 = disabled) | +| `CODESEARCH_MAX_CHUNKS_PER_BATCH` | Hard cap on chunks held in memory per refresh batch; splits the batch early (default: 0 = disabled) | | `CODESEARCH_SCIP_CSHARP` | Override path to `scip-csharp` helper | | `CODESEARCH_EXTENSION_MAP` | Path to the extension→language map (default: `~/.codesearch/extensions.json`) — see [Extension map](#extension-map) | | `RUST_LOG` | Log level (e.g. `codesearch=debug`) | +### Indexing limits + +A serve hub with many repos used to start one full refresh/reindex/HNSW-build +pass per repo as soon as that repo had work. A mass `git pull`, a burst of +first queries, or several `POST /repos/:alias/reindex` calls therefore ran +many CPU-heavy passes at once, held many file/chunk windows in memory, and +queued the (already serialised) ONNX inference behind all of them. These knobs +bound that work; they are read at startup, so a shell or systemd unit can set +them without editing anything: + +| Knob | Default | Effect | +|------|---------|--------| +| `CODESEARCH_INDEX_JOBS` | `1` | Max repos doing a refresh / reindex / HNSW build at once. Queries never take this slot. | +| `CODESEARCH_EMBED_THREADS` | unset (all cores) | ONNX intra-op threads per embedding session. | +| `CODESEARCH_INCREMENTAL_BATCH_SIZE` | `200` | Files read/chunked per refresh batch. | +| `CODESEARCH_MAX_CHUNKS_PER_BATCH` | `0` (off) | Split a batch early once it reaches this many chunks (a few huge files can produce more memory than many small ones). | +| `CODESEARCH_EMBED_PAUSE_MS` | `0` | Sleep this long between background embedding mini-batches (duty cycle). | +| `CODESEARCH_MIN_FREE_MB` | `0` (off) | A job waits (bounded to 30 s, then proceeds with a warning) until this much memory is free. | + +`codesearch serve` also accepts `--index-jobs`, `--embed-threads`, +`--embed-pause-ms` and `--min-free-mb`, which set the corresponding variable +for that process (flags win over the environment). Standalone `codesearch +index` and `codesearch mcp` runs have no job gate and keep their previous +behaviour; the embedding and batching variables still apply to them. + +Example for a large shared machine (128 cores, ~30 repos, dozens of repos +indexed at once are the failure mode): + +```ini +# ~/.config/systemd/user/codesearch.service.d/limits.conf +[Service] +Environment=CODESEARCH_INDEX_JOBS=1 +Environment=CODESEARCH_EMBED_THREADS=8 +Environment=CODESEARCH_INCREMENTAL_BATCH_SIZE=100 +Environment=CODESEARCH_MAX_CHUNKS_PER_BATCH=5000 +Environment=CODESEARCH_EMBED_PAUSE_MS=25 +Environment=CODESEARCH_MIN_FREE_MB=4096 +# C# SCIP helpers are bounded separately; lower it for the same reason. +Environment=CSHARP_SCIP_CONCURRENCY=1 +``` + ### `.codesearchignore` Place in repo root. Gitignore syntax. Excludes paths from indexing: diff --git a/build.ps1 b/build.ps1 index 50acac0c..3e1fb376 100644 --- a/build.ps1 +++ b/build.ps1 @@ -4,7 +4,7 @@ Builds CodeSearch. .DESCRIPTION - Always runs `cargo build` (debug or release). Cargo's own incremental + Always runs `cargo build` (debug). Cargo's own incremental compilation decides what actually needs to be recompiled — this script no longer tries to second-guess it with git-diff heuristics. @@ -18,11 +18,7 @@ .EXAMPLE .\build.ps1 - Builds in debug mode - -.EXAMPLE - .\build.ps1 -Release - Builds in release mode + Builds in debug mode (the only supported mode) #> param( @@ -31,6 +27,10 @@ param( $ErrorActionPreference = "Stop" +if ($Release) { + throw "-Release is not supported (debug-only policy, issue #192). Release artifacts are built by CI (.github/workflows/release.yml)." +} + # Change to script directory (where Cargo.toml is located) $ScriptDir = $PSScriptRoot Set-Location $ScriptDir @@ -67,8 +67,8 @@ if ($orphans.Count -gt 0) { Write-Host " [warn] if this build hangs on 'Blocking waiting for file lock', terminate the stale process manually." -ForegroundColor Yellow } -# Determine build mode -$BuildMode = if ($Release) { "release" } else { "debug" } +# Debug only (see -Release guard above) +$BuildMode = "debug" Write-Host "Building in $BuildMode mode..." -ForegroundColor Yellow @@ -81,11 +81,7 @@ $TmpDir = Join-Path $ScriptDir ".tmp" New-Item -ItemType Directory -Force -Path $TmpDir | Out-Null $LogFile = Join-Path $TmpDir "build-$BuildMode.log" -if ($Release) { - & cargo build --release 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 -} else { - & cargo build 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 -} +& cargo build 2>&1 | Out-File -FilePath $LogFile -Encoding utf8 $BuildExit = $LASTEXITCODE Get-Content $LogFile | ForEach-Object { Write-Host $_ } diff --git a/helpers/csharp/Program.cs b/helpers/csharp/Program.cs index e8539efc..bc0e3deb 100644 --- a/helpers/csharp/Program.cs +++ b/helpers/csharp/Program.cs @@ -1,5 +1,7 @@ +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Locator; +using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.MSBuild; namespace ScipCsharp; @@ -309,18 +311,7 @@ await Console.Error.WriteLineAsync( /// private static MSBuildWorkspace CreateTolerantWorkspace() { - var properties = new Dictionary - { - // Tell Roslyn to skip projects it cannot load instead of crashing. - { "BuildingInsideVisualStudio", "true" }, - // Design-time build: prevents auto-generated files in obj/ (e.g. - // .AssemblyAttributes.cs, .AssemblyInfo.cs) from being included as - // explicit Compile items. Without this, SDK-style projects produce - // duplicate Compile items (auto-include + obj/ generated), which - // causes MSBuildWorkspace to fail loading the project. - { "DesignTimeBuild", "true" }, - { "SkipCompilerExecution", "true" }, - }; + var properties = WorkspaceLoad.GlobalProperties(); // If MSBUILD_EXE_PATH is set, pass it to the workspace so the BuildHost // uses the same MSBuild we registered via MSBuildLocator (typically the .NET SDK). @@ -334,10 +325,92 @@ private static MSBuildWorkspace CreateTolerantWorkspace() var workspace = MSBuildWorkspace.Create(properties); workspace.WorkspaceFailed += (_, e) => + { + // Warning-severity diagnostics are benign resolution notes (e.g. + // "Found project reference without a matching metadata reference" + // for excluded or external projects): they do not block indexing + // and would pin the repo's index warning forever. Only true + // failures belong in the warning channel. + if (e.Diagnostic.Kind == WorkspaceDiagnosticKind.Warning) + { + Console.Error.WriteLine($"[INFO] Workspace note: {e.Diagnostic}"); + return; + } + // NuGet's vulnerability audit surfaces as an Msbuild "failure" but + // is a security advisory, not a load failure — the index is + // unaffected and emitting it would pin the repo's index warning + // until the package is bumped. + if (WorkspaceNoise.IsNuGetAuditNoise(e.Diagnostic?.Message)) + { + return; + } Console.Error.WriteLine($"[WARN] Workspace error: {e.Diagnostic}"); + }; return workspace; } + /// + /// Restores the solution before the design-time load. MSBuildWorkspace + /// does not restore: with stale or missing obj/project.assets.json, + /// package-typed references fail to resolve and the load drowns in CS0246 + /// cascades even though `dotnet build` (which restores first) succeeds. + /// Gated by ; best effort: a + /// failed or missing `dotnet` is logged and the load proceeds — degraded + /// indexing beats no indexing. The wait is bounded, but the restore is + /// never killed: on timeout it continues in the background and the next + /// reindex picks up the fresh assets. + /// + private static void RestoreSolution(string solutionPath) + { + var timeoutSeconds = RestorePolicy.RestoreTimeoutSeconds(); + try + { + var psi = new ProcessStartInfo + { + FileName = "dotnet", + // --ignore-failed-sources: an unreachable private feed must not + // block indexing when the packages are already in the cache. + Arguments = $"restore \"{solutionPath}\" --nologo -v q --ignore-failed-sources", + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + }; + using var process = Process.Start(psi); + if (process is null) + { + return; + } + // Drain both pipes in the background: the child inherits these + // handles, so without readers a chatty restore could block on a + // full pipe, and the serve process reads stderr until EOF. + _ = process.StandardOutput.ReadToEndAsync(); + _ = process.StandardError.ReadToEndAsync(); + // Bounded WAIT, never a kill: a killed restore wastes all of its + // work and leaves assets missing, so every reindex would pay the + // same wait and stay degraded. On timeout the restore keeps + // running in the background (the child outlives this process); + // this load proceeds degraded and the next reindex finds the + // fresh assets and loads clean. Timeout is env-tunable for large + // solutions on slow feeds. + if (!process.WaitForExit(timeoutSeconds * 1000)) + { + Console.Error.WriteLine(RestorePolicy.StillRunningWarning(timeoutSeconds)); + return; + } + if (process.ExitCode != 0) + { + Console.Error.WriteLine( + $"[WARN] dotnet restore exited {process.ExitCode} for '{solutionPath}' — continuing; " + + "package references may be unresolved (degraded symbols)."); + } + } + catch (Exception ex) + { + Console.Error.WriteLine( + $"[WARN] dotnet restore could not start ({ex.GetType().Name}: {ex.Message}) — continuing without it."); + } + } + // ── Solution filtering ─────────────────────────────────────────── /// @@ -364,6 +437,16 @@ private static async Task OpenSolutionFilteredAsync(MSBuildWorkspace workspace, .Where(p => UnsupportedProjectExtensions.Contains(Path.GetExtension(p.RelativePath))) .ToList(); + // Restore only when the evidence says assets are missing or stale: the + // design-time build below does not restore, and missing package assets + // cascade into CS0246 noise that would otherwise pin the repo's index + // warning. But restoring on every reindex is far too heavy for + // multi-repo hubs — fresh assets mean no restore at all. + if (RestorePolicy.NeedsRestore(supported.Select(p => p.RelativePath), solutionDir)) + { + RestoreSolution(solutionPath); + } + // Log skipped projects foreach (var skip in unsupported) { diff --git a/helpers/csharp/RestorePolicy.cs b/helpers/csharp/RestorePolicy.cs new file mode 100644 index 00000000..88bf83f4 --- /dev/null +++ b/helpers/csharp/RestorePolicy.cs @@ -0,0 +1,139 @@ +using System.Diagnostics; + +namespace ScipCsharp; + +/// +/// Restore policy for the design-time workspace load: MSBuildWorkspace does +/// not restore, but running dotnet restore on every reindex is far too +/// heavy for multi-repo hubs. Restore only when the evidence says assets are +/// missing or stale. Compiled into both scip-csharp and the test project via +/// a linked Compile item (same pattern as WorkspaceLoad.cs). +/// +internal static class RestorePolicy +{ + /// Env override for the restore kill timeout, in seconds. + public const string TimeoutEnvName = "CODESEARCH_RESTORE_TIMEOUT_SECS"; + private const string RestoreTimeoutEnv = TimeoutEnvName; + private const int DefaultRestoreTimeoutSeconds = 300; + + /// + /// True when at least one project lacks obj/project.assets.json, has + /// one older than its .csproj (a package reference or target framework was + /// edited since the last restore), or has one older than an applicable + /// shared restore input (see ). False + /// means every project has fresh assets and restore can be skipped entirely + /// — the steady state of an actively developed repo. + /// + public static bool NeedsRestore(IEnumerable projectRelativePaths, string solutionDir) + { + foreach (var relative in projectRelativePaths) + { + var projectPath = Path.GetFullPath(Path.Combine(solutionDir, relative)); + var projectDir = Path.GetDirectoryName(projectPath) ?? solutionDir; + var assetsPath = Path.Combine(projectDir, "obj", "project.assets.json"); + if (!File.Exists(assetsPath)) + { + return true; + } + var assetsWritten = File.GetLastWriteTimeUtc(assetsPath); + if (File.GetLastWriteTimeUtc(projectPath) > assetsWritten) + { + return true; + } + if (AnySharedInputNewerThan(projectDir, assetsWritten)) + { + return true; + } + } + return false; + } + + /// + /// True when a shared restore input that applies to + /// was modified after the assets were written. + /// MSBuild/NuGet discover these by walking up from the project directory, + /// and editing one changes the resolved dependency graph without touching + /// the .csproj — under central package management or imported build config + /// a .csproj-only comparison would treat the assets as fresh, skip restore, + /// and load a stale asset graph. Absent files never trigger a restore. + /// + private static bool AnySharedInputNewerThan(string projectDir, DateTime assetsWritten) + { + // Per-project lock file sits beside the .csproj. + if (IsNewerThan(Path.Combine(projectDir, PackagesLockName), assetsWritten)) + { + return true; + } + + // Directory.Build.props / Directory.Packages.props: MSBuild and NuGet + // import the NEAREST one found walking up, so once a name is matched a + // higher directory can no longer supply it. + var nearestPending = new HashSet(NearestInputNames, StringComparer.OrdinalIgnoreCase); + + for (var dir = new DirectoryInfo(projectDir); dir is not null; dir = dir.Parent) + { + // NuGet.config files are merged from every ancestor directory. + if (IsNewerThan(Path.Combine(dir.FullName, NuGetConfigName), assetsWritten)) + { + return true; + } + + foreach (var name in NearestInputNames) + { + if (!nearestPending.Contains(name) || !File.Exists(Path.Combine(dir.FullName, name))) + { + continue; + } + nearestPending.Remove(name); + if (IsNewerThan(Path.Combine(dir.FullName, name), assetsWritten)) + { + return true; + } + } + } + + return false; + } + + private static bool IsNewerThan(string path, DateTime reference) => + File.Exists(path) && File.GetLastWriteTimeUtc(path) > reference; + + private const string NuGetConfigName = "NuGet.config"; + private const string PackagesLockName = "packages.lock.json"; + + private static readonly string[] NearestInputNames = + { + "Directory.Packages.props", + "Directory.Build.props", + }; + + /// + /// Restore wait timeout in seconds: CODESEARCH_RESTORE_TIMEOUT_SECS + /// when set to a positive integer, otherwise 300. Unparseable values fall + /// back to the default rather than failing the load. + /// + public static int RestoreTimeoutSeconds() + { + var raw = Environment.GetEnvironmentVariable(RestoreTimeoutEnv); + if (int.TryParse(raw, out var seconds) && seconds > 0) + { + return seconds; + } + return DefaultRestoreTimeoutSeconds; + } + + /// + /// The [WARN] line emitted when the bounded wait expires while + /// dotnet restore is still running. The restore is NOT killed: it + /// keeps running in the background (the child outlives the helper), this + /// load proceeds degraded, and the next reindex finds the fresh assets — + /// a killed restore would waste all of its work and re-pay the same wait + /// on every subsequent reindex. Pinned by test so a "killed" wording + /// cannot sneak back in. + /// + public static string StillRunningWarning(int seconds) => + $"[WARN] dotnet restore still running after {seconds}s — proceeding without it " + + "(degraded symbols this run); it keeps running in the background and the next " + + "reindex picks up the fresh assets. " + + $"Raise with {TimeoutEnvName} if this solution legitimately needs longer."; +} diff --git a/helpers/csharp/SymbolIndexer.cs b/helpers/csharp/SymbolIndexer.cs index 8a49e4e9..1a34b2df 100644 --- a/helpers/csharp/SymbolIndexer.cs +++ b/helpers/csharp/SymbolIndexer.cs @@ -48,6 +48,10 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj Console.Error.WriteLine($"Compiling {totalProjects} project(s)..."); var compileSw = System.Diagnostics.Stopwatch.StartNew(); + // (error count, summary line) per erroring project — emitted after + // the loop, loudest first, so the warnings-channel cap can never + // crowd out the worst project: that is the one the user needs named. + var erroringProjects = new List<(int Count, string Line)>(); for (int i = 0; i < totalProjects; i++) { var project = projectList[i]; @@ -59,17 +63,31 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj continue; } - // Report diagnostics but don't abort + // Report diagnostics but don't abort. A single broken reference + // resolution cascades into tens of thousands of errors per + // project (CS0518 "System.Object not defined" turns every type + // use into CS0246) — that is ONE root cause, not N problems, so + // the warnings channel carries ONE aggregated line per project + // (count + dominant code + first specimen). Symbols below are + // still collected: a Roslyn compilation with errors yields a + // complete symbol tree — the same reason IntelliSense works + // while the build is red. var diagnostics = compilation.GetDiagnostics() - .Where(d => d.Severity == DiagnosticSeverity.Error); - foreach (var diag in diagnostics) + .Where(d => d.Severity == DiagnosticSeverity.Error) + .ToList(); + if (diagnostics.Count > 0) { - Console.Error.WriteLine($"[WARN] Compilation error in {project.Name}: {diag}"); + erroringProjects.Add((diagnostics.Count, + WarningLineFor(project.Name, diagnostics))); } CollectSymbols(compilation.GlobalNamespace, symbolMap); } compileSw.Stop(); + foreach (var (_, line) in erroringProjects.OrderByDescending(e => e.Count)) + { + Console.Error.WriteLine(line); + } Console.Error.WriteLine($"Compiled {totalProjects} project(s) in {compileSw.Elapsed.TotalSeconds:F1}s"); Console.Error.WriteLine($"Collected {symbolMap.Count} project-internal symbols — building definition index..."); @@ -130,6 +148,32 @@ public async Task IndexAsync(MSBuildWorkspace workspace, string? proj return index; } + /// One-line aggregation of a project's compilation errors for the + /// index-level warnings channel: count, dominant error code, first + /// specimen (capped — the full diagnostic can run very long). Wrapped + /// with the [WARN] marker the Rust capture keys on by + /// . One broken reference resolution + /// cascades into tens of thousands of diagnostics; the channel must + /// carry the root cause, not the cascade. + /// + internal static string CompilationErrorSummaryLine( + string project, IReadOnlyList errors) + { + var byCode = errors.GroupBy(d => d.Id) + .OrderByDescending(g => g.Count()).ToList(); + var first = errors[0].ToString(); + if (first.Length > 160) first = first[..160] + "…"; + return + $"Compilation errors in {project}: {errors.Count} error(s) — dominant " + + $"{byCode[0].Key} x{byCode[0].Count()}; first: {first}"; + } + + /// The channel-emittable form. The `[WARN] ` prefix is LOAD-BEARING: + /// the Rust capture (`is_helper_warning_line`) keys on it, so a dropped + /// prefix does not flood the channel — it silently empties it. + internal static string WarningLineFor(string project, IReadOnlyList errors) => + "[WARN] " + CompilationErrorSummaryLine(project, errors); + internal static void CollectSymbols(INamespaceSymbol ns, Dictionary map) { foreach (var child in ns.GetMembers()) diff --git a/helpers/csharp/WorkspaceLoad.cs b/helpers/csharp/WorkspaceLoad.cs new file mode 100644 index 00000000..e22dc096 --- /dev/null +++ b/helpers/csharp/WorkspaceLoad.cs @@ -0,0 +1,32 @@ +namespace ScipCsharp; + +/// +/// Global MSBuild properties for every workspace load. Non-negotiables: +/// DesignTimeBuild + SkipCompilerExecution and NO warnings-as-errors — a +/// repo that escalates warnings (own settings or Directory.Build.props) +/// would turn benign load-time diagnostics (the NU1603 version-unification +/// class) into MSBuild project-load failures, dropping the project — and +/// every cross-project reference to it — from the index. Everything else +/// about the project keeps its own settings; we only refuse the escalation. +/// Lives outside Program.cs so the test assembly can compile it in +/// (Program carries the Main entrypoint and is deliberately excluded there). +/// +internal static class WorkspaceLoad +{ + internal static System.Collections.Generic.Dictionary GlobalProperties() => new() + { + // Tell Roslyn to skip projects it cannot load instead of crashing. + { "BuildingInsideVisualStudio", "true" }, + // Design-time build: prevents auto-generated files in obj/ (e.g. + // .AssemblyAttributes.cs, .AssemblyInfo.cs) from being included as + // explicit Compile items. Without this, SDK-style projects produce + // duplicate Compile items (auto-include + obj/ generated), which + // causes MSBuildWorkspace to fail loading the project. + { "DesignTimeBuild", "true" }, + { "SkipCompilerExecution", "true" }, + // Warnings stay warnings during the load — never fatal (NU1603 etc.). + { "TreatWarningsAsErrors", "false" }, + { "WarningsAsErrors", "" }, + { "MSBuildTreatWarningsAsErrors", "false" }, + }; +} diff --git a/helpers/csharp/WorkspaceNoise.cs b/helpers/csharp/WorkspaceNoise.cs new file mode 100644 index 00000000..54160f68 --- /dev/null +++ b/helpers/csharp/WorkspaceNoise.cs @@ -0,0 +1,21 @@ +namespace ScipCsharp; + +/// +/// Classifies workspace-load noise that must not surface as index warnings. +/// Compiled into both scip-csharp and the test project via a linked Compile +/// item (same pattern as WorkspaceLoad.cs). +/// +internal static class WorkspaceNoise +{ + /// + /// True when a workspace-failure diagnostic is a NuGet audit advisory + /// ("Package 'X' has a known high severity vulnerability, ...") rather + /// than a load failure — emitting it would pin the repo's index warning + /// until the package is bumped, while the index itself is unaffected. + /// Pure and null-tolerant so it is unit-testable. + /// + public static bool IsNuGetAuditNoise(string? message) => + message is not null + && message.Contains("has a known", StringComparison.OrdinalIgnoreCase) + && message.Contains("vulnerability", StringComparison.OrdinalIgnoreCase); +} diff --git a/helpers/csharp/tests/IndexerTests.cs b/helpers/csharp/tests/IndexerTests.cs index fa4e93bb..b7a5cc52 100644 --- a/helpers/csharp/tests/IndexerTests.cs +++ b/helpers/csharp/tests/IndexerTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Microsoft.CodeAnalysis; using Xunit; using ScipCsharp; @@ -214,4 +215,58 @@ public void OutputWriter_ProducesValidJson() // Verify indented output (has newlines) Assert.Contains('\n', json); } + + /// + /// A broken reference resolution cascades into tens of thousands of + /// compilation errors per project (one missing assembly → every type use + /// errors). The warnings channel must carry ONE aggregated line per + /// project — count + dominant code + first specimen — not the cascade. + /// + [Fact] + public void CompilationErrorSummary_AggregatesCascadePerProject() + { + static Diagnostic Err(string id, string message) + { + var descriptor = new DiagnosticDescriptor( + id, title: message, messageFormat: message, category: "Test", + DiagnosticSeverity.Error, isEnabledByDefault: true); + return Diagnostic.Create(descriptor, Location.None); + } + + var errors = new List + { + Err("CS0518", "Predefined type 'System.Object' is not defined or referenced"), + Err("CS0518", "Predefined type 'System.Object' is not defined or referenced"), + Err("CS0246", "The type or namespace name 'Dam' could not be found"), + }; + + var line = SymbolIndexer.CompilationErrorSummaryLine("Acme.Catalog", errors); + + Assert.StartsWith("Compilation errors in Acme.Catalog:", line); + Assert.Contains("3 error(s)", line); + Assert.Contains("dominant CS0518 x2", line); + Assert.Contains("first:", line); + + // The channel-emittable form carries the [WARN] marker the Rust + // capture keys on — a dropped prefix would silently EMPTY the + // index-warnings channel instead of flooding it. + var warning = SymbolIndexer.WarningLineFor("Acme.Catalog", errors); + Assert.StartsWith("[WARN] Compilation errors in", warning); + } + + /// The specimen is capped: a full Roslyn diagnostic string can run + /// hundreds of chars, and the warnings channel caps the whole list. + [Fact] + public void CompilationErrorSummary_CapsTheSpecimen() + { + var descriptor = new DiagnosticDescriptor( + "CS0001", title: new string('x', 400), messageFormat: new string('x', 400), + category: "Test", DiagnosticSeverity.Error, isEnabledByDefault: true); + var err = Diagnostic.Create(descriptor, Location.None); + + var line = SymbolIndexer.CompilationErrorSummaryLine("Proj", [err]); + + Assert.True(line.Length < 300, $"summary must be capped, got {line.Length} chars"); + Assert.Contains("…", line); + } } diff --git a/helpers/csharp/tests/RestorePolicyTests.cs b/helpers/csharp/tests/RestorePolicyTests.cs new file mode 100644 index 00000000..0f988ba4 --- /dev/null +++ b/helpers/csharp/tests/RestorePolicyTests.cs @@ -0,0 +1,146 @@ +using ScipCsharp; +using Xunit; + +namespace ScipCsharp.Tests; + +/// +/// Unit tests for the restore gating policy: a repo whose projects all carry +/// fresh obj/project.assets.json must skip restore entirely (the steady state +/// of an actively developed repo), while a missing or stale assets file +/// triggers it. +/// +public class RestorePolicyTests : IDisposable +{ + private readonly string _solutionDir; + + public RestorePolicyTests() + { + _solutionDir = Path.Combine(Path.GetTempPath(), "cs-restore-policy-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_solutionDir); + } + + public void Dispose() + { + Directory.Delete(_solutionDir, recursive: true); + } + + private string AddProject(string relativePath, bool withAssets, bool staleAssets = false) + { + var projectPath = Path.Combine(_solutionDir, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(projectPath)!); + File.WriteAllText(projectPath, ""); + if (withAssets) + { + var objDir = Path.Combine(Path.GetDirectoryName(projectPath)!, "obj"); + Directory.CreateDirectory(objDir); + var assetsPath = Path.Combine(objDir, "project.assets.json"); + File.WriteAllText(assetsPath, "{}"); + if (staleAssets) + { + // Make the assets look OLDER than the project file. + File.SetLastWriteTimeUtc(assetsPath, File.GetLastWriteTimeUtc(projectPath).AddMinutes(-5)); + } + } + return projectPath; + } + + [Fact] + public void FreshAssetsOnEveryProjectSkipsRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + AddProject(@"src\Acme.Web\Acme.Web.csproj", withAssets: true); + + Assert.False(RestorePolicy.NeedsRestore( + new[] { @"src\Acme.Core\Acme.Core.csproj", @"src\Acme.Web\Acme.Web.csproj" }, _solutionDir)); + } + + [Fact] + public void MissingAssetsTriggersRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + AddProject(@"src\Acme.Web\Acme.Web.csproj", withAssets: false); + + Assert.True(RestorePolicy.NeedsRestore( + new[] { @"src\Acme.Core\Acme.Core.csproj", @"src\Acme.Web\Acme.Web.csproj" }, _solutionDir)); + } + + [Fact] + public void StaleAssetsOlderThanProjectTriggerRestore() + { + AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true, staleAssets: true); + + Assert.True(RestorePolicy.NeedsRestore(new[] { @"src\Acme.Core\Acme.Core.csproj" }, _solutionDir)); + } + + /// + /// Central package management / imported build config can change the + /// resolved dependency graph without touching any .csproj, so a shared + /// input newer than the assets must force a restore even when every .csproj + /// is older than its obj/project.assets.json. + /// + [Fact] + public void StaleSharedRestoreInputTriggersRestore() + { + var projectPath = AddProject(@"src\Acme.Core\Acme.Core.csproj", withAssets: true); + var assetsPath = Path.Combine(Path.GetDirectoryName(projectPath)!, "obj", "project.assets.json"); + + // A Directory.Packages.props at the solution root, newer than the assets. + var packagesProps = Path.Combine(_solutionDir, "Directory.Packages.props"); + File.WriteAllText(packagesProps, ""); + File.SetLastWriteTimeUtc(packagesProps, File.GetLastWriteTimeUtc(assetsPath).AddMinutes(5)); + + Assert.True(RestorePolicy.NeedsRestore(new[] { @"src\Acme.Core\Acme.Core.csproj" }, _solutionDir)); + } + + [Fact] + public void EmptyProjectListSkipsRestore() + { + Assert.False(RestorePolicy.NeedsRestore(Array.Empty(), _solutionDir)); + } + + [Fact] + public void RestoreTimeoutDefaultsTo300AndParsesEnvOverride() + { + Assert.Equal(300, RestorePolicy.RestoreTimeoutSeconds()); + + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, "600"); + try + { + Assert.Equal(600, RestorePolicy.RestoreTimeoutSeconds()); + } + finally + { + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, null); + } + + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, "not-a-number"); + try + { + Assert.Equal(300, RestorePolicy.RestoreTimeoutSeconds()); + } + finally + { + Environment.SetEnvironmentVariable(RestorePolicy.TimeoutEnvName, null); + } + } + + /// + /// Pins the timeout contract: the wait expiring must announce a restore + /// that is STILL RUNNING in the background, never a killed one — a kill + /// would waste all of the restore's work and re-pay the same wait on + /// every subsequent reindex (the poison loop for the biggest solutions). + /// + [Fact] + public void TimeoutWarningAnnouncesBackgroundContinuationNeverAKill() + { + var warning = RestorePolicy.StillRunningWarning(300); + + Assert.StartsWith("[WARN]", warning); + Assert.Contains("still running after 300s", warning); + Assert.Contains("background", warning); + Assert.Contains("next", warning); + Assert.Contains("fresh assets", warning); + Assert.DoesNotContain("killed", warning, StringComparison.OrdinalIgnoreCase); + Assert.Contains(RestorePolicy.TimeoutEnvName, warning); + } +} diff --git a/helpers/csharp/tests/ScipCsharp.Tests.csproj b/helpers/csharp/tests/ScipCsharp.Tests.csproj index 96712ece..7a57d219 100644 --- a/helpers/csharp/tests/ScipCsharp.Tests.csproj +++ b/helpers/csharp/tests/ScipCsharp.Tests.csproj @@ -28,6 +28,9 @@ + + + diff --git a/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs b/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs new file mode 100644 index 00000000..60cda4ff --- /dev/null +++ b/helpers/csharp/tests/WorkspaceLoadPropertiesTests.cs @@ -0,0 +1,35 @@ +using Xunit; +using ScipCsharp; + +namespace ScipCsharp.Tests; + +/// +/// Pins the workspace load properties. The contract that matters: the +/// symbol load NEVER escalates warnings to errors — a repo with +/// TreatWarningsAsErrors (own settings or Directory.Build.props) must still +/// load its projects, because benign load-time diagnostics (NU1603 class) +/// would otherwise drop the project and every cross-project reference to it. +/// Reverting any of the three demotion properties must fail this test. +/// +public class WorkspaceLoadPropertiesTests +{ + [Fact] + public void Load_NeverEscalatesWarningsToErrors() + { + var properties = WorkspaceLoad.GlobalProperties(); + + Assert.Equal("false", properties["TreatWarningsAsErrors"]); + Assert.Equal("", properties["WarningsAsErrors"]); + Assert.Equal("false", properties["MSBuildTreatWarningsAsErrors"]); + } + + [Fact] + public void Load_KeepsDesignTimeBuildContract() + { + var properties = WorkspaceLoad.GlobalProperties(); + + Assert.Equal("true", properties["BuildingInsideVisualStudio"]); + Assert.Equal("true", properties["DesignTimeBuild"]); + Assert.Equal("true", properties["SkipCompilerExecution"]); + } +} diff --git a/helpers/csharp/tests/WorkspaceNoiseTests.cs b/helpers/csharp/tests/WorkspaceNoiseTests.cs new file mode 100644 index 00000000..792fdb8e --- /dev/null +++ b/helpers/csharp/tests/WorkspaceNoiseTests.cs @@ -0,0 +1,47 @@ +using ScipCsharp; +using Xunit; + +namespace ScipCsharp.Tests; + +/// +/// Unit tests for the workspace-noise policy: NuGet audit advisories must not +/// surface as index warnings (they are security notes, not load failures), +/// while genuine Msbuild load failures still do. +/// +public class WorkspaceNoiseTests +{ + [Fact] + public void NuGetAuditAdvisoryIsFiltered() + { + var message = + "Msbuild failed when processing the file 'Acme.Core.csproj' with message: " + + "Package 'Microsoft.Extensions.Caching.Memory' 8.0.0 has a known high severity " + + "vulnerability, https://github.com/advisories/GHSA-qj66-m88j-hmgj"; + Assert.True(WorkspaceNoise.IsNuGetAuditNoise(message)); + } + + [Fact] + public void GenuineLoadFailureIsKept() + { + var message = + "Msbuild failed when processing the file 'Acme.Core.csproj' with message: " + + "The type or namespace name 'Catalog' does not exist in the namespace 'Acme'"; + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(message)); + } + + [Fact] + public void NullAndUnrelatedMessagesAreKept() + { + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(null)); + Assert.False(WorkspaceNoise.IsNuGetAuditNoise(string.Empty)); + Assert.False(WorkspaceNoise.IsNuGetAuditNoise("Invalid project file, unsupported project type")); + } + + [Theory] + [InlineData("has a KNOWN vulnerability")] + [InlineData("HAS A KNOWN VULNERABILITY")] + public void MatchingIsCaseInsensitive(string message) + { + Assert.True(WorkspaceNoise.IsNuGetAuditNoise(message)); + } +} diff --git a/integrations/claude-code/README.md b/integrations/claude-code/README.md index 1831f031..4e01bb3d 100644 --- a/integrations/claude-code/README.md +++ b/integrations/claude-code/README.md @@ -40,7 +40,8 @@ that make the preference *structural* instead of advisory: Windows-style roots, so POSIX absolute paths were still resolved against the cwd). Coverage is decided by **registration**: the target's git root must be one of the repos in the hub's `~/.codesearch/repos.json` - (honoring `CODESEARCH_REPOS_CONFIG`), which also carves out nested + (honoring `CODESEARCH_REPOS_CONFIG`; `CODESEARCH_HOME` relocates the + root), which also carves out nested repos for free — an unregistered clone inside a registered repo resolves to its own git root and is treated as uncovered. Grep is auto-allowed **only** when codesearch is genuinely @@ -73,7 +74,8 @@ that make the preference *structural* instead of advisory: everything else — making the caller-aware-editing protocol structural. Coverage uses the same registration model as grep-guard (shared `codesearch-common` helpers): the file's git root must be listed in - `~/.codesearch/repos.json` (or `CODESEARCH_SERVER` is set); unregistered + `~/.codesearch/repos.json` (relocatable via `CODESEARCH_HOME`; or + `CODESEARCH_SERVER` is set); unregistered repos and non-git paths are never blocked. - **`edit-guard-post`** — a `PostToolUse` hook on @@ -182,7 +184,7 @@ whose commands point at `hooks/codesearch/`) from `settings.json`, and delete **registration with the serve hub**: the target's git root must be one of the repos listed in `~/.codesearch/repos.json` (the same registration list the hub itself resolves queries by, honoring the - `CODESEARCH_REPOS_CONFIG` override), or an explicit `CODESEARCH_SERVER` + `CODESEARCH_REPOS_CONFIG` / `CODESEARCH_HOME` overrides), or an explicit `CODESEARCH_SERVER` env var for pure remote-serve setups with no local registration. A local `.codesearch.db` directory is deliberately **not** a coverage signal anymore (#199): a stale db from a since-unregistered repo used diff --git a/integrations/claude-code/hooks/codesearch-common.ps1 b/integrations/claude-code/hooks/codesearch-common.ps1 index 00c01d89..35b25501 100644 --- a/integrations/claude-code/hooks/codesearch-common.ps1 +++ b/integrations/claude-code/hooks/codesearch-common.ps1 @@ -5,12 +5,14 @@ # PowerShell twin of codesearch-common.sh — keep the two behaviorally # equivalent. Coverage model: a repo is covered when its git root is # REGISTERED in ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides -# the location), or when CODESEARCH_SERVER opts a pure remote-serve setup +# the location, CODESEARCH_HOME relocates the whole global root), or when +# CODESEARCH_SERVER opts a pure remote-serve setup # in. Everything here fails open: an unresolvable coverage question must # allow, never deny. function Get-CodesearchReposConfigFile { if ($env:CODESEARCH_REPOS_CONFIG) { return $env:CODESEARCH_REPOS_CONFIG } + if ($env:CODESEARCH_HOME) { return (Join-Path $env:CODESEARCH_HOME 'repos.json') } return (Join-Path $HOME '.codesearch/repos.json') } diff --git a/integrations/claude-code/hooks/codesearch-common.sh b/integrations/claude-code/hooks/codesearch-common.sh index a77e51ac..3ea79549 100644 --- a/integrations/claude-code/hooks/codesearch-common.sh +++ b/integrations/claude-code/hooks/codesearch-common.sh @@ -5,7 +5,8 @@ # # Coverage model (shared with the Rust serve hub, src/db_discovery/repos.rs): # a repo is covered when its git root is REGISTERED in -# ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides the location), +# ~/.codesearch/repos.json (CODESEARCH_REPOS_CONFIG overrides the location, +# CODESEARCH_HOME relocates the whole global root), # or when CODESEARCH_SERVER opts a pure remote-serve setup in. Everything # here fails open: an unresolvable coverage question must allow, never deny. # Requires: jq @@ -17,11 +18,14 @@ jq_str() { tr -d '\r' } -# repos.json location — mirrors src/db_discovery/repos.rs `config_path()`: -# CODESEARCH_REPOS_CONFIG override > ~/.codesearch/repos.json. +# repos.json location — mirrors src/db_discovery/repos.rs `config_path()` +# and constants.rs `codesearch_home()`: +# CODESEARCH_REPOS_CONFIG > $CODESEARCH_HOME/repos.json > ~/.codesearch/repos.json. repos_config_file() { if [ -n "${CODESEARCH_REPOS_CONFIG:-}" ]; then printf '%s' "$CODESEARCH_REPOS_CONFIG" + elif [ -n "${CODESEARCH_HOME:-}" ]; then + printf '%s' "${CODESEARCH_HOME%/}/repos.json" else printf '%s' "${HOME:-}/.codesearch/repos.json" fi diff --git a/integrations/claude-code/hooks/edit-guard.ps1 b/integrations/claude-code/hooks/edit-guard.ps1 index a45b0af6..066d5b9d 100644 --- a/integrations/claude-code/hooks/edit-guard.ps1 +++ b/integrations/claude-code/hooks/edit-guard.ps1 @@ -3,8 +3,8 @@ # two behaviorally equivalent. # # When the edited file's repo is codesearch-registered (git root listed in -# ~/.codesearch/repos.json, honoring CODESEARCH_REPOS_CONFIG, or a -# CODESEARCH_SERVER opt-in), every touched file needs a marker proving the +# ~/.codesearch/repos.json, honoring CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME, +# or a CODESEARCH_SERVER opt-in), every touched file needs a marker proving the # agent consulted codesearch for exactly that path within the last 5 # minutes: mcp__codesearch__find_impact for SCIP-backed languages # (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for diff --git a/integrations/claude-code/hooks/edit-guard.sh b/integrations/claude-code/hooks/edit-guard.sh index c37da6f5..45b8a027 100644 --- a/integrations/claude-code/hooks/edit-guard.sh +++ b/integrations/claude-code/hooks/edit-guard.sh @@ -5,7 +5,8 @@ # # When the edited file's repo is codesearch-registered (shared coverage model # with grep-guard: git root listed in ~/.codesearch/repos.json, honoring -# CODESEARCH_REPOS_CONFIG, or a CODESEARCH_SERVER opt-in), every touched file +# CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME, or a CODESEARCH_SERVER opt-in), +# every touched file # needs a marker proving the agent consulted codesearch for exactly that path # within the last 5 minutes: mcp__codesearch__find_impact for SCIP-backed # languages (.cs .ts .tsx .mts .cts), mcp__codesearch__find kind="usages" for diff --git a/integrations/claude-code/hooks/grep-guard.ps1 b/integrations/claude-code/hooks/grep-guard.ps1 index ee919d8b..cdc732de 100644 --- a/integrations/claude-code/hooks/grep-guard.ps1 +++ b/integrations/claude-code/hooks/grep-guard.ps1 @@ -82,7 +82,8 @@ $targetRoot = Resolve-TargetGitRoot $path if (-not $targetRoot) { exit 0 } # Coverage: the TARGET repo's git root is REGISTERED with the serve hub -# (~/.codesearch/repos.json — the same list the hub itself resolves by), +# (~/.codesearch/repos.json, honoring the CODESEARCH_REPOS_CONFIG / +# CODESEARCH_HOME overrides — the same list the hub itself resolves by), # matching the bash twin and edit-guard; a running serve hub alone is NOT a # signal (it covers many repos and being alive says nothing about this one). # CODESEARCH_SERVER stays the explicit opt-in for pure remote-serve setups. diff --git a/integrations/claude-code/hooks/grep-guard.sh b/integrations/claude-code/hooks/grep-guard.sh index cd24b3b1..ce310553 100644 --- a/integrations/claude-code/hooks/grep-guard.sh +++ b/integrations/claude-code/hooks/grep-guard.sh @@ -59,8 +59,9 @@ path=$(echo "$raw" | jq -r '.tool_input.path // empty' | jq_str) # every absolute path. # # Coverage signal (#199): the target's git root is REGISTERED with the -# local serve hub (~/.codesearch/repos.json — the same registration list -# the hub itself resolves queries by). A `.codesearch.db` directory at +# local serve hub (~/.codesearch/repos.json, honoring the +# CODESEARCH_REPOS_CONFIG / CODESEARCH_HOME overrides — the same +# registration list the hub itself resolves queries by). A `.codesearch.db` directory at # the git root was only ever a proxy for that and is wrong in both # directions: a stale db from a since-unregistered repo denied Grep while # the hub could not actually answer for it (unknown alias), and a diff --git a/integrations/claude-code/hooks/run-tests.sh b/integrations/claude-code/hooks/run-tests.sh index e536bdc3..31b6a7d0 100644 --- a/integrations/claude-code/hooks/run-tests.sh +++ b/integrations/claude-code/hooks/run-tests.sh @@ -8,10 +8,11 @@ # # Coverage is simulated without a real index: repoA is "registered" via a # temp repos.json passed through CODESEARCH_REPOS_CONFIG (the same override -# the guards honour at runtime), repoB is not listed. The serve hub is only -# contacted by the grep-guard smoke case, which forces a dead port so the -# answer is deterministic. State lives in a temp TMPDIR; the env overrides -# are process-scoped and die with this script. +# the guards honour at runtime), repoB is not listed. One case pins the +# CODESEARCH_HOME relocation fallback with REPOS_CONFIG unset. The serve hub +# is only contacted by the grep-guard smoke case, which forces a dead port +# so the answer is deterministic. State lives in a temp TMPDIR; the env +# overrides are process-scoped and die with this script. set -u @@ -25,7 +26,7 @@ STATE_DIR="$TMP_ROOT/state" mkdir -p "$STATE_DIR" cleanup() { - unset CODESEARCH_REPOS_CONFIG CODESEARCH_SERVER TMPDIR + unset CODESEARCH_REPOS_CONFIG CODESEARCH_SERVER CODESEARCH_HOME TMPDIR rm -rf "$TMP_ROOT" } trap cleanup EXIT @@ -152,6 +153,23 @@ case_grep_guard_smoke() { fi } +case_covered_via_codesearch_home() { + # Relocation pin: with REPOS_CONFIG unset, guards must resolve repos.json + # from $CODESEARCH_HOME (mirrors constants.rs codesearch_home()) — and + # only from there, so a dev's real global registry cannot leak in. + local home="$TMP_ROOT/cs-home" saved_cfg="${CODESEARCH_REPOS_CONFIG:-}" + mkdir -p "$home" + jq -n --arg a "$ROOT_A" '{repos: {repoA: $a}}' > "$home/repos.json" + unset CODESEARCH_REPOS_CONFIG + export CODESEARCH_HOME="$home" + run_edit "$(edit_event "$REPO_A/x.cs")" + expect_deny "CODESEARCH_HOME registry: covered .cs denied" "find_impact" + run_edit "$(edit_event "$REPO_B/b.cs")" + expect_allow "CODESEARCH_HOME registry: unregistered repo fails open" + unset CODESEARCH_HOME + export CODESEARCH_REPOS_CONFIG="$saved_cfg" +} + case_no_coverage_allows case_covered_cs_denied case_covered_ts_denied @@ -163,6 +181,7 @@ case_multiedit_partial_marks_denied case_find_usages_marker_allows case_window_expiry_denies_again case_grep_guard_smoke +case_covered_via_codesearch_home echo echo "edit-guard self-tests: $PASS passed, $FAIL failed" diff --git a/integrations/claude-code/hooks/web-guard.ps1 b/integrations/claude-code/hooks/web-guard.ps1 index 6dddaac1..d519068c 100644 --- a/integrations/claude-code/hooks/web-guard.ps1 +++ b/integrations/claude-code/hooks/web-guard.ps1 @@ -76,6 +76,7 @@ if ([string]::IsNullOrWhiteSpace($q)) { exit 0 } # names — the opt-in allowlist). No mounts -> nothing to prefer -> allow. # ------------------------------------------------------------------ $config = if ($env:CODESEARCH_REPOS_CONFIG) { $env:CODESEARCH_REPOS_CONFIG } + elseif ($env:CODESEARCH_HOME) { Join-Path $env:CODESEARCH_HOME 'repos.json' } else { Join-Path $HOME '.codesearch/repos.json' } if (-not (Test-Path $config)) { exit 0 } diff --git a/integrations/claude-code/hooks/web-guard.sh b/integrations/claude-code/hooks/web-guard.sh index 6c72a633..31eff40c 100644 --- a/integrations/claude-code/hooks/web-guard.sh +++ b/integrations/claude-code/hooks/web-guard.sh @@ -54,7 +54,7 @@ q=$(echo "$raw" | jq -r '.tool_input.query // .tool_input.url // empty') # names — the opt-in allowlist). No mounts -> nothing to prefer -> allow the # web call unimpeded. # ------------------------------------------------------------------ -config="${CODESEARCH_REPOS_CONFIG:-$HOME/.codesearch/repos.json}" +config="${CODESEARCH_REPOS_CONFIG:-${CODESEARCH_HOME:-$HOME/.codesearch}/repos.json}" [ -f "$config" ] || exit 0 # NOTE: strip CR. On Windows/Git Bash jq emits CRLF, and a trailing \r silently diff --git a/integrations/cloud/README.md b/integrations/cloud/README.md index 24999902..0352d6dd 100644 --- a/integrations/cloud/README.md +++ b/integrations/cloud/README.md @@ -42,6 +42,14 @@ consumed by the serve replica). There is no shared filesystem or database betwee environment, a storage account, and container app/job resources. - The `codesearch` container image published to a registry the platform can pull from. +> ⚠️ **Build locally, not via `az acr build`, on a Basic/Standard-SKU registry.** The +> builder stage compiles the release binary, then downloads and warms up the embedding +> model before tarring it — memory/disk-heavy on top of the compile. ACR's default build +> agent (Basic/Standard SKU; Agent Pools with bigger VMs need Premium) reliably fails at +> the `COPY --from=builder /models.tar.gz` step with `failed to create image: failed to +> get layer sha256:...: layer does not exist`, after a ~25 min compile — confirmed twice, +> not a one-off. Build with local Docker Desktop and `docker push` to the registry instead. + ## Provisioning (generic) Replace every `<...>` placeholder with your own values. diff --git a/integrations/opencode/.gitignore b/integrations/opencode/.gitignore new file mode 100644 index 00000000..504afef8 --- /dev/null +++ b/integrations/opencode/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/integrations/opencode/README.md b/integrations/opencode/README.md new file mode 100644 index 00000000..b120ad28 --- /dev/null +++ b/integrations/opencode/README.md @@ -0,0 +1,477 @@ +# OpenCode integration: making codesearch structural + +## The problem + +codesearch publishes usage instructions to every MCP client on connect (the +`initialize` handshake, see main [README § Agent Guidance](../../README.md#agent-guidance-making-agents-use-codesearch-not-grep)). +OpenCode surfaces those instructions and the model follows them reasonably +well. But they are **advisory** and **connection-scoped**: they cannot see the +host state that decides whether the preference actually holds up in practice. + +In OpenCode that shows up as several recurring failure modes: + +1. **Scope friction.** In multi-repo serve mode every tool call needs + `project=` or `group=`. The model has to guess the alias from the directory + name; when it guesses wrong it burns a turn on `scope_required`, and when it + guesses right it still may have picked the wrong project. +2. **Grep wins by default.** `grep`/`glob` are always loaded, zero-friction and + local. The instruction "prefer codesearch" loses to convenience, and an + **empty grep** (the exact case where the index would help) simply dead-ends + the turn. +3. **Worktrees and clones are invisible.** A git worktree is a separate + indexable repo. The model cannot tell that the current directory is + unregistered, so it greps a tree codesearch has never seen. +4. **The hub can be down.** `codesearch serve` is a local daemon. If it is not + running (or an MCP connect failed), the model keeps trying codesearch tools + and gets transport errors instead of falling back gracefully. +5. **Fresh edits lag the index.** The file watcher reindexes shortly after a + write; a query made in that window can miss the change and send the model + back to grep. +6. **Compaction loses structure.** A long session compacts, and the file + outlines that made navigation cheap are summarised away. + +The plugin below fixes these by adding what only the host knows: the working +directory, the session lifecycle, tool calls, and file edits. + +## The fix + +A self-contained [OpenCode v2](https://opencode.ai/docs/plugins/) plugin +(`codesearch.ts`, no imports, no runtime dependencies) that turns the +preference into behaviour: + +| Feature | Default | What it does | +|---|---|---| +| **Scope resolution** | on | Maps the session directory to a registered project alias (and its groups), from `~/.codesearch/repos.json` (relocatable via `CODESEARCH_HOME`) or a live `status` snapshot. | +| **Scope-aware guidance** | once per session | Injects a compact strategy block plus the resolved scope into system context, so the first tool call already knows `project=`. | +| **Zero-hit rescue** | on | When `grep`/`glob` returns nothing in an indexed repo, runs a literal codesearch query and appends the index hits to the tool result. | +| **Code-search-first guards** | `nudge` | `off`, `nudge` (one tip per session), `prune` (hide `grep`/`glob` from the model), or `block` (deny with the exact codesearch call to use). | +| **Hub health** | on | Probes `/healthz`, asks OpenCode to reload failed MCP servers, and can auto-start a local `codesearch serve` (opt-in). | +| **Unindexed warning** | on | Flags a registered repo with an empty or model-less index, so "no results" is not mistaken for "not there". | +| **Freshness note** | on | Tracks `file.edited` events and reminds the model that a just-edited file may lag the watcher's refresh. | +| **Compaction outlines** | on | Before compaction, captures `explore` outlines of recently edited files and injects them into the summary prompt. | +| **Commands** | on | `/codesearch `, `/codesearch-status`, `/codesearch-index`. | +| **`codesearch_scope` tool** | on | Lets the model ask which project/groups the current directory maps to, and whether the index is healthy. | +| **Skill** | on | Registers a `codesearch` skill with the tool-routing playbook and worked examples. | +| **Auto-recall** | off | Opt-in: runs a semantic search for the user message and injects the top chunks as background context. | + +Everything is **fail-open**: every hook is wrapped, guards only engage while +the hub is confirmed reachable and the repo is registered, and a plugin or +network failure degrades to plain OpenCode behaviour. Nothing here can block a +session or wedge the model. + +## Requirements + +- **OpenCode v2** (tested with 2.0.22). V1 is not supported. +- **A codesearch serve hub** reachable over HTTP (streamable MCP). The default + is `http://127.0.0.1:39725/mcp`; remote serve is supported with an API key. +- Nothing else: the plugin has no npm dependencies and does not need Node or + Bun on PATH (it uses the host's runtime and, when enabled, spawns the same + `codesearch` binary that runs your hub). + +Stdio-only setups (`codesearch mcp` without `serve`) do not expose the HTTP +endpoint this plugin needs. The plugin stays installed and inert there (and +says so in its logs); guards never fire, and the MCP server's own instructions +remain in effect. + +## Install + +The plugin is a single file. Drop it into OpenCode's plugin directory and +restart, or point at it from `opencode.jsonc`. + +### User scope (applies to every project) + +```bash +# macOS / Linux +mkdir -p ~/.config/opencode/plugins +cp integrations/opencode/codesearch.ts ~/.config/opencode/plugins/ +``` + +```powershell +# Windows / PowerShell +New-Item -ItemType Directory -Force "$env:USERPROFILE\.config\opencode\plugins" +Copy-Item integrations\opencode\codesearch.ts "$env:USERPROFILE\.config\opencode\plugins\" +``` + +### Project scope (this repository only) + +```bash +mkdir -p .opencode/plugins +cp integrations/opencode/codesearch.ts .opencode/plugins/ +``` + +### Or reference the path from config + +```jsonc +// ~/.config/opencode/opencode.jsonc +{ + "plugins": ["/absolute/path/to/codesearch.ts"] +} +``` + +OpenCode watches the plugin directories and reloads on change. If the plugin +does not appear, restart the service: + +```sh +opencode service restart +``` + +### Verify + +```sh +opencode plugin list # should list: codesearch local .../codesearch.ts +``` + +Then, inside a session, run `/codesearch-status`. A loaded plugin answers with +the hub URL, health, the directory's resolved project and its index state. + +### Uninstall + +Delete `codesearch.ts` (and optionally `~/.config/opencode/codesearch.json`) +and restart OpenCode. The plugin never installs or edits anything else. + +## Configuration + +Defaults live under `~/.config/opencode/codesearch.json` (override the path +with `CODESEARCH_CONFIG`). Values may use `{env:NAME}` placeholders. +Precedence, lowest to highest: + +1. built-in defaults +2. plugin options in `opencode.jsonc` (object form) +3. the config file +4. environment variables + +```jsonc +{ + // Optional. Usually auto-resolved (see below). + "url": "http://127.0.0.1:39725/mcp", + // Optional bearer token for a remote / authenticated serve. + "token": "{env:CODESEARCH_API_KEY}", + // Name of the MCP server entry in opencode.jsonc to mirror. + "mcpServer": "codesearch", + // Optional override for the registry (default: ~/.codesearch/repos.json, + // relocated wholesale by CODESEARCH_HOME). + "reposConfig": "", + "debug": false, + + "scope": { + "enabled": true, + // How often the scope+guidance block is injected: + // "session" (first message), "message" (every message), "off". + "inject": "session", + "includeGroups": true, + "siblings": true, + "maxSiblings": 12, + "warnUnindexed": true + }, + + "guidance": { + "enabled": true, + // Optional Markdown file (or inline text) replacing the built-in guidance. + "file": "", + "text": "" + }, + + "guards": { + // "off" | "nudge" | "prune" | "block" (see the guard section below). + "mode": "nudge", + "tools": ["grep", "glob"], + "rescue": true, + "rescueLimit": 5, + "rescueTimeoutMs": 3000 + }, + + "health": { + "enabled": true, + "intervalMs": 90000, + "initialDelayMs": 60000, + "maxBackoffMs": 900000, + "reconnect": true, + // Opt-in: start a local serve hub when it is not running. + "autoStart": false, + "startCommand": "codesearch serve --quiet" + }, + + "freshness": { "enabled": true, "maxFiles": 8 }, + "compaction": { "enabled": true, "maxFiles": 3, "timeoutMs": 4000, "budgetChars": 2400 }, + + // Opt-in auto-recall. + "recall": { "enabled": false, "limit": 3, "timeoutMs": 2500, "budgetChars": 1800 }, + + "commands": { "enabled": true, "indexCommand": "codesearch index" }, + "tools": { "scope": true }, + "skill": { "enabled": true, "autoinvoke": true }, + + "log": { "scope": false, "guidance": false, "guards": true, "rescue": true, "health": true, "recall": false, "commands": true } +} +``` + +### Environment variables + +| Variable | Default | Meaning | +|---|---|---| +| `CODESEARCH_URL` | – | Full MCP URL (`.../mcp`). Wins over every discovery step. | +| `CODESEARCH_SERVER` | – | Serve **base** URL (e.g. `http://127.0.0.1:39725`), same meaning as the Claude Code guard hooks use. `/mcp` is appended. | +| `CODESEARCH_API_KEY` | – | Bearer token for an authenticated/remote serve. | +| `CODESEARCH_CONFIG` | `~/.config/opencode/codesearch.json` | Config file path. | +| `CODESEARCH_REPOS_CONFIG` | `$CODESEARCH_HOME/repos.json` (`~/.codesearch` when unset) | Registry path (shared with the hub and the Claude Code hooks). | +| `CODESEARCH_MCP_SERVER` | `codesearch` | MCP entry name to mirror for URL/token. | +| `CODESEARCH_PLUGIN_GUARDS` | `nudge` | `off` / `nudge` / `prune` / `block`. | +| `CODESEARCH_PLUGIN_SCOPE` | `session` | `session` / `message` / `off`. | +| `CODESEARCH_PLUGIN_RESCUE` | `true` | Zero-hit rescue on/off. | +| `CODESEARCH_PLUGIN_RECALL` | `false` | Auto-recall on/off. | +| `CODESEARCH_PLUGIN_COMPACTION` | `true` | Compaction outlines on/off. | +| `CODESEARCH_PLUGIN_COMMANDS` | `true` | Slash commands on/off. | +| `CODESEARCH_PLUGIN_SKILL` | `true` | Skill registration on/off. | +| `CODESEARCH_PLUGIN_HEALTH` | `true` | Health probing on/off. | +| `CODESEARCH_PLUGIN_DEBUG` | `false` | Verbose plugin logs. | + +Settings are resolved **once, at plugin setup** (the host calls `setup()` when +the plugin loads). Editing `codesearch.json` or a `CODESEARCH_*` variable +mid-session therefore does not take effect until OpenCode restarts or the +plugin is reloaded. + +### Hub discovery order + +The plugin resolves the MCP endpoint in this order, first non-empty wins: + +1. `url` from config / `CODESEARCH_URL` +2. `CODESEARCH_SERVER` +3. the `url` of the MCP entry named by `mcpServer` in `opencode.jsonc` + (its `Authorization` header is used as the token, minus `Bearer`) +4. `~/.codesearch/serve_url`, or `$CODESEARCH_HOME/serve_url` when the global root is relocated (written by `codesearch serve`) +5. `http://127.0.0.1:39725/mcp` + +A base URL without `/mcp` is normalised by appending it, so pointing +`CODESEARCH_SERVER` at the hub root is enough. + +## Guards and rescue in detail + +`guards.mode` controls how strongly codesearch replaces `grep`/`glob` in a +**covered** directory (one whose path maps to a registered, indexed repo): + +- **`off`** — hooks are not even registered. Nothing changes. +- **`nudge`** (default) — greps work normally. The first successful + `grep`/`glob` in a session gets a one-line tip appended to its result, and an + **empty** grep/glob is rescued (below). +- **`prune`** — `grep` and `glob` are removed from the tool list for covered + sessions, so the model must use the codesearch tools. The scope block says + so explicitly. Everything else is untouched, and reading/editing files is + unaffected. +- **`block`** — `grep`/`glob` calls are denied with a message naming the exact + `project=` value and the codesearch calls to use instead. Denial happens + **only** while the hub is confirmed reachable (`/healthz` answered) and the + repo is registered; in any other state the call proceeds. This mirrors the + Claude Code `grep-guard` semantics and fails open the same way. + +**Zero-hit rescue** (part of `nudge` and `prune`, and independently +controlled by `guards.rescue`) is the highest-value, lowest-risk piece: it +only ever fires when a tool already returned nothing. For a plain-text pattern +it runs `search` with `mode="literal", phrase=true`; for a regex-shaped +pattern it uses `regex=true`. Hits are appended as metadata-style lines +(path/line/kind/signature), not full code, and the model is pointed at +`get_chunk` for the body: + +```text +[codesearch] grep returned nothing, but the index has 2 match(es) for "retryUpload" in project "my-service": + - src/uploader.ts:42 (Function) function retryUpload(job: Job, attempt: number) + - src/queue.ts:13 (Class) class UploadQueue +Use search/find/get_chunk for the full result. +``` + +Glob patterns that are pure filesystem wildcards (`**/*.ts`) are skipped; +there is nothing semantic to look up. + +## Commands + +| Command | What it does | +|---|---| +| `/codesearch ` | Runs a semantic search in the resolved project and injects the top hits as a prompt, with a pointer to `get_chunk`. Falls back to guidance if the directory is unregistered. | +| `/codesearch-status` | Probes health, refreshes the status snapshot, and reports hub URL, health, directory, resolved project, chunk/file counts, model and lock state, groups and hub repo count. | +| `/codesearch-index` | Runs `commands.indexCommand` (`codesearch index` by default) in the current directory, detached, then reports that indexing started. Use this to register an unregistered clone or worktree. | + +## The `codesearch_scope` tool and the skill + +`codesearch_scope` returns JSON for the current directory: + +```json +{ + "directory": "/work/my-service", + "project": "my-service", + "groups": ["platform"], + "registered": true, + "siblings": [], + "indexed": true, + "chunks": 4123, + "model": "embeddinggemma-q4", + "lockStatus": "warm", + "serveUrl": "http://127.0.0.1:39725/mcp", + "serveHealth": "ok" +} +``` + +Use it when the directory is unusual (a worktree, a multi-repo parent) or when +you want to confirm the index health before trusting a result. It is the one +piece of state the MCP server itself cannot see. + +The plugin also registers a **`codesearch` skill** containing the same routing +playbook as the guidance plus worked examples, so a session can pull the +strategy in deliberately. Disable with `skill.enabled: false`. + +## Health and auto-start + +The health runner probes the serve hub's unauthenticated `/healthz` endpoint +(any HTTP response counts as reachable; only connection-level failures count +as down), with exponential backoff while down. On failure it asks OpenCode to +reload its MCP servers (which re-establishes a failed `codesearch` MCP entry). +With `health.autoStart: true` and a loopback URL it also runs +`codesearch serve --quiet` once, detached, and keeps probing. + +While the hub is down, the injected scope block says so and explicitly allows +`grep` as a fallback; rescue skips its lookup (no timeout cost). + +## Compaction outlines (and optional recall) + +Before a session is compacted, the plugin collects the recently edited files +(tracked from `file.edited` events), fetches `explore kind="outline"` for up +to `compaction.maxFiles`, and appends a compact symbol list to the +compaction prompt, so file structure survives the summary: + +```text +# codesearch outlines (captured before compaction) +- src/uploader.ts: function retryUpload(job: Job, attempt: number):41; class UploadManager:88 +- src/queue.ts: class UploadQueue:13; function enqueue(job: Job):55 +``` + +**Auto-recall** (`recall.enabled`, default `false`) is the opt-in mirror of +the guidance injection: for each new user message that looks like a code +question, it runs a semantic search and injects the top `recall.limit` chunks, +truncated to `budgetChars`. It costs a lookup plus context on every turn, which +is why it is off by default. Label it as background context; it is code from +the index, not instructions. + +## Examples + +**1. Multi-repo scope, no detour.** A workspace registers `my-service`, +`billing-api` and `docs`; the session opens in `my-service`. + +```text +user> where does upload retry backoff live? +# plugin injects: project "my-service", group "platform" (cross-repo fan-out) +model> search(project:"my-service", query:"upload retry backoff") -> get_chunk(...) +``` + +Without the plugin, the first call is a guess: `project:"my_service"` or no +scope at all, and the turn pays for a `scope_required` round-trip. + +**2. Empty grep becomes a hit.** The model greps for an old function name: + +```text +model> grep(pattern:"retryUpload") +tool> [codesearch] grep returned nothing, but the index has 2 match(es) for "retryUpload" in project "my-service": ... +model> get_chunk(chunk_id: 7) +``` + +**3. Guard modes.** `prune` removes `grep`/`glob` from the tool list in +covered sessions; `block` returns: + +```text +codesearch: grep is disabled here — this repository is indexed as project "my-service". +Use the codesearch MCP tools instead: + - search { project: "my-service", query: "" } ... +``` + +Both fail open when the hub is down or the directory is unregistered. + +**4. A dead hub recovers.** `codesearch serve` stops or an MCP connect fails: + +```text +[codesearch] hub unreachable at http://127.0.0.1:39725/mcp: fetch failed +[codesearch] asked OpenCode to reload MCP servers +``` + +With `health.autoStart: true` the plugin starts the hub and logs when it comes +back; the scope block tells the model grep is acceptable until then. + +**5. A new worktree.** The session opens in an unregistered worktree of a +registered repo. The scope block says it is unregistered and points at +`/codesearch-index`; one command registers and indexes it (`codesearch index`, +which produces a separate index per worktree by design). + +**6. Compaction keeps the map.** After a long refactor, the compaction summary +carries the outline block above instead of losing every symbol location. + +## Why this exists alongside the MCP instructions + +The MCP instructions tell the model *what* to do. The plugin makes it +*true* in the host: + +- The server cannot know the session directory, so it cannot resolve + `project=`. The plugin can, because it runs inside the host. +- The server cannot see a `grep` call, let alone an empty one. +- The server cannot recover its own MCP connection or restart itself. +- The server cannot see file edits or pending compaction. + +Together they behave like the Claude Code hooks from +[`integrations/claude-code/`](../claude-code/README.md), but implemented in +the host's own plugin API instead of shell guards: no `jq`, no `curl`, no +per-call process spawn, and coverage decisions come from the same registry the +hub uses. + +## Troubleshooting + +- **Plugin not listed** — confirm the file is `~/.config/opencode/plugins/codesearch.ts` + (or referenced in `opencode.jsonc`), then `opencode service restart`. Look + for `failed to load plugin` and `codesearch` in OpenCode's log + (`~/.local/share/opencode/log/opencode.log`). +- **No scope / `project=null`** — the directory is not registered. `/codesearch-index` + runs `codesearch index`; verify with `codesearch index list` and inspect + `~/.codesearch/repos.json` (or your `CODESEARCH_REPOS_CONFIG`; `CODESEARCH_HOME` relocates the default). +- **Rescue never fires** — check `guards.rescue`, `guards.tools`, and health + (`/codesearch-status`). Rescue is skipped while the hub is down. +- **Block mode too strict** — `guards.mode: "nudge"` (or `prune`) is the + intended middle ground. Blocking deliberately requires a live hub. +- **Top-level `scope` tool missing / other plugins conflict** — the tool is + registered as `codesearch_scope` precisely to avoid generic names. +- **Remote serve** — set `url` (or `CODESEARCH_SERVER`) and `token` (or + `CODESEARCH_API_KEY`). `/healthz` stays probeable behind auth. +- **Skill/commands not visible** — they need an OpenCode build with skill and + command transforms (2.x). Registration failures are logged and non-fatal; + everything else keeps working. + +## Security notes + +- The plugin reads `opencode.jsonc`, its own config, and `repos.json`. It + sends network requests only to the resolved serve URL (`/mcp`, `/healthz`). +- **Auto-recall sends the current user message to the serve hub** and injects + retrieved code into the model context. It is off by default; enable it only + where that is acceptable. Treat injected chunks as data, not instructions. +- **Auto-start** is off by default. When enabled it runs + `health.startCommand` with your privileges; keep it pointed at + `codesearch serve`. +- Guards fail open. `block` mode never denies while the hub is unreachable or + the repo is unregistered, so a misconfigured guard cannot trap a session. +- No secrets are stored by the plugin; use `{env:...}` placeholders or the + environment for tokens, and keep the config file readable only by you. + +## Development + +The plugin is dependency-free at runtime. For type checking and the tests +(unit tests for the pure helpers and the health runner, plus an end-to-end +smoke test against a mock OpenCode host and MCP server): + +```bash +cd integrations/opencode +npm install # dev-only: typescript + @types/node +npm run typecheck # tsc --noEmit +npm run unit # node test/unit.ts — stripJsonc + HealthRunner (Node >= 23.6) +npm run smoke # node test/smoke.ts — end-to-end vs mock host/MCP (or: bun) +npm test # typecheck + unit + smoke +``` + +`test/unit.ts` covers the JSONC scanner (comments, string escapes, trailing +commas including comment-separated ones) and the health runner (probe +transitions, backoff/reconnect, recovery callback, stop, disabled mode). +`test/smoke.ts` covers scope resolution, guidance injection and deduplication, +zero-hit rescue, the one-time nudge, prune and block modes (including +fail-open for unregistered directories and an unreachable hub), compaction +outlines, the commands, the scope tool and the skill registration. diff --git a/integrations/opencode/codesearch.ts b/integrations/opencode/codesearch.ts new file mode 100644 index 00000000..36e6395c --- /dev/null +++ b/integrations/opencode/codesearch.ts @@ -0,0 +1,1855 @@ +/** + * codesearch — OpenCode plugin + * ============================ + * + * Makes the codesearch MCP server *structural* inside OpenCode instead of + * advisory: resolves the current directory to an indexed project, injects the + * usage guidance once per session, nudges (or prunes/blocks) grep/glob, rescues + * empty greps with index results, keeps the serve hub healthy, and adds slash + * commands, a scope tool, a skill and compaction assistance. + * + * The MCP server already publishes usage instructions on connect, but those are + * advisory and connection-scoped. This plugin adds what only the host knows: + * the working directory, the session lifecycle, tool calls and file edits. + * + * Design constraints (see README.md in this directory): + * - Self-contained: no imports, no runtime dependencies. The host executes + * this file as-is; fs/child_process come from `process.getBuiltinModule`. + * - Fail open: every hook is wrapped so a plugin or network failure can never + * break a session. Guards only block while the hub is confirmed reachable. + * - Generic: no assumption about repo names, groups or layout. Everything + * comes from `~/.codesearch/repos.json`, the serve hub, or the config file. + * + * Targets OpenCode v2 (`opencode` 2.x). Install: copy this file to + * `~/.config/opencode/plugins/codesearch.ts` and restart OpenCode. + */ + +/* ------------------------------------------------------------------ * + * Types (structural views; no @opencode/plugin import so the file runs + * as-is under the host's module loader) + * ------------------------------------------------------------------ */ + +type Loose = Record + +interface Registration { + dispose(): Promise +} + +interface SystemPart { + type: "text" + text: string + [key: string]: unknown +} + +interface PluginContext { + app?: { version?: string } + location?: { + directory?: string + workspaceID?: string + project?: { id?: string; canonical?: string } + } + options?: Record + session?: { + hook( + name: string, + callback: (event: Loose) => Promise | void, + options?: { providerID?: string }, + ): Promise + prompt?(input: Loose): Promise + context?(input: { sessionID: string }): Promise + } + tool?: { + hook(name: string, callback: (event: Loose) => Promise | void): Promise + transform(callback: (editor: ToolEditor) => void): Promise + } + command?: { + transform(callback: (editor: CommandEditor) => void): Promise + } + skill?: { + transform(callback: (editor: SkillEditor) => void): Promise + } + mcp?: { + list(): Promise + reload?(): Promise + } + event?: { + subscribe(options?: { signal?: AbortSignal }): AsyncIterable + } +} + +interface ToolEditor { + namespace(namespace: { name: string; description: string }): void + add(tool: Loose): void + get?(id: string): Loose | undefined + update?(id: string, update: (tool: Loose) => void): void + remove?(id: string): void +} + +interface CommandEditor { + add(definition: Loose): void +} + +interface SkillEditor { + add(skill: Loose): void + get?(id: string): Loose | undefined + update?(id: string, update: (skill: Loose) => void): void +} + +type GuardMode = "off" | "nudge" | "prune" | "block" + +interface CodeSearchSettings { + url: string + token: string + mcpServer: string + reposConfig: string + debug: boolean + scope: { + enabled: boolean + inject: "session" | "message" | "off" + includeGroups: boolean + siblings: boolean + maxSiblings: number + warnUnindexed: boolean + } + guidance: { enabled: boolean; file: string; text: string } + guards: { + mode: GuardMode + tools: string[] + rescue: boolean + rescueLimit: number + rescueTimeoutMs: number + } + health: { + enabled: boolean + intervalMs: number + initialDelayMs: number + maxBackoffMs: number + reconnect: boolean + autoStart: boolean + startCommand: string + } + freshness: { enabled: boolean; maxFiles: number } + compaction: { enabled: boolean; maxFiles: number; timeoutMs: number; budgetChars: number } + recall: { enabled: boolean; limit: number; timeoutMs: number; budgetChars: number } + commands: { enabled: boolean; indexCommand: string } + tools: { scope: boolean } + skill: { enabled: boolean; autoinvoke: boolean } + log: { + scope: boolean + guidance: boolean + guards: boolean + rescue: boolean + health: boolean + recall: boolean + commands: boolean + } +} + +interface ScopeInfo { + directory: string + project?: string + projectPath?: string + groups: string[] + siblings: string[] + registered: boolean + hasGit: boolean +} + +interface StatusRepo { + alias: string + project_path?: string + total_chunks?: number + total_files?: number + model?: string + lock_status?: string +} + +/* ------------------------------------------------------------------ * + * Constants + * ------------------------------------------------------------------ */ + +const DEFAULTS: CodeSearchSettings = { + url: "", + token: "", + mcpServer: "codesearch", + reposConfig: "", + debug: false, + scope: { + enabled: true, + inject: "session", + includeGroups: true, + siblings: true, + maxSiblings: 12, + warnUnindexed: true, + }, + guidance: { enabled: true, file: "", text: "" }, + guards: { + mode: "nudge", + tools: ["grep", "glob"], + rescue: true, + rescueLimit: 5, + rescueTimeoutMs: 3000, + }, + health: { + enabled: true, + intervalMs: 90_000, + initialDelayMs: 60_000, + maxBackoffMs: 900_000, + reconnect: true, + autoStart: false, + startCommand: "codesearch serve --quiet", + }, + freshness: { enabled: true, maxFiles: 8 }, + compaction: { enabled: true, maxFiles: 3, timeoutMs: 4000, budgetChars: 2400 }, + recall: { enabled: false, limit: 3, timeoutMs: 2500, budgetChars: 1800 }, + commands: { enabled: true, indexCommand: "codesearch index" }, + tools: { scope: true }, + skill: { enabled: true, autoinvoke: true }, + log: { + scope: false, + guidance: false, + guards: true, + rescue: true, + health: true, + recall: false, + commands: true, + }, +} + +const DEFAULT_SERVE_URL = "http://127.0.0.1:39725/mcp" +const PROTOCOL_VERSION = "2025-06-18" +const FALLBACK_PROTOCOL_VERSION = "2024-11-05" +const MAX_SEEN = 20_000 + +/** + * Built-in guidance. Deliberately strategy-only: per-tool parameter detail + * lives in the MCP tool schemas and is injected by the server's `initialize` + * instructions, so it is not duplicated here. + */ +const DEFAULT_GUIDANCE = `codesearch indexes this workspace and serves semantic code search over MCP. Prefer it over grep/glob for anything conceptual, cross-file, or symbol-shaped. + +Tool routing: +- search — concepts, unknown locations, mixed natural language + identifiers. Default mode is semantic (hybrid vector + BM25); use mode="literal" with regex=true or phrase=true for exact syntax. +- find — where a symbol is defined (kind="definition"), its call-sites (kind="usages"), a file's imports (kind="imports"), or a module's dependents (kind="dependents"). +- find_impact — precise transitive call sites before a rename or edit (SCIP: C#, TypeScript; on a "no backend" answer fall back to find kind="usages"). +- explore — outline a file's top-level symbols (kind="outline") or find chunks similar to a known hit (kind="similar"). +- get_chunk — read the code behind a search hit, without loading the whole file. +- status — index health and registered repos. + +Rules: +- Multi-repo serve mode requires a scope on every call except status: pass project="" (returned paths are the server's paths; read content via get_chunk, not the local filesystem), or group="" to fan out. +- Start broad with search, narrow with find/explore, read with get_chunk. Keep compact=true (the default) until you know which chunk you need. +- The serve hub watches files and refreshes indexes automatically; if a just-made change seems missing, retry the query once before falling back to grep.` + +/** Skill playbook: the same strategy plus worked examples. */ +const SKILL_CONTENT = `${DEFAULT_GUIDANCE} + +Examples: +- "Where is the upload retried?" -> search(project=..., query="upload retry backoff") -> get_chunk(top hit). +- "Who calls validateToken?" in a TypeScript repo -> find_impact(project=..., symbol_name="validateToken") -> rename/edit with the full caller list. +- "Add a field to the settings struct" -> find(project=..., symbol="Settings", kind="definition") -> find(project=..., symbol="Settings", kind="usages"). +- Exact syntax like "Vec" or "foo = null" -> search(project=..., mode="literal", regex=true, query="Vec<"). +- "How does this file relate to the rest?" -> explore(project=..., kind="outline", target=) -> explore(kind="similar", target=). +- Freshly edited file returning stale hits -> wait a moment, rerun the query; the watcher is reindexing.` + +/* ------------------------------------------------------------------ * + * Small utilities + * ------------------------------------------------------------------ */ + +function bool(value: string | undefined, fallback: boolean): boolean { + if (value === undefined || value === "") return fallback + return !/^(0|false|no|off)$/i.test(value) +} + +function truncate(input: string, max: number): string { + if (input.length <= max) return input + return input.slice(0, Math.max(0, max - 1)) + "…" +} + +function shortHash(input: string): string { + let h = 5381 + for (let i = 0; i < input.length; i++) h = ((h << 5) + h + input.charCodeAt(i)) | 0 + return (h >>> 0).toString(36) +} + +function isPlainObject(value: unknown): value is Loose { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +/** Deep-merge plain objects; arrays and scalars replace. */ +function mergeDeep(base: T, override: unknown): T { + if (!isPlainObject(override)) return base + const out: Loose = { ...(base as unknown as Loose) } + for (const [key, value] of Object.entries(override)) { + const current = out[key] + if (isPlainObject(current) && isPlainObject(value)) out[key] = mergeDeep(current, value) + else out[key] = value + } + return out as T +} + +function expandEnv(value: string): string { + return value.replace(/\{env:([A-Za-z_][A-Za-z0-9_]*)\}/g, (_m, name: string) => process.env[name] ?? "") +} + +function expandEnvDeep(value: T): T { + if (typeof value === "string") return expandEnv(value) as unknown as T + if (Array.isArray(value)) return value.map((v) => expandEnvDeep(v)) as unknown as T + if (isPlainObject(value)) { + const out: Loose = {} + for (const [k, v] of Object.entries(value)) out[k] = expandEnvDeep(v) + return out as unknown as T + } + return value +} + +/** Inline builtin loader: works under Node (getBuiltinModule) and Bun (require). */ +function builtin(name: string): any { + const p = process as unknown as { getBuiltinModule?: (n: string) => unknown } + if (typeof p.getBuiltinModule === "function") { + try { + const mod = p.getBuiltinModule(name) + if (mod) return mod + } catch { + /* fall through */ + } + } + const g = globalThis as unknown as { require?: (n: string) => unknown } + if (typeof g.require === "function") { + try { + return g.require(name) + } catch { + /* fall through */ + } + } + return undefined +} + +function homeDir(): string { + return process.env.HOME || process.env.USERPROFILE || "" +} + +function configDir(): string { + return process.env.XDG_CONFIG_HOME || `${homeDir()}/.config` +} + +/** Global codesearch root: $CODESEARCH_HOME, else ~/.codesearch (mirrors constants.rs `codesearch_home()`). */ +function codesearchHome(): string { + return process.env.CODESEARCH_HOME || `${homeDir()}/.codesearch` +} + +/** + * Strip `//` and block comments plus trailing commas from JSONC, honouring + * string literals. Enough for OpenCode's config files. + */ +/** + * Strip JSONC comments and trailing commas. Exported for `test/unit.ts`; the + * host consumes the default export only. + */ +export function stripJsonc(text: string): string { + let out = "" + let inString = false + let escaped = false + for (let i = 0; i < text.length; i++) { + const ch = text[i] + const next = text[i + 1] + if (inString) { + out += ch + if (escaped) escaped = false + else if (ch === "\\") escaped = true + else if (ch === '"') inString = false + continue + } + if (ch === '"') { + inString = true + out += ch + continue + } + if (ch === "/" && next === "/") { + while (i < text.length && text[i] !== "\n") i++ + out += "\n" + continue + } + if (ch === "/" && next === "*") { + i += 2 + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++ + i++ + continue + } + if (ch === ",") { + // A trailing comma may sit before whitespace AND comments, e.g. + // "a": 1, // keep this member last + // } + // Skip both while looking ahead for the closing bracket. + let j = i + 1 + for (;;) { + while (j < text.length && /\s/.test(text[j])) j++ + if (text[j] === "/" && text[j + 1] === "/") { + while (j < text.length && text[j] !== "\n") j++ + continue + } + if (text[j] === "/" && text[j + 1] === "*") { + j += 2 + while (j < text.length && !(text[j] === "*" && text[j + 1] === "/")) j++ + j += 2 // skip the closing */ + continue + } + break + } + if (text[j] === "}" || text[j] === "]") continue // drop trailing comma + } + out += ch + } + return out +} + +function readJsonFile(path: string): Loose | undefined { + const fs = builtin("node:fs") + if (!fs || !path) return undefined + try { + const raw = fs.readFileSync(path, "utf8") + const parsed = JSON.parse(stripJsonc(raw)) + return isPlainObject(parsed) ? parsed : undefined + } catch { + return undefined + } +} + +function fileExists(path: string): boolean { + const fs = builtin("node:fs") + if (!fs || !path) return false + try { + fs.statSync(path) + return true + } catch { + return false + } +} + +function readTextFile(path: string): string | undefined { + const fs = builtin("node:fs") + if (!fs || !path) return undefined + try { + return fs.readFileSync(path, "utf8") + } catch { + return undefined + } +} + +/* ------------------------------------------------------------------ * + * Path helpers + * ------------------------------------------------------------------ */ + +function normalizePath(input: string): string { + let p = (input || "").trim() + if (!p) return "" + // Windows UNC prefix (\\?\C:\...) and backslashes. + p = p.replace(/^\\\\\?\\/, "") + p = p.replace(/\\/g, "/") + p = p.replace(/\/+$/, "") + // Collapse duplicate slashes (keep the leading two for UNC-style roots). + p = p.replace(/([^:])\/{2,}/g, "$1/") + if (process.platform === "win32") p = p.toLowerCase() + return p +} + +/** True when `child` is `parent` or lives under it. */ +function isUnder(child: string, parent: string): boolean { + const c = normalizePath(child) + const p = normalizePath(parent) + if (!c || !p) return false + return c === p || c.startsWith(p + "/") +} + +/** Path relative to `root`, or the original when not under it. */ +function relativeTo(path: string, root: string): string { + const p = normalizePath(path) + const r = normalizePath(root) + if (!p || !r) return path + if (p === r) return "." + if (p.startsWith(r + "/")) return p.slice(r.length + 1) + return path +} + +/* ------------------------------------------------------------------ * + * Configuration resolution + * ------------------------------------------------------------------ */ + +function opencodeConfigPath(): string { + const candidates = [ + `${configDir()}/opencode/opencode.jsonc`, + `${configDir()}/opencode/opencode.json`, + ] + for (const candidate of candidates) if (fileExists(candidate)) return candidate + return candidates[0] +} + +/** + * Read the `mcp` entry named `serverName` from the OpenCode config, supporting + * both the v2 shape (`mcp.servers.`) and the v1 shape (`mcp.`). + */ +function readMcpEntry(serverName: string): { url: string; authorization: string } { + const config = readJsonFile(opencodeConfigPath()) + if (!config) return { url: "", authorization: "" } + const mcp = isPlainObject(config.mcp) ? config.mcp : undefined + if (!mcp) return { url: "", authorization: "" } + const servers = isPlainObject(mcp.servers) ? mcp.servers : mcp + const entry = isPlainObject(servers[serverName]) ? servers[serverName] : undefined + if (!entry) return { url: "", authorization: "" } + const url = typeof entry.url === "string" ? expandEnv(entry.url) : "" + const headers = isPlainObject(entry.headers) ? entry.headers : undefined + const rawAuth = headers && typeof headers.Authorization === "string" ? expandEnv(headers.Authorization) : "" + const authorization = rawAuth.replace(/^Bearer\s+/i, "") + return { url, authorization } +} + +function resolveSettings(ctx: PluginContext): CodeSearchSettings { + let settings = mergeDeep(DEFAULTS, ctx.options ?? {}) + + const configPath = + process.env.CODESEARCH_CONFIG || `${configDir()}/opencode/codesearch.json` + const fileConfig = readJsonFile(configPath) + if (fileConfig) settings = mergeDeep(settings, expandEnvDeep(fileConfig)) + + const env = process.env + settings.url = env.CODESEARCH_URL ?? settings.url + settings.token = env.CODESEARCH_API_KEY ?? settings.token + settings.mcpServer = env.CODESEARCH_MCP_SERVER ?? settings.mcpServer + settings.reposConfig = env.CODESEARCH_REPOS_CONFIG ?? settings.reposConfig + settings.debug = bool(env.CODESEARCH_PLUGIN_DEBUG, settings.debug) + + if (env.CODESEARCH_PLUGIN_GUARDS) settings.guards.mode = env.CODESEARCH_PLUGIN_GUARDS as GuardMode + if (env.CODESEARCH_PLUGIN_SCOPE) settings.scope.inject = env.CODESEARCH_PLUGIN_SCOPE as CodeSearchSettings["scope"]["inject"] + if (env.CODESEARCH_PLUGIN_RESCUE) settings.guards.rescue = bool(env.CODESEARCH_PLUGIN_RESCUE, settings.guards.rescue) + if (env.CODESEARCH_PLUGIN_RECALL) settings.recall.enabled = bool(env.CODESEARCH_PLUGIN_RECALL, settings.recall.enabled) + if (env.CODESEARCH_PLUGIN_COMPACTION) settings.compaction.enabled = bool(env.CODESEARCH_PLUGIN_COMPACTION, settings.compaction.enabled) + if (env.CODESEARCH_PLUGIN_COMMANDS) settings.commands.enabled = bool(env.CODESEARCH_PLUGIN_COMMANDS, settings.commands.enabled) + if (env.CODESEARCH_PLUGIN_SKILL) settings.skill.enabled = bool(env.CODESEARCH_PLUGIN_SKILL, settings.skill.enabled) + if (env.CODESEARCH_PLUGIN_HEALTH) settings.health.enabled = bool(env.CODESEARCH_PLUGIN_HEALTH, settings.health.enabled) + + if (!["off", "nudge", "prune", "block"].includes(settings.guards.mode)) settings.guards.mode = "nudge" + if (!["session", "message", "off"].includes(settings.scope.inject)) settings.scope.inject = "session" + settings.guards.tools = (settings.guards.tools || []).map((t) => String(t).toLowerCase()) + + return settings +} + +function normalizeMcpUrl(value: string): string { + let v = (value || "").trim() + if (!v) return "" + if (!/^https?:\/\//i.test(v)) v = `http://${v}` + v = v.replace(/\/+$/, "") + if (!/\/mcp$/i.test(v)) v += "/mcp" + return v +} + +/** Resolve the serve URL: config/env > CODESEARCH_SERVER > MCP entry > serve_url file > default. */ +function resolveServeUrl(settings: CodeSearchSettings): string { + const mcpEntry = readMcpEntry(settings.mcpServer) + const serveUrlFile = readTextFile(`${codesearchHome()}/serve_url`)?.trim() ?? "" + const candidates = [ + settings.url, + process.env.CODESEARCH_SERVER ?? "", + mcpEntry.url, + serveUrlFile, + DEFAULT_SERVE_URL, + ] + for (const candidate of candidates) { + const normalized = normalizeMcpUrl(candidate) + if (normalized) return normalized + } + return DEFAULT_SERVE_URL +} + +function resolveToken(settings: CodeSearchSettings): string { + if (settings.token) return settings.token + const mcpEntry = readMcpEntry(settings.mcpServer) + return mcpEntry.authorization || "" +} + +/* ------------------------------------------------------------------ * + * MCP client (streamable HTTP, JSON-RPC) + * ------------------------------------------------------------------ */ + +interface CallOptions { + timeoutMs?: number + retry?: boolean +} + +class McpClient { + private sessionId: string | null = null + private connected = false + private connecting: Promise | null = null + private idSeq = 0 + private chain: Promise = Promise.resolve() + private protocolVersion = PROTOCOL_VERSION + private readonly url: string + private readonly token: string + private readonly debug: boolean + private readonly label: string + + constructor(url: string, token: string, debug: boolean, label: string) { + this.url = url + this.token = token + this.debug = debug + this.label = label + } + + private log(...args: unknown[]): void { + if (this.debug) console.log(`[codesearch:${this.label}]`, ...args) + } + + /** Serialise requests per client so reconnect/session state never races. */ + private serial(fn: () => Promise): Promise { + const run = this.chain.then(fn, fn) + this.chain = run.then( + () => undefined, + () => undefined, + ) + return run + } + + async call(tool: string, args: Record, options: CallOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 30_000 + const retry = options.retry ?? true + return this.serial(async () => { + await this.connect() + const payload = { jsonrpc: "2.0", id: ++this.idSeq, method: "tools/call", params: { name: tool, arguments: args } } + try { + return this.resultOf(tool, await this.post(payload, timeoutMs)) + } catch (err) { + if (!retry) throw err + const message = err instanceof Error ? err.message : String(err) + if (/network|fetch failed|session|ECONN|socket|timed? ?out|abort|4[0-9][0-9]/i.test(message)) { + this.connected = false + this.sessionId = null + await this.connect() + return this.resultOf(tool, await this.post(payload, timeoutMs)) + } + throw err + } + }) + } + + private async connect(): Promise { + if (this.connected) return + if (this.connecting) return this.connecting + this.connecting = this.doConnect().finally(() => { + this.connecting = null + }) + return this.connecting + } + + private async doConnect(): Promise { + const attempt = async (version: string): Promise => { + const init = { + jsonrpc: "2.0", + id: 0, + method: "initialize", + params: { + protocolVersion: version, + capabilities: {}, + clientInfo: { name: "opencode-codesearch-plugin", version: "1.0.0" }, + }, + } + const { session, messages } = await this.postRaw(init, 15_000) + const first = messages.find((m) => (m as Loose).id === 0) as Loose | undefined + if (!first?.result) return false + this.sessionId = session + this.protocolVersion = version + await this.postRaw({ jsonrpc: "2.0", method: "notifications/initialized", params: {} }, 15_000) + return true + } + if (!(await attempt(this.protocolVersion)) && this.protocolVersion !== FALLBACK_PROTOCOL_VERSION) { + if (!(await attempt(FALLBACK_PROTOCOL_VERSION))) { + throw new Error(`codesearch initialize failed at ${this.url}`) + } + } + this.connected = true + this.log("connected", this.sessionId ? `session ${this.sessionId}` : "(stateless)") + } + + private async post(payload: Record, timeoutMs: number): Promise { + const { messages } = await this.postRaw(payload, timeoutMs) + return messages + } + + private async postRaw( + payload: Record, + timeoutMs: number, + ): Promise<{ session: string | null; messages: Loose[] }> { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + } + if (this.token) headers.authorization = `Bearer ${this.token}` + if (this.sessionId) headers["mcp-session-id"] = this.sessionId + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), timeoutMs) + // The timer (and its abort signal) must stay armed through the body read: + // `response.text()` is as capable of hanging as the response headers are + // (undici's ~300 s bodyTimeout), and clearing the timer early would leave + // the serial call chain blocked far past the requested timeout. + try { + const response = await fetch(this.url, { + method: "POST", + headers, + body: JSON.stringify(payload), + signal: controller.signal, + }) + if (!response.ok) throw new Error(`codesearch HTTP ${response.status} ${response.statusText}`) + const session = response.headers.get("mcp-session-id") + const body = await response.text() + const messages = parseMcpBody(body) as Loose[] + if (messages.length === 0 && payload.id !== undefined) { + throw new Error(`codesearch returned no JSON-RPC message for ${String(payload.method)}`) + } + return { session, messages } + } finally { + clearTimeout(timer) + } + } + + private resultOf(tool: string, messages: Loose[]): unknown { + const message = messages.find((m) => m.result || m.error) + if (!message) throw new Error(`No result for ${tool}`) + if (message.error) throw new Error(`codesearch ${tool}: ${message.error.message ?? message.error.code ?? "error"}`) + const result = message.result as Loose | undefined + if (result?.isError) { + const text = result.content?.[0]?.text || "unknown tool error" + throw new Error(`codesearch ${tool} failed: ${String(text).slice(0, 500)}`) + } + const text = (result?.content ?? []) + .filter((c: Loose) => c.type === "text" && typeof c.text === "string") + .map((c: Loose) => c.text as string) + .join("\n") + if (text.trim() === "") return undefined + try { + return JSON.parse(text) + } catch { + return text + } + } +} + +/** Parse a streamable-HTTP body (plain JSON or SSE `data:` lines). */ +function parseMcpBody(body: string): unknown[] { + const out: unknown[] = [] + const trimmed = body.trim() + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + try { + out.push(JSON.parse(trimmed)) + return out + } catch { + /* fall through to SSE parsing */ + } + } + for (const line of body.split(/\r?\n/)) { + const text = line.trim() + if (!text.startsWith("data:")) continue + const raw = text.slice(5).trim() + if (!raw) continue + try { + out.push(JSON.parse(raw)) + } catch { + /* ignore malformed frames */ + } + } + return out +} + +/* ------------------------------------------------------------------ * + * Registry + status cache + * ------------------------------------------------------------------ */ + +class RepoRegistry { + private repos = new Map() + private groups = new Map() + private loadedAt = 0 + private signature = "" + private readonly path: string + + constructor(path: string) { + this.path = path + } + + load(force = false): void { + const now = Date.now() + if (!force && now - this.loadedAt < 30_000) return + this.loadedAt = now + const data = readJsonFile(this.path) + const signature = data ? JSON.stringify(data) : "" + if (signature === this.signature && !force) return + this.signature = signature + const repos = new Map() + const groups = new Map() + if (data) { + const rawRepos = isPlainObject(data.repos) ? data.repos : {} + for (const [alias, path] of Object.entries(rawRepos)) { + if (typeof path === "string" && path) repos.set(alias, normalizePath(path)) + } + const rawGroups = isPlainObject(data.groups) ? data.groups : {} + for (const [name, members] of Object.entries(rawGroups)) { + if (Array.isArray(members)) groups.set(name, members.map(String)) + } + } + this.repos = repos + this.groups = groups + } + + get size(): number { + return this.repos.size + } + + /** Registered project whose path contains (or equals) `directory`. */ + projectFor(directory: string): { alias: string; path: string } | undefined { + this.load() + const dir = normalizePath(directory) + if (!dir) return undefined + let best: { alias: string; path: string } | undefined + for (const [alias, path] of this.repos) { + if (!isUnder(dir, path)) continue + if (!best || path.length > best.path.length) best = { alias, path } + } + return best + } + + /** Registered repos nested under `directory` (multi-repo workspace roots). */ + siblingsFor(directory: string): string[] { + this.load() + const dir = normalizePath(directory) + if (!dir) return [] + const out: string[] = [] + for (const [alias, path] of this.repos) { + if (isUnder(path, dir) && normalizePath(path) !== dir) out.push(alias) + } + return out.sort() + } + + groupsFor(alias: string): string[] { + this.load() + const out: string[] = [] + for (const [name, members] of this.groups) if (members.includes(alias)) out.push(name) + return out.sort() + } + + findAliasByPath(path: string): string | undefined { + this.load() + const target = normalizePath(path) + for (const [alias, repoPath] of this.repos) if (repoPath === target) return alias + return undefined + } +} + +/** Cached `status(kind="projects")` payload, used for scope fallback and health. */ +class StatusCache { + private repos: StatusRepo[] = [] + private groups: Record = {} + private fetchedAt = 0 + + async refresh(client: McpClient, timeoutMs = 10_000): Promise { + const result = (await client.call("status", { kind: "projects" }, { timeoutMs, retry: false })) as Loose | undefined + if (!result) return + if (Array.isArray(result.repos)) this.repos = result.repos as StatusRepo[] + if (isPlainObject(result.groups)) this.groups = result.groups as Record + this.fetchedAt = Date.now() + } + + get ageMs(): number { + return Date.now() - this.fetchedAt + } + + get isFresh(): boolean { + return this.fetchedAt > 0 + } + + get all(): StatusRepo[] { + return this.repos + } + + repo(alias: string): StatusRepo | undefined { + return this.repos.find((r) => r.alias === alias) + } + + projectFor(directory: string): { alias: string; path: string } | undefined { + const dir = normalizePath(directory) + if (!dir) return undefined + let best: { alias: string; path: string } | undefined + for (const repo of this.repos) { + if (!repo.project_path) continue + const path = normalizePath(repo.project_path) + if (!isUnder(dir, path)) continue + if (!best || path.length > best.path.length) best = { alias: repo.alias, path } + } + return best + } + + groupsFor(alias: string): string[] { + const out: string[] = [] + for (const [name, members] of Object.entries(this.groups)) if (members.includes(alias)) out.push(name) + return out.sort() + } + + siblingsFor(directory: string): string[] { + const dir = normalizePath(directory) + if (!dir) return [] + const out: string[] = [] + for (const repo of this.repos) { + if (!repo.project_path) continue + const path = normalizePath(repo.project_path) + if (isUnder(path, dir) && path !== dir) out.push(repo.alias) + } + return out.sort() + } +} + +/* ------------------------------------------------------------------ * + * Message helpers (new-user-message detection) + * ------------------------------------------------------------------ */ + +function messageText(message: Loose): string { + const parts = message?.parts + if (Array.isArray(parts)) { + return parts + .map((p: Loose) => (p && typeof p.text === "string" ? p.text : "")) + .filter(Boolean) + .join("\n") + } + if (typeof message?.content === "string") return message.content + if (Array.isArray(message?.content)) { + return message.content + .map((c: Loose) => (c && typeof c.text === "string" ? c.text : "")) + .filter(Boolean) + .join("\n") + } + return "" +} + +function roleOf(message: Loose): string { + return String(message?.info?.role ?? message?.role ?? "") +} + +function lastUserMessage(messages: Loose[]): { id: string; text: string } | null { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if (roleOf(message) !== "user") continue + const id = String(message?.info?.id ?? message?.id ?? "") + const text = messageText(message) + if (!text.trim()) continue + return { id: id || shortHash(text), text } + } + return null +} + +/** Heuristic for "this user message is a question about code" (recall gate). */ +function isLikelyCodeQuestion(text: string): boolean { + const trimmed = text.trim() + if (trimmed.length < 20 || trimmed.length > 4000) return false + if (/^\//.test(trimmed)) return false + return ( + /\b(where|how|why|what|who|which|find|locate|implement|handle|call|use|flow|work|refactor|fix|add|change|rename|test)\b/i.test(trimmed) || + /[A-Za-z_][A-Za-z0-9_]*\(|::|->|\.\w+\b|_[a-z]/.test(trimmed) + ) +} + +/* ------------------------------------------------------------------ * + * Guidance and scope text + * ------------------------------------------------------------------ */ + +function buildScopePrelude(scope: ScopeInfo, extra: string[]): string { + const lines: string[] = [] + if (scope.project) { + lines.push(`- Current directory is indexed by codesearch as project "${scope.project}"${scope.projectPath ? ` (${scope.projectPath})` : ""}.`) + if (scope.groups.length) { + lines.push(`- Project group${scope.groups.length > 1 ? "s" : ""}: ${scope.groups.map((g) => `"${g}"`).join(", ")} (use group= for cross-repo fan-out).`) + } + } else if (scope.registered && scope.siblings.length) { + lines.push(`- This workspace contains ${scope.siblings.length} registered codesearch project(s): ${scope.siblings.join(", ")}. Pass project= for one, or group= to fan out.`) + } else if (scope.hasGit) { + lines.push(`- This git repository is not registered with codesearch. Register it with \`codesearch index\` (or the /codesearch-index command) to enable semantic search here.`) + } + for (const line of extra) lines.push(line) + return lines.length ? `codesearch scope:\n${lines.join("\n")}` : "" +} + +/* ------------------------------------------------------------------ * + * Health runner + * ------------------------------------------------------------------ */ + +/** + * Periodic `/healthz` probe with backoff and optional MCP reload. Exported for + * `test/unit.ts`; the host consumes the default export only. + */ +export class HealthRunner { + private timer: ReturnType | null = null + private stopped = false + private failures = 0 + state: "unknown" | "ok" | "down" = "unknown" + lastError = "" + private readonly ctx: PluginContext + private readonly url: string + private readonly settings: CodeSearchSettings + private readonly onRecovered: () => void + private readonly log: (...args: unknown[]) => void + private readonly warn: (...args: unknown[]) => void + + constructor( + ctx: PluginContext, + url: string, + settings: CodeSearchSettings, + onRecovered: () => void, + log: (...args: unknown[]) => void, + warn: (...args: unknown[]) => void, + ) { + this.ctx = ctx + this.url = url + this.settings = settings + this.onRecovered = onRecovered + this.log = log + this.warn = warn + } + + start(): void { + if (!this.settings.health.enabled) return + this.schedule(this.settings.health.initialDelayMs) + } + + stop(): void { + this.stopped = true + if (this.timer) clearTimeout(this.timer) + this.timer = null + } + + private schedule(ms: number): void { + if (this.stopped) return + const timer = (this.timer = setTimeout(() => void this.tick(), ms)) + ;(timer as unknown as { unref?: () => void }).unref?.() + } + + private async tick(): Promise { + if (this.stopped) return + const before = this.state + await this.probe() + if (before === "down" && this.state === "ok") { + this.log(`hub recovered at ${this.url}`) + this.onRecovered() + } + let delay = this.settings.health.intervalMs + if (this.state === "down") { + this.failures += 1 + delay = Math.min(this.settings.health.intervalMs * 2 ** (this.failures - 1), this.settings.health.maxBackoffMs) + if (this.settings.health.reconnect) await this.tryReconnect() + if (this.settings.health.autoStart && this.failures === 1) await this.tryAutoStart() + } else { + this.failures = 0 + } + this.schedule(delay) + } + + /** Liveness probe: any HTTP response means reachable; only network errors mean down. */ + async probe(): Promise { + const base = this.url.replace(/\/mcp\/?$/i, "") + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), 2000) + try { + await fetch(`${base}/healthz`, { signal: controller.signal }) + if (this.state !== "ok") this.log("hub reachable") + this.state = "ok" + this.lastError = "" + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (this.state !== "down") this.warn(`hub unreachable at ${this.url}: ${message}`) + this.state = "down" + this.lastError = message + } finally { + clearTimeout(timer) + } + } + + private async tryReconnect(): Promise { + if (!this.settings.health.reconnect) return + try { + if (this.ctx.mcp && typeof this.ctx.mcp.reload === "function") { + await this.ctx.mcp.reload() + this.log("asked OpenCode to reload MCP servers") + } + } catch (err) { + this.warn(`MCP reload failed: ${err instanceof Error ? err.message : err}`) + } + } + + private async tryAutoStart(): Promise { + if (!/^https?:\/\/(127\.0\.0\.1|localhost|\[::1\])/i.test(this.url)) return + const childProcess = builtin("node:child_process") + if (!childProcess || !this.settings.health.startCommand.trim()) return + try { + const child = childProcess.spawn(this.settings.health.startCommand, { + cwd: this.ctx.location?.directory || process.cwd(), + shell: true, + detached: true, + stdio: "ignore", + }) + child.unref?.() + this.log(`started: ${this.settings.health.startCommand}`) + } catch (err) { + this.warn(`auto-start failed: ${err instanceof Error ? err.message : err}`) + } + } +} + +/* ------------------------------------------------------------------ * + * Guard helpers + * ------------------------------------------------------------------ */ + +function resultOfToolEvent(event: Loose): { output?: string; metadata?: Loose } { + const result = isPlainObject(event?.result) ? event.result : undefined + if (!result) return {} + const output = typeof result.output === "string" ? result.output : undefined + const metadata = isPlainObject(result.metadata) ? result.metadata : undefined + return { output, metadata } +} + +function looksEmptyResult(event: Loose): boolean { + const { output, metadata } = resultOfToolEvent(event) + if (metadata) { + for (const key of ["matches", "count", "numMatches", "total"]) { + if (typeof metadata[key] === "number" && metadata[key] === 0) return true + } + } + if (output === undefined) return false + const trimmed = output.trim() + if (trimmed === "") return true + return /^(no (files|matches|results) found|0 (matches|results)|no matches)\b/i.test(trimmed) +} + +function extractPattern(event: Loose): string { + const input = isPlainObject(event?.input) ? event.input : undefined + if (!input) return "" + for (const key of ["pattern", "query", "search"]) { + const value = input[key] + if (typeof value === "string" && value.trim()) return value.trim() + } + return "" +} + +const REGEX_META = /[\\^$.*+?()[\]{}|]/ + +function formatSearchHits(result: Loose | undefined, limit: number): string[] { + const results = Array.isArray(result?.results) ? (result!.results as Loose[]) : [] + const lines: string[] = [] + for (const hit of results.slice(0, limit)) { + const path = typeof hit.path === "string" ? hit.path : "?" + const line = typeof hit.start_line === "number" ? `:${hit.start_line + 1}` : "" + const kind = typeof hit.kind === "string" ? ` (${hit.kind})` : "" + const signature = typeof hit.signature === "string" && hit.signature ? ` ${truncate(hit.signature, 90)}` : "" + lines.push(` - ${path}${line}${kind}${signature}`) + } + return lines +} + +/* ------------------------------------------------------------------ * + * Plugin + * ------------------------------------------------------------------ */ + +export default { + id: "codesearch", + + async setup(ctx: PluginContext) { + const settings = resolveSettings(ctx) + const log = (...args: unknown[]) => console.log("[codesearch]", ...args) + const warn = (...args: unknown[]) => console.warn("[codesearch]", ...args) + const directory = normalizePath(ctx.location?.directory ?? "") + + const url = resolveServeUrl(settings) + const token = resolveToken(settings) + const registry = new RepoRegistry( + settings.reposConfig || `${codesearchHome()}/repos.json`, + ) + registry.load() + const status = new StatusCache() + + // Interactive client (rescue/commands/recall) and a background client + // (health/status/compaction assistance) so a slow background call can + // never queue ahead of an interactive one. + const client = new McpClient(url, token, settings.debug, "client") + const background = new McpClient(url, token, settings.debug, "background") + + const disposables: Registration[] = [] + const timers: Array> = [] + const controller = new AbortController() + + const logGuard = (...args: unknown[]) => { + if (settings.log.guards) log(...args) + } + const logRescue = (...args: unknown[]) => { + if (settings.log.rescue) log(...args) + } + const logScope = (...args: unknown[]) => { + if (settings.log.scope) log(...args) + } + + if (settings.debug) log(`loaded in ${directory || "(unknown directory)"}; serve ${url}`) + + /* ---------------- scope resolution ---------------- */ + + const scopeCache = { at: 0, value: undefined as ScopeInfo | undefined } + + function resolveScope(): ScopeInfo { + const now = Date.now() + if (scopeCache.value && now - scopeCache.at < 15_000) return scopeCache.value + + registry.load() + const dir = directory + let project = dir ? registry.projectFor(dir) : undefined + if (!project && status.isFresh && dir) project = status.projectFor(dir) + + let groups: string[] = [] + if (project) { + groups = registry.groupsFor(project.alias) + if (!groups.length && status.isFresh) groups = status.groupsFor(project.alias) + } + + let siblings: string[] = [] + if (!project && dir) { + siblings = registry.siblingsFor(dir) + if (!siblings.length && status.isFresh) siblings = status.siblingsFor(dir) + siblings = siblings.slice(0, settings.scope.maxSiblings) + } + + const hasGit = dir ? fileExists(`${dir}/.git`) : false + const value: ScopeInfo = { + directory: dir, + project: project?.alias, + projectPath: project?.path, + groups: settings.scope.includeGroups ? groups : [], + siblings: settings.scope.siblings ? siblings : [], + registered: !!project, + hasGit, + } + scopeCache.at = now + scopeCache.value = value + return value + } + + /** Repo alias coverage: for guards and rescue. Uses the plugin's location. */ + function coveredAlias(): string | undefined { + return resolveScope().project + } + + /* ---------------- health + status refresh ---------------- */ + + const refreshStatus = async (): Promise => { + try { + await status.refresh(background) + scopeCache.at = 0 // pick up registry changes discovered via status + } catch (err) { + if (settings.debug) warn(`status refresh failed: ${err instanceof Error ? err.message : err}`) + } + } + + const health = new HealthRunner( + ctx, + url, + settings, + () => { + void refreshStatus() + }, + log, + warn, + ) + + // Startup probe + first status refresh, off the critical path. + void health.probe().then(() => void refreshStatus()) + + /* ---------------- freshness (file edits) ---------------- */ + + const editedPaths = new Map() + const recordEdited = (path: string): void => { + const normalized = normalizePath(path) + if (!normalized) return + if (!/\./.test(normalized)) return // skip directories / odd values + editedPaths.set(normalized, Date.now()) + while (editedPaths.size > 64) { + const oldest = editedPaths.keys().next().value as string | undefined + if (oldest === undefined) break + editedPaths.delete(oldest) + } + } + + const collectEditedFromEvent = (event: Loose): void => { + const data = isPlainObject(event?.data) ? event.data : event + for (const key of ["file", "path", "filePath", "filename"]) { + const value = data?.[key] + if (typeof value === "string") recordEdited(value) + } + } + + if (settings.freshness.enabled && ctx.event) { + const loop = (async () => { + try { + for await (const event of ctx.event!.subscribe({ signal: controller.signal })) { + try { + const type = String((event as Loose)?.type ?? "") + if (type.startsWith("file.")) collectEditedFromEvent(event as Loose) + } catch { + /* ignore malformed events */ + } + } + } catch (err) { + if (!(err instanceof Error && err.name === "AbortError")) { + if (settings.debug) warn(`event loop ended: ${err instanceof Error ? err.message : err}`) + } + } + })() + void loop + } + + function recentEdits(max: number): string[] { + const entries = [...editedPaths.entries()].sort((a, b) => b[1] - a[1]) + return entries.map(([path]) => path).slice(0, max) + } + + /* ---------------- guidance text ---------------- */ + + function loadGuidance(): string { + if (!settings.guidance.enabled) return "" + const inline = settings.guidance.text.trim() + if (inline) return inline + const path = expandEnv(settings.guidance.file).trim() + if (path) { + const text = readTextFile(path)?.trim() + if (text) return text + } + return DEFAULT_GUIDANCE + } + + const guidanceText = loadGuidance() + + /* ---------------- context hook (scope, guidance, prune, recall) ---------------- */ + + const seenMessages = new Set() + const seenSessions = new Set() + const nudged = new Set() + + function markSeen(set: Set, key: string): boolean { + if (set.has(key)) return false + set.add(key) + if (set.size > MAX_SEEN) set.clear() + return true + } + + async function onContext(event: Loose): Promise { + try { + const sessionID = String(event?.sessionID ?? "unknown") + const userMessage = lastUserMessage((event?.messages ?? []) as Loose[]) + if (!userMessage || !userMessage.text.trim()) return + const messageKey = `${sessionID}:${userMessage.id}` + if (!markSeen(seenMessages, messageKey)) return + const firstForSession = !seenSessions.has(sessionID) + seenSessions.add(sessionID) + if (seenSessions.size > MAX_SEEN) seenSessions.clear() + + const scope = resolveScope() + const system = Array.isArray(event?.system) ? (event.system as SystemPart[]) : undefined + if (!system) return + + // 1. Opt-in auto-recall: inject index context for the user message. + if (settings.recall.enabled && scope.project && isLikelyCodeQuestion(userMessage.text)) { + await injectRecall(system, sessionID, userMessage.text, scope) + } + + // 2. Prune guard: hide grep/glob from the model when enforcement is + // structural, so the first move is a codesearch call. Like block + // mode, only engage while the hub is confirmed reachable — + // otherwise the removed tools would contradict the "grep/glob is an + // acceptable fallback" guidance injected below. + const pruneActive = + settings.guards.mode === "prune" && !!scope.project && health.state === "ok" + if (pruneActive && isPlainObject(event.tools)) { + for (const tool of settings.guards.tools) delete (event.tools as Loose)[tool] + } + + // 3. Guidance + scope, once per session (or per message). + const injectGuidance = settings.scope.enabled || guidanceText + const wantsThisMessage = + settings.scope.inject === "message" || (settings.scope.inject === "session" && firstForSession) + if (injectGuidance && wantsThisMessage) { + const extra: string[] = [] + if (pruneActive) { + extra.push(`- grep and glob are intentionally unavailable in this project (codesearch guard mode "prune"); use the codesearch tools for discovery.`) + } + if (health.state === "down" && !pruneActive) { + extra.push(`- The codesearch serve hub is unreachable right now; grep/glob is an acceptable fallback until it recovers.`) + } + if (freshUnindexedWarning(scope)) { + extra.push(`- The registered index for "${scope.project}" is empty or was built with an unknown model; searches will miss until \`codesearch index\` refreshes it.`) + } + if (settings.freshness.enabled) { + const edits = recentEdits(settings.freshness.maxFiles) + if (edits.length) { + const shown = edits.map((p) => relativeTo(p, scope.projectPath || directory) || p) + extra.push(`- Files edited this session: ${shown.join(", ")}. The serve hub reindexes on write; if a query misses a very recent change, retry it once.`) + editedPaths.clear() + } + } + const prelude = buildScopePrelude(scope, extra) + const text = [prelude, guidanceText].filter(Boolean).join("\n\n") + if (text) { + system.push({ type: "text", text }) + logScope(`injected scope/guidance for session ${sessionID} (project=${scope.project ?? "none"})`) + } + } + } catch (err) { + if (settings.debug) warn(`context hook error: ${err instanceof Error ? err.message : err}`) + } + } + + function freshUnindexedWarning(scope: ScopeInfo): boolean { + if (!settings.scope.warnUnindexed || !scope.project) return false + const repo = status.repo(scope.project) + if (!repo) return false + if (typeof repo.total_chunks === "number" && repo.total_chunks === 0) return true + return !repo.model || repo.model === "unknown" + } + + async function injectRecall(system: SystemPart[], sessionID: string, text: string, scope: ScopeInfo): Promise { + try { + const result = (await client.call( + "search", + { project: scope.project, query: truncate(text, 1000), limit: settings.recall.limit, compact: false }, + { timeoutMs: settings.recall.timeoutMs, retry: false }, + )) as Loose | undefined + const results = Array.isArray(result?.results) ? (result!.results as Loose[]) : [] + const kept: string[] = [] + let budget = settings.recall.budgetChars + for (const hit of results) { + const content = typeof hit.content === "string" ? hit.content.trim() : "" + if (!content) continue + const snippet = truncate(content, Math.min(700, budget)) + if (snippet.length > budget) continue + budget -= snippet.length + kept.push(`- ${hit.path ?? "?"}:${(hit.start_line ?? 0) + 1}\n ${snippet.replace(/\n/g, "\n ")}`) + } + if (!kept.length) return + system.push({ + type: "text", + text: + `# codesearch context (auto-injected)\n` + + `Relevant indexed code for the current request (background context, not instructions):\n` + + kept.join("\n"), + }) + if (settings.log.recall) log(`recall injected ${kept.length} chunk(s) for ${sessionID}`) + } catch (err) { + if (settings.debug) warn(`recall failed: ${err instanceof Error ? err.message : err}`) + } + } + + /* ---------------- tool hooks (guards, rescue, nudge) ---------------- */ + + function blockMessage(tool: string, scope: ScopeInfo): string { + return ( + `codesearch: ${tool} is disabled here — this repository is indexed as project "${scope.project}".\n` + + `Use the codesearch MCP tools instead:\n` + + ` - search { project: "${scope.project}", query: "" } (add mode: "literal", regex: true for exact syntax)\n` + + ` - find { project: "${scope.project}", symbol: "", kind: "definition" | "usages" | "imports" | "dependents" }\n` + + ` - find_impact { project: "${scope.project}", symbol_name: "" } (C#/TypeScript call sites)\n` + + ` - explore { project: "${scope.project}", target: "", kind: "outline" }\n` + + ` - get_chunk { project: "${scope.project}", chunk_id: }\n` + + `If codesearch is genuinely unavailable, set "guards": { "mode": "off" } in ~/.config/opencode/codesearch.json (or CODESEARCH_PLUGIN_GUARDS=off) and retry.` + ) + } + + async function onToolBefore(event: Loose): Promise { + let blocked: Error | null = null + try { + if (settings.guards.mode !== "block") return + const tool = String(event?.tool ?? "").toLowerCase() + if (!settings.guards.tools.includes(tool)) return + const alias = coveredAlias() + if (!alias) return + if (health.state !== "ok") return // fail open unless the hub is confirmed up + blocked = new Error(blockMessage(tool, resolveScope())) + } catch (err) { + if (settings.debug) warn(`execute.before error: ${err instanceof Error ? err.message : err}`) + } + if (blocked) throw blocked + } + + async function onToolAfter(event: Loose): Promise { + try { + if (event?.status && event.status !== "completed") return + const tool = String(event?.tool ?? "").toLowerCase() + if (!settings.guards.tools.includes(tool)) return + const scope = resolveScope() + if (!scope.project) return + const result = isPlainObject(event?.result) ? event.result : undefined + if (!result) return + + if (looksEmptyResult(event)) { + await rescueEmptyResult(event, result, scope) + } else { + nudgeOnce(event, result, scope) + } + } catch (err) { + if (settings.debug) warn(`execute.after error: ${err instanceof Error ? err.message : err}`) + } + } + + async function rescueEmptyResult(event: Loose, result: Loose, scope: ScopeInfo): Promise { + if (!settings.guards.rescue) return + if (health.state === "down") return + const pattern = extractPattern(event) + if (!pattern || pattern.length > 500) return + // Glob patterns are not search queries: skip when the pattern is just a + // filesystem wildcard ("**/*.ts") — there is nothing semantic to look up. + if (String(event?.tool ?? "").toLowerCase() === "glob" && /[*?[\]{}]/.test(pattern)) return + const plain = !REGEX_META.test(pattern) + try { + const hits = (await client.call( + "search", + { + project: scope.project, + query: pattern, + mode: "literal", + ...(plain ? { phrase: true } : { regex: true }), + limit: settings.guards.rescueLimit, + compact: true, + }, + { timeoutMs: settings.guards.rescueTimeoutMs, retry: false }, + )) as Loose | undefined + const lines = formatSearchHits(hits, settings.guards.rescueLimit) + if (!lines.length) return + const notice = + `\n[codesearch] ${String(event?.tool ?? "grep")} returned nothing, but the index has ` + + `${lines.length >= settings.guards.rescueLimit ? `${settings.guards.rescueLimit}+` : lines.length} ` + + `match(es) for ${plain ? `"${pattern}"` : `/${pattern}/`} in project "${scope.project}":\n` + + lines.join("\n") + + `\nUse search/find/get_chunk for the full result.` + const existing = typeof result.output === "string" ? result.output : "" + result.output = existing + notice + logRescue(`rescued empty ${String(event?.tool ?? "grep")} (${scope.project})`) + } catch (err) { + if (settings.debug) warn(`rescue failed: ${err instanceof Error ? err.message : err}`) + } + } + + function nudgeOnce(event: Loose, result: Loose, scope: ScopeInfo): void { + if (settings.guards.mode === "off") return + const sessionID = String(event?.sessionID ?? "unknown") + if (!markSeen(nudged, sessionID)) return + const existing = typeof result.output === "string" ? result.output : "" + result.output = + existing + + `\n[codesearch] tip: this repository is indexed as project "${scope.project}" — ` + + `codesearch search is faster and more token-efficient than ${String(event?.tool ?? "grep")} for cross-file or symbol lookups.` + logGuard(`nudged session ${sessionID} after ${String(event?.tool ?? "grep")}`) + } + + /* ---------------- commands ---------------- */ + + async function promptSession(sessionID: string, prompt: Loose, text: string): Promise { + if (!ctx.session?.prompt) { + warn("session.prompt is unavailable; cannot deliver command output") + return + } + await ctx.session.prompt({ ...prompt, sessionID, text, delivery: (prompt as Loose)?.delivery ?? "queue" }) + } + + function formatStatusText(scope: ScopeInfo): string { + const lines: string[] = [] + lines.push(`codesearch serve: ${url}`) + lines.push(`health: ${health.state}${health.lastError ? ` (${health.lastError})` : ""}`) + lines.push(`directory: ${directory || "?"}`) + if (scope.project) { + const repo = status.repo(scope.project) + lines.push( + `project: ${scope.project}` + + (repo + ? ` — ${repo.total_chunks ?? "?"} chunks, ${repo.total_files ?? "?"} files, model=${repo.model ?? "?"}, state=${repo.lock_status ?? "?"}` + : " (not in the latest status snapshot)"), + ) + } else if (scope.siblings.length) { + lines.push(`workspace projects: ${scope.siblings.join(", ")}`) + } else { + lines.push(`project: not registered${scope.hasGit ? " (git repo — run /codesearch-index to register)" : ""}`) + } + if (scope.groups.length) lines.push(`groups: ${scope.groups.join(", ")}`) + if (status.isFresh) lines.push(`hub repos: ${status.all.length}`) + return lines.join("\n") + } + + async function registerCommands(): Promise { + if (!settings.commands.enabled || !ctx.command?.transform) return + const registration = await ctx.command.transform((editor) => { + editor.add({ + name: "codesearch", + description: "Semantic code search (codesearch) — query the index and inject the top hits", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + const query = String(prompt.text ?? invocation?.arguments ?? "").trim() + const scope = resolveScope() + if (!query) { + await promptSession(sessionID, prompt, "Usage: /codesearch ") + return + } + if (!scope.project) { + await promptSession( + sessionID, + prompt, + `codesearch: this directory is not registered with a codesearch project, so there is nothing to search yet. Run /codesearch-index first.`, + ) + return + } + try { + const hits = (await client.call( + "search", + { project: scope.project, query, limit: 8 }, + { timeoutMs: 8_000, retry: false }, + )) as Loose | undefined + const lines = formatSearchHits(hits, 8) + await promptSession( + sessionID, + prompt, + lines.length + ? `codesearch results for "${query}" (project "${scope.project}"):\n${lines.join("\n")}\n\nFetch full code with get_chunk, or refine with find/explore.` + : `codesearch returned no results for "${query}" in project "${scope.project}". Try different wording, or mode literal with regex=true for exact syntax.`, + ) + if (settings.log.commands) log(`/codesearch "${truncate(query, 80)}" -> ${lines.length} hit(s)`) + } catch (err) { + await promptSession( + sessionID, + prompt, + `codesearch search failed: ${err instanceof Error ? err.message : String(err)}`, + ) + } + }, + }) + + editor.add({ + name: "codesearch-status", + description: "Show codesearch serve health, current project scope and index state", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + await health.probe() + await refreshStatus() + scopeCache.at = 0 + await promptSession(sessionID, prompt, formatStatusText(resolveScope())) + }, + }) + + editor.add({ + name: "codesearch-index", + description: "Register and index the current directory with codesearch (runs in the background)", + execute: async (invocation: Loose) => { + const sessionID = String(invocation?.sessionID ?? "") + const prompt = isPlainObject(invocation?.prompt) ? invocation.prompt : {} + const childProcess = builtin("node:child_process") + const command = settings.commands.indexCommand.trim() + if (!command) { + await promptSession(sessionID, prompt, "codesearch: commands.indexCommand is empty; nothing to run.") + return + } + if (!childProcess) { + await promptSession(sessionID, prompt, `codesearch: cannot spawn processes here. Run \`${command}\` in ${directory} manually.`) + return + } + try { + const child = childProcess.spawn(command, { + cwd: directory || process.cwd(), + shell: true, + detached: true, + stdio: "ignore", + }) + child.unref?.() + await promptSession( + sessionID, + prompt, + `codesearch indexing started in ${directory} (\`${command}\`). It runs in the background; check /codesearch-status in a minute (large repos can take longer).`, + ) + if (settings.log.commands) log(`/codesearch-index started: ${command}`) + } catch (err) { + await promptSession(sessionID, prompt, `codesearch index failed to start: ${err instanceof Error ? err.message : String(err)}`) + } + }, + }) + }) + disposables.push(registration) + } + + /* ---------------- scope tool ---------------- */ + + async function registerScopeTool(): Promise { + if (!settings.tools.scope || !ctx.tool?.transform) return + const registration = await ctx.tool.transform((editor) => { + editor.add({ + name: "codesearch_scope", + description: + "Return the codesearch scope for the current directory: project alias, groups, index state, serve health. " + + "Use it when you are unsure which project= value to pass to codesearch MCP tools.", + input: { type: "object", properties: {}, additionalProperties: false }, + execute: async () => { + await refreshStatus().catch(() => undefined) + scopeCache.at = 0 + const scope = resolveScope() + const repo = scope.project ? status.repo(scope.project) : undefined + const info = { + directory: scope.directory, + project: scope.project ?? null, + groups: scope.groups, + registered: scope.registered, + siblings: scope.siblings, + indexed: repo ? (repo.total_chunks ?? 0) > 0 : undefined, + chunks: repo?.total_chunks, + model: repo?.model, + lockStatus: repo?.lock_status, + serveUrl: url, + serveHealth: health.state, + } + return { content: JSON.stringify(info, null, 2) } + }, + }) + }) + disposables.push(registration) + } + + /* ---------------- skill ---------------- */ + + async function registerSkill(): Promise { + if (!settings.skill.enabled || !ctx.skill?.transform) return + const registration = await ctx.skill.transform((editor) => { + editor.add({ + id: "codesearch", + name: "codesearch navigation", + description: + "Use the codesearch MCP tools (search, find, find_impact, explore, get_chunk) to navigate code: " + + "project scope rules, tool routing, and exact-syntax patterns.", + path: "codesearch-plugin/skill.md", + content: SKILL_CONTENT, + }) + if (settings.skill.autoinvoke && typeof editor.get === "function" && typeof editor.update === "function") { + const existing = editor.get("codesearch") + if (existing) editor.update("codesearch", (skill: Loose) => void (skill.autoinvoke = true)) + } + }) + disposables.push(registration) + } + + /* ---------------- compaction assist ---------------- */ + + function formatOutline(target: string, result: unknown): string { + const entries = Array.isArray(result) ? (result as Loose[]) : [] + const symbols = entries + .filter((entry) => entry && typeof entry.signature === "string" && entry.signature) + .filter((entry) => !/^imports\b/i.test(String(entry.signature))) + .slice(0, 12) + .map((entry) => `${truncate(String(entry.signature), 100)}:${(entry.start_line ?? 0) + 1}`) + const label = target.replace(/\\/g, "/") + return symbols.length ? `- ${label}: ${symbols.join("; ")}` : "" + } + + async function onCompaction(event: Loose): Promise { + try { + if (!settings.compaction.enabled) return + const scope = resolveScope() + if (!scope.project || !Array.isArray(event?.system)) return + const candidates: string[] = [] + for (const path of recentEdits(settings.compaction.maxFiles * 2)) { + if (!candidates.includes(path)) candidates.push(path) + } + if (!candidates.length) return + const deadline = Date.now() + settings.compaction.timeoutMs + const outlines: string[] = [] + let budget = settings.compaction.budgetChars + for (const path of candidates) { + if (outlines.length >= settings.compaction.maxFiles || Date.now() >= deadline || budget <= 0) break + try { + // Strip the project root the same way `relativeTo` does for the + // freshness notice: it normalizes both sides (including the + // win32 lowercase rule), so absolute paths, alias-prefixed paths + // and already-relative paths all resolve consistently. Stripping + // a literal alias prefix here silently produced absolute targets + // on Windows, where the case-normalized path never matched. + const target = relativeTo(path, scope.projectPath || directory) || path + const result = await background.call( + "explore", + { project: scope.project, target, kind: "outline" }, + { timeoutMs: Math.max(500, deadline - Date.now()), retry: false }, + ) + const line = formatOutline(target, result) + if (line) { + outlines.push(line) + budget -= line.length + } + } catch { + /* best effort */ + } + } + if (!outlines.length) return + ;(event.system as SystemPart[]).push({ + type: "text", + text: + `# codesearch outlines (captured before compaction)\n` + + `Pre-compaction structure of recently edited files (from the code index):\n` + + outlines.join("\n"), + }) + if (settings.log.scope) log(`compaction outlines: ${outlines.length} file(s)`) + } catch (err) { + if (settings.debug) warn(`compaction hook error: ${err instanceof Error ? err.message : err}`) + } + } + + /* ---------------- register everything (fail soft) ---------------- */ + + const safeRegister = async (name: string, fn: () => Promise): Promise => { + try { + await fn() + if (settings.debug) log(`registered: ${name}`) + } catch (err) { + warn(`could not register ${name}: ${err instanceof Error ? err.message : err}`) + } + } + + if (ctx.session?.hook) { + await safeRegister("session context hook", async () => { + disposables.push(await ctx.session!.hook("context", onContext)) + }) + } + if (ctx.tool?.hook && settings.guards.mode !== "off") { + await safeRegister("tool execute.before hook", async () => { + disposables.push(await ctx.tool!.hook("execute.before", onToolBefore)) + }) + await safeRegister("tool execute.after hook", async () => { + disposables.push(await ctx.tool!.hook("execute.after", onToolAfter)) + }) + } + if (settings.compaction.enabled && ctx.session?.hook) { + await safeRegister("compaction hook", async () => { + disposables.push(await ctx.session!.hook("compaction", onCompaction)) + }) + } + await safeRegister("commands", registerCommands) + await safeRegister("scope tool", registerScopeTool) + await safeRegister("skill", registerSkill) + + /* ---------------- timers ---------------- */ + + if (settings.health.enabled) { + health.start() + const statusTimer = setInterval(() => void refreshStatus(), 10 * 60_000) + ;(statusTimer as unknown as { unref?: () => void }).unref?.() + timers.push(statusTimer) + } + + log(`ready (project=${resolveScope().project ?? "none"}, guards=${settings.guards.mode}, serve=${url})`) + + return async () => { + controller.abort() + health.stop() + for (const timer of timers) clearTimeout(timer) + for (const registration of disposables) await registration.dispose().catch(() => undefined) + } + }, +} diff --git a/integrations/opencode/package.json b/integrations/opencode/package.json new file mode 100644 index 00000000..0ff721e0 --- /dev/null +++ b/integrations/opencode/package.json @@ -0,0 +1,18 @@ +{ + "name": "opencode-codesearch-plugin", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "OpenCode v2 plugin that makes codesearch structural: project scope resolution, usage guidance, grep/glob guards with zero-hit rescue, serve health, commands, a scope tool, a skill and compaction assistance.", + "license": "Apache-2.0", + "scripts": { + "typecheck": "tsc --noEmit", + "unit": "node test/unit.ts", + "smoke": "node test/smoke.ts", + "test": "npm run typecheck && npm run unit && npm run smoke" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "typescript": "^5.6.0" + } +} diff --git a/integrations/opencode/test/smoke.ts b/integrations/opencode/test/smoke.ts new file mode 100644 index 00000000..1e555b0a --- /dev/null +++ b/integrations/opencode/test/smoke.ts @@ -0,0 +1,460 @@ +/** + * Smoke test for the codesearch OpenCode plugin. + * + * Runs the plugin against a mock OpenCode context and a mock codesearch MCP + * server (streamable HTTP) so scope resolution, guidance injection, zero-hit + * rescue, the nudge/prune/block guards (including prune fail-open while the + * hub is down), compaction outlines, commands, the scope tool and the skill + * registration are exercised end-to-end without a real model session. + * + * Run (Node >= 23.6 strips types natively): + * node test/smoke.ts + * or: + * bun test/smoke.ts + */ + +import assert from "node:assert/strict" +import http from "node:http" +import { mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "../codesearch.ts" + +type Loose = Record +type Hook = (event: Loose) => any + +const HITS = [ + { + chunk_id: 7, + path: "my-service/src/uploader.ts", + start_line: 41, + end_line: 63, + kind: "Function", + signature: "function retryUpload(job: Job, attempt: number)", + }, + { + chunk_id: 9, + path: "my-service/src/queue.ts", + start_line: 12, + end_line: 30, + kind: "Class", + signature: "class UploadQueue", + }, +] + +const STATUS = { + repos: [ + { + alias: "my-service", + project_path: "/work/my-service", + total_chunks: 120, + total_files: 30, + model: "minilm-l6-q", + lock_status: "warm", + }, + { + alias: "sibling-api", + project_path: "/work/sibling-api", + total_chunks: 0, + total_files: 0, + model: "unknown", + lock_status: "available", + }, + ], + groups: { platform: ["my-service", "sibling-api"] }, + hub: { indexed: true, status: "ready" }, +} + +function startMockServer(): Promise<{ url: string; close: () => Promise; calls: Loose[] }> { + const calls: Loose[] = [] + const server = http.createServer((req, res) => { + let body = "" + req.on("data", (chunk) => (body += chunk)) + req.on("end", () => { + if (req.url === "/healthz") { + res.writeHead(200, { "content-type": "application/json" }) + res.end('{"status":"ok"}') + return + } + if (req.url !== "/mcp" || req.method !== "POST") { + res.writeHead(404) + res.end() + return + } + let message: Loose + try { + message = JSON.parse(body) as Loose + } catch { + res.writeHead(400) + res.end() + return + } + const reply = (payload: Loose, session = false): void => { + res.writeHead(200, { + "content-type": "application/json", + ...(session ? { "mcp-session-id": "smoke-session" } : {}), + }) + res.end(JSON.stringify(payload)) + } + if (message.method === "initialize") { + reply( + { + jsonrpc: "2.0", + id: message.id, + result: { + protocolVersion: message.params?.protocolVersion ?? "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "mock-codesearch", version: "smoke" }, + }, + }, + true, + ) + return + } + if (message.method === "notifications/initialized") { + res.writeHead(202) + res.end() + return + } + if (message.method === "tools/call") { + const name = String(message.params?.name ?? "") + const args = (message.params?.arguments ?? {}) as Loose + calls.push({ name, args }) + let payload: unknown = {} + if (name === "status") { + payload = STATUS + } else if (name === "search") { + const query = String(args.query ?? "").toLowerCase() + payload = query.includes("none-at-all") ? { results: [] } : { results: HITS } + } else if (name === "explore") { + payload = [ + { chunk_id: 1, kind: "Function", signature: "function retryUpload(job: Job)", start_line: 40, end_line: 63 }, + ] + } + reply({ + jsonrpc: "2.0", + id: message.id, + result: { content: [{ type: "text", text: JSON.stringify(payload) }] }, + }) + return + } + reply({ jsonrpc: "2.0", id: message.id, result: {} }) + }) + }) + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() + const port = typeof address === "object" && address ? address.port : 0 + resolve({ + url: `http://127.0.0.1:${port}/mcp`, + close: () => + new Promise((done) => { + server.close(() => done(undefined)) + }), + calls, + }) + }) + }) +} + +function makeCtx(directory: string) { + const hooks: Record = {} + const toolHooks: Record = {} + const tools: Loose[] = [] + const commands: Loose[] = [] + const skills: Loose[] = [] + const prompts: string[] = [] + const eventQueue: Loose[] = [] + let eventWake: (() => void) | undefined + + const context: Loose = { + app: { version: "smoke" }, + location: { directory }, + session: { + hook: async (name: string, callback: Hook) => { + ;(hooks[name] ||= []).push(callback) + return { dispose: async () => {} } + }, + prompt: async (input: Loose) => { + prompts.push(String(input.text ?? "")) + return {} + }, + }, + tool: { + hook: async (name: string, callback: Hook) => { + ;(toolHooks[name] ||= []).push(callback) + return { dispose: async () => {} } + }, + transform: async (callback: (editor: Loose) => void) => { + callback({ + namespace: () => {}, + add: (tool: Loose) => tools.push(tool), + get: () => undefined, + update: () => {}, + }) + return { dispose: async () => {} } + }, + }, + command: { + transform: async (callback: (editor: Loose) => void) => { + callback({ add: (command: Loose) => commands.push(command) }) + return { dispose: async () => {} } + }, + }, + skill: { + transform: async (callback: (editor: Loose) => void) => { + callback({ + add: (skill: Loose) => skills.push(skill), + get: () => undefined, + update: () => {}, + }) + return { dispose: async () => {} } + }, + }, + mcp: { list: async () => [] }, + event: { + subscribe: async function* (options?: { signal?: AbortSignal }) { + const signal = options?.signal + while (!signal?.aborted) { + if (!eventQueue.length) { + await new Promise((resolve) => { + eventWake = resolve + signal?.addEventListener("abort", () => resolve(), { once: true }) + }) + } + while (eventQueue.length) yield eventQueue.shift() as Loose + } + }, + }, + } + + return { + context, + hooks, + toolHooks, + tools, + commands, + skills, + prompts, + emit: (event: Loose) => { + eventQueue.push(event) + const wake = eventWake + eventWake = undefined + wake?.() + }, + } +} + +async function setupPlugin(directory: string) { + const app = makeCtx(directory) + const cleanup = await plugin.setup(app.context as any) + return { app, cleanup } +} + +function userMessage(sessionID: string, messageID: string, text: string): Loose { + return { + sessionID, + system: [], + tools: { grep: {}, glob: {}, read: {} }, + messages: [{ info: { role: "user", id: messageID }, parts: [{ type: "text", text }] }], + } +} + +async function main(): Promise { + const mock = await startMockServer() + const tmp = mkdtempSync(join(tmpdir(), "codesearch-plugin-smoke-")) + + const reposPath = join(tmp, "repos.json") + writeFileSync( + reposPath, + JSON.stringify({ + repos: { "my-service": "/work/my-service", "sibling-api": "/work/sibling-api" }, + groups: { platform: ["my-service", "sibling-api"] }, + }), + ) + const configPath = join(tmp, "codesearch.json") + writeFileSync( + configPath, + JSON.stringify({ + url: "", + log: { guards: false, rescue: false, health: false, commands: false, scope: false, recall: false }, + }), + ) + + process.env.CODESEARCH_CONFIG = configPath + process.env.CODESEARCH_REPOS_CONFIG = reposPath + process.env.CODESEARCH_URL = mock.url + process.env.CODESEARCH_PLUGIN_HEALTH = "0" + delete process.env.CODESEARCH_PLUGIN_GUARDS + + /* ---------------- nudge mode (default) ---------------- */ + + const one = await setupPlugin("/work/my-service") + assert.ok(one.app.hooks.context?.length === 1, "context hook registered") + assert.ok(one.app.toolHooks["execute.before"]?.length === 1, "execute.before hook registered") + assert.ok(one.app.toolHooks["execute.after"]?.length === 1, "execute.after hook registered") + assert.ok(one.app.hooks.compaction?.length === 1, "compaction hook registered") + assert.deepEqual( + one.app.commands.map((c) => c.name).sort(), + ["codesearch", "codesearch-index", "codesearch-status"], + "commands registered", + ) + assert.equal(one.app.tools.length, 1, "scope tool registered") + assert.equal(one.app.tools[0].name, "codesearch_scope") + assert.equal(one.app.skills.length, 1, "skill registered") + assert.equal(one.app.skills[0].id, "codesearch") + + // Guidance + scope injection, once per session. + const first = userMessage("s1", "m1", "Where is the upload retried after a failure?") + await one.app.hooks.context[0](first) + const injected = first.system.map((p: Loose) => p.text).join("\n") + assert.match(injected, /project "my-service"/, "scope line names the resolved project") + assert.match(injected, /group: "platform"/, "scope line lists groups") + assert.match(injected, /codesearch/, "guidance is injected") + assert.ok(first.tools.grep, "nudge mode keeps grep visible") + + const second = userMessage("s1", "m2", "And where is it queued?") + await one.app.hooks.context[0](second) + assert.equal(second.system.length, 0, "session mode injects once per session") + + const duplicate = userMessage("s1", "m1", "Where is the upload retried after a failure?") + await one.app.hooks.context[0](duplicate) + assert.equal(duplicate.system.length, 0, "the same user message never injects twice") + + // Zero-hit rescue turns an empty grep into codesearch hits. + const rescueEvent: Loose = { + tool: "grep", + status: "completed", + sessionID: "s1", + callID: "c1", + input: { pattern: "retryUpload" }, + result: { output: "" }, + } + await one.app.toolHooks["execute.after"][0](rescueEvent) + assert.match(String(rescueEvent.result.output), /codesearch/, "rescue appends a codesearch section") + assert.match(String(rescueEvent.result.output), /uploader\.ts/, "rescue includes index hits") + + // A successful grep gets a one-time nudge per session. + const nudgeEvent: Loose = { + tool: "grep", + status: "completed", + sessionID: "s1", + callID: "c2", + input: { pattern: "queue" }, + result: { output: "src/queue.ts:1:const queue = []", metadata: { count: 1 } }, + } + await one.app.toolHooks["execute.after"][0](nudgeEvent) + assert.match(String(nudgeEvent.result.output), /tip:/, "first successful grep is nudged") + const nudgeAgain: Loose = { ...nudgeEvent, callID: "c3", result: { output: "hit", metadata: { count: 1 } } } + await one.app.toolHooks["execute.after"][0](nudgeAgain) + assert.doesNotMatch(String(nudgeAgain.result.output), /tip:/, "the nudge fires once per session") + + // Compaction assist: recently edited files become explore outlines, and the + // target sent to the server must be project-relative. Regression guard: an + // alias-prefix strip left absolute targets that could never match on + // Windows, where the plugin lowercases paths but the server strips them + // case-sensitively — a silent compaction no-op. + one.app.emit({ type: "file.edited", data: { file: "/work/my-service/src/uploader.ts" } }) + await new Promise((resolve) => setTimeout(resolve, 150)) + const compactionEvent: Loose = { system: [] } + await one.app.hooks.compaction[0](compactionEvent) + const compacted = compactionEvent.system.map((p: Loose) => p.text).join("\n") + assert.match(compacted, /src\/uploader\.ts/, "compaction outline names the relative file") + const exploreCall = mock.calls.filter((c) => c.name === "explore").at(-1) + assert.ok(exploreCall, "compaction asked the server for an outline") + assert.equal( + exploreCall!.args.target, + "src/uploader.ts", + "compaction passes the project-relative target, not an absolute/alias-prefixed path", + ) + + // Scope tool reports the resolved project and index state. + const scopeResult = (await one.app.tools[0].execute({}, {})) as { content: string } + const scopeInfo = JSON.parse(scopeResult.content) + assert.equal(scopeInfo.project, "my-service") + assert.equal(scopeInfo.indexed, true) + assert.equal(scopeInfo.chunks, 120) + + // Commands resolve and deliver results through session.prompt. + const searchCommand = one.app.commands.find((c) => c.name === "codesearch") + assert.ok(searchCommand) + await searchCommand.execute({ sessionID: "s1", prompt: { text: "upload retry" } }) + assert.match(one.app.prompts.at(-1) ?? "", /uploader\.ts/, "/codesearch injects hits") + + const statusCommand = one.app.commands.find((c) => c.name === "codesearch-status") + assert.ok(statusCommand) + await statusCommand.execute({ sessionID: "s1", prompt: { text: "" } }) + assert.match(one.app.prompts.at(-1) ?? "", /project: my-service/, "/codesearch-status reports scope") + + await one.cleanup() + + /* ---------------- prune mode ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "prune" + const pruned = await setupPlugin("/work/my-service") + // Prune only engages once the startup health probe has confirmed the hub is + // reachable; poll with fresh sessions instead of racing the probe (the + // guidance is injected once per session, so each attempt needs its own). + let prunedEvent: Loose | undefined + for (let i = 0; i < 30 && !prunedEvent; i++) { + const candidate = userMessage(`s2-${i}`, `p${i}`, "How does the uploader work?") + await pruned.app.hooks.context[0](candidate) + if (candidate.tools.grep === undefined) prunedEvent = candidate + else await new Promise((r) => setTimeout(r, 100)) + } + assert.ok(prunedEvent, "prune engages once the hub is reachable") + assert.equal(prunedEvent!.tools.grep, undefined, "prune removes grep") + assert.equal(prunedEvent!.tools.glob, undefined, "prune removes glob") + assert.ok(prunedEvent!.tools.read, "prune keeps unrelated tools") + assert.match(prunedEvent!.system.map((p: Loose) => p.text).join("\n"), /intentionally unavailable/) + await pruned.cleanup() + + /* ---------------- block mode (fail-open everywhere else) ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "block" + const blocked = await setupPlugin("/work/my-service") + const before = blocked.app.toolHooks["execute.before"][0] + let threw = "" + for (let i = 0; i < 30 && !threw; i++) { + try { + await before({ tool: "grep", input: { pattern: "x" }, sessionID: "s3" }) + } catch (err) { + threw = String(err) + } + if (!threw) await new Promise((r) => setTimeout(r, 100)) + } + assert.match(threw, /codesearch/, "block mode denies grep once the hub is reachable") + assert.match(threw, /project "my-service"/, "denial names the project scope to use") + + const uncovered = await setupPlugin("/work/unregistered") + await uncovered.app.toolHooks["execute.before"][0]({ tool: "grep", input: {}, sessionID: "s4" }) + await uncovered.cleanup() + await blocked.cleanup() + + /* ---------------- prune fails open while the hub is down ---------------- */ + + process.env.CODESEARCH_PLUGIN_GUARDS = "prune" + process.env.CODESEARCH_URL = "http://127.0.0.1:1/mcp" + const downPrune = await setupPlugin("/work/my-service") + // Let the startup probe fail (connection refused) so health settles on down. + await new Promise((resolve) => setTimeout(resolve, 300)) + const downEvent = userMessage("s5", "d1", "Where is the uploader?") + await downPrune.app.hooks.context[0](downEvent) + assert.ok(downEvent.tools.grep, "prune keeps grep visible while the hub is down") + assert.ok(downEvent.tools.glob, "prune keeps glob visible while the hub is down") + assert.match( + downEvent.system.map((p: Loose) => p.text).join("\n"), + /unreachable|acceptable fallback/, + "guidance names the grep/glob fallback while the hub is down", + ) + await downPrune.cleanup() + process.env.CODESEARCH_URL = mock.url + + delete process.env.CODESEARCH_PLUGIN_GUARDS + // Give any in-flight startup probe a moment before the mock hub disappears. + await new Promise((resolve) => setTimeout(resolve, 200)) + await mock.close() + console.log("smoke: OK") +} + +await main() diff --git a/integrations/opencode/test/unit.ts b/integrations/opencode/test/unit.ts new file mode 100644 index 00000000..814335c0 --- /dev/null +++ b/integrations/opencode/test/unit.ts @@ -0,0 +1,230 @@ +/** + * Unit tests for the codesearch OpenCode plugin's pure helpers and the health + * runner. Complements test/smoke.ts, which exercises the plugin end-to-end + * against a mock OpenCode host and a mock codesearch MCP server. + * + * Run (Node >= 23.6 strips types natively): + * node test/unit.ts + * or: + * npm run unit + */ + +import assert from "node:assert/strict" +import http from "node:http" +import type { AddressInfo } from "node:net" +import { HealthRunner, stripJsonc } from "../codesearch.ts" + +type Loose = Record + +/* ------------------------------------------------------------------ * + * stripJsonc + * ------------------------------------------------------------------ */ + +function testStripJsonc(): void { + // Plain JSON passes through unchanged and still parses. + const plain = '{"a":1,"b":[1,2],"c":"x"}' + assert.equal(stripJsonc(plain), plain, "plain JSON is unchanged") + assert.deepEqual(JSON.parse(stripJsonc(plain)), JSON.parse(plain)) + + // Line comments (leading, trailing, at EOF) are removed. + assert.deepEqual(JSON.parse(stripJsonc('{\n // note\n "a": 1 // tail\n}\n')), { a: 1 }) + assert.deepEqual(JSON.parse(stripJsonc('{"a":1} // done')), { a: 1 }, "comment at EOF without newline") + + // Block comments, including inline and multi-line. + assert.deepEqual(JSON.parse(stripJsonc('{/* one */"a":/* two\nlines */1}')), { a: 1 }) + + // Comment markers inside strings are data, not comments. + assert.deepEqual( + JSON.parse(stripJsonc('{"url":"http://example.com/a//b","glob":"/* literal */"}')), + { url: "http://example.com/a//b", glob: "/* literal */" }, + ) + + // Escaped quotes do not terminate the string early. + assert.deepEqual(JSON.parse(stripJsonc('{"s":"a\\"//not-a-comment"}')), { s: 'a"//not-a-comment' }) + + // Trailing commas in objects and arrays are dropped. + assert.deepEqual(JSON.parse(stripJsonc('{\n "a": [1, 2,],\n "b": {"c": 3,},\n}')), { + a: [1, 2], + b: { c: 3 }, + }) + + // Regression: a trailing comma may be followed by a comment before the + // closing bracket ("a": 1, // last member \n }). Comments must be skipped + // while looking ahead, not mistaken for the next member. + assert.deepEqual(JSON.parse(stripJsonc('{"a": 1, // last member\n}')), { a: 1 }) + assert.deepEqual(JSON.parse(stripJsonc('{"a": [1, /* c */ ]}')), { a: [1] }) + + // A comma followed by a real member is preserved. + assert.deepEqual(JSON.parse(stripJsonc('{"a":1, "b":2}')), { a: 1, b: 2 }) + + // CRLF line comments terminate at the newline, not at a literal \r. + assert.deepEqual(JSON.parse(stripJsonc('{\r\n // note\r\n "a": 1\r\n}\r\n')), { a: 1 }) +} + +/* ------------------------------------------------------------------ * + * HealthRunner + * ------------------------------------------------------------------ */ + +function healthSettings(overrides: Loose = {}): Loose { + return { + health: { + enabled: true, + intervalMs: 40, + initialDelayMs: 10, + maxBackoffMs: 80, + reconnect: true, + autoStart: false, + startCommand: "", + ...overrides, + }, + } +} + +async function waitFor(predicate: () => boolean, what: string, timeoutMs = 4000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`timed out waiting for ${what}`) +} + +/** Minimal /healthz server that counts probes, can stop/restart in place, and + * can hold responses so a probe can be kept in flight deterministically. */ +async function startHealthServer(port = 0) { + let count = 0 + let hold = false + let pending: Array<() => void> = [] + const make = () => + http.createServer((_req, res) => { + count += 1 + const respond = () => { + res.writeHead(200, { "content-type": "application/json" }) + res.end('{"status":"ok"}') + } + if (hold) pending.push(respond) + else respond() + }) + let server = make() + await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)) + const actual = (server.address() as AddressInfo).port + return { + url: `http://127.0.0.1:${actual}/mcp`, + hits: () => count, + pendingProbes: () => pending.length, + hold: (value: boolean) => { + hold = value + }, + release: () => { + const waiting = pending + pending = [] + for (const respond of waiting) respond() + }, + close: () => new Promise((resolve) => server.close(() => resolve())), + restart: async () => { + server = make() + await new Promise((resolve) => server.listen(actual, "127.0.0.1", resolve)) + }, + } +} + +async function testHealthRunner(): Promise { + const hub = await startHealthServer() + const logs: string[] = [] + const warnings: string[] = [] + const log = (...args: unknown[]) => logs.push(args.join(" ")) + const warn = (...args: unknown[]) => warnings.push(args.join(" ")) + + // probe() reflects reachability in state and lastError. + { + const runner = new HealthRunner({} as any, hub.url, healthSettings() as any, () => {}, log, warn) + await runner.probe() + assert.equal(runner.state, "ok", "probe marks a reachable hub ok") + assert.equal(runner.lastError, "", "ok state clears lastError") + await hub.close() + await runner.probe() + assert.equal(runner.state, "down", "probe marks an unreachable hub down") + assert.ok(runner.lastError.length > 0, "down state records the error") + assert.ok( + warnings.some((line) => /hub unreachable/.test(line)), + "the first down transition is warned about", + ) + runner.stop() + } + + // start() probes after the initial delay, backs off while down, asks the + // host to reload failed MCP servers, and reports recovery exactly once. + { + await hub.restart() + let reloads = 0 + let recovered = 0 + const ctx = { + mcp: { + reload: async () => { + reloads += 1 + }, + }, + } + const runner = new HealthRunner( + ctx as any, + hub.url, + healthSettings() as any, + () => { + recovered += 1 + }, + log, + warn, + ) + runner.start() + await waitFor(() => runner.state === "ok", "the initial probe to mark the hub ok") + await hub.close() + await waitFor(() => runner.state === "down", "the hub to be detected down") + await waitFor(() => reloads >= 1, "the reconnect reload call") + await hub.restart() + await waitFor(() => runner.state === "ok", "the hub to be detected recovered") + await waitFor(() => recovered === 1, "the recovery callback") + assert.ok( + logs.some((line) => /hub recovered/.test(line)), + "recovery is logged", + ) + // stop() must latch: even a probe that is already in flight must not + // reschedule another one afterwards. + hub.hold(true) + await waitFor(() => hub.pendingProbes() >= 1, "an in-flight probe to hold") + const hitsAtStop = hub.hits() + runner.stop() + hub.release() + await new Promise((resolve) => setTimeout(resolve, 250)) + assert.equal(hub.hits(), hitsAtStop, "stop() halts probing, including from an in-flight tick") + hub.hold(false) + hub.release() + } + + // start() is a no-op when health probing is disabled. + { + const hits = hub.hits() + const runner = new HealthRunner( + {} as any, + hub.url, + healthSettings({ enabled: false }) as any, + () => {}, + log, + warn, + ) + runner.start() + await new Promise((resolve) => setTimeout(resolve, 120)) + assert.equal(hub.hits(), hits, "disabled health never probes") + assert.equal(runner.state, "unknown", "state stays unknown while disabled") + runner.stop() + } + + await hub.close() +} + +async function main(): Promise { + testStripJsonc() + await testHealthRunner() + console.log("unit: OK") +} + +await main() diff --git a/integrations/opencode/tsconfig.json b/integrations/opencode/tsconfig.json new file mode 100644 index 00000000..393d8e8a --- /dev/null +++ b/integrations/opencode/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "noUnusedLocals": true, + "noUnusedParameters": false + }, + "include": ["codesearch.ts", "test/smoke.ts"] +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 64a052e0..6c065e81 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -460,6 +460,26 @@ pub enum Commands { #[arg(long)] idle_suspend_secs: Option, + /// Max concurrent repo indexing jobs (refresh / reindex / HNSW build) + /// this serve runs. Overrides CODESEARCH_INDEX_JOBS (default 1). + #[arg(long)] + index_jobs: Option, + + /// ONNX intra-op threads per embedding session. Overrides + /// CODESEARCH_EMBED_THREADS (unset = ONNX default, all cores). + #[arg(long)] + embed_threads: Option, + + /// Milliseconds to pause between background embedding mini-batches. + /// Overrides CODESEARCH_EMBED_PAUSE_MS (default 0). + #[arg(long)] + embed_pause_ms: Option, + + /// Minimum free memory (MB) before a heavy indexing job may start. + /// Overrides CODESEARCH_MIN_FREE_MB (default 0 = disabled). + #[arg(long)] + min_free_mb: Option, + /// For `tui` action: serve URL to connect to #[arg(long, default_value = DEFAULT_SERVE_URL)] url: String, @@ -508,7 +528,7 @@ pub enum Commands { /// Download embedding models Setup { - /// Model to download (defaults to mxbai-embed-xsmall-v1) + /// Model to download (defaults to minilm-l6-q) #[arg(long)] model: Option, }, @@ -985,19 +1005,28 @@ fn warn_if_heavier_model(model_type: ModelType) { } } +/// Resolve a CLI `--model` flag to an optional [`ModelType`]. +/// +/// `None` (flag absent) stays `None` so runtime paths keep their per-query +/// model resolution; an unknown name is rejected with the full valid-model +/// list. Shared by every command that takes `--model`. +pub(crate) fn parse_model_arg(model: Option<&str>) -> Result> { + match model { + Some(name) => ModelType::parse(name).map(Some).ok_or_else(|| { + anyhow::anyhow!( + "Unknown model '{name}' (valid models: {})", + ModelType::valid_short_names() + ) + }), + None => Ok(None), + } +} + pub async fn run(cancel_token: CancellationToken) -> Result<()> { let cli = Cli::parse(); // Parse model from CLI flag - let model_type = cli.model.as_ref().and_then(|m| ModelType::parse(m)); - if cli.model.is_some() && model_type.is_none() { - eprintln!( - "Unknown model: '{}'. Available models:", - cli.model.as_deref().unwrap_or_default() - ); - eprintln!(" {}", ModelType::valid_short_names()); - std::process::exit(1); - } + let model_type = parse_model_arg(cli.model.as_deref())?; // Set quiet mode if requested if cli.quiet { @@ -1087,18 +1116,7 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Some(peer_name) = &remote { run_remote_add(peer_name, add_path).await } else { - let mt = model - .as_deref() - .and_then(|m| { - let parsed = ModelType::parse(m); - if parsed.is_none() { - eprintln!("Unknown model: '{}'. Available models:", m); - eprintln!(" {}", ModelType::valid_short_names()); - std::process::exit(1); - } - parsed - }) - .or(model_type); + let mt = parse_model_arg(model.as_deref())?.or(model_type); if let Some(mt) = mt { warn_if_heavier_model(mt); } @@ -1234,6 +1252,10 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { no_tui, keep_warm_url, idle_suspend_secs, + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, url, api_key, } => { @@ -1258,6 +1280,24 @@ pub async fn run(cancel_token: CancellationToken) -> Result<()> { if let Some(mt) = model_type { warn_if_heavier_model(mt); } + // CLI overrides for the heavy-work limits. Serve is a single + // long-lived process, so exporting the overrides here keeps a + // single source of truth (the environment) for every + // component that reads them (job gate, embedder, refresh + // batching) without threading a config struct through each + // constructor. Unset flags change nothing. + if let Some(n) = index_jobs { + std::env::set_var(crate::limits::INDEX_JOBS_ENV, n.to_string()); + } + if let Some(n) = embed_threads { + std::env::set_var(crate::limits::EMBED_THREADS_ENV, n.to_string()); + } + if let Some(ms) = embed_pause_ms { + std::env::set_var(crate::limits::EMBED_PAUSE_MS_ENV, ms.to_string()); + } + if let Some(mb) = min_free_mb { + std::env::set_var(crate::limits::MIN_FREE_MB_ENV, mb.to_string()); + } crate::serve::run_serve( host, port, @@ -1779,8 +1819,9 @@ fn codesearch_hook_block() -> String { # Only react to branch/worktree checkouts ($3 = 1). `git worktree add` fires # post-checkout with flag 1; a file checkout (`git checkout -- path`) fires with # flag 0 and must not re-register (it changes no repo location, just wastes a POST). -if [ "$3" = "1" ] && [ -f "$HOME/.codesearch/serve_url" ]; then - __cs_url=$(cat "$HOME/.codesearch/serve_url") +__cs_home="${CODESEARCH_HOME:-$HOME/.codesearch}" +if [ "$3" = "1" ] && [ -f "$__cs_home/serve_url" ]; then + __cs_url=$(cat "$__cs_home/serve_url") if [ -n "$__cs_url" ]; then # Git Bash `pwd` yields an msys path (/c/...) that codesearch serve # cannot canonicalize (HTTP 400); `pwd -W` yields a native C:/ path. @@ -2017,6 +2058,21 @@ mod tests { ); } + #[test] + fn test_hook_block_resolves_serve_url_via_codesearch_home() { + let block = codesearch_hook_block(); + // serve_url lives under the relocatable global root: the hook must + // fall back through CODESEARCH_HOME, mirroring codesearch_home(). + assert!( + block.contains("${CODESEARCH_HOME:-$HOME/.codesearch}"), + "hook must honour CODESEARCH_HOME before ~/.codesearch" + ); + assert!( + !block.contains("$HOME/.codesearch/serve_url"), + "no direct ~/.codesearch/serve_url path may remain" + ); + } + #[test] fn test_full_hook_has_posix_shebang() { let hook = codesearch_full_hook(); @@ -2094,6 +2150,58 @@ mod tests { assert_eq!(after_second.matches(HOOK_BEGIN).count(), 1); } + #[test] + fn test_serve_heavy_work_limit_flags_parse() { + let cli = Cli::try_parse_from([ + "codesearch", + "serve", + "--index-jobs", + "2", + "--embed-threads", + "4", + "--embed-pause-ms", + "25", + "--min-free-mb", + "4096", + ]) + .expect("cli parse should succeed"); + match cli.command { + Commands::Serve { + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, + .. + } => { + assert_eq!(index_jobs, Some(2)); + assert_eq!(embed_threads, Some(4)); + assert_eq!(embed_pause_ms, Some(25)); + assert_eq!(min_free_mb, Some(4096)); + } + _ => panic!("expected Serve command"), + } + } + + #[test] + fn test_serve_heavy_work_limit_flags_default_to_none() { + let cli = Cli::try_parse_from(["codesearch", "serve"]).expect("cli parse should succeed"); + match cli.command { + Commands::Serve { + index_jobs, + embed_threads, + embed_pause_ms, + min_free_mb, + .. + } => { + assert_eq!(index_jobs, None, "unset flags must leave env/defaults"); + assert_eq!(embed_threads, None); + assert_eq!(embed_pause_ms, None); + assert_eq!(min_free_mb, None); + } + _ => panic!("expected Serve command"), + } + } + #[test] fn test_mcp_create_index_defaults_to_true() { let cli = Cli::try_parse_from(["codesearch", "mcp"]).expect("cli parse should succeed"); diff --git a/src/cli/setup.rs b/src/cli/setup.rs index 2fcd8c67..5bc7315f 100644 --- a/src/cli/setup.rs +++ b/src/cli/setup.rs @@ -1,12 +1,91 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; + +use crate::embed::ModelType; + +/// Resolve the CLI `--model` argument to a [`ModelType`]. +/// +/// Thin setup-specific wrapper over [`super::parse_model_arg`]: setup wants +/// the default model when the flag is absent, where other commands keep +/// `None` for per-query resolution. +fn resolve_model(model: Option<&str>) -> Result { + Ok(super::parse_model_arg(model)?.unwrap_or_default()) +} pub async fn run(model: Option) -> Result<()> { - let model_name = model.unwrap_or_else(|| "mxbai-embed-xsmall-v1".to_string()); + let model_type = resolve_model(model.as_deref())?; + + let cache_dir = crate::constants::get_global_models_cache_dir()?; + + println!( + "📦 Downloading embedding model: {} ({})", + model_type.name(), + model_type.short_name() + ); + println!(" Cache dir: {}", cache_dir.display()); - println!("📦 Downloading embedding model: {}", model_name); + // Unlike the runtime paths, setup shows the download progress bar: it + // pulls hundreds of MB on purpose and a silent run looks like a hang. + let mut embedder = crate::embed::FastEmbedder::with_cache_dir_progress( + model_type, + Some(&cache_dir), + true, + ) + .map_err(|e| { + anyhow!( + "Failed to initialize embedding model: {e}\n\ + Hint: if huggingface.co is unreachable, set HF_ENDPOINT to a HuggingFace mirror and retry." + ) + })?; - // TODO: Download model from HuggingFace Hub + // Probe inference so a broken download fails here, not mid-indexing. + let dimensions = embedder.embed_query("setup probe")?.len(); + if dimensions != model_type.dimensions() { + return Err(anyhow!( + "Model returned {dimensions} dims, expected {}", + model_type.dimensions() + )); + } - println!("✅ Setup complete!"); + println!( + "✅ Setup complete! Model '{}' ({dimensions} dims) ready at {}", + model_type.short_name(), + cache_dir.display() + ); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_model_is_rejected_with_valid_list() { + let err = resolve_model(Some("not-a-model")).expect_err("unknown model must be rejected"); + let msg = format!("{err:#}"); + assert!( + msg.contains("not-a-model"), + "error must echo the bad name: {msg}" + ); + assert!( + msg.contains(ModelType::valid_short_names().as_str()), + "error must list valid models: {msg}" + ); + } + + #[test] + fn none_resolves_to_default_model() { + assert_eq!(resolve_model(None).unwrap(), ModelType::default()); + } + + #[test] + fn known_short_names_resolve() { + assert_eq!( + resolve_model(Some("embeddinggemma-q4")).unwrap(), + ModelType::EmbeddingGemma300MQ4 + ); + assert_eq!( + resolve_model(Some("minilm-l6-q")).unwrap(), + ModelType::default() + ); + } +} diff --git a/src/constants.rs b/src/constants.rs index 6d783876..d07e36d2 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -59,18 +59,50 @@ pub const DEFAULT_LOG_MAX_FILES: usize = 5; /// Default log retention period in days pub const DEFAULT_LOG_RETENTION_DAYS: u64 = 5; -/// Get the global models cache directory (~/.codesearch/models/). +/// Environment variable relocating the entire global codesearch root. /// -/// This centralizes embedding model downloads so they are shared across all -/// databases instead of being duplicated per-project. The directory is created -/// if it does not exist. +/// When set to a non-empty absolute path, every global artifact — repos.json, +/// the embedding-models cache, serve logs, serve_url, the global +/// .codesearchignore and extensions.json — resolves under it instead of +/// `~/.codesearch`. This is the single knob for machines that keep large +/// artifacts off the home directory. Per-file overrides such as +/// [`REPOS_CONFIG_ENV`] and [`EXTENSION_MAP_ENV`] still win when set. +pub const CODESEARCH_HOME_ENV: &str = "CODESEARCH_HOME"; + +/// Resolve the global codesearch root directory. /// -/// Falls back to a temp directory if the home directory cannot be determined. -pub fn get_global_models_cache_dir() -> anyhow::Result { - let base = - dirs::home_dir().ok_or_else(|| anyhow::anyhow!("Could not determine home directory"))?; +/// Returns `$CODESEARCH_HOME` when set and non-empty (it must be absolute — a +/// relative value would silently resolve against the current working +/// directory and fragment state between runs), otherwise `~/.codesearch`. +/// All global-path accessors derive from this single point, so one hook +/// relocates everything. +pub fn codesearch_home() -> anyhow::Result { + if let Ok(dir) = std::env::var(CODESEARCH_HOME_ENV) { + let dir = dir.trim(); + if !dir.is_empty() { + let path = PathBuf::from(dir); + if !path.is_absolute() { + anyhow::bail!( + "{} must be an absolute path, got: {}", + CODESEARCH_HOME_ENV, + dir + ); + } + return Ok(path); + } + } + dirs::home_dir() + .map(|home| home.join(CONFIG_DIR_NAME)) + .ok_or_else(|| anyhow::anyhow!("Could not determine home directory")) +} - let models_dir = base.join(CONFIG_DIR_NAME).join(MODELS_SUBDIR); +/// Get the global models cache directory (`/models/`). +/// +/// This centralizes embedding model downloads so they are shared across all +/// databases instead of being duplicated per-project. Relocated wholesale via +/// [`CODESEARCH_HOME_ENV`]. The directory is created if it does not exist. +pub fn get_global_models_cache_dir() -> anyhow::Result { + let models_dir = codesearch_home()?.join(MODELS_SUBDIR); if !models_dir.exists() { std::fs::create_dir_all(&models_dir).map_err(|e| { @@ -85,13 +117,17 @@ pub fn get_global_models_cache_dir() -> anyhow::Result { Ok(models_dir) } -/// Get the global cache directory (~/.codesearch/). +/// Get the global cache directory (`/`). /// /// Used for client/auto mode logging when no local DB is available. -/// The directory is created if it does not exist. +/// Relocated wholesale via [`CODESEARCH_HOME_ENV`]. The directory is created +/// if it does not exist. pub fn get_global_cache_dir() -> PathBuf { - let base = dirs::home_dir().unwrap_or_else(|| std::env::current_dir().unwrap_or_default()); - let cache_dir = base.join(CONFIG_DIR_NAME); + let cache_dir = codesearch_home().unwrap_or_else(|_| { + std::env::current_dir() + .unwrap_or_default() + .join(CONFIG_DIR_NAME) + }); if !cache_dir.exists() { let _ = std::fs::create_dir_all(&cache_dir); } @@ -101,18 +137,19 @@ pub fn get_global_cache_dir() -> PathBuf { /// Name of the global codesearchignore file in ~/.codesearch/ pub const GLOBAL_CODESEARCHIGNORE_FILE: &str = ".codesearchignore"; -/// Get the path to the global .codesearchignore file (~/.codesearch/.codesearchignore). +/// Get the path to the global .codesearchignore file +/// (`/.codesearchignore`). /// /// This file uses the same gitignore syntax and is applied to all indexed repos, /// providing a way to set ignore rules without modifying repo-local files. /// Repo-local `.codesearchignore` and `.gitignore` take precedence. +/// Relocated wholesale via [`CODESEARCH_HOME_ENV`]. /// -/// Returns `None` only if the home directory cannot be determined. +/// Returns `None` only if the root cannot be resolved. pub fn global_codesearchignore_path() -> Option { - dirs::home_dir().map(|home| { - home.join(CONFIG_DIR_NAME) - .join(GLOBAL_CODESEARCHIGNORE_FILE) - }) + codesearch_home() + .ok() + .map(|root| root.join(GLOBAL_CODESEARCHIGNORE_FILE)) } /// Name of the global extension→language map file in ~/.codesearch/ @@ -132,15 +169,17 @@ pub const EXTENSION_MAP_ENV: &str = "CODESEARCH_EXTENSION_MAP"; /// the binary. User overrides take precedence over the built-in extension table. /// /// The path resolves to `$CODESEARCH_EXTENSION_MAP` when set and non-empty, -/// otherwise `~/.codesearch/extensions.json`. Returns `None` only when neither -/// the env var nor the home directory is available. +/// otherwise `/extensions.json`. Returns `None` only when +/// neither the env var nor a resolvable root is available. pub fn global_extension_map_path() -> Option { if let Ok(p) = std::env::var(EXTENSION_MAP_ENV) { if !p.is_empty() { return Some(PathBuf::from(p)); } } - dirs::home_dir().map(|home| home.join(CONFIG_DIR_NAME).join(GLOBAL_EXTENSION_MAP_FILE)) + codesearch_home() + .ok() + .map(|root| root.join(GLOBAL_EXTENSION_MAP_FILE)) } /// Name of the repos configuration file @@ -504,14 +543,17 @@ pub const REMOTE_ROW_REFRESH_SECS: u64 = 5; /// cancellation between insert and remove silently leaks the entry, leaving /// the TUI stuck on "Indexing" forever. /// -/// Rather than chase every leak path, entries older than this threshold are +/// Rather than chase every leak path, a marker older than this threshold is /// treated as absent by all read sites (`repo_statuses_lightweight`, -/// `evict_idle_repos`, and the reindex 409 guard) and lazily evicted. This -/// makes the system self-healing regardless of the leak cause. +/// `evict_idle_repos`, and the reindex 409 guard) and lazily evicted — but +/// only when no tracked indexing task for the alias is still alive. A live +/// task (queued on the job gate, or in a long uninterruptible build) renews +/// its marker instead, so a backlog drains rather than being cancelled as +/// "leaked"; the system stays self-healing for genuinely dead tasks without +/// killing slow ones. /// /// 30 minutes is deliberately generous: a force reindex of a very large repo /// or a full scip-csharp solution rebuild can legitimately take many minutes. -/// Any session stuck beyond that is almost certainly leaked. /// Override with `CODESEARCH_MAX_INDEXING_SECS`. pub const MAX_INDEXING_SECS: u64 = 30 * 60; // 30 minutes @@ -655,6 +697,23 @@ pub const SCIP_HEAD_SHA_KEY: &str = "head_sha"; /// old-format keys being served as fresh. pub const SCIP_KEY_FORMAT_KEY: &str = "key_format"; +/// LMDB metadata key holding a JSON array of index-level completeness +/// warnings from the last rebuild (workspace/MSBuild failures the helper +/// survived while still producing an index). Absent or empty = built clean. +/// Surfaced on every `find_impact` answer from that index — a reference +/// list out of a partially loaded solution may be missing callers. +pub const SCIP_INDEX_WARNINGS_KEY: &str = "index_warnings"; + +/// LMDB metadata key stamping which codesearch build produced the symbol +/// index (full `CARGO_PKG_VERSION_FULL`, so every rebuilt binary differs). +/// The C# rebuild gate treats an index whose stamp is absent or differs from +/// [`INDEX_BUILDER_VERSION`] as stale: after a deploy, serve rebuilds the +/// indexes itself at startup instead of waiting for a manual command. +pub const SCIP_INDEX_BUILDER_VERSION_KEY: &str = "index_builder_version"; + +/// The value written for [`SCIP_INDEX_BUILDER_VERSION_KEY`] by this binary. +pub const INDEX_BUILDER_VERSION: &str = env!("CARGO_PKG_VERSION_FULL"); + /// Current value written for [`SCIP_KEY_FORMAT_KEY`]. Bump whenever the /// canonical SCIP symbol key format produced by a language helper changes /// shape (B4: C# generic arity / containing-type path / fully qualified @@ -891,8 +950,12 @@ pub const ALWAYS_SKIP_FILENAME_SUFFIXES: &[&str] = &[ ".orig", ]; -/// Directories and files that should always be excluded from indexing -/// These are added to both .gitignore and .codesearchignore automatically +/// Directories and files that should always be excluded from indexing. +/// +/// This list is applied purely in memory: the file walker filters these names +/// during traversal and the watcher skips paths containing them. Nothing is +/// written to `.gitignore` or `.codesearchignore`, and previously indexed +/// content keeps appearing in search results until the next reindex. pub const ALWAYS_EXCLUDED: &[&str] = &[ // Codesearch databases ".codesearch", @@ -930,6 +993,26 @@ pub const ALWAYS_EXCLUDED: &[&str] = &[ "coverage", ".nyc_output", ".cache", + // Dependency lock files — generated by package managers, never source. + // Matched by exact filename so the walker and watcher exclude them even + // when the extension alone is not skipped (pnpm-lock.yaml, + // package-lock.json, npm-shrinkwrap.json, bun.lockb). + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "bun.lock", + "bun.lockb", + "deno.lock", + "Cargo.lock", + "go.sum", + "poetry.lock", + "Pipfile.lock", + "uv.lock", + "pdm.lock", + "composer.lock", + "Gemfile.lock", + "flake.lock", ]; #[cfg(test)] @@ -949,6 +1032,9 @@ mod tests { } #[test] + // Reads global-path accessors that resolve via CODESEARCH_HOME: must not + // overlap the serial tests that set/clear that env var in this binary. + #[serial_test::serial] fn global_codesearchignore_path_returns_home_codesearch_dir() { let path = global_codesearchignore_path(); assert!(path.is_some(), "Should return Some when home dir exists"); @@ -964,4 +1050,96 @@ mod tests { "Filename should match GLOBAL_CODESEARCHIGNORE_FILE constant" ); } + + #[test] + #[serial_test::serial] + fn codesearch_home_relocates_all_global_paths() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + // A developer machine with either per-file override exported would + // otherwise win over CODESEARCH_HOME and fail this test spuriously. + let _clear_specific = + crate::testing::EnvRestore::remove(&[REPOS_CONFIG_ENV, EXTENSION_MAP_ENV]); + let _set = + crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, root.to_str().unwrap())]); + + assert_eq!(codesearch_home().unwrap(), root); + assert_eq!( + get_global_models_cache_dir().unwrap(), + root.join(MODELS_SUBDIR), + "models cache must live under CODESEARCH_HOME" + ); + assert!(root.join(MODELS_SUBDIR).exists()); + assert_eq!(get_global_cache_dir(), root); + assert_eq!( + global_codesearchignore_path(), + Some(root.join(GLOBAL_CODESEARCHIGNORE_FILE)) + ); + assert_eq!( + global_extension_map_path(), + Some(root.join(GLOBAL_EXTENSION_MAP_FILE)) + ); + assert_eq!( + crate::db_discovery::repos::config_path().unwrap(), + root.join(REPOS_CONFIG_FILE), + "repos.json default must follow CODESEARCH_HOME" + ); + } + + #[test] + #[serial_test::serial] + fn relative_codesearch_home_is_rejected() { + let _set = crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, "relative/dir")]); + + let err = codesearch_home().unwrap_err(); + assert!( + err.to_string().contains("absolute"), + "error must mention the absolute-path requirement, got: {err}" + ); + assert!( + get_global_models_cache_dir().is_err(), + "a bad root must not silently fall back to the default models dir" + ); + assert_eq!( + global_codesearchignore_path(), + None, + "a bad root must not silently resolve the global ignore file" + ); + } + + #[test] + #[serial_test::serial] + fn empty_or_unset_codesearch_home_falls_back_to_default() { + let expected = dirs::home_dir().unwrap().join(CONFIG_DIR_NAME); + + let _empty = crate::testing::EnvRestore::set(&[(CODESEARCH_HOME_ENV, "")]); + assert_eq!( + codesearch_home().unwrap(), + expected, + "an empty override must fall back to ~/.codesearch" + ); + + let _unset = crate::testing::EnvRestore::remove(&[CODESEARCH_HOME_ENV]); + assert_eq!(codesearch_home().unwrap(), expected); + } + + #[test] + #[serial_test::serial] + fn specific_overrides_win_over_codesearch_home() { + let tmp = tempfile::tempdir().unwrap(); + let map_path = tmp.path().join("my-extensions.json"); + let repos_path = tmp.path().join("my-repos.json"); + let _set = crate::testing::EnvRestore::set(&[ + (CODESEARCH_HOME_ENV, tmp.path().to_str().unwrap()), + (EXTENSION_MAP_ENV, map_path.to_str().unwrap()), + (REPOS_CONFIG_ENV, repos_path.to_str().unwrap()), + ]); + + assert_eq!(global_extension_map_path(), Some(map_path)); + assert_eq!( + crate::db_discovery::repos::config_path().unwrap(), + repos_path, + "CODESEARCH_REPOS_CONFIG must win over CODESEARCH_HOME" + ); + } } diff --git a/src/db_discovery/repos.rs b/src/db_discovery/repos.rs index 36ce1bd2..318c8208 100644 --- a/src/db_discovery/repos.rs +++ b/src/db_discovery/repos.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::{Path, PathBuf}; use crate::cache::{normalize_user_path, safe_canonicalize, strip_unc_prefix}; -use crate::constants::{CONFIG_DIR_NAME, REPOS_CONFIG_FILE}; +use crate::constants::REPOS_CONFIG_FILE; /// A remote `codesearch serve` peer that can be queried for federation. /// @@ -1147,8 +1147,7 @@ impl ReposConfig { } pub fn config_dir() -> Result { - let home_dir = dirs::home_dir().ok_or_else(|| anyhow::anyhow!("No home directory found"))?; - Ok(home_dir.join(CONFIG_DIR_NAME)) + crate::constants::codesearch_home() } pub fn config_path() -> Result { diff --git a/src/embed/embedder.rs b/src/embed/embedder.rs index 75472d15..96065ebf 100644 --- a/src/embed/embedder.rs +++ b/src/embed/embedder.rs @@ -318,6 +318,27 @@ impl FastEmbedder { pub fn with_cache_dir( model_type: ModelType, cache_dir: Option<&std::path::Path>, + ) -> Result { + Self::init(model_type, cache_dir, false) + } + + /// Create a new embedder with the fastembed download progress bar visible. + /// + /// Only `codesearch setup` wants this: it downloads hundreds of MB on + /// purpose, and a silent run looks like a hang. Runtime paths keep the + /// bar off so it does not corrupt TUI/serve output. + pub fn with_cache_dir_progress( + model_type: ModelType, + cache_dir: Option<&std::path::Path>, + show_download_progress: bool, + ) -> Result { + Self::init(model_type, cache_dir, show_download_progress) + } + + fn init( + model_type: ModelType, + cache_dir: Option<&std::path::Path>, + show_download_progress: bool, ) -> Result { // Set cache directory via environment variable if provided // Note: fastembed library uses FASTEMBED_CACHE_DIR (not FASTEMBED_CACHE_PATH) @@ -332,31 +353,50 @@ impl FastEmbedder { // Arena allocator provides fast memory reuse during inference. let cpu_ep = CPU::default().with_arena_allocator(true).build(); - let model = TextEmbedding::try_new( - TextInitOptions::new(model_type.to_fastembed_model()) - .with_show_download_progress(false) - .with_execution_providers(vec![cpu_ep]), - ) - .map_err(|e| anyhow!("Failed to initialize embedding model: {}", e))?; + let options = TextInitOptions::new(model_type.to_fastembed_model()) + .with_show_download_progress(show_download_progress) + .with_execution_providers(vec![cpu_ep]); + + // Cap ONNX intra-op threads when CODESEARCH_EMBED_THREADS is set. + // Unset leaves fastembed/ONNX Runtime at its default (all cores): one + // inference alone can saturate a large machine, which starves every + // other task on a shared box. See `crate::limits`. + let options = match crate::limits::embed_threads() { + Some(threads) => options.with_intra_threads(threads), + None => options, + }; + + let model = TextEmbedding::try_new(options) + .map_err(|e| anyhow!("Failed to initialize embedding model: {}", e))?; Ok(Self { model, model_type }) } - /// Embed a batch of texts (processes in mini-batches to avoid OOM) - /// Uses adaptive batch size based on model dimensions - /// Can be overridden with CODESEARCH_BATCH_SIZE environment variable - pub fn embed_batch(&mut self, texts: Vec) -> Result>> { + /// Mini-batch size for a single ONNX `embed` call: `CODESEARCH_BATCH_SIZE` + /// when set, otherwise adaptive by model dimensions. + /// + /// Shared by [`Self::embed_batch`] and the serve pool's interleaved batch + /// loop (`EmbeddingServicePool::embed_chunks_yielding`), so both agree on + /// how long one ONNX call (and therefore one lock acquisition) lasts. + pub fn effective_batch_size(model_type: ModelType) -> usize { // Check for env var override (tune with CODESEARCH_BATCH_SIZE=N) - let batch_size = if let Ok(env_size) = std::env::var("CODESEARCH_BATCH_SIZE") { + if let Ok(env_size) = std::env::var("CODESEARCH_BATCH_SIZE") { env_size.parse().unwrap_or(256) } else { // Adaptive batch size: without arena allocator, ONNX frees buffers after each batch // so larger batches are faster without accumulating memory. - match self.model_type.dimensions() { + match model_type.dimensions() { d if d <= 384 => 256, // Small models (MiniLM etc.) d if d <= 768 => 128, // Medium models (BGE-base, Jina etc.) _ => 64, // Large models (BGE-large, MxBai etc.) } - }; + } + } + + /// Embed a batch of texts (processes in mini-batches to avoid OOM) + /// Uses adaptive batch size based on model dimensions + /// Can be overridden with CODESEARCH_BATCH_SIZE environment variable + pub fn embed_batch(&mut self, texts: Vec) -> Result>> { + let batch_size = Self::effective_batch_size(self.model_type); self.embed_batch_chunked(texts, batch_size) } diff --git a/src/embed/mod.rs b/src/embed/mod.rs index bc0a497f..b653e7ba 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -341,6 +341,47 @@ impl EmbeddingServicePool { guard.insert(model, arc.clone()); Ok(arc) } + + /// Embed `chunks`, releasing the service lock between ONNX mini-batches. + /// + /// The per-model `Mutex` is what serialises embedding + /// between repos and queries. Holding it for a whole refresh batch (one + /// `CODESEARCH_INCREMENTAL_BATCH_SIZE` window can hold thousands of + /// chunks) made an interactive query embedding queue behind minutes of + /// background inference. Each lock acquisition here covers exactly one + /// ONNX mini-batch ([`FastEmbedder::effective_batch_size`]), followed by + /// the optional `pause` duty cycle, so a long background pass yields the + /// model between mini-batches. + pub fn embed_chunks_yielding( + &self, + model: ModelType, + chunks: Vec, + pause: std::time::Duration, + ) -> Result> { + if chunks.is_empty() { + return Ok(Vec::new()); + } + let service = self.get(model)?; + let group_size = FastEmbedder::effective_batch_size(model).max(1); + let mut out: Vec = Vec::with_capacity(chunks.len()); + for group in chunks.chunks(group_size) { + { + // Recover from poisoning: a panicked embed service still yields + // a valid guard; the batch result carries correctness, and a + // poison here must not hard-fail every later batch for the + // process lifetime. + let mut guard = service + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut embedded = guard.embed_chunks(group.to_vec())?; + out.append(&mut embedded); + } + if !pause.is_zero() { + std::thread::sleep(pause); + } + } + Ok(out) + } } #[cfg(test)] @@ -353,6 +394,47 @@ mod tests { assert_eq!(model.dimensions(), 384); } + /// Mini-batch size drives how long one lock acquisition lasts; pin the + /// dimension-adaptive default and the env override. + #[test] + #[serial_test::serial] + fn effective_batch_size_defaults_by_dimensions_and_env_overrides() { + let _unset = crate::testing::EnvRestore::remove(&["CODESEARCH_BATCH_SIZE"]); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::AllMiniLML6V2Q), + 256, + "384-dim models use 256-text mini-batches" + ); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::EmbeddingGemma300MQ4), + 128, + "768-dim models use 128-text mini-batches" + ); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::BGELargeENV15), + 64, + ">768-dim models use 64-text mini-batches" + ); + + let _override = crate::testing::EnvRestore::set(&[("CODESEARCH_BATCH_SIZE", "7")]); + assert_eq!( + FastEmbedder::effective_batch_size(ModelType::EmbeddingGemma300MQ4), + 7, + "CODESEARCH_BATCH_SIZE must win over the adaptive default" + ); + } + + /// The interleaved embed path must not even load the model for an empty + /// batch; this is its no-op arm and the only one testable without ONNX. + #[test] + fn embed_chunks_yielding_empty_is_a_noop() { + let pool = EmbeddingServicePool::new(None); + let out = pool + .embed_chunks_yielding(ModelType::default(), Vec::new(), std::time::Duration::ZERO) + .expect("empty embed must succeed"); + assert!(out.is_empty()); + } + /// The index-metadata reader must invert `write_metadata_fields`, and must /// report "no answer" (None) for unknown/missing names rather than silently /// claiming the default — callers decide the fallback. diff --git a/src/file/mod.rs b/src/file/mod.rs index b98c3d1d..6cfa21e3 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -322,6 +322,25 @@ mod tests { assert_eq!(files[0].path.file_name().unwrap(), "index.js"); } + /// Dependency lock files are excluded by exact filename: their extensions + /// (.yaml/.json) are indexable on their own, so without the name match the + /// walker would index every churn commit of a generated lock file. + #[test] + fn test_excluded_lock_files() { + let dir = TempDir::new().unwrap(); + + fs::write(dir.path().join("pnpm-lock.yaml"), "lockVersion: '9.0'").unwrap(); + fs::write(dir.path().join("package-lock.json"), "{}").unwrap(); + fs::write(dir.path().join("yarn.lock"), "# yarn lockfile").unwrap(); + fs::write(dir.path().join("main.rs"), "fn main() {}").unwrap(); + + let walker = FileWalker::new(dir.path()); + let (files, _) = walker.walk().unwrap(); + + assert_eq!(files.len(), 1); + assert_eq!(files[0].path.file_name().unwrap(), "main.rs"); + } + /// A root whose own name matches an `ALWAYS_EXCLUDED` entry (e.g. `.git`) /// must be rejected at `walk()` time — otherwise the depth==0 short-circuit /// in `filter_entry` would let every internal file be indexed. diff --git a/src/index/manager.rs b/src/index/manager.rs index ff190f91..57442b60 100644 --- a/src/index/manager.rs +++ b/src/index/manager.rs @@ -48,8 +48,12 @@ use super::Result; pub enum SymbolRebuildSignal { /// A rebuild is about to run (helper available and project applies). Started, - /// Rebuild finished successfully. - Succeeded, + /// Rebuild finished successfully. Carries the index-level warnings the + /// rebuild survived (empty = clean) so the serve layer can render a + /// degraded index red instead of green — the watcher path is the common + /// path during active development, and a green indicator on an index + /// with missing cross-project references reads as "safe". + Succeeded { index_warnings: Vec }, /// Rebuild failed with the given message. Failed(String), } @@ -200,9 +204,57 @@ pub struct SharedStores { /// Counter for number of file changes processed (indexed + removed) since serve start. /// Incremented by FSW batches and incremental refreshes. Read by TUI/dashboard. pub changes_count: std::sync::atomic::AtomicU64, + /// Process-wide heavy-job gate. `Some` in serve mode (attached by + /// `ServeState::try_open_stores`), `None` in standalone CLI/MCP processes. + /// Every whole-repo refresh/reindex/build takes one permit from it; see + /// [`crate::limits`]. + pub job_gate: Option>, } impl SharedStores { + /// Attach a process-wide heavy-job gate to these stores (serve mode). + pub fn with_job_gate(mut self, gate: Arc) -> Self { + self.job_gate = Some(gate); + self + } + + /// Acquire one heavy-job permit, or `None` when no gate is attached + /// (standalone processes keep their previous unbounded behaviour). + /// + /// Hold the returned permit for the whole job: that is what bounds how + /// many repos chunk/embed/build at once. Queries never take this permit. + pub async fn acquire_job_permit(&self) -> Option { + match self.job_gate.as_ref() { + Some(gate) => Some(gate.acquire().await), + None => None, + } + } + + /// Cancellation-aware variant of [`Self::acquire_job_permit`]. + /// + /// Returns [`crate::limits::JobAcquire::Cancelled`] when `cancel_token` + /// fires (or already fired) before a slot is granted, so the caller can + /// abort the job promptly and release its stores instead of parking on + /// the gate for the permit holder's whole run. + pub async fn acquire_job_permit_or_cancel( + &self, + cancel_token: &CancellationToken, + ) -> crate::limits::JobAcquire { + match self.job_gate.as_ref() { + Some(gate) => match gate.acquire_or_cancel(cancel_token).await { + Some(permit) => crate::limits::JobAcquire::Ready(Some(permit)), + None => crate::limits::JobAcquire::Cancelled, + }, + None => { + if cancel_token.is_cancelled() { + crate::limits::JobAcquire::Cancelled + } else { + crate::limits::JobAcquire::Ready(None) + } + } + } + } + /// Create new shared stores from the database path (read-write mode). /// /// This acquires a writer lock. If another process already has the lock, @@ -238,6 +290,7 @@ impl SharedStores { writer_lock: lock, readonly: false, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, }) } @@ -257,6 +310,7 @@ impl SharedStores { writer_lock: None, readonly: true, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, }) } @@ -339,6 +393,18 @@ fn embed_service_choice(pool: Option<&Arc>) -> EmbeddingSo } } +/// True when the current refresh batch should stop accumulating files before +/// reading the next one. +/// +/// `consumed > 0` guarantees at least one file per batch, so a single file +/// larger than `max_chunks` still makes progress; `max_chunks == 0` disables +/// the cap entirely (the file-count batch size remains the only bound). +/// Extracted as a pure predicate so its edges are unit-testable without +/// loading ONNX. +fn chunk_cap_reached(consumed: usize, accumulated_chunks: usize, max_chunks: usize) -> bool { + consumed > 0 && max_chunks > 0 && accumulated_chunks >= max_chunks +} + impl IndexManager { /// Cancellation guard shared by every cancellable indexing function. /// @@ -600,6 +666,43 @@ impl IndexManager { cancel_token: &CancellationToken, embedding_pool: Option<&Arc>, heartbeat: Option<&IndexingHeartbeat>, + ) -> Result<()> { + // Serialise this whole refresh behind the process-wide heavy-job gate + // (when one is attached): the permit is held across the file walk, the + // chunk/embed batches and the final HNSW build, so at most + // `CODESEARCH_INDEX_JOBS` repos do this at once. Queries never take it. + // Cancellation-aware: a queued job that was cancelled aborts here and + // drops its stores instead of parking for the holder's whole run. + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; + Self::perform_incremental_refresh_inner( + codebase_path, + db_path, + stores, + cancel_token, + embedding_pool, + heartbeat, + ) + .await + } + + /// Gate-free body of [`Self::perform_incremental_refresh_with_stores`]. + /// + /// Split out for `force_reindex_with_stores`, which holds the gate permit + /// across the destructive clear phase as well and must call this directly: + /// re-entering the public wrapper would re-acquire the gate and deadlock + /// under `CODESEARCH_INDEX_JOBS=1`. + async fn perform_incremental_refresh_inner( + codebase_path: &Path, + db_path: &Path, + stores: &SharedStores, + cancel_token: &CancellationToken, + embedding_pool: Option<&Arc>, + heartbeat: Option<&IndexingHeartbeat>, ) -> Result<()> { use crate::cache::FileMetaStore; use crate::chunker::SemanticChunker; @@ -799,24 +902,43 @@ impl IndexManager { // sync. Batching bounds peak memory to O(batch), not O(total delta), // so a delta of any size is now safe — it just takes longer, spread // across sequential batches. See `INCREMENTAL_REFRESH_BATCH_SIZE`. + // + // A batch is additionally cut short once its accumulated chunks reach + // `CODESEARCH_MAX_CHUNKS_PER_BATCH` (when set): file count alone is a + // poor memory proxy when a few files can produce tens of thousands of + // chunks. if !changed_files.is_empty() { let batch_size = std::env::var(crate::constants::INCREMENTAL_REFRESH_BATCH_SIZE_ENV) .ok() .and_then(|s| s.parse::().ok()) .filter(|&n| n > 0) .unwrap_or(crate::constants::INCREMENTAL_REFRESH_BATCH_SIZE); - let total_batches = changed_files.len().div_ceil(batch_size); + let max_chunks = crate::limits::max_chunks_per_batch(); + let embed_pause = crate::limits::embed_pause(); + // Upper-bound estimate for the log line; the chunk cap can split + // this into more batches than the file count alone implies. + let estimated_batches = changed_files.len().div_ceil(batch_size); info!( - "🔄 Processing {} changed files in {} batch(es) of up to {} file(s) each...", + "🔄 Processing {} changed files in ~{} batch(es) of up to {} file(s) each \ + (chunk cap per batch: {})...", changed_files.len(), - total_batches, - batch_size + estimated_batches, + batch_size, + if max_chunks == 0 { + "none".to_string() + } else { + max_chunks.to_string() + } ); let cache_dir = crate::constants::get_global_models_cache_dir()?; let mut total_indexed = 0usize; + let mut cursor = 0usize; + let mut batch_idx = 0usize; - for (batch_idx, file_batch) in changed_files.chunks(batch_size).enumerate() { + while cursor < changed_files.len() { + let take = (changed_files.len() - cursor).min(batch_size); + let files_for_embed = changed_files[cursor..cursor + take].to_vec(); // Abort between batches if the repo was removed mid-index. Self::ensure_indexing_active(cancel_token)?; // A batch of a large delta can run for minutes — renew the @@ -832,7 +954,11 @@ impl IndexManager { // so it never runs on a tokio worker thread. The `EmbeddingService` // and `SemanticChunker` are built inside the closure because they // are not needed on the async side and may not be `Send`. - let files_for_embed = file_batch.to_vec(); + // + // The closure consumes as many of the offered files as fit the + // chunk cap (always at least one, so every iteration makes + // progress) and reports how many it took, so the outer loop's + // bookkeeping covers exactly the files that were processed. let cache_dir_for_batch = cache_dir.clone(); let root_for_batch = codebase_path.to_path_buf(); // Clone the token into the blocking closure so a cancel arriving @@ -840,18 +966,30 @@ impl IndexManager { // per-file, not only once the whole batch returns. let batch_cancel = cancel_token.clone(); let embed_source = embed_service_choice(embedding_pool); - let embedded_chunks = tokio::task::spawn_blocking( - move || -> Result> { + let (embedded_chunks, consumed) = tokio::task::spawn_blocking( + move || -> Result<(Vec, usize)> { let mut chunker = SemanticChunker::new(100, 2000, 10); let mut all_chunks = Vec::new(); + let mut consumed = 0usize; for file in &files_for_embed { + // Stop once this batch has reached its chunk budget. + // `consumed > 0` guarantees at least one file per + // iteration even if that one file alone exceeds the + // cap, so the outer loop always advances. + if chunk_cap_reached(consumed, all_chunks.len(), max_chunks) { + break; + } // Mid-embed cancellation point: abort inside the // spawn_blocking task so we stop reading/chunking/ // embedding further files in this batch promptly. if batch_cancel.is_cancelled() { return Err(anyhow::anyhow!("indexing cancelled")); } + // Count the file before the read: a read failure + // still advances the outer cursor, so the loop can + // never stall on an unreadable file. + consumed += 1; let content = match std::fs::read_to_string(&file.path) { Ok(c) => c, Err(_) => continue, @@ -867,54 +1005,46 @@ impl IndexManager { } if all_chunks.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), consumed)); } - // NOTE: embed_chunks runs a single ONNX inference over - // the whole batch atomically, so it is not interruptible - // mid-call. Worst-case cancel latency is bounded to one - // batch's embed (INCREMENTAL_REFRESH_BATCH_SIZE=200 - // files); the per-file check above bounds the read/chunk - // phase that precedes it. - match embed_source { - // Serve process: the pool already holds the - // persistent cache's LMDB env open — a fresh - // EmbeddingService here is refused by the - // process-global registry and would re-embed - // everything without cache. + // Serve process: the pool already holds the persistent + // cache's LMDB env open — a fresh `EmbeddingService` + // here is refused by the process-global registry and + // would re-embed everything without cache. + // + // `embed_chunks_yielding` holds the per-model lock for + // one ONNX mini-batch at a time (plus the configured + // pause), so an interactive query embedding is not stuck + // behind this whole file batch. + let embedded = match embed_source { EmbeddingSource::Pool(pool) => { - let service = pool.get(embed_model)?; - // Recover from poisoning: a panicked embed - // service still yields a valid guard (the - // batch's embed_chunks result is what carries - // correctness); a poison here must not - // hard-fail every later batch for the - // process lifetime. - let mut guard = service - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.embed_chunks(all_chunks) + pool.embed_chunks_yielding(embed_model, all_chunks, embed_pause)? } EmbeddingSource::Fresh => { let mut embedding_service = EmbeddingService::with_cache_dir( embed_model, Some(cache_dir_for_batch.as_path()), )?; - embedding_service.embed_chunks(all_chunks) + embedding_service.embed_chunks(all_chunks)? } - } + }; + Ok((embedded, consumed)) }, ) .await .map_err(|e| { - anyhow::anyhow!( - "chunk+embed task panicked (batch {}/{}): {}", - batch_idx + 1, - total_batches, - e - ) + anyhow::anyhow!("chunk+embed task panicked (batch {}): {}", batch_idx + 1, e) })??; + // The closure always consumes at least one file; guard the + // cursor arithmetic against a future regression that would + // otherwise spin forever. + debug_assert!(consumed > 0, "refresh batch made no progress"); + let file_batch = &changed_files[cursor..cursor + consumed]; + cursor += consumed; + batch_idx += 1; + // A cancel arriving after embed completed but before we commit // the batch to the stores must skip the insert + the final // build_index, so a removed repo never receives fresh data. @@ -922,9 +1052,8 @@ impl IndexManager { if !embedded_chunks.is_empty() { info!( - "📦 Batch {}/{}: embedding {} chunks with model {}...", - batch_idx + 1, - total_batches, + "📦 Batch {}: embedding {} chunks with model {}...", + batch_idx, embedded_chunks.len(), embed_model.short_name() ); @@ -1007,7 +1136,7 @@ impl IndexManager { info!( "✅ Indexed {} chunks across {} batch(es)", - total_indexed, total_batches + total_indexed, batch_idx ); } @@ -1079,6 +1208,22 @@ impl IndexManager { // Bail before clearing any store data if the repo was already removed. Self::ensure_indexing_active(cancel_token)?; + // Hold the process-wide heavy-job gate across the destructive clear AND + // the full reindex. The nested call goes to the gate-free inner body so + // the permit is not re-acquired (which would deadlock under + // `CODESEARCH_INDEX_JOBS=1`). The wait is cancellation-aware so a repo + // removed while queued aborts before the destructive clear. + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; + // Re-check after the (possibly long) wait: a cancellation that landed + // while queued must not start the clear, and the stores are still + // intact at this point. + Self::ensure_indexing_active(cancel_token)?; + // ── Step 0: Read and preserve metadata BEFORE clearing anything ── // This is defensive: the DB may be incomplete (no metadata.json at all), // or clear() may indirectly remove it. We preserve model info so we can @@ -1187,7 +1332,7 @@ impl IndexManager { info!("✅ Stores cleared, metadata preserved. Starting full reindex..."); // ── Step 5: Reindex — all files treated as "changed" since metadata is empty ── - Self::perform_incremental_refresh_with_stores( + Self::perform_incremental_refresh_inner( codebase_path, db_path, stores, @@ -1338,7 +1483,9 @@ impl IndexManager { summary.duration_ms ); if let Some(n) = notifier { - n(SymbolRebuildSignal::Succeeded); + n(SymbolRebuildSignal::Succeeded { + index_warnings: summary.index_warnings.clone(), + }); } } Err(e) => { @@ -1786,6 +1933,11 @@ impl IndexManager { // cleaned up from LMDB regardless of origin project. let total_groups = groups.len(); let mut last_error: Option = None; + // Index-level warnings survived by any + // group's rebuild — deduped across groups + // (MSBuild repeats per project). + let mut index_warnings: Vec = Vec::new(); + let mut seen_warnings = std::collections::HashSet::new(); for (i, (csproj, files)) in groups.into_iter().enumerate() { let csproj_name = csproj .file_name() @@ -1806,6 +1958,11 @@ impl IndexManager { }; match indexer.rebuild(&rp, &dp, scope) { Ok(summary) => { + for w in &summary.index_warnings { + if seen_warnings.insert(w.clone()) { + index_warnings.push(w.clone()); + } + } if summary.symbols_indexed == 0 { warn!( "⚠️ [{}/{}] Symbol rebuild returned 0 symbols for '{}' — \ @@ -1844,7 +2001,9 @@ impl IndexManager { // Notify serve layer about overall outcome if let Some(ref n) = notifier { match last_error { - None => n(SymbolRebuildSignal::Succeeded), + None => { + n(SymbolRebuildSignal::Succeeded { index_warnings }) + } Some(msg) => n(SymbolRebuildSignal::Failed(msg)), } } @@ -1970,6 +2129,16 @@ impl IndexManager { // Bail before touching any store if the repo was removed. Self::ensure_indexing_active(cancel_token)?; + // One permit for the whole watcher batch: a batch can hold hundreds of + // files, each re-embedded and followed by an HNSW rebuild, and every + // repo's watcher would otherwise run one at the same time. + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; + // Enable quiet mode during FSW batch processing to suppress verbose embedding output set_quiet(true); @@ -2131,6 +2300,14 @@ impl IndexManager { // Abort before the filesystem walk if the repo was already removed. Self::ensure_indexing_active(cancel_token)?; + // One permit for the whole branch-refresh pass (walk, delete, re-index). + let _job_permit = match stores.acquire_job_permit_or_cancel(cancel_token).await { + crate::limits::JobAcquire::Ready(permit) => permit, + crate::limits::JobAcquire::Cancelled => { + return Err(anyhow::anyhow!("indexing cancelled")); + } + }; + let result: Result<()> = async { // Phase 1: Discover current files on disk. // `walk()` is synchronous + I/O-heavy — offload off the async executor. @@ -2469,11 +2646,11 @@ impl IndexManager { // registry and would degrade this path to cache-less re-embedding. let embedded_chunks = match embed_service_choice(embedding_pool) { EmbeddingSource::Pool(pool) => { - let service = pool.get(embed_model)?; - let mut guard = service - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.embed_chunks(chunks)? + // The FSW paths hold a job permit (`process_batch_with_stores` + // / `refresh_index_with_stores`); interleave the model lock per + // mini-batch so a concurrent query embedding is not stuck + // behind this file. + pool.embed_chunks_yielding(embed_model, chunks, std::time::Duration::ZERO)? } EmbeddingSource::Fresh => { let cache_dir = crate::constants::get_global_models_cache_dir()?; @@ -2712,6 +2889,7 @@ mod tests { writer_lock: None, readonly: false, changes_count: std::sync::atomic::AtomicU64::new(0), + job_gate: None, } } @@ -2753,6 +2931,174 @@ mod tests { assert!(db_path.join(WRITER_LOCK_FILE).exists()); } + /// The chunk-cap predicate is the batch-splitting decision; pin its edges + /// here because the loop that uses it runs inside `spawn_blocking` and + /// would otherwise need the ONNX embedding model. + #[test] + fn chunk_cap_predicate_table() { + let cases = [ + // (consumed, accumulated_chunks, cap, expected) + (0, 10_000, 5_000, false), // first file of a batch always proceeds + (1, 4_999, 5_000, false), // below the cap + (1, 5_000, 5_000, true), // exactly at the cap + (1, 5_001, 5_000, true), // one file overshot the cap + (7, 10_000, 0, false), // cap disabled + (1, 0, 0, false), // cap disabled, nothing accumulated + ]; + for (consumed, accumulated, cap, expected) in cases { + assert_eq!( + chunk_cap_reached(consumed, accumulated, cap), + expected, + "chunk_cap_reached(consumed={consumed}, accumulated={accumulated}, cap={cap})" + ); + } + } + + /// Standalone CLI / `codesearch mcp` stores carry no gate and must keep + /// their previous (unbounded) behaviour. + #[tokio::test] + async fn acquire_job_permit_is_none_for_ungated_stores() { + let temp = tempdir().unwrap(); + let stores = create_test_stores(&temp.path().join("db"), 4).await; + assert!(stores.acquire_job_permit().await.is_none()); + } + + /// Cancellation-aware acquisition: ungated stores must distinguish + /// "proceed without a gate" from "cancelled", and a waiter cancelled while + /// the only slot is held must abort promptly instead of parking. + #[tokio::test] + async fn acquire_job_permit_or_cancel_distinguishes_cancel_from_ungated() { + let temp = tempdir().unwrap(); + let stores = create_test_stores(&temp.path().join("db"), 4).await; + + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert!(matches!( + stores.acquire_job_permit_or_cancel(&cancelled).await, + crate::limits::JobAcquire::Cancelled + )); + assert!(matches!( + stores + .acquire_job_permit_or_cancel(&CancellationToken::new()) + .await, + crate::limits::JobAcquire::Ready(None) + )); + + // Gated: a waiter cancelled while the only slot is held aborts. + let gate = crate::limits::JobGate::with_config(1, 0); + let gated = Arc::new( + create_test_stores(&temp.path().join("db2"), 4) + .await + .with_job_gate(Arc::clone(&gate)), + ); + let held = gate.acquire().await; + let waiter = CancellationToken::new(); + let canceller = waiter.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + canceller.cancel(); + }); + assert!(matches!( + gated.acquire_job_permit_or_cancel(&waiter).await, + crate::limits::JobAcquire::Cancelled + )); + drop(held); + } + + /// A gated refresh must (a) wait while the only slot is held and (b) + /// release the slot when it finishes. The repo has NO changed files, so the + /// refresh completes without loading the ONNX model. + #[tokio::test] + async fn gated_refresh_waits_for_a_free_slot_and_releases_it() { + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + + let gate = crate::limits::JobGate::with_config(1, 0); + let held = gate.acquire().await; + let stores = Arc::new( + create_test_stores(&db_path, 4) + .await + .with_job_gate(Arc::clone(&gate)), + ); + + let mut refresh = tokio::spawn({ + let stores = Arc::clone(&stores); + let codebase_path = codebase_path.clone(); + let db_path = db_path.clone(); + async move { + IndexManager::perform_incremental_refresh_with_stores( + &codebase_path, + &db_path, + &stores, + &CancellationToken::new(), + None, + None, + ) + .await + } + }); + + assert!( + tokio::time::timeout(std::time::Duration::from_millis(150), &mut refresh) + .await + .is_err(), + "the refresh must wait while the only job slot is held" + ); + + drop(held); + let result = tokio::time::timeout(std::time::Duration::from_secs(10), refresh) + .await + .expect("refresh must complete once the slot is free") + .expect("refresh task must not panic"); + assert!(result.is_ok(), "refresh failed: {:?}", result.err()); + + // The permit was released: the gate can be acquired again immediately. + let _again = tokio::time::timeout(std::time::Duration::from_secs(1), gate.acquire()) + .await + .expect("the refresh must release its job permit"); + } + + /// Regression for the wrapper/inner split: `force_reindex_with_stores` + /// holds the gate permit across the destructive clear and then calls the + /// gate-free inner body. Calling the public wrapper again would re-acquire + /// the only permit and hang forever under `CODESEARCH_INDEX_JOBS=1` (this + /// test times out instead of hanging only because of the explicit + /// `timeout`). + #[tokio::test] + async fn force_reindex_under_gate_does_not_reacquire_the_permit() { + let temp = tempdir().unwrap(); + let codebase_path = temp.path().join("codebase"); + let db_path = temp.path().join("db"); + std::fs::create_dir_all(&codebase_path).unwrap(); + std::fs::create_dir_all(&db_path).unwrap(); + create_metadata_json(&db_path, 4); + + let gate = crate::limits::JobGate::with_config(1, 0); + let stores = create_test_stores(&db_path, 4) + .await + .with_job_gate(Arc::clone(&gate)); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + IndexManager::force_reindex_with_stores( + &codebase_path, + &db_path, + &stores, + None, + &CancellationToken::new(), + None, + ), + ) + .await + .expect("force reindex must not deadlock on its own job permit"); + + assert!(result.is_ok(), "force reindex failed: {:?}", result.err()); + } + #[tokio::test] async fn test_refresh_no_metadata_early_return() { // When metadata.json doesn't exist, refresh should return Ok early diff --git a/src/lib.rs b/src/lib.rs index f50f2ae3..7539e797 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod federation; pub mod file; pub mod fts; pub mod index; +pub mod limits; pub mod lmdb_registry; pub mod logger; pub mod mcp; diff --git a/src/limits.rs b/src/limits.rs new file mode 100644 index 00000000..68ebee2e --- /dev/null +++ b/src/limits.rs @@ -0,0 +1,264 @@ +//! Process-wide limits for heavy indexing work. +//! +//! `codesearch serve` hosts many repos, and each repo's refresh/reindex pass +//! is CPU- and memory-heavy: file reads, tree-sitter chunking, ONNX +//! embedding, HNSW graph construction. Without a process-wide budget, N repos +//! with work start N concurrent passes — every core is saturated, and N +//! file/chunk windows are held in memory at once even though ONNX inference +//! itself already serialises on the per-model mutex. +//! +//! This module provides one shared budget ([`JobGate`]) plus the per-inference +//! and per-batch knobs that bound what a single job costs: +//! +//! | Variable | Default | Meaning | +//! |---|---|---| +//! | `CODESEARCH_INDEX_JOBS` | `1` | Max concurrent heavy repo jobs per process | +//! | `CODESEARCH_EMBED_THREADS` | unset (ONNX default = all cores) | ONNX intra-op threads per session | +//! | `CODESEARCH_EMBED_PAUSE_MS` | `0` | Pause between background embedding mini-batches | +//! | `CODESEARCH_MIN_FREE_MB` | `0` (off) | Free-memory floor before a job may start | +//! | `CODESEARCH_MAX_CHUNKS_PER_BATCH` | `0` (off) | Hard cap on chunks held per refresh batch | +//! +//! All values are read from the environment so a systemd unit or shell can set +//! them without a config file. See the README section "Indexing limits" for +//! recommended values on shared machines. +//! +//! The gate is attached to [`crate::index::SharedStores`] at open time in +//! serve mode; standalone processes (CLI index, `codesearch mcp`) leave it +//! `None` and behave exactly as before. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::sync::CancellationToken; + +/// Max concurrent heavy repo jobs in this process (>= 1). +pub const INDEX_JOBS_ENV: &str = "CODESEARCH_INDEX_JOBS"; +/// ONNX intra-op thread count per embedding session (>= 1; unset = all cores). +pub const EMBED_THREADS_ENV: &str = "CODESEARCH_EMBED_THREADS"; +/// Milliseconds to pause between background embedding mini-batches. +pub const EMBED_PAUSE_MS_ENV: &str = "CODESEARCH_EMBED_PAUSE_MS"; +/// Free-memory floor (MB) before a heavy job may start (0 = disabled). +pub const MIN_FREE_MB_ENV: &str = "CODESEARCH_MIN_FREE_MB"; +/// Hard cap on chunks held in memory per refresh batch (0 = disabled). +pub const MAX_CHUNKS_PER_BATCH_ENV: &str = "CODESEARCH_MAX_CHUNKS_PER_BATCH"; + +/// Default for [`index_jobs`]: strictly one heavy repo job at a time. +/// +/// Embedding is already serialised by the per-model mutex, so running another +/// repo's refresh concurrently buys little throughput while multiplying peak +/// memory (each queued job holds its own chunk window) and store-lock +/// contention. Raise it deliberately, with `CODESEARCH_EMBED_THREADS` set, on +/// machines with cores and memory to spare. +pub const DEFAULT_INDEX_JOBS: usize = 1; + +const DEFAULT_EMBED_PAUSE_MS: u64 = 0; +const DEFAULT_MIN_FREE_MB: u64 = 0; +const DEFAULT_MAX_CHUNKS_PER_BATCH: usize = 0; + +/// How often the memory admission check re-reads available memory. +const MEMORY_POLL_INTERVAL: Duration = Duration::from_millis(500); +/// How long a job waits for memory before proceeding with a warning. A +/// permanent wait would be a deadlock (the running job's ONNX arena does not +/// return memory to the OS), so this is deliberately bounded. +const MEMORY_WAIT_MAX: Duration = Duration::from_secs(30); +/// Log when a job waited at least this long for a free slot. +const SLOW_ACQUIRE_LOG_AFTER: Duration = Duration::from_secs(2); + +fn env_parsed(name: &str) -> Option { + std::env::var(name) + .ok() + .and_then(|s| s.trim().parse::().ok()) +} + +/// Max concurrent heavy repo jobs. Values below 1 fall back to the default. +pub fn index_jobs() -> usize { + env_parsed::(INDEX_JOBS_ENV) + .filter(|&n| n >= 1) + .unwrap_or(DEFAULT_INDEX_JOBS) +} + +/// ONNX intra-op threads per embedding session. +/// +/// `None` (unset, or a value below 1) leaves fastembed/ONNX Runtime at its +/// default, which uses every available core. Set a value to cap one inference +/// on a shared machine. +pub fn embed_threads() -> Option { + env_parsed::(EMBED_THREADS_ENV).filter(|&n| n >= 1) +} + +/// Duty-cycle pause between background embedding mini-batches. +pub fn embed_pause() -> Duration { + Duration::from_millis(env_parsed::(EMBED_PAUSE_MS_ENV).unwrap_or(DEFAULT_EMBED_PAUSE_MS)) +} + +/// Free-memory floor in MB for starting a heavy job (0 = disabled). +pub fn min_free_mb() -> u64 { + env_parsed::(MIN_FREE_MB_ENV).unwrap_or(DEFAULT_MIN_FREE_MB) +} + +/// Hard cap on chunks held in memory per refresh batch (0 = disabled). +pub fn max_chunks_per_batch() -> usize { + env_parsed::(MAX_CHUNKS_PER_BATCH_ENV).unwrap_or(DEFAULT_MAX_CHUNKS_PER_BATCH) +} + +/// Available (reclaimable) system memory in MB, read fresh on each call. +pub fn available_memory_mb() -> u64 { + let mut sys = sysinfo::System::new(); + sys.refresh_memory(); + sys.available_memory() / (1024 * 1024) +} + +/// Bounds how many heavy repo jobs run concurrently in one process. +/// +/// A "heavy job" is a whole refresh / reindex / vector-index build pass, not a +/// single inference or a query. Query embeddings never take this gate, so +/// interactive search stays responsive while a background job runs. +#[derive(Debug)] +pub struct JobGate { + permits: Arc, + index_jobs: usize, + min_free_mb: u64, + memory_wait: Duration, +} + +impl JobGate { + /// Build a gate from the environment (the normal serve path). + pub fn from_env() -> Arc { + Self::with_config(index_jobs(), min_free_mb()) + } + + /// Build a gate with an explicit job count and memory floor. + pub fn with_config(index_jobs: usize, min_free_mb: u64) -> Arc { + Self::with_config_and_wait(index_jobs, min_free_mb, MEMORY_WAIT_MAX) + } + + /// Build a gate with a custom memory wait ceiling. Test seam: production + /// callers use [`Self::from_env`]. + pub fn with_config_and_wait( + index_jobs: usize, + min_free_mb: u64, + memory_wait: Duration, + ) -> Arc { + let index_jobs = index_jobs.max(1); + Arc::new(Self { + permits: Arc::new(Semaphore::new(index_jobs)), + index_jobs, + min_free_mb, + memory_wait, + }) + } + + /// Number of concurrent job slots this gate allows. + pub fn index_jobs(&self) -> usize { + self.index_jobs + } + + /// Acquire one job slot, then (when enabled) wait, bounded, for the free + /// memory floor. The caller holds the returned permit for the whole job. + /// + /// Prefer [`Self::acquire_or_cancel`] for jobs that carry a cancellation + /// token; this variant waits unbounded because there is nothing to cancel + /// it with. + pub async fn acquire(&self) -> JobPermit { + self.acquire_or_cancel(&CancellationToken::new()) + .await + .expect("a never-cancelled token can never abort the acquire") + } + + /// Acquire one job slot, aborting promptly when `cancel` fires while + /// queued. + /// + /// Returns `None` without consuming a slot when the token was already + /// cancelled or fires before a slot is granted. A job cancelled while + /// queued must abort rather than park: parking kept a removed or cancelled + /// job's `Arc` (and its LMDB env) alive for the permit + /// holder's whole run, leaving "locked by another process" / double-open + /// failures behind with nothing left to release them. + pub async fn acquire_or_cancel(&self, cancel: &CancellationToken) -> Option { + let wait_start = Instant::now(); + if cancel.is_cancelled() { + return None; + } + let permit = tokio::select! { + biased; + _ = cancel.cancelled() => return None, + permit = self.permits.clone().acquire_owned() => { + permit.expect("job gate semaphore is never closed") + } + }; + let waited = wait_start.elapsed(); + if waited >= SLOW_ACQUIRE_LOG_AFTER { + tracing::info!( + "⏳ Indexing job waited {:.1}s for a free slot ({} slot(s) total; CODESEARCH_INDEX_JOBS)", + waited.as_secs_f64(), + self.index_jobs + ); + } + if self.min_free_mb > 0 { + self.wait_for_memory().await; + } + // The memory wait above is bounded; a cancellation that lands during + // it still wins — release the slot rather than start cancelled work. + if cancel.is_cancelled() { + return None; + } + Some(JobPermit { _permit: permit }) + } + + async fn wait_for_memory(&self) { + let free = available_memory_mb(); + if free >= self.min_free_mb { + return; + } + tracing::info!( + "⏳ Free memory is {} MB, below the {} MB floor (CODESEARCH_MIN_FREE_MB); \ + waiting up to {:.0}s before starting the job", + free, + self.min_free_mb, + self.memory_wait.as_secs_f64() + ); + let deadline = Instant::now() + self.memory_wait; + loop { + tokio::time::sleep(MEMORY_POLL_INTERVAL).await; + let free = available_memory_mb(); + if free >= self.min_free_mb { + return; + } + if Instant::now() >= deadline { + tracing::warn!( + "⚠️ Free memory still below CODESEARCH_MIN_FREE_MB \ + ({} MB available, {} MB required); starting the job anyway", + free, + self.min_free_mb + ); + return; + } + } + } +} + +/// Held for the duration of one heavy job; dropping it releases the slot. +#[derive(Debug)] +pub struct JobPermit { + _permit: OwnedSemaphorePermit, +} + +/// Cancellation-aware acquisition outcome for a heavy job. +/// +/// `#[must_use]`: dropping a [`Self::Ready`] value releases the job slot +/// immediately (the job would run ungated), and ignoring [`Self::Cancelled`] +/// means starting work that was already cancelled. Always match on it. +#[derive(Debug)] +#[must_use] +pub enum JobAcquire { + /// The job may proceed. The permit is `None` when no gate is attached + /// (standalone processes keep their previous unbounded behaviour). + Ready(Option), + /// The cancellation token fired before a slot was granted; no slot was + /// consumed and the job must abort promptly, dropping its stores. + Cancelled, +} + +#[cfg(test)] +#[path = "limits_tests.rs"] +mod tests; diff --git a/src/limits_tests.rs b/src/limits_tests.rs new file mode 100644 index 00000000..1c0fd3f2 --- /dev/null +++ b/src/limits_tests.rs @@ -0,0 +1,178 @@ +use super::*; +use crate::testing::EnvRestore; +use serial_test::serial; + +/// Remove every limit env var so an inherited shell value cannot change the +/// default assertions. The process environment is shared, so every test that +/// touches it is `#[serial]` and guarded by `EnvRestore`. +fn clear_env() -> EnvRestore { + EnvRestore::remove(&[ + INDEX_JOBS_ENV, + EMBED_THREADS_ENV, + EMBED_PAUSE_MS_ENV, + MIN_FREE_MB_ENV, + MAX_CHUNKS_PER_BATCH_ENV, + ]) +} + +#[test] +#[serial] +fn defaults_are_conservative() { + let _env = clear_env(); + assert_eq!(index_jobs(), DEFAULT_INDEX_JOBS); + assert_eq!( + embed_threads(), + None, + "unset must leave ONNX at its default (all cores)" + ); + assert_eq!(embed_pause(), Duration::ZERO); + assert_eq!(min_free_mb(), 0, "memory floor is opt-in"); + assert_eq!(max_chunks_per_batch(), 0, "chunk cap is opt-in"); +} + +#[test] +#[serial] +fn env_overrides_parse() { + let _env = clear_env(); + let _set = EnvRestore::set(&[ + (INDEX_JOBS_ENV, "3"), + (EMBED_THREADS_ENV, "4"), + (EMBED_PAUSE_MS_ENV, "25"), + (MIN_FREE_MB_ENV, "4096"), + (MAX_CHUNKS_PER_BATCH_ENV, "5000"), + ]); + assert_eq!(index_jobs(), 3); + assert_eq!(embed_threads(), Some(4)); + assert_eq!(embed_pause(), Duration::from_millis(25)); + assert_eq!(min_free_mb(), 4096); + assert_eq!(max_chunks_per_batch(), 5000); +} + +#[test] +#[serial] +fn invalid_values_fall_back() { + let _env = clear_env(); + let _set = EnvRestore::set(&[ + (INDEX_JOBS_ENV, "0"), + (EMBED_THREADS_ENV, "0"), + (EMBED_PAUSE_MS_ENV, "soon"), + (MIN_FREE_MB_ENV, "lots"), + (MAX_CHUNKS_PER_BATCH_ENV, "-1"), + ]); + assert_eq!( + index_jobs(), + DEFAULT_INDEX_JOBS, + "0 job slots is meaningless" + ); + assert_eq!(embed_threads(), None, "0 threads means 'ONNX default'"); + assert_eq!(embed_pause(), Duration::ZERO); + assert_eq!(min_free_mb(), 0); + assert_eq!(max_chunks_per_batch(), 0); +} + +#[tokio::test] +async fn gate_serialises_jobs_when_configured_to_one() { + let gate = JobGate::with_config(1, 0); + let first = gate.acquire().await; + assert!( + tokio::time::timeout(Duration::from_millis(50), gate.acquire()) + .await + .is_err(), + "a second job must wait while the only slot is held" + ); + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), gate.acquire()) + .await + .expect("dropping the permit must release the slot"); + drop(second); +} + +#[tokio::test] +async fn gate_allows_the_configured_number_of_parallel_jobs() { + let gate = JobGate::with_config(2, 0); + let first = gate.acquire().await; + let second = gate.acquire().await; + assert!( + tokio::time::timeout(Duration::from_millis(50), gate.acquire()) + .await + .is_err(), + "a third job must wait once both slots are held" + ); + drop(first); + drop(second); +} + +#[tokio::test] +async fn memory_floor_zero_never_waits() { + let gate = JobGate::with_config_and_wait(1, 0, Duration::from_secs(3600)); + let permit = tokio::time::timeout(Duration::from_millis(200), gate.acquire()) + .await + .expect("no memory wait when the floor is disabled"); + drop(permit); +} + +#[tokio::test] +async fn memory_wait_is_bounded() { + // A floor no machine can meet: acquire must return after the (shortened) + // memory wait instead of blocking forever. + let gate = JobGate::with_config_and_wait(1, u64::MAX, Duration::from_millis(50)); + let start = Instant::now(); + let permit = tokio::time::timeout(Duration::from_secs(5), gate.acquire()) + .await + .expect("the memory wait must be bounded"); + assert!(start.elapsed() >= Duration::from_millis(50)); + drop(permit); +} + +#[tokio::test] +async fn cancelled_gate_waiter_aborts_without_consuming_a_slot() { + // Regression: a reindex queued behind a long-running job parked on the + // gate holding its store handles. A cancellation while queued must abort + // the wait promptly and leave the slot untouched. + let gate = JobGate::with_config(1, 0); + let held = gate.acquire().await; + let token = CancellationToken::new(); + let canceller = token.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + canceller.cancel(); + }); + + let start = Instant::now(); + let outcome = gate.acquire_or_cancel(&token).await; + assert!( + outcome.is_none(), + "a cancelled waiter must not receive a permit" + ); + assert!( + start.elapsed() < Duration::from_secs(2), + "cancellation must abort the wait promptly" + ); + + drop(held); + // The slot is still free: the cancelled wait never consumed it. + assert!(gate + .acquire_or_cancel(&CancellationToken::new()) + .await + .is_some()); +} + +#[tokio::test] +async fn pre_cancelled_token_never_takes_a_slot() { + let gate = JobGate::with_config(1, 0); + let token = CancellationToken::new(); + token.cancel(); + assert!(gate.acquire_or_cancel(&token).await.is_none()); + assert!(gate + .acquire_or_cancel(&CancellationToken::new()) + .await + .is_some()); +} + +#[tokio::test] +async fn uncancelled_token_proceeds_through_the_gate() { + let gate = JobGate::with_config(2, 0); + let token = CancellationToken::new(); + assert!(gate.acquire_or_cancel(&token).await.is_some()); + assert!(gate.acquire_or_cancel(&token).await.is_some()); +} diff --git a/src/main.rs b/src/main.rs index a08f1cb1..24035e65 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ mod federation; mod file; mod fts; mod index; +mod limits; mod lmdb_registry; mod logger; mod mcp; diff --git a/src/mcp/find_impact.rs b/src/mcp/find_impact.rs index 035a1957..5faf63bf 100644 --- a/src/mcp/find_impact.rs +++ b/src/mcp/find_impact.rs @@ -93,6 +93,38 @@ fn dedupe_references( out } +/// What find_impact actually did (or could not do) about a missing symbol +/// index — drives the wording of [`missing_index_warning`] so the answer +/// never claims a rebuild that is not running. +pub(crate) enum SymbolIndexHeal { + /// Serve mode with a routed alias: the self-heal task was spawned. + BackgroundStarted, + /// No serve / no routed alias: nothing runs automatically; the caller + /// must build the index explicitly. + Manual, + /// The indexer does not apply to this repository (e.g. no top-level + /// tsconfig.json for TypeScript) — no rebuild can ever start here. + NotApplicable(&'static str), +} + +/// Caller-facing warning for a find_impact answer against a project whose +/// symbol index was never built. The empty reference list such an answer +/// carries means UNKNOWN — stamping this on the payload is what keeps it from +/// passing for "no callers" while the status says what will actually happen. +pub(crate) fn missing_index_warning(lang: &str, heal: SymbolIndexHeal) -> String { + match heal { + SymbolIndexHeal::BackgroundStarted => format!( + "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. A background rebuild was started; retry this call once it completes." + ), + SymbolIndexHeal::Manual => format!( + "No {lang} symbol index exists for this project — an empty reference list means UNKNOWN, not 'no references'. Background rebuilds only run in serve mode; build it with `codesearch index symbol ` and retry." + ), + SymbolIndexHeal::NotApplicable(reason) => format!( + "No {lang} symbol index exists for this project and none can be built automatically ({reason}) — an empty reference list means UNKNOWN, not 'no references'." + ), + } +} + #[tool_router(router = find_impact_router, vis = "pub(crate)")] impl CodesearchService { /// Symbol impact analysis — returns transitive call-sites of a symbol with file/line precision. @@ -249,6 +281,50 @@ impl CodesearchService { )])); } + // Self-heal + honesty: a project whose symbol index was never built + // answers EVERY query with NotFound/empty references — visually + // identical to "no callers", the exact trap that reads as "safe to + // remove" when it is not. Start a background rebuild (serve mode + // only) whenever the definitions index is absent; the *warning* + // below rides only on answers that actually come back empty + // (NotFound / resolution failure) — a successful resolution proves + // usable data exists and keeps its payload to the persisted + // warnings channel. Drift (index_head_sha vs current) is + // deliberately NOT healed — reindexing on every branch switch would + // thrash (see `SymbolIndexer::index_head_sha`). + // + // The warning wording follows the real status: a repo the indexer + // does not apply to (no top-level tsconfig.json / no .sln) can never + // get a rebuilt index, so saying "a rebuild was started" there was a + // promise nothing would fulfil. + let index_missing = !indexer.has_index(&db_path); + let mut heal_spawned = false; + let heal_warning: Vec = if index_missing { + let heal = if !indexer.applies_to(&project_root) { + SymbolIndexHeal::NotApplicable(indexer.applicability_hint()) + } else if self.serve_state.is_some() && ctx.project_alias.is_some() { + heal_spawned = true; + SymbolIndexHeal::BackgroundStarted + } else { + SymbolIndexHeal::Manual + }; + vec![missing_index_warning(indexer.language(), heal)] + } else { + Vec::new() + }; + if heal_spawned { + if let (Some(ref serve_state), Some(ref alias)) = + (&self.serve_state, &ctx.project_alias) + { + let st = std::sync::Arc::clone(serve_state); + let alias_heal = alias.clone(); + let lang_heal = indexer.language().to_string(); + tokio::spawn(async move { + crate::serve::heal_missing_symbol_index(&st, &alias_heal, &lang_heal).await; + }); + } + } + // Perform the lookup under an internal wall-clock budget. // // `find_references_for_key` may invoke `scip-csharp find-refs` on a cache miss @@ -341,6 +417,10 @@ impl CodesearchService { // helper invocation — as a typed candidates answer; the server never // silently picks among the query's matches (the old behaviour took // the shortest fuzzy candidate and answered about the wrong symbol). + // The same read pass collects the index-level rebuild warnings: a + // partially loaded solution produces an index whose reference lists + // are incomplete EVEN WHEN the lookup succeeds, and that honesty + // rides on every answer from this index. let registry_for_resolve = self.symbol_registry.clone(); let language_for_resolve = language_for_lookup.clone(); let db_path_for_resolve = db_path.clone(); @@ -354,7 +434,9 @@ impl CodesearchService { language_for_resolve ) })?; - indexer.resolve_query(&db_path_for_resolve, &query_for_resolve) + let resolution = indexer.resolve_query(&db_path_for_resolve, &query_for_resolve)?; + let index_warnings = indexer.index_warnings(&db_path_for_resolve); + Ok((resolution, index_warnings)) }) .await { @@ -364,13 +446,33 @@ impl CodesearchService { // (classified stale/failed by the index age) instead of `?`. Err(e) => Err(anyhow::anyhow!("symbol resolve task failed: {e:#}")), }; + // Both are read from the SAME env/txn pass above; on a JoinError the + // warnings are unknown and the failure path below speaks for itself. + let (resolution, index_warnings) = match resolution { + Ok((resolution, index_warnings)) => (Ok(resolution), index_warnings), + Err(e) => (Err(e), Vec::new()), + }; + // Warnings assembled per answer path: persisted per-symbol warnings + // (set at the lookup sites) + index-level rebuild warnings. The + // missing-index self-heal warning does NOT belong here — a successful + // resolution proves usable data exists (see the NotFound arm, the + // only payload that carries it). + let answer_warnings = |mut persisted: Vec| -> Vec { + persisted.extend(index_warnings.iter().cloned()); + persisted + }; let canonical = match resolution { Err(e) => { - let failure = crate::symbols::SymbolLookupFailure::classify( + let mut failure = crate::symbols::SymbolLookupFailure::classify( format!("{e:#}"), indexer.index_age(&db_path), ); + if index_missing { + failure.hint_for_agent.push_str( + " A background rebuild of the missing index was started — retry this call once it completes.", + ); + } let json = serde_json::to_string(&failure).unwrap_or_else(|_| failure.error.clone()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); @@ -406,8 +508,12 @@ impl CodesearchService { } Ok(crate::symbols::KeyMatch::NotFound) => { // Preserve the historical contract for fuzzy queries: an - // unresolvable name/position answers empty references. - let impact = build_impact(Vec::new(), None, Vec::new()); + // unresolvable name/position answers empty references. With + // a missing index that answer also carries the self-heal + // warning — empty must not pass for "no callers". + let mut warnings = answer_warnings(Vec::new()); + warnings.extend(heal_warning.iter().cloned()); + let impact = build_impact(Vec::new(), None, warnings); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); } @@ -450,7 +556,11 @@ impl CodesearchService { // The warm retry must carry the same honesty as a fresh // answer: read the persisted warnings for this key. let warnings = indexer.lookup_warnings(&db_path, &canonical); - let impact = build_impact(references, Some(canonical.clone()), warnings); + let impact = build_impact( + references, + Some(canonical.clone()), + answer_warnings(warnings), + ); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); return Ok(CallToolResult::success(vec![ContentBlock::text(json)])); } @@ -506,7 +616,7 @@ impl CodesearchService { // Surface what the lookup survived: a partial answer must // say so in its own payload, not only in a log line. let warnings = indexer.lookup_warnings(&db_path, &canonical); - let impact = build_impact(references, Some(canonical), warnings); + let impact = build_impact(references, Some(canonical), answer_warnings(warnings)); let json = serde_json::to_string(&impact).unwrap_or_else(|_| "{}".to_string()); Ok(CallToolResult::success(vec![ContentBlock::text(json)])) } diff --git a/src/mcp/find_impact_tests.rs b/src/mcp/find_impact_tests.rs index a07b7927..60726ce2 100644 --- a/src/mcp/find_impact_tests.rs +++ b/src/mcp/find_impact_tests.rs @@ -7,7 +7,10 @@ //! resolution is tested separately against `resolve_find_impact_budget_secs` //! with `#[serial]` + `EnvRestore` per the repo rule for env mutation. -use super::{find_impact_with_budget, resolve_find_impact_budget_secs, ImpactLookupOutcome}; +use super::{ + find_impact_with_budget, missing_index_warning, resolve_find_impact_budget_secs, + ImpactLookupOutcome, SymbolIndexHeal, +}; use crate::constants::{DEFAULT_FIND_IMPACT_BUDGET_SECS, FIND_IMPACT_BUDGET_SECS_ENV}; use crate::symbols::{SymbolLookupBusy, SymbolReference}; use std::path::PathBuf; @@ -283,6 +286,36 @@ fn fingerprint_fields_present_when_set_and_omitted_when_none() { assert!(!keys.contains(&"resolved_symbol"), "keys: {keys:?}"); } +#[test] +fn index_age_sentinel_serializes_as_null_and_real_ages_as_numbers() { + use crate::symbols::{FindImpactResult, SymbolReference}; + + let result = |age: u64| FindImpactResult { + symbol: "FieldDefinition.Validate".to_string(), + resolved_symbol: None, + references: vec![SymbolReference { + file: PathBuf::from("a.cs"), + start_line: 1, + end_line: 1, + kind: "definition".to_string(), + }], + warnings: Vec::new(), + index_age_seconds: age, + language: "csharp".to_string(), + scope: "project:p".to_string(), + index_head_sha: None, + current_head_sha: None, + }; + + // u64::MAX is `index_age`'s unreadable-index sentinel; a consumer must + // see "unknown" (null), not 18446744073709551615 seconds. + let unknown: serde_json::Value = serde_json::to_value(result(u64::MAX)).unwrap(); + assert_eq!(unknown["index_age_seconds"], serde_json::Value::Null); + + let known: serde_json::Value = serde_json::to_value(result(12)).unwrap(); + assert_eq!(known["index_age_seconds"], serde_json::json!(12)); +} + #[test] fn ambiguity_envelope_serializes_the_four_documented_fields() { let ambiguity = crate::symbols::SymbolAmbiguity { @@ -712,6 +745,24 @@ fn warnings_bytes(warnings: &[&str]) -> Vec { bytes } +/// Persist index-level rebuild warnings the way a C# rebuild does (meta +/// key, JSON array) — the fixture for answers served from an index built +/// out of a partially loaded solution. +fn write_index_warnings_meta(db_path: &std::path::Path, warnings: &[String]) { + let env = crate::symbols::get_shared_scip_env(db_path).expect("shared env"); + let mut wtxn = env.write_txn().expect("wtxn"); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + serde_json::to_string(warnings).unwrap().as_str(), + ) + .unwrap(); + wtxn.commit().unwrap(); +} + /// One resolved symbol with persisted warnings, one without. fn populate_warnings_fixture(db_path: &std::path::Path) -> (String, String) { let warned_key = "csharp Ns . W#Warned().".to_string(); @@ -811,3 +862,159 @@ async fn clean_resolved_answer_omits_the_warnings_field() { "a clean answer must OMIT warnings (skip_serializing_if), got: {out}" ); } + +/// The missing-index warning is what keeps a never-built symbol index's +/// empty reference list from passing for "no callers": whatever the heal +/// status, it must name the language and state UNKNOWN explicitly. The +/// wording per status must also stay honest — "a background rebuild was +/// started" is only allowed when one actually was. +#[test] +fn missing_index_warning_names_language_and_marks_answer_unknown() { + let warning = missing_index_warning("csharp", SymbolIndexHeal::BackgroundStarted); + assert!( + warning.contains("csharp"), + "must name the language, got: {warning}" + ); + assert!( + warning.contains("UNKNOWN"), + "must mark the empty list as UNKNOWN, got: {warning}" + ); + assert!( + warning.contains("retry"), + "must tell the caller to retry after the rebuild, got: {warning}" + ); +} + +#[test] +fn missing_index_warning_manual_names_the_build_command() { + let warning = missing_index_warning("typescript", SymbolIndexHeal::Manual); + assert!(warning.contains("UNKNOWN"), "got: {warning}"); + assert!( + warning.contains("codesearch index symbol"), + "must tell the caller how to build the index, got: {warning}" + ); + assert!( + !warning.contains("rebuild was started"), + "nothing was started — must not claim it, got: {warning}" + ); +} + +#[test] +fn missing_index_warning_not_applicable_explains_why_and_promises_nothing() { + let reason = "the scip-typescript adapter requires a top-level tsconfig.json (monorepo layouts are not resolved yet)"; + let warning = missing_index_warning("typescript", SymbolIndexHeal::NotApplicable(reason)); + assert!(warning.contains("UNKNOWN"), "got: {warning}"); + assert!( + warning.contains("tsconfig.json"), + "must surface the indexer's own reason, got: {warning}" + ); + assert!( + !warning.contains("rebuild was started"), + "a non-applicable repo can never be rebuilt here — must not claim a rebuild, got: {warning}" + ); +} + +/// A never-built index must not answer "no references" silently: the empty +/// NotFound answer carries the self-heal warning. Fails without the fix — +/// the pre-fix answer is byte-identical to a genuine "no callers", the exact +/// trap that reads as "safe to remove" (observed live on a runtime-registered +/// worktree whose SCIP index was never built). +#[tokio::test] +#[serial_test::serial] +async fn never_built_index_warns_on_the_empty_answer() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + // No fixture population: a valid-but-empty store — neither symbols nor a + // rebuild timestamp. This is the never-built state. + let (service, _root) = build_service(); + + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("ResolveLinkedActivityAsync".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!( + v["references"].as_array().is_some_and(|a| a.is_empty()), + "the never-built index answers with empty references, got: {out}" + ); + let warnings = v["warnings"].as_array().expect("self-heal warning present"); + assert_eq!( + warnings.len(), + 1, + "exactly the missing-index warning, got: {out}" + ); + assert!( + warnings[0].as_str().is_some_and(|w| w.contains("UNKNOWN")), + "the warning must mark the answer UNKNOWN, got: {out}" + ); +} + +/// Index-level rebuild warnings must ride on EVERY answer from an index +/// built out of a partially loaded solution — resolved answers AND empty +/// NotFound answers. This pins the live defect reported on a customer +/// worktree: MSBuild project-load failures (broken NuGet restore) produced +/// an index with definitions but no cross-project references, and every +/// answer passed as clean while an orphan-check read the empty lists as +/// "no callers". +#[tokio::test] +#[serial_test::serial] +async fn index_level_rebuild_warnings_ride_on_resolved_and_empty_answers() { + let helper_root = tempfile::tempdir().unwrap(); + let _guard = make_helper_available(&helper_root); + let (service, project) = build_service(); + let db = project.path().join(".codesearch.db"); + let (_warned_key, clean_key) = populate_warnings_fixture(&db); + let index_warnings = vec![ + "The symbol index was built while the C# workspace reported 2 distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex." + .to_string(), + "App.Dam.csproj: depends on Analyzer.X (>= 1.2.3) but it was not found".to_string(), + ]; + write_index_warnings_meta(&db, &index_warnings); + + // A RESOLVED answer from the partial index carries the index warnings. + let out = tool_text( + &service, + FindImpactRequest { + symbol_key: Some(clean_key.clone()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!( + v["resolved_symbol"], clean_key, + "the fixture query must resolve, got: {out}" + ); + let warnings = v["warnings"].as_array().expect("warnings present"); + assert_eq!( + serde_json::to_value(warnings).unwrap(), + serde_json::to_value(&index_warnings).unwrap(), + "a resolved answer from a partially built index carries exactly the index warnings: {out}" + ); + + // The EMPTY NotFound answer carries them too — an empty reference list + // out of a partial index must never pass for "no callers". + let out = tool_text( + &service, + FindImpactRequest { + symbol_name: Some("NotARealSymbolAnywhere".to_string()), + ..find_impact_request() + }, + ) + .await; + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert!( + v["references"].as_array().is_some_and(|a| a.is_empty()), + "unknown symbol answers empty, got: {out}" + ); + assert!( + v["warnings"].as_array().is_some_and( + |w| !w.is_empty() && w[0].as_str().is_some_and(|s| s.contains("cross-project")) + ), + "the empty answer must carry the index-level warning, got: {out}" + ); +} diff --git a/src/mcp/helpers.rs b/src/mcp/helpers.rs index 02fbdcd6..77b5efbf 100644 --- a/src/mcp/helpers.rs +++ b/src/mcp/helpers.rs @@ -119,24 +119,6 @@ pub(crate) fn prefix_path_with_alias( } } -/// Prefix a result path with the matching repo alias from a set of aliases and their roots. -/// Used by handlers that have alias/root info but not a full `MultiStoreContext`. -pub(crate) fn prefix_path_multi( - path: &str, - aliases: &[String], - alias_roots: &std::collections::HashMap, -) -> String { - let normalized = crate::cache::normalize_path_str(path); - for alias in aliases { - if let Some(root) = alias_roots.get(alias) { - if normalized.starts_with(root.as_str()) { - return prefix_path_with_alias(path, Some(alias), root); - } - } - } - normalized -} - /// Pick the project root to relativise a result path against for a `filter_path` /// prefix match, so `filter_path` is interpreted **relative to the repo root** /// in every routing mode: diff --git a/src/mcp/mod.rs b/src/mcp/mod.rs index ac0a38a7..a57ae7d7 100644 --- a/src/mcp/mod.rs +++ b/src/mcp/mod.rs @@ -37,6 +37,29 @@ fn store_lock_wait() -> std::time::Duration { .unwrap_or_else(|| std::time::Duration::from_secs(crate::constants::STORE_LOCK_WAIT_SECS)) } +/// Build the structured `project_unresolved` refusal JSON for an unknown +/// `project=` alias (NEE-P-CONSISTENT: a refusal names its class and remedy). +/// Pure + unit-testable; the caller supplies the serve hub's enumeration. +fn unknown_project_error_json( + alias: &str, + projects: Vec, + groups: Vec, + project_groups: std::collections::HashMap>, +) -> String { + let payload = serde_json::json!({ + "error_code": "project_unresolved", + "message": format!("Unknown project alias '{}'.", alias), + "available_projects": projects, + "available_groups": groups, + "project_groups": project_groups, + "hint_for_agent": format!( + "'{}' is not a registered project. Re-call with one of available_projects, or ask the user which repository to search. project_groups maps each project to its group(s) — prefer group= when the target is listed there.", + alias + ) + }); + payload.to_string() +} + /// Acquire the vector-store read lock with a bounded wait. /// /// A handler queued on this lock keeps its `Arc` — and with it @@ -1145,8 +1168,16 @@ impl CodesearchService { types::validate_project_group(project, group, true)?; if let Some(ref alias) = project { - let stores = serve_state.get_or_open_stores(alias, true).await?; - return Ok(Some((vec![stores], vec![alias.clone()]))); + // A bad alias is a NAMED refusal, not a bare string: surface the same + // available-projects envelope the unscoped case uses (P1, codesearch + // tuning round 2026-10-16) so an agent corrects without guessing. + return match serve_state.get_or_open_stores(alias, true).await { + Ok(stores) => Ok(Some((vec![stores], vec![alias.clone()]))), + Err(e) if e.contains("Unknown alias") => { + Err(self.format_unknown_project_error(alias)) + } + Err(e) => Err(e), + }; } if let Some(ref group_name) = group { @@ -1258,25 +1289,7 @@ impl CodesearchService { /// that picking a single project would miss sibling repos in the same group /// (e.g. a separate config / import-data repo). fn format_scope_error(&self) -> String { - let (projects, mut groups, project_groups) = if let Some(ref serve_state) = self.serve_state - { - let cfg = serve_state.config_snapshot(); - let mut projects: Vec = cfg.repos.keys().cloned().collect(); - // Include opt-in mounted remote projects so an agent can discover and - // route to them by name (they are first-class `project=` targets). - projects.extend( - cfg.mounted_remote_projects() - .into_iter() - .map(|(name, _)| name), - ); - projects.sort(); - projects.dedup(); - let mut groups: Vec = cfg.groups.keys().cloned().collect(); - groups.sort(); - (projects, groups, cfg.project_groups()) - } else { - (vec![], vec![], std::collections::HashMap::new()) - }; + let (projects, mut groups, project_groups) = self.project_scope_payload(); // The "all" virtual group is always available when there are projects to // search — advertise it so agents discover the cross-repo shortcut. // (scope_required only fires when >1 repo is registered, so this is safe.) @@ -1297,6 +1310,45 @@ impl CodesearchService { payload.to_string() } + /// The `(available_projects, available_groups, project_groups)` triple for the + /// active serve hub (empty in local mode). One home for the enumeration shared + /// by the scope and unknown-project refusals. + fn project_scope_payload( + &self, + ) -> ( + Vec, + Vec, + std::collections::HashMap>, + ) { + if let Some(ref serve_state) = self.serve_state { + let cfg = serve_state.config_snapshot(); + let mut projects: Vec = cfg.repos.keys().cloned().collect(); + // Include opt-in mounted remote projects so an agent can discover and + // route to them by name (they are first-class `project=` targets). + projects.extend( + cfg.mounted_remote_projects() + .into_iter() + .map(|(name, _)| name), + ); + projects.sort(); + projects.dedup(); + let mut groups: Vec = cfg.groups.keys().cloned().collect(); + groups.sort(); + (projects, groups, cfg.project_groups()) + } else { + (vec![], vec![], std::collections::HashMap::new()) + } + } + + /// Build a structured `project_unresolved` error JSON for an unknown + /// `project=` alias (NEE-P-CONSISTENT: a refusal names its class and remedy). + /// Previously a bare `Unknown alias 'x'` string reached the agent with no + /// valid-project list — the exact shape that forces a guessing round-trip. + fn format_unknown_project_error(&self, alias: &str) -> String { + let (projects, groups, project_groups) = self.project_scope_payload(); + unknown_project_error_json(alias, projects, groups, project_groups) + } + /// Execute a read-only action against the vector store with an explicit store override. /// /// If `store_override` is provided (from project/group routing), it takes precedence. diff --git a/src/mcp/search.rs b/src/mcp/search.rs index 5ac0200a..c47d324e 100644 --- a/src/mcp/search.rs +++ b/src/mcp/search.rs @@ -499,6 +499,16 @@ impl CodesearchService { let mode = request.mode.as_deref().unwrap_or("auto"); let structural_intent = detect_structural_intent(&request.query); + // Repo ownership is carried WITH each result, not reconstructed from a + // chunk-id-only map. Every fan-out hit and every chunk resolved from a + // store has its alias baked into its path the moment the owning store + // (alias) is known. Chunk ids are allocated per store and collide + // across repos, so keying ownership by `u32` alone silently mislabels + // (or drops) the result of one repo when two share a local id — the + // reason a `group=` response could point at the wrong repo. Attributing + // at creation keeps group results ("alias/path") consistent with the + // `project=` routing without ever trusting a non-unique id. + // === Lexical mode: FTS only across all stores === if mode == "lexical" { // Lexical has no second backend, so a failed store here is invisible @@ -545,22 +555,21 @@ impl CodesearchService { .unwrap_or(std::cmp::Ordering::Equal) }); - let results = self + let mut results = self .resolve_fts_to_search_results_multi( &all_fts, limit, &stores, aliases, + alias_roots, &mut lexical_warnings, ) .await; if let Some(target_kind) = structural_intent { - // We need mutable results but we have them as vectordb::SearchResult - let mut mutable_results = results; - boost_kind(&mut mutable_results, target_kind); + boost_kind(&mut results, target_kind); return self.build_semantic_response( - mutable_results, + results, request, compact, !identifiers.is_empty(), @@ -643,9 +652,19 @@ impl CodesearchService { "internal error: no query embedding resolved for repo '{alias}'" ) })?; - store + let mut found = store .search(embedding, limit * 5) - .context("Error searching vector store") + .context("Error searching vector store")?; + // Attribute each hit to its own repo here, while the owning + // store (alias) is known: chunk ids are only unique within a + // store, so deferring to a chunk-id-keyed map would collapse + // hits from different repos that share a local id. + if let Some(root) = alias_roots.get(alias) { + for r in &mut found { + r.path = prefix_path_with_alias(&r.path, Some(alias), root); + } + } + Ok(found) }, stores.clone(), aliases, @@ -806,6 +825,7 @@ impl CodesearchService { f.rrf_score, &stores, aliases, + alias_roots, &mut search_warnings, ) .await @@ -832,12 +852,14 @@ impl CodesearchService { } /// Resolve a single chunk from multiple stores (used for FTS-only hits in multi-store fusion). + #[allow(clippy::too_many_arguments)] async fn resolve_chunk_from_stores( &self, chunk_id: u32, score: f32, stores: &[Arc], aliases: &[String], + alias_roots: &std::collections::HashMap, warnings: &mut Vec, ) -> Option { for (idx, store_arc) in stores.iter().enumerate() { @@ -853,10 +875,21 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { + // Attribute the path to the store this content was actually + // read from — the probe order here is what decided the owner. + let path = match ( + aliases.get(idx), + aliases.get(idx).and_then(|a| alias_roots.get(a)), + ) { + (Some(alias), Some(root)) => { + prefix_path_with_alias(&chunk.path, Some(alias.as_str()), root) + } + _ => chunk.path, + }; return Some(crate::vectordb::SearchResult { id: chunk_id, content: chunk.content, - path: chunk.path, + path, start_line: chunk.start_line, end_line: chunk.end_line, kind: chunk.kind, @@ -875,12 +908,14 @@ impl CodesearchService { } /// Resolve FTS results to SearchResult using multiple stores. + #[allow(clippy::too_many_arguments)] async fn resolve_fts_to_search_results_multi( &self, fts_results: &[crate::fts::FtsResult], limit: usize, stores: &[Arc], aliases: &[String], + alias_roots: &std::collections::HashMap, warnings: &mut Vec, ) -> Vec { let mut results = Vec::new(); @@ -903,10 +938,21 @@ impl CodesearchService { note_store_failure(warnings, aliases, idx, "chunk lookup", e); } if let Ok(Some(chunk)) = looked_up { + // Same as resolve_chunk_from_stores: attribute the path to + // the store this content was actually read from. + let path = match ( + aliases.get(idx), + aliases.get(idx).and_then(|a| alias_roots.get(a)), + ) { + (Some(alias), Some(root)) => { + prefix_path_with_alias(&chunk.path, Some(alias.as_str()), root) + } + _ => chunk.path, + }; results.push(crate::vectordb::SearchResult { id: fts.chunk_id, content: chunk.content, - path: chunk.path, + path, start_line: chunk.start_line, end_line: chunk.end_line, kind: chunk.kind, @@ -1090,7 +1136,10 @@ impl CodesearchService { }) .collect(); - // Prefix paths with alias for multi-repo / single-project identification + // `project=` routing prefixes paths with the routed alias here. Group + // fan-out results arrive already alias-prefixed (semantic_search_multi + // recorded each chunk's repo), so this pass only normalizes them; + // stdio without alias roots passes paths through untouched. for item in &mut items { if let Some(alias) = project_alias { if let Some(root) = alias_roots.get(alias) { @@ -1099,7 +1148,7 @@ impl CodesearchService { item.path = crate::cache::normalize_path_str(&item.path); } } else if !alias_roots.is_empty() { - item.path = prefix_path_multi(&item.path, &[], alias_roots); + item.path = crate::cache::normalize_path_str(&item.path); } } diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index 79408b7d..586be728 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -565,6 +565,140 @@ fn test_relative_path_empty_alias() { assert_eq!(result, "docs/guide.md"); } +// === group fan-out alias attribution (semantic_search_multi) === + +#[tokio::test] +async fn group_lexical_search_attributes_each_hit_to_its_repo() { + use rmcp::model::ContentBlock; + + // Two repos with disjoint chunk ids. A group query must answer with + // alias-prefixed paths ("alpha/…", "beta/…") exactly like a project= + // query — before the fix the fan-out lost the store of origin and both + // hits came back bare, making a cross-repo response unattributable. + let root_a = tempfile::tempdir().expect("tempdir a"); + let stores_a = std::sync::Arc::new( + crate::index::SharedStores::new(&root_a.path().join(".codesearch.db"), 2) + .expect("stores a"), + ); + { + let mut vs = stores_a.vector_store.write().await; + let chunk = crate::chunker::Chunk::new( + "fn gadget() {}".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/lib.rs".to_string(), + ); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + chunk, + vec![0.0, 1.0], + )]) + .expect("insert chunk a"); + vs.build_index().expect("build index a"); + } + { + let mut fts = stores_a.fts_store.write().await; + fts.add_chunk(0, "fn gadget() {}", "src/lib.rs", None, "Function") + .expect("fts doc a"); + fts.commit().expect("commit a"); + } + + let root_b = tempfile::tempdir().expect("tempdir b"); + let stores_b = std::sync::Arc::new( + crate::index::SharedStores::new(&root_b.path().join(".codesearch.db"), 2) + .expect("stores b"), + ); + { + // insert_chunks assigns ids from 0 per store, so a filler chunk keeps + // the real one at id 1 and the two test repos never share a chunk id. + let mut vs = stores_b.vector_store.write().await; + let filler = crate::chunker::Chunk::new( + "filler".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/filler.rs".to_string(), + ); + let chunk = crate::chunker::Chunk::new( + "gadget helper".to_string(), + 0, + 0, + crate::chunker::ChunkKind::Function, + "src/util.rs".to_string(), + ); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + filler, + vec![0.5, 0.5], + )]) + .expect("insert filler b"); + vs.insert_chunks(vec![crate::embed::EmbeddedChunk::new( + chunk, + vec![1.0, 0.0], + )]) + .expect("insert chunk b"); + vs.build_index().expect("build index b"); + } + { + let mut fts = stores_b.fts_store.write().await; + fts.add_chunk(1, "gadget helper", "src/util.rs", None, "Function") + .expect("fts doc b"); + fts.commit().expect("commit b"); + } + + let service = super::CodesearchService::new_with_stores( + Some(root_a.path().to_path_buf()), + Some(stores_a.clone()), + ) + .expect("service"); + + let request = super::SemanticSearchRequest { + query: "gadget".to_string(), + limit: Some(10), + compact: Some(false), + filter_path: None, + mode: Some("lexical".to_string()), + project: None, + group: Some("all".to_string()), + }; + let mut alias_roots = std::collections::HashMap::new(); + alias_roots.insert( + "alpha".to_string(), + crate::cache::normalize_path_str(&root_a.path().to_string_lossy()), + ); + alias_roots.insert( + "beta".to_string(), + crate::cache::normalize_path_str(&root_b.path().to_string_lossy()), + ); + + let result = service + .semantic_search_multi( + &request, + &[], + 10, + false, + vec![stores_a.clone(), stores_b.clone()], + &["alpha".to_string(), "beta".to_string()], + &alias_roots, + ) + .await + .expect("group search must succeed"); + + let json = match &result.content[0] { + ContentBlock::Text(t) => t.text.clone(), + other => panic!("expected text content, got {other:?}"), + }; + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid json response"); + let paths: Vec<&str> = parsed["results"] + .as_array() + .expect("results array") + .iter() + .map(|r| r["path"].as_str().expect("path string")) + .collect(); + assert_eq!(paths.len(), 2, "both repos must answer, paths: {paths:?}"); + assert!(paths.contains(&"alpha/src/lib.rs"), "paths: {paths:?}"); + assert!(paths.contains(&"beta/src/util.rs"), "paths: {paths:?}"); +} + #[test] fn test_normalize_tool_path_relativizes_absolute() { let root = std::path::Path::new("/tmp/proj"); @@ -2877,3 +3011,28 @@ fn single_index_status_requires_a_built_graph_to_report_ready() { let (status, _) = super::single_index_status(0, false); assert_eq!(status, "building"); } + +#[test] +fn test_unknown_project_error_is_classified() { + // P1 (codesearch tuning round 2026-10-16): a bad `project=` alias must be a + // NAMED refusal carrying the available-project list, not a bare + // `Unknown alias 'x'` string (NEE-P-CONSISTENT: a refusal names its class + // and remedy so the agent corrects without a guessing round-trip). + let mut project_groups = std::collections::HashMap::new(); + project_groups.insert( + "nexartis".to_string(), + vec!["alpha".to_string(), "beta".to_string()], + ); + let json = super::unknown_project_error_json( + "nope", + vec!["alpha".to_string(), "beta".to_string()], + vec!["all".to_string(), "nexartis".to_string()], + project_groups, + ); + let v: serde_json::Value = serde_json::from_str(&json).expect("valid json"); + assert_eq!(v["error_code"], "project_unresolved"); + assert_eq!(v["available_projects"][0], "alpha"); + assert_eq!(v["available_projects"][1], "beta"); + assert_eq!(v["available_groups"][1], "nexartis"); + assert!(v["hint_for_agent"].as_str().unwrap().contains("nope")); +} diff --git a/src/serve/mod.rs b/src/serve/mod.rs index 5ea6d09d..eb3c86cb 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use axum::response::{IntoResponse, Json as AxumJson}; use colored::Colorize; -use dashmap::DashMap; +use dashmap::{DashMap, DashSet}; use rmcp::transport::{ streamable_http_server::session::local::LocalSessionManager, StreamableHttpServerConfig, StreamableHttpService, @@ -48,6 +48,7 @@ use crate::index::{ CSharpRebuildNotifier, IndexManager, IndexingHeartbeat, IndexingStatusCallback, SharedStores, SymbolRebuildSignal, }; +use crate::limits::JobAcquire; use crate::mcp::types::HealthResponse; use crate::symbols::{csharp, RebuildScope, SymbolIndexerRegistry}; @@ -92,12 +93,35 @@ pub(crate) enum CSharpIndexStatus { None, /// Helper available and index built successfully. Ready, + /// Index built, but the helper survived workspace/MSBuild failures — + /// definitions are present while cross-project references may be + /// missing. Rendered red (`C#⚠`) in the TUI; the failures are in the + /// info panel and on every find_impact answer from this index. + Partial, /// Index exists but had errors or is stale. Error, /// Symbol index is currently being built. Indexing, } +/// Outcome of a SUCCESSFUL C# symbol rebuild, derived from the rebuild's +/// index warnings: clean → Ready (error cleared), survived failures → +/// Partial with a bounded detail line for the TUI info panel. Pure so the +/// status contract is testable without running a helper. +fn csharp_success_outcome(index_warnings: &[String]) -> (CSharpIndexStatus, Option) { + if index_warnings.is_empty() { + (CSharpIndexStatus::Ready, None) + } else { + let detail = index_warnings + .iter() + .take(3) + .cloned() + .collect::>() + .join(" | "); + (CSharpIndexStatus::Partial, Some(detail)) + } +} + /// Lightweight repo status derived from DashMap state only (no DB opens). pub(crate) struct RepoStatusInfo { pub(crate) status: RepoStateLabel, @@ -198,6 +222,19 @@ pub(crate) enum IndexingOwner { Symbol, } +/// A tracked background indexing task. +/// +/// `db_path` travels with the entry so the DB directory of a repo that was +/// removed mid-index can be cleaned up after the task finally exits, and +/// `started_at` lets the periodic sweeper report tasks that have been alive +/// past the stale threshold (a wedge signal for operators). +struct IndexTask { + handle: tokio::task::JoinHandle<()>, + token: CancellationToken, + db_path: PathBuf, + started_at: Instant, +} + /// Shared state for the serve mode. pub(crate) struct ServeState { /// Repo alias → opened stores (or conflicted marker). @@ -232,22 +269,28 @@ pub(crate) struct ServeState { /// `stop_fsw`) so the LMDB `Environment` drops and releases the file /// handles BEFORE the DB directory is deleted. See `await_fsw_shutdown`. fsw_tasks: DashMap>, - /// Repo alias → `(JoinHandle, CancellationToken)` of its background - /// *indexing* task (the heavy `add_repo`/`reindex` embed pass), separate - /// from `fsw_tasks` because `restart_fsw` reuses the `fsw_tasks` slot for - /// the continuous watcher loop. + /// Repo alias → its tracked background *indexing* task (the heavy + /// `add_repo`/`reindex` embed pass), separate from `fsw_tasks` because + /// `restart_fsw` reuses the `fsw_tasks` slot for the continuous watcher + /// loop. /// /// This exists to fix the index-cancellation no-op (BUG1): `add_repo` and /// `reindex` used to spawn detached, untracked `tokio::spawn` tasks that /// neither observed the cancel token nor could be awaited, so `remove_repo` /// reported success while a full-corpus embed pass kept running (and /// writing) on the removed alias — 6 GB / 52% CPU runaway. Registering the - /// handle here lets `await_index_task` (called from `remove_repo`) cancel - /// the token AND await the task before the DB directory is deleted, so the - /// task's `Arc` (and the LMDB mmap handles it keeps alive) - /// drop first. The token is stored alongside the handle so `remove_repo` - /// can cancel regardless of the repo's `RepoState` variant. - index_tasks: DashMap, CancellationToken)>, + /// task here lets `await_index_task` (called from `remove_repo`) cancel the + /// token AND wait for the task, so the task's `Arc` (and the + /// LMDB mmap handles it keeps alive) drop first. The entry is *never* + /// dropped while the task may still run: when the cooperative wait cannot + /// join it (uninterruptible `build_index`), the handle goes back into this + /// map so [`Self::sweep_index_tasks`] can reap it — and clean up a removed + /// repo's DB directory — once it finally exits. + index_tasks: DashMap, + /// Repo alias → last time [`Self::is_indexing`] / [`Self::sweep_index_tasks`] + /// warned that a *live* task's marker crossed `MAX_INDEXING_SECS`. + /// Throttles the wedge warning to once per alias per few minutes. + stale_live_warned: DashMap, /// Aliases detected with LMDB storage-format corruption (e.g. /// `MDB_BAD_VALSIZE` after a storage-layer major upgrade such as /// arroy 0.5→0.8 / heed 0.20→0.22), queued for a wipe + full force @@ -319,6 +362,12 @@ pub(crate) struct ServeState { /// with the model of the repo it targets. Mirrors the `symbol_registry` /// pattern. embedding_pool: Arc, + /// Process-wide heavy-job gate (`CODESEARCH_INDEX_JOBS`): bounds how many + /// repos run a refresh / reindex / HNSW build at once. It is attached to + /// every [`SharedStores`] this state opens, so cold opens, warmup and + /// file-watcher refreshes all draw from the same budget. Queries never + /// take it. See [`crate::limits`]. + job_gate: Arc, /// Serve-wide default embedding model for newly created indexes /// (`codesearch serve --model `), or `None` for the built-in default. /// @@ -344,6 +393,17 @@ pub(crate) struct ServeState { /// Per-repo C# symbol index last error message (set when status is Error, cleared on success). /// Wrapped in `Arc` for the same reason as `csharp_index_status`. csharp_index_error: Arc>, + /// Symbol rebuilds in flight, keyed `"{lang}:{alias}"` → claim Instant. + /// Guards against double-triggered full rebuilds (find_impact self-heal + /// racing phase-2/reindex/watcher): two concurrent rebuilds interleave + /// their LMDB clear/write passes. Stale claims self-heal past the + /// indexing timeout, like `active_reindexes` markers. + symbol_rebuild_in_flight: DashMap, + /// Aliases whose degraded index (index_warnings in meta) already got its + /// automatic retry this serve process. Without this, a still-broken + /// build environment would rebuild on every evaluation — the next serve + /// start retries once more, which is the operator-visible bound. + degraded_rebuild_attempted: DashSet, /// Debounced deadline for persisting repos config metadata (unix millis). persist_deadline_unix_ms: AtomicU64, /// Ensures only one debounce worker task runs. @@ -375,6 +435,9 @@ enum RebuildDecision { Fresh, /// No `.sln` file found; C# indexing is not applicable for this repo. NoSolutionFile, + /// The repo lacks this language's entrypoint (e.g. no `tsconfig.json`) — + /// the generic per-language gate's counterpart of `NoSolutionFile`. + NotApplicable, /// The scip-csharp helper binary is not available. HelperUnavailable, /// An indexing task for this alias is already running. @@ -403,6 +466,7 @@ impl ServeState { open_locks: DashMap::new(), fsw_tasks: DashMap::new(), index_tasks: DashMap::new(), + stale_live_warned: DashMap::new(), format_recovery_queue: std::sync::Mutex::new(std::collections::VecDeque::new()), format_recovery_done: DashMap::new(), format_recovery_worker_started: std::sync::atomic::AtomicBool::new(false), @@ -420,11 +484,14 @@ impl ServeState { embedding_pool: Arc::new(crate::embed::EmbeddingServicePool::new( crate::constants::get_global_models_cache_dir().ok(), )), + job_gate: crate::limits::JobGate::from_env(), default_model: None, legacy_model_warned: DashMap::new(), tool_call_counts: DashMap::new(), csharp_index_status: Arc::new(DashMap::new()), csharp_index_error: Arc::new(DashMap::new()), + symbol_rebuild_in_flight: DashMap::new(), + degraded_rebuild_attempted: DashSet::new(), persist_deadline_unix_ms: AtomicU64::new(0), persist_worker_started: AtomicBool::new(false), #[cfg(test)] @@ -612,9 +679,17 @@ impl ServeState { // watcher-triggered rebuild, matching `trigger_symbol_rebuild`. status_map.insert(alias_key.clone(), CSharpIndexStatus::Indexing); } - SymbolRebuildSignal::Succeeded => { - status_map.insert(alias_key.clone(), CSharpIndexStatus::Ready); - error_map.remove(&alias_key); + SymbolRebuildSignal::Succeeded { index_warnings } => { + // Same contract as trigger_symbol_rebuild's success arm: a + // rebuild that survived workspace failures is Partial (red + // C#⚠) with the failures as the info detail — never green. + let (status, detail) = csharp_success_outcome(&index_warnings); + status_map.insert(alias_key.clone(), status); + if let Some(detail) = detail { + error_map.insert(alias_key.clone(), detail); + } else { + error_map.remove(&alias_key); + } } SymbolRebuildSignal::Failed(msg) => { error_map.insert(alias_key.clone(), msg); @@ -652,18 +727,37 @@ impl ServeState { } /// Mark `alias` as actively indexing for `owner`, returning `true` if the - /// caller may proceed. Returns `false` when **any** owner holds a - /// non-stale marker (i.e. another indexing run is genuinely in progress) — - /// in that case the caller should return HTTP 409. Stale markers from - /// leaked/crashed tasks are silently dropped first. + /// caller may proceed. Returns `false` when **any** owner holds a live + /// marker (i.e. another indexing run is genuinely in progress) — in that + /// case the caller should return HTTP 409. Stale markers from leaked + /// tasks are dropped first, but a marker whose tracked task is still + /// alive is never treated as stale: queued-behind-the-gate jobs can wait + /// far past `MAX_INDEXING_SECS` without being leaks. /// /// This is the guard used by `reindex_handler`, `add_repo_handler`, and /// `spawn_force_reindex` to reject concurrent reindexes. fn begin_indexing(&self, alias: &str, owner: IndexingOwner) -> bool { let now = Instant::now(); let max = self.indexing_timeout(); + // Liveness is read before locking the marker map so the two DashMaps + // are never locked in a nested order. + let task_live = self.index_task_is_live(alias); let mut owners = self.active_reindexes.entry(alias.to_string()).or_default(); - owners.retain(|_, ts| now.duration_since(*ts) < max); + owners.retain(|owner, ts| { + if now.duration_since(*ts) < max { + return true; + } + if task_live && *owner == IndexingOwner::Reindex { + // Not a leak: renew rather than drop, or a concurrent begin + // could double-run the repo. Only the tracked index task + // (Reindex) supplies liveness evidence; other owners' stale + // markers are dropped as before. + *ts = now; + true + } else { + false + } + }); if owners.is_empty() { owners.insert(owner, now); true @@ -803,7 +897,18 @@ impl ServeState { self.repos.remove(alias); self.last_access.remove(alias); self.await_fsw_shutdown(alias).await; - self.await_index_task(alias).await; + if !self.await_index_task(alias).await { + // The task is still alive (typically parked in the uninterruptible + // build_index). Wiping the DB dir from under it is unsafe — it can + // wedge the task while it holds the store handles and the + // process-wide job permit. Refuse and let the next corruption + // detection retry once the task exits. + return Err(format!( + "in-flight indexing task for '{}' did not stop within the cooperative \ + budget; refusing to wipe its DB dir", + alias + )); + } let deadline = Instant::now() + Duration::from_secs(crate::constants::DB_DELETE_RETRY_BUDGET_SECS); @@ -904,7 +1009,7 @@ impl ServeState { let cancelled = self .index_tasks .get(alias) - .is_some_and(|entry| entry.value().1.is_cancelled()); + .is_some_and(|entry| entry.value().token.is_cancelled()); if cancelled { return Err(format!( "rebuild for '{}' was cancelled before it completed (stale indexing marker); \ @@ -915,14 +1020,50 @@ impl ServeState { Ok(()) } - /// Returns `true` if `alias` is currently (non-stale) indexing for any - /// owner. + /// True when a tracked indexing task for `alias` has not finished. /// - /// Stale markers — those older than [`MAX_INDEXING_SECS`] — are lazily - /// evicted here. This is the self-healing mechanism: even if a - /// fire-and-forget background task panics or is cancelled between - /// `begin_indexing` and `end_indexing`, its marker eventually expires and - /// the TUI returns to the correct state without a server restart. + /// Live-task evidence overrides marker age: a task queued behind the + /// process-wide job gate can wait far longer than `MAX_INDEXING_SECS` + /// without being a leak, and treating it as one (evict marker + cancel) + /// cancels healthy queued work whenever the backlog drains slower than the + /// threshold. + fn index_task_is_live(&self, alias: &str) -> bool { + self.index_tasks + .get(alias) + .is_some_and(|entry| !entry.value().handle.is_finished()) + } + + /// Throttled gate for the "live task past the stale threshold" warning: + /// true at most once per alias per five minutes. + fn stale_warn_due(&self, alias: &str) -> bool { + const WARN_EVERY: Duration = Duration::from_secs(300); + let now = Instant::now(); + let due = match self.stale_live_warned.get_mut(alias) { + Some(mut last) => { + if now.duration_since(*last) < WARN_EVERY { + false + } else { + *last = now; + true + } + } + None => true, + }; + if due { + self.stale_live_warned.insert(alias.to_string(), now); + } + due + } + + /// Returns `true` if `alias` is currently indexing for any owner. + /// + /// Markers older than [`MAX_INDEXING_SECS`] are aged out unless their + /// owner carries liveness evidence: a `Reindex` marker backed by a live + /// tracked index task is renewed instead. That keeps the self-healing + /// behavior for panicked or crashed tasks while never cancelling + /// live-but-slow work — a task queued on the process-wide job gate or a + /// long `build_index` routinely exceeds the threshold, and a backlog of + /// queued reindexes must drain instead of being cancelled as "leaked". /// /// Eviction runs under the shard's write lock (`get_mut` + `retain`), so a /// concurrent `begin_indexing`/`renew_indexing` that refreshed a marker in @@ -932,16 +1073,26 @@ impl ServeState { /// predicate + removal). pub(crate) fn is_indexing(&self, alias: &str) -> bool { let max = self.indexing_timeout(); + // Liveness is read before locking the marker map so the two DashMaps + // are never locked in a nested order. + let task_live = self.index_task_is_live(alias); let mut evicted: Vec = Vec::new(); + let mut renewed: Vec = Vec::new(); let mut any_left = false; if let Some(mut owners) = self.active_reindexes.get_mut(alias) { let now = Instant::now(); owners.retain(|owner, ts| { - let fresh = now.duration_since(*ts) < max; - if !fresh { + if now.duration_since(*ts) < max { + return true; + } + if task_live && *owner == IndexingOwner::Reindex { + *ts = now; + renewed.push(*owner); + true + } else { evicted.push(*owner); + false } - fresh }); any_left = !owners.is_empty(); } @@ -950,56 +1101,123 @@ impl ServeState { // raced this eviction keeps its fresh marker. self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); } + if !renewed.is_empty() && self.stale_warn_due(alias) { + tracing::warn!( + "⏳ Indexing marker(s) for '{}' exceeded {}s but the tracked task is still \ + live (queued on the job gate, or a long-running build) — renewing, not \ + cancelling. Owners: {:?}", + alias, + max.as_secs(), + renewed + ); + } if !evicted.is_empty() { tracing::warn!( "🧹 Evicted stale indexing marker(s) for '{}' (older than {}s, owners \ - {:?}) — likely a leaked/crashed background task", + {:?}) — no tracked task is alive; reaping", alias, max.as_secs(), evicted ); - self.cancel_stale_index_task(alias); + self.reap_finished_index_task(alias); } any_left } - /// Cancel the background index task still registered for `alias` after its - /// indexing marker was evicted as stale. + /// Reap the tracked index task for `alias` once its handle has finished. /// - /// Dropping the marker only fixes what the TUI and the reindex guard - /// *believe*; the task itself keeps running, and with it the - /// `Arc` it captured — so the LMDB env and the - /// `.writer.lock` stay held for the process lifetime. Every later write - /// (reindex, format recovery, `POST /repos`) then fails with "Database is - /// locked by another process" even though the repo looks idle and closed. - /// Cancelling the task's token releases those handles at its next - /// cancellation point. + /// Called after stale markers are evicted (there is no live task by + /// definition) and from the periodic [`Self::sweep_index_tasks`]. A + /// running task is never cancelled here, however old its marker: only + /// [`Self::remove_repo`] cancels, and only for its own alias. + fn reap_finished_index_task(&self, alias: &str) { + let Some(entry) = self.index_tasks.get(alias) else { + return; + }; + if !entry.value().handle.is_finished() { + return; + } + let db_path = entry.value().db_path.clone(); + drop(entry); + if self + .index_tasks + .remove_if(alias, |_, v| v.handle.is_finished()) + .is_some() + { + self.stale_live_warned.remove(alias); + // A finished task that left markers behind panicked or errored + // without `end_indexing`; with no task left they are stale state. + // Only the Reindex owner belongs to the index task — the other + // owners (Warmup, Watcher, Symbol) manage their own lifecycle. + if !self.index_tasks.contains_key(alias) { + if let Some(mut owners) = self.active_reindexes.get_mut(alias) { + owners.remove(&IndexingOwner::Reindex); + } + self.active_reindexes.remove_if(alias, |_, m| m.is_empty()); + } + // Covers the removed-mid-build case where the task exited via a + // path that never reached its post-build self-clean. + self.self_clean_if_unregistered(alias, &db_path); + } + } + + /// Periodic sweep of tracked index tasks (runs on the idle-reaper tick). /// - /// Cooperative only: the handle is never aborted (see - /// [`Self::await_index_task`] — an abort would detach the blocking - /// `build_index` that owns its own store clone and drop the post-build - /// self-cleanup), and the entry stays registered so `remove_repo` can - /// still join it. A task that already finished is reaped here instead. - fn cancel_stale_index_task(&self, alias: &str) { - let finished = match self.index_tasks.get(alias) { - Some(entry) => { - let (handle, token) = entry.value(); - if handle.is_finished() { - true - } else { - token.cancel(); - tracing::warn!( - "🧹 Cancelled the leaked index task for '{}' — releasing its store \ - handles and writer lock", - alias - ); - false + /// Finishes what cooperative removal could not: reaps entries whose tasks + /// finally exited (clearing leftover markers and deleting a removed + /// repo's DB directory), re-asserts cancellation for tasks whose repo is + /// gone, and loudly reports tasks alive far past the stale threshold — + /// the wedge signal an operator needs. Live tasks are never cancelled + /// here. + fn sweep_index_tasks(&self) { + let entries: Vec<(String, bool, Duration)> = self + .index_tasks + .iter() + .map(|e| { + let v = e.value(); + ( + e.key().clone(), + v.handle.is_finished(), + v.started_at.elapsed(), + ) + }) + .collect(); + let max = self.indexing_timeout(); + for (alias, finished, age) in entries { + if finished { + self.reap_finished_index_task(&alias); + continue; + } + let registered = self + .config + .read() + .map(|c| c.resolve(&alias).is_some()) + .unwrap_or(true); + if !registered { + // A removed alias's task has no reason to keep going. The + // token was cancelled by remove_repo; re-assert it + // (idempotent, cooperative) so the entry can be reaped at its + // next cancellation point. + if let Some(entry) = self.index_tasks.get(&alias) { + if !entry.value().token.is_cancelled() { + entry.value().token.cancel(); + tracing::warn!( + "🧹 Index task for removed repo '{}' is still live — \ + cancellation re-asserted", + alias + ); + } } + continue; + } + if age > max && self.stale_warn_due(&alias) { + tracing::warn!( + "⏳ Index task for '{}' has been live for {}s with no completion — \ + possible wedge; it stays tracked and will be reaped when it exits", + alias, + age.as_secs() + ); } - None => return, - }; - if finished { - self.index_tasks.remove(alias); } } @@ -1216,6 +1434,25 @@ impl ServeState { return RebuildDecision::NoIndex; } + // Fresh-by-timestamp is not correct: an index built by an older + // binary predates both the warning capture and the helper's + // load-behaviour fixes (the deployed binary must never serve an + // index it did not produce), and an index built WITH surviving + // workspace failures carries those warnings in its meta. Both + // rebuild automatically — the version path on every binary change, + // the degraded path once per serve process so a still-broken + // environment cannot loop; the next serve start retries once. + if indexer.index_builder_version(db_path).as_deref() + != Some(crate::constants::INDEX_BUILDER_VERSION) + { + return RebuildDecision::ChangedSinceLastBuild; + } + if !indexer.index_warnings(db_path).is_empty() + && self.degraded_rebuild_attempted.insert(alias.to_string()) + { + return RebuildDecision::ChangedSinceLastBuild; + } + if last_changed > last_scip { RebuildDecision::ChangedSinceLastBuild } else { @@ -1223,6 +1460,73 @@ impl ServeState { } } + /// Claim the `(alias, language)` symbol-rebuild slot. Returns `false` + /// when a fresh claim exists (a rebuild is genuinely running); a claim + /// older than the indexing timeout is taken over (leaked by a crashed + /// task, same self-heal as [`Self::begin_indexing`]). + fn begin_symbol_rebuild(&self, alias: &str, lang: &str) -> bool { + let now = std::time::Instant::now(); + let max = self.indexing_timeout(); + let key = format!("{lang}:{alias}"); + match self.symbol_rebuild_in_flight.entry(key) { + dashmap::mapref::entry::Entry::Occupied(mut occupied) => { + if now.duration_since(*occupied.get()) < max { + return false; + } + *occupied.get_mut() = now; + true + } + dashmap::mapref::entry::Entry::Vacant(vacant) => { + vacant.insert(now); + true + } + } + } + + /// Release the `(alias, language)` symbol-rebuild slot claimed by + /// [`Self::begin_symbol_rebuild`]. + fn end_symbol_rebuild(&self, alias: &str, lang: &str) { + self.symbol_rebuild_in_flight + .remove(&format!("{lang}:{alias}")); + } + + /// Generic per-language rebuild gate. C# keeps its dedicated evaluator + /// (last_scip/last_changed bookkeeping, TUI status map); every other + /// language rebuilds only a *missing* index — drift stays + /// watcher/branch-change driven, mirroring the deliberate no-thrash + /// policy documented on `SymbolIndexer::index_head_sha`. + fn evaluate_symbol_rebuild( + self: &Arc, + lang: &str, + alias: &str, + repo_path: &Path, + db_path: &Path, + ) -> RebuildDecision { + if lang.eq_ignore_ascii_case(LANG_CSHARP) { + return self.evaluate_csharp_rebuild(alias, repo_path, db_path); + } + let Some(indexer) = self.symbol_registry.get(lang) else { + return RebuildDecision::HelperUnavailable; + }; + // Applicability before availability: the entrypoint check is cheap and + // deterministic (mirrors `evaluate_csharp_rebuild`'s .sln-first order), + // while helper availability depends on the machine. + if !indexer.applies_to(repo_path) { + return RebuildDecision::NotApplicable; + } + if !indexer.is_available() { + return RebuildDecision::HelperUnavailable; + } + if !self.begin_symbol_rebuild(alias, lang) { + return RebuildDecision::AlreadyInFlight; + } + self.end_symbol_rebuild(alias, lang); + if !indexer.has_index(db_path) { + return RebuildDecision::NoIndex; + } + RebuildDecision::Fresh + } + pub(crate) fn schedule_persist_repos_config(self: &Arc) { let deadline = Self::now_unix_millis() + (PERSIST_DEBOUNCE_SECS * 1000); self.persist_deadline_unix_ms @@ -1411,10 +1715,15 @@ impl ServeState { } } - /// Phase 2: semaphore-bounded concurrent C# SCIP rebuilds, sorted by recency. - pub(crate) async fn run_phase_2_csharp_scip(self: &Arc) { + /// Phase 2: semaphore-bounded concurrent SCIP rebuilds across all + /// installed languages, sorted by recency. C# keeps its dedicated gate + /// (`evaluate_csharp_rebuild`); other languages rebuild only a missing + /// index (no-thrash: drift is watcher/branch-change driven). + pub(crate) async fn run_phase_2_symbols(self: &Arc) { let aliases = self.aliases(); - let mut candidates: Vec<(String, i64)> = Vec::new(); + let langs = self.symbol_registry.installed_languages(); + // (alias, lang, last_changed) + let mut candidates: Vec<(String, String, i64)> = Vec::new(); for alias in &aliases { let path = match self.config.read().ok().and_then(|c| c.resolve(alias)) { @@ -1422,70 +1731,75 @@ impl ServeState { None => continue, }; let db_path = path.join(DB_DIR_NAME); - // evaluate_csharp_rebuild may spawn a git subprocess and walk the - // filesystem — offload to the blocking pool so the async runtime - // stays responsive while processing all candidates. - let state2 = self.clone(); - let alias2 = alias.clone(); - let path2 = path.clone(); - let db_path2 = db_path.clone(); - let decision = match tokio::task::spawn_blocking(move || { - state2.evaluate_csharp_rebuild(&alias2, &path2, &db_path2) - }) - .await - { - Ok(d) => d, - Err(e) => { - warn!( - "phase-2: evaluate_csharp_rebuild panicked for '{}': {:?}", - alias, e - ); - continue; - } - }; - if !decision.needs_rebuild() { - // If the SCIP index exists and is fresh, mark C# status as Ready - // so the TUI shows the C# indicator (e.g. "C#·") instead of None. - if decision == RebuildDecision::Fresh { - let mut status = self - .csharp_index_status - .get(alias) - .map(|e| *e.value()) - .unwrap_or(CSharpIndexStatus::None); - if matches!(status, CSharpIndexStatus::None) { - status = CSharpIndexStatus::Ready; + for lang in &langs { + // evaluate_symbol_rebuild may spawn a git subprocess and walk + // the filesystem — offload to the blocking pool so the async + // runtime stays responsive while processing all candidates. + let state2 = self.clone(); + let alias2 = alias.clone(); + let lang2 = lang.clone(); + let path2 = path.clone(); + let db_path2 = db_path.clone(); + let decision = match tokio::task::spawn_blocking(move || { + state2.evaluate_symbol_rebuild(&lang2, &alias2, &path2, &db_path2) + }) + .await + { + Ok(d) => d, + Err(e) => { + warn!( + "phase-2: symbol evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + continue; } - self.csharp_index_status.insert(alias.to_string(), status); + }; + if !decision.needs_rebuild() { + // A fresh C# index flips the TUI C# indicator to Ready; + // other languages have no status map, nothing to update. + if decision == RebuildDecision::Fresh && lang.as_str() == LANG_CSHARP { + let mut status = self + .csharp_index_status + .get(alias) + .map(|e| *e.value()) + .unwrap_or(CSharpIndexStatus::None); + if matches!(status, CSharpIndexStatus::None) { + status = CSharpIndexStatus::Ready; + } + self.csharp_index_status.insert(alias.to_string(), status); + } + info!("phase-2: skip '{}' ({}) — {:?}", alias, lang, decision); + continue; } - info!("phase-2: skip '{}' — {:?}", alias, decision); - continue; + let last_changed = self + .config + .read() + .ok() + .and_then(|c| c.meta(alias).last_changed_unix) + .unwrap_or(0); + info!( + "phase-2: queued '{}' ({}) — {:?} (last_changed={})", + alias, lang, decision, last_changed + ); + candidates.push((alias.clone(), lang.clone(), last_changed)); } - let last_changed = self - .config - .read() - .ok() - .and_then(|c| c.meta(alias).last_changed_unix) - .unwrap_or(0); - info!( - "phase-2: queued '{}' — {:?} (last_changed={})", - alias, decision, last_changed - ); - candidates.push((alias.clone(), last_changed)); } - candidates.sort_by_key(|b| std::cmp::Reverse(b.1)); + candidates.sort_by_key(|c| std::cmp::Reverse(c.2)); if candidates.is_empty() { info!("phase-2 complete: 0 candidates"); return; } - // Pre-mark all queued candidates as C# Indexing so the TUI C# indicator + // Pre-mark queued C# candidates as Indexing so the TUI C# indicator // reflects pending rebuilds immediately, even before each repo acquires // its semaphore slot. trigger_symbol_rebuild will overwrite this with the // same value (no-op) and eventually with Ready or Error on completion. - for (alias, _) in &candidates { - self.csharp_index_status - .insert(alias.clone(), CSharpIndexStatus::Indexing); + for (alias, lang, _) in &candidates { + if lang.as_str() == LANG_CSHARP { + self.csharp_index_status + .insert(alias.clone(), CSharpIndexStatus::Indexing); + } } let concurrency = Self::csharp_scip_concurrency(); @@ -1497,7 +1811,7 @@ impl ServeState { let sem = Arc::new(Semaphore::new(concurrency)); let mut handles = Vec::with_capacity(candidates.len()); - for (alias, _) in candidates { + for (alias, lang, _) in candidates { let sem = sem.clone(); let state = self.clone(); handles.push(tokio::spawn(async move { @@ -1505,7 +1819,7 @@ impl ServeState { Ok(p) => p, Err(_) => return, }; - info!("phase-2: starting '{}'", alias); + info!("phase-2: starting '{}' ({})", alias, lang); let path = match state.config.read().ok().and_then(|c| c.resolve(&alias)) { Some(p) => p, None => { @@ -1526,7 +1840,7 @@ impl ServeState { return; } let db_path = path.join(DB_DIR_NAME); - trigger_symbol_rebuild(&alias, &path, &db_path, &state).await; + trigger_symbol_rebuild(&alias, &path, &db_path, &state, &lang).await; drop(permit); })); } @@ -1798,6 +2112,21 @@ impl ServeState { /// This is the shared logic used by both the HTTP `DELETE /repos/{alias}` handler /// and the TUI confirmation flow. pub(crate) async fn remove_repo(&self, alias: &str) -> Result { + self.remove_repo_with_budget( + alias, + std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), + ) + .await + } + + /// [`Self::remove_repo`] with an injectable cooperative budget for the + /// index-task wait (tests use a short one; production passes + /// `BG_TASK_COOPERATIVE_TIMEOUT_SECS`). + pub(crate) async fn remove_repo_with_budget( + &self, + alias: &str, + index_task_budget: Duration, + ) -> Result { // 1. Resolve project path from config let project_path = { let config = self @@ -1831,9 +2160,15 @@ impl ServeState { // above cancelled a token nobody listened to, this await found nothing // to wait on, and the embed pass kept running (writing chunks, holding // the LMDB mmap open) long after remove_repo reported success. - // await_index_task cancels the task's OWN token and awaits its exit, so - // its Arc drops before the DB delete below. - self.await_index_task(alias).await; + // await_index_task cancels the task's OWN token and waits for its + // exit, so its Arc drops before the DB delete below. It + // returns false when the task could not be joined within the + // cooperative budget (e.g. parked in an uninterruptible build): the DB + // delete is then DEFERRED instead of pulling the directory out from + // under the live build. + let index_task_exited = self + .await_index_task_with_budget(alias, index_task_budget) + .await; // 3. Unregister from repos.json { @@ -1872,7 +2207,24 @@ impl ServeState { // report honestly. let mut db_deleted = !db_path.exists(); let mut db_delete_error: Option = None; - if db_path.exists() { + if db_path.exists() && !index_task_exited { + // The index task is still alive (typically parked in the + // uninterruptible `build_index`). Deleting the directory out from + // under it can wedge a serve process: the task's + // mmapped LMDB/arroy/Tantivy files vanished mid-build, it never + // reached a cancellation point, and it then held its + // `Arc` plus the process-wide job permit for the + // process lifetime. Defer instead: the task's post-build guard + // (`self_clean_if_unregistered`) deletes the dir once it observes + // the removal, and [`Self::sweep_index_tasks`] retries if a panic + // skips that path. + tracing::warn!( + "Database dir for '{}' left in place: its index task is still running \ + (cleanup deferred to the task's post-build guard / sweeper)", + alias + ); + db_delete_error = Some("index task still running; DB cleanup deferred".to_string()); + } else if db_path.exists() { // Deadline-bounded exponential-backoff retry. We ONLY retry on // lock-class errors (sharing/lock violation or access-denied on // Windows, or a message hinting the dir is in use) — a genuine @@ -2102,58 +2454,85 @@ impl ServeState { } } - /// Cancel and await the background *indexing* task for `alias` - /// (`add_repo`/`reindex` embed pass), if one is registered in + /// Cancel and wait (bounded) for the background *indexing* task for + /// `alias` (`add_repo`/`reindex` embed pass), if one is registered in /// [`Self::index_tasks`]. /// - /// Cancels the task's token first (so an in-flight embed pass aborts at the - /// next batch/phase boundary), then awaits its `JoinHandle` with a 5s - /// timeout. The await is what guarantees the task's `Arc` — - /// and the LMDB mmap handles it keeps alive on Windows — have actually - /// dropped before `remove_repo` deletes the DB directory. Without this, - /// `remove_repo` would delete `repos.json` while the detached task kept - /// writing chunks into a soon-to-be-orphaned `.codesearch.db`. - async fn await_index_task(&self, alias: &str) { - if let Some((_, (handle, token))) = self.index_tasks.remove(alias) { - token.cancel(); - // Bounded cooperative join. We deliberately do NOT abort the task - // on timeout. An indexing task can be parked inside `build_index`'s - // synchronous arroy HNSW build, which runs on a `spawn_blocking` - // thread and has no cancellation point Tokio can interrupt. - // Aborting the OUTER `JoinHandle` would only detach that blocking - // task (it keeps its own `Arc>` clone, so the - // LMDB mmap stays open regardless) AND drop the post-build - // continuation — including the self-cleanup that deletes the - // orphaned `.codesearch.db` dir once the build finishes. So on - // timeout we detach the outer task ON PURPOSE: its post-build guard - // (`remove_orphaned_db_dir`) releases the handles and self-cleans - // the directory. The deadline-bounded delete retry in `remove_repo` - // covers builds that finish within its budget; a serve restart reaps - // anything left over. - match tokio::time::timeout( - std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), - handle, - ) - .await - { - Ok(Ok(())) => { - tracing::debug!("Index task for '{}' exited cleanly", alias); - } - Ok(Err(join_err)) => { - tracing::warn!( - "Index task for '{}' panicked during shutdown: {}", - alias, - join_err - ); - } - Err(_) => { - tracing::warn!( - "Index task for '{}' still in an uninterruptible build_index after {}s; \ - detaching — its post-build guard will self-clean the DB dir", - alias, - crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS, - ); - } + /// Cancels the task's token first (so an in-flight embed pass aborts at + /// the next batch/phase boundary), then waits up to the cooperative + /// budget for its exit. Returns `true` when the task exited within the + /// budget, `false` when it is still running (typically parked inside the + /// uninterruptible `build_index`). + /// + /// The entry is never lost: on timeout the still-pending handle goes back + /// into [`Self::index_tasks`]. The old code removed the entry before + /// waiting and dropped the handle on timeout, so a detached task held its + /// `Arc` (and the process-wide job permit) invisibly for the + /// process lifetime — the leak class this tracking exists to prevent. Keeping + /// the handle tracked lets [`Self::sweep_index_tasks`] reap it and clean + /// up a removed repo's DB directory once it finally exits. + async fn await_index_task(&self, alias: &str) -> bool { + self.await_index_task_with_budget( + alias, + std::time::Duration::from_secs(crate::constants::BG_TASK_COOPERATIVE_TIMEOUT_SECS), + ) + .await + } + + /// [`Self::await_index_task`] with an injectable budget (tests use a short + /// one; production passes `BG_TASK_COOPERATIVE_TIMEOUT_SECS`). + async fn await_index_task_with_budget(&self, alias: &str, budget: Duration) -> bool { + let Some((_, task)) = self.index_tasks.remove(alias) else { + return true; + }; + task.token.cancel(); + let IndexTask { + mut handle, + token, + db_path, + started_at, + } = task; + // Bounded cooperative wait; never abort. An indexing task can be + // parked inside `build_index`'s synchronous arroy HNSW build, which + // runs on a `spawn_blocking` thread and has no cancellation point + // Tokio can interrupt. Aborting the OUTER `JoinHandle` would only + // detach that blocking task (it keeps its own + // `Arc>` clone, so the LMDB mmap stays open + // regardless) AND drop the post-build continuation — including the + // self-cleanup that deletes the orphaned `.codesearch.db` dir once the + // build finishes. + match tokio::time::timeout(budget, &mut handle).await { + Ok(Ok(())) => { + tracing::debug!("Index task for '{}' exited cleanly", alias); + true + } + Ok(Err(join_err)) => { + tracing::warn!( + "Index task for '{}' panicked during shutdown: {}", + alias, + join_err + ); + true + } + Err(_) => { + tracing::warn!( + "Index task for '{}' did not exit within {}s; keeping it tracked — the \ + stale-task sweeper will reap it and clean up its DB dir once it exits", + alias, + budget.as_secs() + ); + // `timeout(&mut handle)` leaves the handle owned; put it back + // so the task can never become invisible. + self.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token, + db_path, + started_at, + }, + ); + false } } } @@ -2262,8 +2641,8 @@ impl ServeState { .unwrap_or(true); if registered { tracing::info!( - "Task for '{}' was cancelled but the repo is still registered — keeping its DB \ - dir (handles released)", + "Index task for '{}' ended but the repo is still registered — keeping its DB \ + dir", alias ); return; @@ -2510,6 +2889,9 @@ impl ServeState { } }; if let Some(total_chunks) = needs_build { + // HNSW construction is one of the heaviest single operations a repo + // can do; draw the process-wide job permit around it. + let _job_permit = stores.acquire_job_permit().await; info!( "Warmup '{}': building vector index ({} existing chunks)", alias, total_chunks @@ -2704,6 +3086,8 @@ impl ServeState { // "Index not built" until the background refresh completes. // build_index() is CPU-heavy — offload to the blocking pool so the async // runtime is not stalled while building the HNSW index for large repos. + // The open itself is part of a heavy job: hold a process-wide permit. + let _job_permit = stores.acquire_job_permit().await; { let vector_store = Arc::clone(&stores.vector_store); let alias_owned = alias.to_string(); @@ -2983,7 +3367,7 @@ impl ServeState { return match SharedStores::new_readonly(db_path, dims) { Ok(s) => { info!("Opened repo in readonly mode (forced by config): {}", alias); - let stores_arc = Arc::new(s); + let stores_arc = Arc::new(s.with_job_gate(Arc::clone(&self.job_gate))); self.repos.insert( alias.to_string(), RepoState::Readonly { @@ -3003,7 +3387,9 @@ impl ServeState { match SharedStores::new(db_path, dims) { Ok(s) => { info!("Opened repo in write mode: {}", alias); - Ok(OpenedStores::Write(Arc::new(s))) + Ok(OpenedStores::Write(Arc::new( + s.with_job_gate(Arc::clone(&self.job_gate)), + ))) } Err(write_err) => { if allow_create { @@ -3015,7 +3401,7 @@ impl ServeState { match SharedStores::new_readonly(db_path, dims) { Ok(s) => { info!("Opened repo in readonly mode: {}", alias); - let stores_arc = Arc::new(s); + let stores_arc = Arc::new(s.with_job_gate(Arc::clone(&self.job_gate))); self.repos.insert( alias.to_string(), RepoState::Readonly { @@ -3227,6 +3613,29 @@ impl ServeState { self.active_sessions.load(Ordering::Relaxed) } + /// Durable symbol-index warnings for the info surfaces: read fresh from + /// the on-disk meta of every symbol language, language-prefixed. + /// Deliberately NOT filtered by installed languages — reading meta needs + /// no helper, and a helper that vanished after a degraded rebuild must + /// not hide the record. Unlike the in-memory C# status this survives a + /// serve restart. + pub(crate) fn collect_symbol_warnings(&self, db_path: &Path) -> Vec { + let mut warnings: Vec = Vec::new(); + for lang in [LANG_CSHARP, LANG_TYPESCRIPT] { + if let Some(indexer) = self.symbol_registry.get(lang) { + let lang_warnings = indexer.index_warnings(db_path); + if !lang_warnings.is_empty() { + warnings.extend( + lang_warnings + .into_iter() + .map(|w| format!("[{}] {w}", indexer.language())), + ); + } + } + } + warnings + } + /// Get lightweight repo statuses WITHOUT opening any databases. /// Returns a list of (alias, status_info) where status is derived from DashMap state only. pub(crate) fn repo_statuses_lightweight(&self) -> Vec<(String, RepoStatusInfo)> { @@ -3307,7 +3716,10 @@ impl ServeState { } }); - let csharp_error = if matches!(csharp_index, CSharpIndexStatus::Error) { + let csharp_error = if matches!( + csharp_index, + CSharpIndexStatus::Error | CSharpIndexStatus::Partial + ) { self.csharp_index_error .get(alias) .map(|e: dashmap::mapref::one::Ref| e.value().clone()) @@ -3506,16 +3918,17 @@ impl ServeState { /// closed. Naming the holders turns a two-day silent failure into one /// warning line. /// - /// Gated on a still-running index task: the reaper only evicts repos that - /// are not indexing, so a live task here means its marker was already - /// evicted as stale — the leak signature. Without that gate the warning - /// would fire on every eviction, because a just-cancelled FSW drains - /// asynchronously and still holds the env for a moment. + /// Gated on a still-running index task: the idle reaper evicts repos only + /// when they are not indexing, so a live task at eviction time means its + /// marker is missing or was dropped by another flow (e.g. a stale + /// non-Reindex marker) — the leak/teardown signature. Without that gate + /// the warning would fire on every eviction, because a just-cancelled FSW + /// drains asynchronously and still holds the env for a moment. fn warn_if_still_held(&self, alias: &str) { let leaked_task = self .index_tasks .get(alias) - .is_some_and(|entry| !entry.value().0.is_finished()); + .is_some_and(|entry| !entry.value().handle.is_finished()); if !leaked_task { return; } @@ -3766,12 +4179,14 @@ async fn status_handler( let csharp_str = match info.csharp_index { CSharpIndexStatus::None => "none", CSharpIndexStatus::Ready => "ready", + CSharpIndexStatus::Partial => "partial", CSharpIndexStatus::Error => "error", CSharpIndexStatus::Indexing => "indexing", }; let ts_str = match info.typescript_index { CSharpIndexStatus::None => "none", CSharpIndexStatus::Ready => "ready", + CSharpIndexStatus::Partial => "partial", CSharpIndexStatus::Error => "error", CSharpIndexStatus::Indexing => "indexing", }; @@ -3995,6 +4410,10 @@ async fn info_handler( let db_size_human = tui::dir_size_human(&db_path); + // Durable symbol-index warnings, read fresh from the on-disk meta + // (shared with the TUI info overlay so the two cannot drift). + let symbol_warnings = state.collect_symbol_warnings(&db_path); + AxumJson(json!({ "path": db_path.display().to_string(), "chunks": chunks, @@ -4005,6 +4424,9 @@ async fn info_handler( "dims": dims, "lock": lock, "index_age": index_age, + // Durable record of a degraded symbol index (see the TUI's C#⚠): + // build-environment failures the helper survived during rebuild. + "symbol_warnings": symbol_warnings, // Is the HNSW graph built and committed? A non-zero `chunks` with // `indexed: false` is a searchable-looking but silently dead index: // `VectorStore::search` refuses to run without the graph. The cloud @@ -4071,38 +4493,59 @@ async fn doctor_handler( AxumJson(json!({ "results": results })).into_response() } -/// Trigger a symbol index rebuild for a repo (C# etc.). +/// Trigger a symbol index rebuild for a repo in one language (C#, TypeScript). /// -/// Reuses the shared `SymbolIndexerRegistry` from `ServeState`, looks up the C# indexer, -/// and runs `rebuild()` in a blocking task. Updates the C# index status on success/failure. +/// Reuses the shared `SymbolIndexerRegistry` from `ServeState`, looks up the +/// indexer for `lang`, and runs `rebuild()` in a blocking task. C#-only +/// bookkeeping (TUI status/error maps, `last_scip` timestamp) stays gated on +/// the language; `IndexingOwner::Symbol` markers apply to every language. +/// +/// A per-`(alias, language)` in-flight slot prevents two concurrent full +/// rebuilds from interleaving their LMDB clear/write passes (find_impact +/// self-heal racing phase-2/reindex/watcher). Languages for one alias run +/// sequentially at multi-language call sites (see `maybe_rebuild_symbols`) — +/// the `IndexingOwner::Symbol` marker is not language-scoped. async fn trigger_symbol_rebuild( alias: &str, project_path: &Path, db_path: &Path, state: &Arc, + lang: &str, ) { + let is_csharp = lang.eq_ignore_ascii_case(LANG_CSHARP); // Skip non-applicable repos BEFORE touching status — otherwise the TUI // would flip C#-indicator red on Rust/Python repos that simply have no - // .sln. The phase-2 gate (`evaluate_csharp_rebuild`) already filters + // .sln. The phase-2 gate (`evaluate_symbol_rebuild`) already filters // these out, but other callers (POST /reindex?symbols=true, - // .cs watcher debounce, future paths) bypass that gate. + // .cs watcher debounce, find_impact self-heal) bypass that gate. let applies = state .symbol_registry - .get(LANG_CSHARP) + .get(lang) .map(|i| i.applies_to(project_path)) .unwrap_or(false); if !applies { tracing::info!( - "🔬 symbol reindex skipped for '{}': not applicable (no .sln)", + "🔬 symbol reindex skipped for '{}': not applicable (no .sln / tsconfig.json)", alias ); return; } - tracing::info!("🔬 symbol reindex triggered for '{}'", alias); - state - .csharp_index_status - .insert(alias.to_string(), CSharpIndexStatus::Indexing); + if !state.begin_symbol_rebuild(alias, lang) { + tracing::info!( + "🔬 symbol reindex skipped for '{}' ({}): already in flight", + alias, + lang + ); + return; + } + + tracing::info!("🔬 symbol reindex triggered for '{}' ({})", alias, lang); + if is_csharp { + state + .csharp_index_status + .insert(alias.to_string(), CSharpIndexStatus::Indexing); + } // Mark as actively indexing so the TUI status column shows "Indexing" // (not just the C# indicator). This mirrors what reindex_handler does. // @@ -4110,17 +4553,25 @@ async fn trigger_symbol_rebuild( // for the same alias simultaneously, each path ends only its own marker, // so neither can flip the TUI back to Warm/Open while the other still // runs. (Stale entries from a crashed task self-heal via `is_indexing`.) - state.begin_indexing(alias, IndexingOwner::Symbol); + // The return value is deliberately ignored: nested callers (the reindex + // and add-repo background tasks) legitimately hold the alias's Reindex + // marker while triggering this. + let _ = state.begin_indexing(alias, IndexingOwner::Symbol); let rp = project_path.to_path_buf(); let dp = db_path.to_path_buf(); let alias_owned = alias.to_string(); + let lang_owned = lang.to_string(); let registry = state.symbol_registry.clone(); match tokio::task::spawn_blocking(move || { - let Some(indexer) = registry.get(LANG_CSHARP) else { - return Err(anyhow::anyhow!("No C# symbol indexer registered")); + let Some(indexer) = registry.get(&lang_owned) else { + return Err(anyhow::anyhow!( + "No symbol indexer registered for '{lang_owned}'" + )); }; if !indexer.is_available() { - return Err(anyhow::anyhow!("scip-csharp helper not available")); + return Err(anyhow::anyhow!( + "symbol helper for '{lang_owned}' not available" + )); } indexer.rebuild(&rp, &dp, RebuildScope::Full) }) @@ -4135,15 +4586,22 @@ async fn trigger_symbol_rebuild( summary.duration_ms ); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_status - .insert(alias_owned.clone(), CSharpIndexStatus::Ready); - state.csharp_index_error.remove(&alias_owned); + if is_csharp { + let (status, detail) = csharp_success_outcome(&summary.index_warnings); + state + .csharp_index_status + .insert(alias_owned.clone(), status); + if let Some(detail) = detail { + state.csharp_index_error.insert(alias_owned.clone(), detail); + } else { + state.csharp_index_error.remove(&alias_owned); + } - if let Ok(mut cfg) = state.config.write() { - cfg.touch_last_scip(&alias_owned, ServeState::now_unix_secs()); + if let Ok(mut cfg) = state.config.write() { + cfg.touch_last_scip(&alias_owned, ServeState::now_unix_secs()); + } + state.schedule_persist_repos_config(); } - state.schedule_persist_repos_config(); } Ok(Err(e)) => { // `{:#}` — the whole chain, not just the outermost context. The @@ -4151,9 +4609,11 @@ async fn trigger_symbol_rebuild( // would hide the `MDB_*` code the classifier below matches on. let msg = format!("{e:#}"); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_error - .insert(alias_owned.clone(), msg.clone()); + if is_csharp { + state + .csharp_index_error + .insert(alias_owned.clone(), msg.clone()); + } if ServeState::is_lmdb_format_corruption(&msg) && state.enqueue_format_recovery(&alias_owned) { @@ -4165,9 +4625,11 @@ async fn trigger_symbol_rebuild( ); // Recovery owns the outcome from here: show in-progress rather // than Error; it flips to Ready on success or Error on failure. - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Indexing); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Indexing); + } } else if ServeState::is_lmdb_format_corruption(&msg) { // A wipe already happened for this alias in this process, so the // data was written by the running binary: the error is write-side @@ -4179,14 +4641,18 @@ async fn trigger_symbol_rebuild( alias_owned, msg ); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } else { tracing::error!("❌ Symbol rebuild failed for '{}': {}", alias_owned, msg); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } } Err(e) => { @@ -4196,14 +4662,107 @@ async fn trigger_symbol_rebuild( e ); state.end_indexing(&alias_owned, IndexingOwner::Symbol); - state - .csharp_index_error - .insert(alias_owned.clone(), format!("Task panicked: {}", e)); - state - .csharp_index_status - .insert(alias_owned, CSharpIndexStatus::Error); + if is_csharp { + state + .csharp_index_error + .insert(alias_owned.clone(), format!("Task panicked: {}", e)); + state + .csharp_index_status + .insert(alias_owned, CSharpIndexStatus::Error); + } } } + state.end_symbol_rebuild(alias, lang); +} + +/// Evaluate every installed symbol language for `alias` and rebuild the ones +/// whose gate says so. `force` (POST /reindex?symbols=true) bypasses the +/// gate, preserving the old unconditional-rebuild behaviour. +/// +/// Sequential per language: `IndexingOwner::Symbol` is not language-scoped, +/// so parallel languages would collide on the marker. Called from the +/// reindex and add-repo background tasks, where the alias already holds a +/// Reindex owner marker — `trigger_symbol_rebuild`'s own marker insert is +/// additive, not a guard, so nesting is safe. +async fn maybe_rebuild_symbols( + alias: &str, + project_path: &Path, + db_path: &Path, + state: &Arc, + force: bool, +) { + let langs = state.symbol_registry.installed_languages(); + for lang in langs { + let decision = if force { + RebuildDecision::ChangedSinceLastBuild + } else { + // evaluate may spawn a git subprocess — offload like phase-2. + let st = Arc::clone(state); + let a = alias.to_string(); + let l = lang.clone(); + let p = project_path.to_path_buf(); + let d = db_path.to_path_buf(); + match tokio::task::spawn_blocking(move || st.evaluate_symbol_rebuild(&l, &a, &p, &d)) + .await + { + Ok(d) => d, + Err(e) => { + warn!( + "symbol evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + continue; + } + } + }; + if decision.needs_rebuild() { + trigger_symbol_rebuild(alias, project_path, db_path, state, &lang).await; + } else { + info!("symbols fresh for '{}' ({}) — {:?}", alias, lang, decision); + } + } +} + +/// Fire-and-forget self-heal invoked by `find_impact` when the target index +/// does not exist: rebuild exactly that language's *missing* index — never on +/// drift (reindexing per query would thrash; see +/// `SymbolIndexer::index_head_sha`). The alias is re-resolved from the serve +/// config so remote-mounted projects (`peer/name`) are never healed locally. +pub(crate) async fn heal_missing_symbol_index(state: &Arc, alias: &str, lang: &str) { + let Some(path) = state.config.read().ok().and_then(|c| c.resolve(alias)) else { + return; + }; + if !path.exists() { + return; + } + let db_path = path.join(DB_DIR_NAME); + // evaluate may spawn a git subprocess (C# last_changed bootstrap) — + // offload so the heal task never blocks an async worker. + let st = Arc::clone(state); + let a = alias.to_string(); + let l = lang.to_string(); + let p = path.clone(); + let d = db_path.clone(); + let decision = + match tokio::task::spawn_blocking(move || st.evaluate_symbol_rebuild(&l, &a, &p, &d)).await + { + Ok(decision) => decision, + Err(e) => { + warn!( + "self-heal evaluate panicked for '{}' ({}): {:?}", + alias, lang, e + ); + return; + } + }; + if decision != RebuildDecision::NoIndex { + return; + } + info!( + "🔬 self-heal: '{}' has no {} symbol index — rebuilding in background (triggered by find_impact)", + alias, lang + ); + trigger_symbol_rebuild(alias, &path, &db_path, state, lang).await; } /// Reload repos config handler: POST /reload @@ -4334,6 +4893,8 @@ async fn reindex_handler( let guard_alias = alias_bg.clone(); let guard_state = state.clone(); + // `symbols=true` forces a symbol rebuild; without it the rebuild is + // gate-driven (missing/stale indexes are still healed). let do_symbols = symbols; if force { @@ -4400,6 +4961,9 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry + // so a removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "Force reindex for '{}': clearing stores and reindexing", @@ -4459,16 +5023,21 @@ async fn reindex_handler( // 3. Restart FSW with fresh IndexManager. g_state.restart_fsw(&g_alias, stores).await; - // 4. Optional symbol index rebuild - if do_symbols { - trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; - } + // 4. Symbol index rebuild — gate-driven for every installed + // language; `symbols=true` forces it (previous behaviour). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &g_state, do_symbols).await; g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); - state - .index_tasks - .insert(alias.to_string(), (handle, reindex_token)); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); } else { // Incremental refresh: ensure the repo is opened, then refresh let stores = match state.get_or_open_stores(&alias, true).await { @@ -4492,6 +5061,9 @@ async fn reindex_handler( let g_alias = guard_alias.clone(); let g_state = guard_state.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry + // so a removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "🔄 Incremental reindex triggered for '{}' via HTTP API", @@ -4533,16 +5105,21 @@ async fn reindex_handler( return; } - // Optional symbol index rebuild - if do_symbols { - trigger_symbol_rebuild(&alias_bg, &project_path, &db_path, &g_state).await; - } + // Symbol index rebuild — gate-driven for every installed + // language; `symbols=true` forces it (previous behaviour). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &g_state, do_symbols).await; g_state.end_indexing(&g_alias, IndexingOwner::Reindex); }); - state - .index_tasks - .insert(alias.to_string(), (handle, reindex_token)); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); } ( @@ -4810,6 +5387,9 @@ async fn add_repo_handler( // indexing task, so cancelling it (stop_fsw) did nothing and the task ran // the full embed pass to completion on a removed alias. let token_for_task = cancel_token.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry so a + // removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let index_handle = tokio::spawn(async move { tracing::info!( @@ -4878,7 +5458,23 @@ async fn add_repo_handler( } // Build vector index from freshly indexed data. - // build_index() is CPU-heavy — offload to the blocking pool. + // build_index() is CPU-heavy — offload to the blocking pool, under the + // same process-wide job permit as every other heavy pass. The wait is + // cancellation-aware: a repo removed while queued behind another build + // must not keep its stores (and a queue position) parked. + let _job_permit = match stores.acquire_job_permit_or_cancel(&token_for_task).await { + JobAcquire::Ready(permit) => permit, + JobAcquire::Cancelled => { + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); + return; + } + }; + // Re-check liveness after the wait: a removal that landed while the + // task was parked must not resurrect the alias's vector index. + if !state_bg.is_alias_live(&alias_bg, &token_for_task) { + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); + return; + } { let vector_store = Arc::clone(&stores.vector_store); let alias_bi = alias_bg.clone(); @@ -4921,16 +5517,28 @@ async fn add_repo_handler( // Start FSW and transition to proper Write state with IndexManager state_bg.restart_fsw(&alias_bg, stores).await; + // Phase-2 equivalence for runtime adds: a repo registered while serve + // runs never sees the startup symbol pass, so its symbol indexes would + // not exist until a restart — build them now (gate-driven). + maybe_rebuild_symbols(&alias_bg, &project_path, &db_path, &state_bg, false).await; + state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); tracing::info!("Repo '{}' fully indexed and ready", alias_bg); }); // Register the indexing task so remove_repo can cancel + await it (BUG1). // Storing the token alongside the handle means remove_repo can cancel - // regardless of the repo's RepoState variant. - state - .index_tasks - .insert(alias.clone(), (index_handle, cancel_token)); + // regardless of the repo's RepoState variant. The DB path travels with + // the entry for deferred cleanup if the repo is removed mid-index. + state.index_tasks.insert( + alias.clone(), + IndexTask { + handle: index_handle, + token: cancel_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); ( StatusCode::ACCEPTED, @@ -5694,6 +6302,22 @@ pub async fn run_serve( // — it never polls a federated peer on a timer at all. let serve_state = Arc::new(ServeState::new(config, None).with_default_model(default_model)); + // One line that answers "why is it slower / less parallel than before?" + // without grepping the code: the effective heavy-work budget. + info!( + "⚙️ Indexing limits: index_jobs={} (CODESEARCH_INDEX_JOBS), embed_threads={} \ + (CODESEARCH_EMBED_THREADS), embed_pause_ms={} (CODESEARCH_EMBED_PAUSE_MS), \ + min_free_mb={} (CODESEARCH_MIN_FREE_MB), max_chunks_per_batch={} \ + (CODESEARCH_MAX_CHUNKS_PER_BATCH)", + serve_state.job_gate.index_jobs(), + crate::limits::embed_threads() + .map(|n| n.to_string()) + .unwrap_or_else(|| "default".to_string()), + crate::limits::embed_pause().as_millis(), + crate::limits::min_free_mb(), + crate::limits::max_chunks_per_batch(), + ); + // Construct the bind address from resolved host + port. // Using `format!` with `parse::()` handles both IPv4 and IPv6. // For IPv6 literals, users should pass e.g. `[::1]` (with brackets). @@ -5884,7 +6508,7 @@ pub async fn run_serve( // ── Startup phase orchestration ── // Phase 1 warms all repos sequentially (text/vector ready). - // Phase 2 runs gated C# SCIP rebuilds ordered by last_changed. + // Phase 2 runs gated per-language SCIP rebuilds ordered by last_changed. { let phase_state = serve_state.clone(); tokio::spawn(async move { @@ -5901,7 +6525,7 @@ pub async fn run_serve( warn!("reconcile: spawn_blocking panicked: {:?}", e); } phase_state.run_phase_1_warmup_all().await; - phase_state.run_phase_2_csharp_scip().await; + phase_state.run_phase_2_symbols().await; phase_state.run_phase_3_prewarm().await; }); } @@ -5918,6 +6542,7 @@ pub async fn run_serve( tokio::select! { _ = tokio::time::sleep(interval) => { reaper_state.evict_idle_repos(); + reaper_state.sweep_index_tasks(); // Dashboard refresh handled by TUI auto-refresh (TTY) or not needed (non-TTY) } _ = reaper_cancel.cancelled() => { diff --git a/src/serve/tests.rs b/src/serve/tests.rs index 1e71a301..235329ba 100644 --- a/src/serve/tests.rs +++ b/src/serve/tests.rs @@ -2,6 +2,33 @@ use super::*; use serial_test::serial; use std::io::Write; +#[tokio::test] +async fn try_open_stores_attaches_the_serve_job_gate() { + // Serve-opened stores must carry the same process-wide gate as the + // ServeState, so cold opens, warmup and FSW refreshes all draw from one + // heavy-job budget (CODESEARCH_INDEX_JOBS). + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("repo").join(DB_DIR_NAME); + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + + let stores = match state + .try_open_stores("repo", &db_path, true, false, None) + .expect("a fresh repo must open in write mode") + { + OpenedStores::Write(s) => s, + OpenedStores::Readonly(_) => panic!("allow_create=true must open write mode"), + }; + + let store_gate = stores + .job_gate + .as_ref() + .expect("serve-opened stores must carry the job gate"); + assert!( + std::sync::Arc::ptr_eq(store_gate, &state.job_gate), + "the store gate must be the ServeState gate" + ); +} + #[test] fn test_api_key_matches() { assert!(api_key_matches("secret-key", "secret-key")); @@ -163,9 +190,15 @@ async fn await_index_task_cancels_and_joins_indexing_task() { } done_clone.store(true, std::sync::atomic::Ordering::SeqCst); }); - state - .index_tasks - .insert("repo-x".to_string(), (handle, token)); + state.index_tasks.insert( + "repo-x".to_string(), + IndexTask { + handle, + token, + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); state.await_index_task("repo-x").await; assert!( !state.index_tasks.contains_key("repo-x"), @@ -302,9 +335,15 @@ async fn remove_repo_during_active_build_self_cleans_db_dir() { // orphaned — self-clean it now that the build's handles are released. ServeState::remove_orphaned_db_dir("buildrepo", &db_path_for_cleanup); }); - state - .index_tasks - .insert("buildrepo".to_string(), (handle, token)); + state.index_tasks.insert( + "buildrepo".to_string(), + IndexTask { + handle, + token, + db_path: db_path.clone(), + started_at: Instant::now(), + }, + ); // remove_repo lands WHILE the spawn_blocking build is still sleeping. let outcome = state @@ -2998,13 +3037,15 @@ async fn a_second_format_recovery_for_the_same_alias_is_refused() { #[tokio::test] #[serial] -async fn stale_indexing_marker_cancels_the_leaked_index_task() { - // Regression: evicting the stale marker used to fix only what the TUI - // believed. The task itself kept running and kept its `Arc` - // — so the LMDB env and `.writer.lock` stayed held and every later write - // failed with "Database is locked by another process" on a repo that - // logged as idle and closed. - let _env = crate::testing::EnvRestore::set(&[(crate::constants::MAX_INDEXING_SECS_ENV, "1")]); +async fn await_index_task_cancels_and_releases_the_task_handles() { + // Regression: a leaked index task used to keep its `Arc` + // after every cooperative path forgot it — the LMDB env and + // `.writer.lock` stayed held, and later writes failed with "Database is + // locked by another process" on a repo that logged as idle. Cancellation + // must be delivered AND the handle joined (not detached), so the stores + // are released. The staleness check no longer cancels live tasks (see + // `stale_marker_with_live_task_is_renewed_not_cancelled`); + // `await_index_task` is the cancellation trigger. let state = Arc::new(ServeState::new(ReposConfig::default(), None)); // Stand-in for the store handles the real task captures. @@ -3016,39 +3057,32 @@ async fn stale_indexing_marker_cancels_the_leaked_index_task() { task_token.cancelled().await; drop(stores_task); }); - state - .index_tasks - .insert("leaky".to_string(), (handle, token.clone())); - state.active_reindexes.insert( + state.index_tasks.insert( "leaky".to_string(), - std::iter::once(( - IndexingOwner::Reindex, - Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("monotonic clock at least 2s old"), - )) - .collect(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, ); assert!( - !state.is_indexing("leaky"), - "a marker older than the timeout must not read as indexing" + state.await_index_task("leaky").await, + "a task observing cancellation must join within the cooperative budget" ); assert!( token.is_cancelled(), - "the leaked task must be cancelled, not merely forgotten" + "the cancelled task must observe cancellation" + ); + assert!( + !state.index_tasks.contains_key("leaky"), + "a joined task must be reaped from index_tasks" ); - - let (handle, _) = state - .index_tasks - .remove("leaky") - .expect("task entry kept") - .1; - handle.await.expect("cancelled task joins cleanly"); assert_eq!( Arc::strong_count(&stores), 1, - "the leaked task must have released its store handle" + "the cancelled task must have released its store handle" ); } @@ -3061,9 +3095,15 @@ async fn fresh_indexing_marker_leaves_its_index_task_running() { let token = CancellationToken::new(); let task_token = token.clone(); let handle = tokio::spawn(async move { task_token.cancelled().await }); - state - .index_tasks - .insert("busy".to_string(), (handle, token.clone())); + state.index_tasks.insert( + "busy".to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); state.begin_indexing("busy", IndexingOwner::Reindex); assert!( @@ -3287,3 +3327,854 @@ async fn stale_eviction_is_scoped_to_the_stale_owner() { "an empty owner map must be removed, not left behind" ); } + +// ── Always-on symbol indexes (features/always-on-symbol-indexes) ────────── + +/// The per-(alias, language) claim must reject a second fresh claim, accept +/// one per language independently, become claimable after release, and take +/// over a claim older than the indexing timeout (leaked by a crashed task). +#[test] +fn symbol_rebuild_claim_rejects_fresh_and_takes_over_stale() { + let state = ServeState::new(ReposConfig::default(), None); + + assert!( + state.begin_symbol_rebuild("repo", "csharp"), + "first claim must succeed" + ); + assert!( + !state.begin_symbol_rebuild("repo", "csharp"), + "a second fresh claim must be rejected" + ); + assert!( + state.begin_symbol_rebuild("repo", "typescript"), + "a different language must claim independently" + ); + + state.end_symbol_rebuild("repo", "csharp"); + assert!( + state.begin_symbol_rebuild("repo", "csharp"), + "a released slot must be claimable again" + ); + + // Stale takeover: a claim past the indexing timeout is taken over. + let max = state.indexing_timeout(); + state.symbol_rebuild_in_flight.insert( + "csharp:stale".to_string(), + std::time::Instant::now() - max - std::time::Duration::from_secs(1), + ); + assert!( + state.begin_symbol_rebuild("stale", "csharp"), + "a stale claim must be taken over" + ); +} + +/// The generic gate must answer NotApplicable for a repo without the +/// language's entrypoint (no tsconfig.json) — deterministically, before the +/// machine-dependent helper check — and route C# through its dedicated +/// evaluator (.sln check → NoSolutionFile). +#[test] +fn evaluate_symbol_rebuild_gates_applicability_before_availability() { + // `evaluate_symbol_rebuild` takes `self: &Arc` (it may be forwarded + // to spawn_blocking by callers), so the probe state lives in an Arc. + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + let tmp = tempfile::tempdir().unwrap(); + let db = tmp.path().join(DB_DIR_NAME); + + assert_eq!( + state.evaluate_symbol_rebuild("typescript", "x", tmp.path(), &db), + RebuildDecision::NotApplicable, + "a repo without tsconfig.json is not applicable, helper or not" + ); + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "x", tmp.path(), &db), + RebuildDecision::NoSolutionFile, + "C# must keep its dedicated .sln-first evaluator" + ); +} + +/// The find_impact self-heal must no-op quietly for an unknown alias (remote +/// mounts, typos) and for a registered repo the language does not apply to — +/// no claim taken, no C# status touched, no panic. +#[tokio::test] +async fn heal_missing_symbol_index_noops_outside_its_remit() { + // Unknown alias. + let state = std::sync::Arc::new(ServeState::new(ReposConfig::default(), None)); + heal_missing_symbol_index(&state, "ghost", "csharp").await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "unknown alias must not take a claim" + ); + + // Registered alias, non-applicable language (no .sln / tsconfig.json). + let (_tmp, _path, bare) = state_with_repo("plainrepo"); + let state = std::sync::Arc::new(bare); + heal_missing_symbol_index(&state, "plainrepo", "csharp").await; + heal_missing_symbol_index(&state, "plainrepo", "typescript").await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "non-applicable repo must not take a claim" + ); + assert!( + !state.csharp_index_status.contains_key("plainrepo"), + "non-applicable repo must not flip the C# indicator" + ); +} + +/// `maybe_rebuild_symbols` on a repo no language applies to must neither +/// panic nor leave claims behind — with and without force (the force path +/// skips the gate but trigger_symbol_rebuild still skips non-applicable +/// repos before claiming). +#[tokio::test] +async fn maybe_rebuild_symbols_skips_non_applicable_repo_under_force() { + let (_tmp, _path, bare) = state_with_repo("plainrepo2"); + let state = std::sync::Arc::new(bare); + let path = state.config.read().unwrap().resolve("plainrepo2").unwrap(); + let db = path.join(DB_DIR_NAME); + maybe_rebuild_symbols("plainrepo2", &path, &db, &state, true).await; + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "force must not claim a slot for a non-applicable repo" + ); +} + +/// A dummy `scip-csharp` helper at the env-override path: available for +/// detection, fails on execution — which flips the C# status to Error while +/// proving a rebuild was actually attempted. The helper file lives in +/// `root`, which the caller keeps alive via its TempDir. +fn dummy_csharp_helper(root: &std::path::Path) -> crate::testing::EnvRestore { + let helper = root.join(if cfg!(windows) { + "scip-csharp.exe" + } else { + "scip-csharp" + }); + std::fs::write(&helper, b"dummy").expect("dummy helper file"); + crate::testing::EnvRestore::set(&[( + crate::constants::SCIP_CSHARP_HELPER_ENV, + helper.to_string_lossy().as_ref(), + )]) +} + +/// Phase-2 composition wiring: a registered repo whose language applies and +/// whose index is missing must flow gate → queue → trigger end-to-end. The +/// dummy helper makes the rebuild FAIL, observable as C# status Error once +/// `run_phase_2_symbols` completes — None would mean the candidate was never +/// queued (the composition gap this test pins). +#[tokio::test] +#[serial] +async fn run_phase_2_symbols_wires_candidate_through_to_trigger() { + let helper_root = tempfile::tempdir().unwrap(); + let _env = dummy_csharp_helper(helper_root.path()); + + let (_tmp, repo_path, bare) = state_with_repo("phase2repo"); + std::fs::write( + repo_path.join("test.sln"), + b"Microsoft Visual Studio Solution File", + ) + .unwrap(); + + let state = std::sync::Arc::new(bare); + state.run_phase_2_symbols().await; + + assert_eq!( + state + .csharp_index_status + .get("phase2repo") + .map(|e| *e.value()), + Some(CSharpIndexStatus::Error), + "the missing C# index must be queued and the rebuild attempted \ + (dummy helper → Error); None means never queued" + ); + assert!( + state.symbol_rebuild_in_flight.is_empty(), + "the claim must be released after the failed rebuild" + ); +} + +/// A successful rebuild WITHOUT survived failures is Ready with the error +/// cleared; one WITH survived workspace failures is Partial (red `C#⚠` in +/// the TUI) with a bounded detail line for the info panel — capped at the +/// first three entries so the detail row stays readable. +#[test] +fn csharp_success_outcome_partial_when_warnings_survived() { + assert_eq!( + csharp_success_outcome(&[]), + (CSharpIndexStatus::Ready, None), + "a clean rebuild must read Ready with no error detail" + ); + + let degraded = [ + "summary".to_string(), + "A.csproj: failure one".to_string(), + "B.csproj: failure two".to_string(), + "C.csproj: failure three".to_string(), + "D.csproj: failure four".to_string(), + ]; + let (status, detail) = csharp_success_outcome(°raded); + assert_eq!(status, CSharpIndexStatus::Partial); + let detail = detail.expect("partial carries detail"); + assert!( + detail.contains("summary") && detail.contains("B.csproj"), + "detail joins the leading entries, got: {detail}" + ); + assert!( + !detail.contains("D.csproj"), + "detail is capped at three entries, got: {detail}" + ); +} + +/// The watcher notifier must apply the SAME status contract as +/// trigger_symbol_rebuild's success arm: Succeeded-with-warnings → Partial +/// (red C#⚠) with bounded detail — never green — and a clean Succeeded → +/// Ready with the detail cleared. Reverting the arm to insert(Ready) ships +/// the round-1 defect (green indicator on a degraded index) with a green +/// suite. +#[test] +fn csharp_notifier_partial_on_succeeded_with_warnings() { + let state = ServeState::new(ReposConfig::default(), None); + let notifier = state.make_csharp_notifier("wrepo"); + + notifier(SymbolRebuildSignal::Started); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Indexing) + ); + + notifier(SymbolRebuildSignal::Succeeded { + index_warnings: vec![ + "summary".to_string(), + "A.csproj: boom".to_string(), + "B.csproj: boom2".to_string(), + "C.csproj: boom3".to_string(), + ], + }); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Partial), + "a degraded watcher rebuild must render Partial, never green" + ); + let detail = state + .csharp_index_error + .get("wrepo") + .map(|e| e.value().clone()) + .expect("detail stored"); + assert!(detail.contains("summary") && !detail.contains("C.csproj")); + + notifier(SymbolRebuildSignal::Succeeded { + index_warnings: Vec::new(), + }); + assert_eq!( + state.csharp_index_status.get("wrepo").map(|e| *e.value()), + Some(CSharpIndexStatus::Ready) + ); + assert!( + !state.csharp_index_error.contains_key("wrepo"), + "a clean rebuild must clear the detail" + ); +} + +/// `repo_statuses_lightweight()` must carry the Partial status AND its +/// detail message — that pair is what renders the red `C#⚠` and the info +/// panel's failure line. +#[test] +fn repo_statuses_lightweight_carries_partial_with_detail() { + let (_tmp, _path, bare) = state_with_repo("partialrepo"); + let state = std::sync::Arc::new(bare); + state + .csharp_index_status + .insert("partialrepo".to_string(), CSharpIndexStatus::Partial); + state.csharp_index_error.insert( + "partialrepo".to_string(), + "summary | A.csproj: failure one".to_string(), + ); + + let statuses = state.repo_statuses_lightweight(); + let (_, info) = statuses + .iter() + .find(|(alias, _)| alias == "partialrepo") + .expect("partialrepo row"); + assert_eq!(info.csharp_index, CSharpIndexStatus::Partial); + assert_eq!( + info.csharp_error.as_deref(), + Some("summary | A.csproj: failure one"), + "the detail message must ride on the status row" + ); +} + +/// The serve-tui `i` overlay reads the DURABLE warnings from the on-disk +/// meta (survives a restart, unlike the in-memory status) and prefixes them +/// with the language. A repo whose meta carries no warnings yields none. +#[test] +fn build_info_overlay_reads_symbol_warnings_from_meta() { + use crate::serve::tui::build_info_overlay; + + let (_tmp, repo_path, bare) = state_with_repo("inforepo"); + let db = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write( + db.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .unwrap(); + // Seed the meta the way a degraded rebuild does (see the roundtrip test + // in csharp_tests for the read side). + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["summary","A.csproj: failure one"]"#, + ) + .unwrap(); + wtxn.commit().unwrap(); + + let state = std::sync::Arc::new(bare); + let statuses = state.repo_statuses_lightweight(); + let idx = statuses + .iter() + .position(|(alias, _)| alias == "inforepo") + .expect("inforepo row"); + let overlay = build_info_overlay(idx, &statuses, &state).expect("overlay"); + match overlay { + crate::serve::tui_common::OverlayState::Info { + symbol_warnings, .. + } => { + assert!( + symbol_warnings.len() == 2 && symbol_warnings[0].starts_with("[csharp] summary"), + "warnings must be language-prefixed from meta, got: {symbol_warnings:?}" + ); + } + other => panic!("expected Info overlay, got {other:?}"), + } +} + +/// The serve-mode find_impact self-heal spawn must reach +/// `trigger_symbol_rebuild` for a registered, applicable repo: after the +/// warned empty answer, the C# status eventually flips to Error (dummy +/// helper) in the serve state the heal captured. +#[tokio::test] +#[serial] +async fn find_impact_self_heal_runs_the_rebuild_in_serve_mode() { + let helper_root = tempfile::tempdir().unwrap(); + let _env = dummy_csharp_helper(helper_root.path()); + + let (_tmp, repo_path, bare) = state_with_repo("healrepo"); + std::fs::write( + repo_path.join("test.sln"), + b"Microsoft Visual Studio Solution File", + ) + .unwrap(); + // The routing layer requires an existing index dir before it will open + // the repo (seed like the find_impact fixtures do). + let db = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db).unwrap(); + std::fs::write( + db.join("metadata.json"), + r#"{"schema_version":1,"dimensions":2,"model_short_name":"minilm-l6-q"}"#, + ) + .unwrap(); + let state = std::sync::Arc::new(bare); + + let request = crate::mcp::types::FindImpactRequest { + symbol_name: Some("ResolveLinkedActivityAsync".to_string()), + file: None, + line: None, + symbol_key: None, + language: Some("csharp".to_string()), + project: Some("healrepo".to_string()), + group: None, + }; + // The REST mirror builds its own serve-mode service from the state — + // the same path the MCP tool takes for a project-scoped query. + let v = crate::mcp::rest_find_impact_handler( + axum::extract::State(state.clone()), + axum::Json(request), + ) + .await + .expect("rest mirror result") + .0; + let out = v.to_string(); + assert!( + v["warnings"] + .as_array() + .is_some_and(|w| !w.is_empty() && w[0].as_str().is_some_and(|s| s.contains("UNKNOWN"))), + "the never-built index answer must carry the UNKNOWN warning, got: {out}" + ); + + // The detached heal task runs the rebuild (dummy helper → Error). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + loop { + if state + .csharp_index_status + .get("healrepo") + .map(|e| *e.value()) + == Some(CSharpIndexStatus::Error) + { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the self-heal never flipped the C# status — spawn branch not wired" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } +} + +// ── Rebuild gate: builder-version stamp + degraded retry ───────────────── + +/// Seeds the SCIP meta for `db` exactly like `rebuild` stamps it (rebuild +/// timestamp + key format), plus the caller's extra key/values — so gate +/// tests can shape a fresh, stale or degraded index without the helper. +fn seed_scip_meta(db: &std::path::Path, extra: &[(&str, &str)]) { + let env = crate::symbols::get_shared_scip_env(db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + meta.put(&mut wtxn, crate::constants::SCIP_REBUILD_TIMESTAMP_KEY, "0") + .unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_KEY_FORMAT_KEY, + crate::constants::SCIP_KEY_FORMAT, + ) + .unwrap(); + for (key, value) in extra { + meta.put(&mut wtxn, *key, *value).unwrap(); + } + wtxn.commit().unwrap(); +} + +/// Shared arrangement: applicable repo (`.sln`), dummy helper, seeded SCIP +/// meta, config timestamps showing a fresh build (last_scip ≥ last_changed) +/// so ONLY the builder-version / degraded-retry checks can move the gate. +/// The helper-env guard and both tempdirs are returned so they outlive the +/// fixture. +#[allow(clippy::type_complexity)] +fn gate_fixture( + alias: &str, + meta_extra: &[(&str, &str)], +) -> ( + crate::testing::EnvRestore, + tempfile::TempDir, + tempfile::TempDir, + std::path::PathBuf, + std::sync::Arc, +) { + let helper_root = tempfile::tempdir().unwrap(); + let env_guard = dummy_csharp_helper(helper_root.path()); + let (tmp, repo_path, bare) = state_with_repo(alias); + std::fs::write(repo_path.join("test.sln"), b"Solution File").unwrap(); + let db = repo_path.join(DB_DIR_NAME); + seed_scip_meta(&db, meta_extra); + let state = std::sync::Arc::new(bare); + let now = ServeState::now_unix_secs(); + { + let mut cfg = state.config.write().unwrap(); + cfg.touch_last_changed(alias, now - 100); + cfg.touch_last_scip(alias, now); + } + (env_guard, helper_root, tmp, repo_path, state) +} + +/// An index this binary never produced (no builder stamp — anything built +/// before the stamp existed) must read as stale: a deployed binary rebuilds +/// the indexes it inherited itself at startup, never waits for a manual +/// command. This is the DRM-24427_Versioning defect class. +#[test] +#[serial] +fn evaluate_rebuilds_an_index_from_an_older_binary() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture("oldbinary", &[]); + + assert_eq!( + state.evaluate_symbol_rebuild( + "csharp", + "oldbinary", + &repo_path, + &repo_path.join(DB_DIR_NAME) + ), + RebuildDecision::ChangedSinceLastBuild, + "absent builder stamp must read as stale" + ); +} + +/// A degraded index (warnings in meta, current builder version) rebuilds +/// exactly once per serve process: the second evaluation is Fresh (no loop +/// while the environment stays broken); a fresh process gets a fresh set. +#[test] +#[serial] +fn evaluate_retries_a_degraded_index_once_per_process() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture( + "degraded", + &[( + crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY, + crate::constants::INDEX_BUILDER_VERSION, + )], + ); + let db = repo_path.join(DB_DIR_NAME); + // Re-seed WITH warnings (the fixture seeded only the version stamp). + seed_scip_meta( + &db, + &[( + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["Broken.csproj: Msbuild failed when processing the file"]"#, + )], + ); + + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "degraded", &repo_path, &db), + RebuildDecision::ChangedSinceLastBuild, + "a degraded index must get its one automatic retry" + ); + assert_eq!( + state.evaluate_symbol_rebuild("csharp", "degraded", &repo_path, &db), + RebuildDecision::Fresh, + "the retry must not loop within one serve process" + ); +} + +/// A clean index with the current builder stamp and fresh timestamps stays +/// Fresh — the gate must not churn indexes that are actually fine. +#[test] +#[serial] +fn evaluate_stays_fresh_when_index_is_current_and_clean() { + let (_env, _h, _tmp, repo_path, state) = gate_fixture( + "cleanidx", + &[( + crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY, + crate::constants::INDEX_BUILDER_VERSION, + )], + ); + + assert_eq!( + state.evaluate_symbol_rebuild( + "csharp", + "cleanidx", + &repo_path, + &repo_path.join(DB_DIR_NAME) + ), + RebuildDecision::Fresh, + "current + clean + fresh timestamps must not rebuild" + ); + assert!( + !state.degraded_rebuild_attempted.contains("cleanidx"), + "a clean index must not consume the degraded retry" + ); +} + +// --------------------------------------------------------------------------- +// Queue-backlog regressions: tracked index tasks, stale-marker liveness, +// deferred DB cleanup. +// --------------------------------------------------------------------------- + +/// Age `alias`'s Reindex marker past the stale threshold. +fn aged_marker(state: &ServeState, alias: &str) { + state + .active_reindexes + .entry(alias.to_string()) + .or_default() + .insert( + IndexingOwner::Reindex, + Instant::now() - Duration::from_secs(MAX_INDEXING_SECS * 2), + ); +} + +/// A handle whose task has already completed (the handle is kept, not awaited). +async fn finished_handle() -> tokio::task::JoinHandle<()> { + let handle = tokio::spawn(async {}); + while !handle.is_finished() { + tokio::task::yield_now().await; + } + handle +} + +fn tracked_task(handle: tokio::task::JoinHandle<()>, db_path: std::path::PathBuf) -> IndexTask { + IndexTask { + handle, + token: CancellationToken::new(), + db_path, + started_at: Instant::now(), + } +} + +#[tokio::test] +#[serial] +async fn stale_marker_with_live_task_is_renewed_not_cancelled() { + // Regression: a reindex task queued behind the process-wide job gate can + // outlive MAX_INDEXING_SECS without being a leak. Evicting its marker and + // cancelling the task cancels healthy queued work whenever a backlog + // outlives the threshold. + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "live-task-repo"; + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + let token = CancellationToken::new(); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + aged_marker(&state, alias); + assert!( + !state.begin_indexing(alias, IndexingOwner::Reindex), + "a live task must block a second run even with a stale marker" + ); + + // A stale marker owned by another flow (Watcher) carries no liveness + // evidence of its own and must still be evicted, even while the index + // task lives. + state + .active_reindexes + .entry(alias.to_string()) + .or_default() + .insert( + IndexingOwner::Watcher, + Instant::now() - Duration::from_secs(MAX_INDEXING_SECS * 2), + ); + aged_marker(&state, alias); + assert!( + state.is_indexing(alias), + "a live task must keep reporting indexing" + ); + assert!( + state + .active_reindexes + .get(alias) + .is_some_and(|owners| !owners.contains_key(&IndexingOwner::Watcher)), + "a stale non-Reindex marker must still be evicted" + ); + assert!( + !token.is_cancelled(), + "the marker check must never cancel a live task" + ); + let renewed = state + .active_reindexes + .get(alias) + .and_then(|owners| owners.get(&IndexingOwner::Reindex).copied()) + .expect("the marker must survive the check"); + assert!( + Instant::now().duration_since(renewed) < Duration::from_secs(5), + "a live marker must be renewed, not evicted" + ); + + { + let entry = state.index_tasks.get(alias); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} + +#[tokio::test] +#[serial] +async fn stale_marker_without_live_task_is_evicted_and_entry_reaped() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "dead-task-repo"; + let handle = finished_handle().await; + state.index_tasks.insert( + alias.to_string(), + tracked_task(handle, std::path::PathBuf::from("/nonexistent-db")), + ); + aged_marker(&state, alias); + + assert!( + !state.is_indexing(alias), + "with no live task the stale marker must be evicted" + ); + assert!( + !state.index_tasks.contains_key(alias), + "the finished entry must be reaped" + ); + assert!( + !state.active_reindexes.contains_key(alias), + "the evicted marker must be cleaned up" + ); +} + +#[tokio::test] +#[serial] +async fn sweeper_reaps_finished_unregistered_task_and_cleans_db() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("gone-repo").join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("metadata.json"), b"{}").unwrap(); + + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let handle = finished_handle().await; + state.index_tasks.insert( + "gone-repo".to_string(), + tracked_task(handle, db_path.clone()), + ); + state + .active_reindexes + .entry("gone-repo".to_string()) + .or_default() + .insert(IndexingOwner::Reindex, Instant::now()); + + state.sweep_index_tasks(); + + assert!(!state.index_tasks.contains_key("gone-repo")); + assert!(!db_path.exists(), "the orphaned DB dir must be deleted"); + assert!(!state.active_reindexes.contains_key("gone-repo")); +} + +#[tokio::test] +#[serial] +async fn await_index_task_timeout_keeps_task_tracked() { + // Regression: the old code removed the entry before waiting and dropped + // the handle on timeout, making the detached task invisible while it held + // its stores and the job permit. + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let alias = "parked-task"; + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + alias.to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + let exited = state + .await_index_task_with_budget(alias, Duration::from_millis(50)) + .await; + assert!(!exited, "a parked task cannot exit within the budget"); + assert!( + state.index_tasks.contains_key(alias), + "the task must stay tracked — never invisible" + ); + assert!(token.is_cancelled(), "cancellation must still be signalled"); + + { + let entry = state.index_tasks.get(alias); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} + +#[tokio::test] +#[serial] +async fn await_index_task_returns_true_when_task_exits() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let handle = tokio::spawn(async {}); + state.index_tasks.insert( + "quick-task".to_string(), + tracked_task(handle, std::path::PathBuf::from("/nonexistent-db")), + ); + assert!( + state + .await_index_task_with_budget("quick-task", Duration::from_secs(5)) + .await + ); + assert!(!state.index_tasks.contains_key("quick-task")); +} + +#[tokio::test] +#[serial] +async fn remove_repo_defers_db_delete_while_index_task_is_still_running() { + // Regression: deleting the DB dir under a live (uninterruptible) index + // task is what can wedge a hub — the task never reaches a cancellation + // point and keeps its stores and the job permit for the process lifetime. + let (_tmp, repo_path, state) = state_with_repo("slowrepo"); + let db_path = repo_path.join(DB_DIR_NAME); + std::fs::create_dir_all(&db_path).unwrap(); + std::fs::write(db_path.join("data.mdb"), "fake").unwrap(); + + // A task that ignores cancellation entirely, like a parked build. + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + "slowrepo".to_string(), + IndexTask { + handle, + token, + db_path: db_path.clone(), + started_at: Instant::now(), + }, + ); + + let outcome = state + .remove_repo_with_budget("slowrepo", Duration::from_millis(100)) + .await + .expect("remove_repo should succeed"); + + assert!( + !outcome.db_deleted, + "the DB dir must NOT be deleted out from under a live task" + ); + assert!(db_path.exists(), "the dir stays until the task exits"); + assert!( + state.index_tasks.contains_key("slowrepo"), + "the running task must stay tracked for the sweeper" + ); + + // Simulate the task finally exiting (aborted here; a real one would run + // its post-build guard): the sweeper is the backstop that reaps the entry + // and deletes the orphaned dir. + if let Some(entry) = state.index_tasks.get("slowrepo") { + entry.value().handle.abort(); + } + loop { + let finished = state + .index_tasks + .get("slowrepo") + .is_some_and(|entry| entry.value().handle.is_finished()); + if finished { + break; + } + tokio::task::yield_now().await; + } + state.sweep_index_tasks(); + + assert!(!state.index_tasks.contains_key("slowrepo")); + assert!( + !db_path.exists(), + "the sweeper must clean the orphaned DB dir" + ); +} + +#[tokio::test] +#[serial] +async fn sweeper_reasserts_cancellation_for_a_removed_live_task() { + let state = Arc::new(ServeState::new(ReposConfig::default(), None)); + let token = CancellationToken::new(); + let handle = tokio::spawn(async { std::future::pending::<()>().await }); + state.index_tasks.insert( + "removed-repo".to_string(), + IndexTask { + handle, + token: token.clone(), + db_path: std::path::PathBuf::from("/nonexistent-db"), + started_at: Instant::now(), + }, + ); + + state.sweep_index_tasks(); + + assert!( + token.is_cancelled(), + "a removed repo's live task must have cancellation re-asserted" + ); + assert!( + state.index_tasks.contains_key("removed-repo"), + "a live task must stay tracked until it exits" + ); + + { + let entry = state.index_tasks.get("removed-repo"); + if let Some(entry) = entry { + entry.value().handle.abort(); + } + } +} diff --git a/src/serve/tui.rs b/src/serve/tui.rs index 497051f7..3678d47d 100644 --- a/src/serve/tui.rs +++ b/src/serve/tui.rs @@ -7,7 +7,7 @@ use std::io; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use ratatui::backend::CrosstermBackend; use ratatui::layout::{Constraint, Layout}; @@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken; use super::tui_common::{ self, KeyAction, OverlayKeyAction, OverlayState, RemoteIndexStats, RemoteStatsState, RepoRow, }; -use super::{IndexingOwner, ServeState}; +use super::{IndexTask, IndexingOwner, ServeState}; use crate::cli::doctor; use crate::constants::{DB_DIR_NAME, LANG_CSHARP, LANG_TYPESCRIPT}; use crate::index::IndexManager; @@ -444,6 +444,7 @@ fn map_repo_rows( let csharp_str = match info.csharp_index { super::CSharpIndexStatus::Ready => "ready", + super::CSharpIndexStatus::Partial => "partial", super::CSharpIndexStatus::Indexing => "indexing", super::CSharpIndexStatus::Error => "error", super::CSharpIndexStatus::None => "none", @@ -452,6 +453,7 @@ fn map_repo_rows( let ts_str = match info.typescript_index { super::CSharpIndexStatus::Ready => "ready", + super::CSharpIndexStatus::Partial => "partial", super::CSharpIndexStatus::Indexing => "indexing", super::CSharpIndexStatus::Error => "error", super::CSharpIndexStatus::None => "none", @@ -737,7 +739,8 @@ fn cpu_usage_str(sys_system: &mut Option) -> String { // --------------------------------------------------------------------------- /// Build an `OverlayState::Info` by gathering live stats from SharedStores or metadata. -fn build_info_overlay( +/// pub(crate) so the serve test module can pin the symbol-warnings wiring. +pub(crate) fn build_info_overlay( idx: usize, repos: &[(String, super::RepoStatusInfo)], state: &Arc, @@ -808,6 +811,10 @@ fn build_info_overlay( // DB size on disk let db_size_human = dir_size_human(&db_path); + // Durable symbol-index warnings, read fresh from the on-disk meta + // (shared with the REST info surface so the two cannot drift). + let symbol_warnings = state.collect_symbol_warnings(&db_path); + Some(OverlayState::Info { alias: alias.clone(), path: db_path.display().to_string(), @@ -819,6 +826,7 @@ fn build_info_overlay( dims, lock, index_age, + symbol_warnings, }) } @@ -1157,6 +1165,9 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei // `index_tasks` so `remove_repo` can cancel + await it (BUG1). let reindex_token = CancellationToken::new(); let reindex_token_task = reindex_token.clone(); + // The task closure moves `db_path`; keep a copy for the tracked entry so a + // removed repo's DB dir can be cleaned up after the task exits. + let task_db_path = db_path.clone(); let handle = tokio::spawn(async move { tracing::info!( "TUI: Force reindex for '{}': clearing stores and reindexing", @@ -1216,7 +1227,15 @@ pub(crate) fn spawn_force_reindex(alias: String, state: &Arc) -> Rei // Remove guard state_bg.end_indexing(&alias_bg, IndexingOwner::Reindex); }); - state.index_tasks.insert(alias, (handle, reindex_token)); + state.index_tasks.insert( + alias, + IndexTask { + handle, + token: reindex_token, + db_path: task_db_path, + started_at: Instant::now(), + }, + ); ReindexLaunch::Started } diff --git a/src/serve/tui_common.rs b/src/serve/tui_common.rs index d098a3e7..e75e80fb 100644 --- a/src/serve/tui_common.rs +++ b/src/serve/tui_common.rs @@ -10,7 +10,7 @@ use ratatui::layout::{Constraint, Layout, Margin, Rect}; use ratatui::style::{Color, Modifier, Style}; -use ratatui::text::{Line, Span}; +use ratatui::text::{Line, Span, Text}; use ratatui::widgets::{Block, Borders, Cell, Row, Table, TableState}; /// Format elapsed time since `started_at` as "Up xxd xxh xxm xxs". @@ -148,6 +148,11 @@ pub enum OverlayState { dims: usize, lock: String, index_age: String, + /// Index-level symbol warnings read fresh from the on-disk meta + /// (`scip_meta[index_warnings]`): build-environment failures the + /// helper survived. Unlike the in-memory C# status these survive a + /// serve restart — the durable record of a degraded index. + symbol_warnings: Vec, }, /// Info modal for a *mounted remote project* (federation peer). Remote /// mounts have no local on-disk index, so the chunk/file/db-size/model stats @@ -316,6 +321,48 @@ pub fn render_header( f.render_widget(ratatui::widgets::Paragraph::new(title_line), centered[0]); } +/// The C# alias-column indicator: `(glyph, style)` when the status carries +/// one. Color belongs to the indicator only — degraded (warnings) reads as +/// yellow ⚠, a general error as fully red `!` — so a table full of +/// warnings-degraded repos does not read as half the fleet being down. +pub fn csharp_indicator(status: &str, pulse: bool) -> Option<(&'static str, Style)> { + match status { + "ready" => Some((" C#·", Style::default().fg(Color::White))), + "partial" => Some((" C#⚠", Style::default().fg(Color::Yellow))), + "error" => Some(( + " C#!", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + )), + "indexing" => { + let style = if pulse { + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD) + } else { + Style::default().fg(Color::DarkGray) + }; + Some((" C#…", style)) + } + _ => None, + } +} + +/// The TypeScript alias-column indicator — same severity language as the C# +/// one (`ts_indicator("partial")` is yellow; TS has no live Partial source +/// yet, but the shared vocabulary must already agree). +pub fn ts_indicator(status: &str) -> Option<(&'static str, Style)> { + match status { + "ready" => Some((" TS·", Style::default().fg(Color::White))), + "partial" => Some((" TS⚠", Style::default().fg(Color::Yellow))), + "error" => Some(( + " TS!", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + )), + "indexing" => Some((" TS…", Style::default().fg(Color::DarkGray))), + _ => None, + } +} + pub fn render_table( f: &mut ratatui::Frame, area: Rect, @@ -341,14 +388,10 @@ pub fn render_table( let max_alias_w = repos .iter() .map(|r| { - // Each indicator (" C#·" / " TS·") is 4 display cols; account for both. - let mut extra = 0usize; - if matches!(r.csharp_index.as_str(), "ready" | "error" | "indexing") { - extra += 4; - } - if matches!(r.typescript_index.as_str(), "ready" | "error" | "indexing") { - extra += 4; - } + // Each indicator is a suffix like " C#·"; ⚠ (the partial glyph) + // is double-width (unicode-width EAW=W) so its variant budgets + // 6 display cols instead of 4. + let extra = indicator_cols(&r.csharp_index) + indicator_cols(&r.typescript_index); r.alias.len() + extra }) .max() @@ -384,57 +427,30 @@ pub fn render_table( }; let lock_cell = lock_cell(&repo.lock_mode); - // Alias text with optional C# indicator suffix, plus its base style. - let (mut alias_text, mut alias_style) = match repo.csharp_index.as_str() { - "ready" => ( - format!("{} C#·", repo.alias), - Style::default().fg(Color::White), - ), - "error" => ( - format!("{} C#!", repo.alias), - Style::default().fg(Color::Red), - ), - "indexing" => { - let s = if pulse_bright() { - Style::default() - .fg(Color::Yellow) - .add_modifier(Modifier::BOLD) - } else { - Style::default().fg(Color::DarkGray) - }; - (format!("{} C#…", repo.alias), s) - } - _ => (repo.alias.clone(), Style::default().fg(Color::White)), + // Alias text stays NEUTRAL — color lives only on the language + // indicators: yellow ⚠ for a degraded (warnings) index, fully red + // ! for a general error. A table with red aliases reads as half + // the fleet being down, which a warnings-degraded index is not. + let base_alias_style = if repo.is_remote { + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::ITALIC) + } else { + Style::default().fg(Color::White) }; + let mut alias_spans: Vec = + vec![Span::styled(repo.alias.clone(), base_alias_style)]; - // Append the TypeScript indicator alongside the C# one when a TS - // index exists. The alias column is the canonical multi-language - // symbol-index indicator (the status cell only carries C#). - match repo.typescript_index.as_str() { - "ready" => alias_text.push_str(" TS·"), - "error" => { - alias_text.push_str(" TS!"); - alias_style = alias_style.fg(Color::Red); - } - "indexing" => alias_text.push_str(" TS…"), - _ => {} - } - - // Red bold alias if the repo is in an error state. - if repo.status == "error" { - alias_style = Style::default().fg(Color::Red).add_modifier(Modifier::BOLD); + if let Some((glyph, style)) = + csharp_indicator(repo.csharp_index.as_str(), pulse_bright()) + { + alias_spans.push(Span::styled(glyph, style)); } - - // Mounted remote projects render italic to signal they live on a - // peer (cyan unless an error already claimed the color). - if repo.is_remote { - alias_style = alias_style.add_modifier(Modifier::ITALIC); - if repo.status != "error" { - alias_style = alias_style.fg(Color::Cyan); - } + if let Some((glyph, style)) = ts_indicator(repo.typescript_index.as_str()) { + alias_spans.push(Span::styled(glyph, style)); } - let alias_cell = Cell::from(alias_text).style(alias_style); + let alias_cell = Cell::from(Text::from(Line::from(alias_spans))); Row::new(vec![ alias_cell, @@ -515,17 +531,12 @@ pub fn render_detail( repo.path.clone() }; - // Mounted remote projects show italic here too (matches the table): cyan - // normally, red when in error state so the color stays consistent with the - // table's error highlight. Local rows are unchanged (white bold). + // Mounted remote projects show italic here too (matches the table): cyan, + // never red — alias text stays neutral on every surface; the status line + // below carries the error severity. Local rows are unchanged (white bold). let alias_style = if repo.is_remote { - let color = if repo.status == "error" { - Color::Red - } else { - Color::Cyan - }; Style::default() - .fg(color) + .fg(Color::Cyan) .add_modifier(Modifier::BOLD | Modifier::ITALIC) } else { Style::default() @@ -592,8 +603,8 @@ pub fn render_detail( let info_line = Line::from(info_spans); - // Optional error line for C# errors - let error_line = if repo.csharp_index == "error" { + // Optional error line for C# errors and degraded (partial) indexes + let error_line = if matches!(repo.csharp_index.as_str(), "error" | "partial") { let err_msg = repo.csharp_error.as_deref().unwrap_or("Unknown error"); const ERR_PREFIX_COLS: usize = 7; let max_err_chars = (area.width as usize).saturating_sub(ERR_PREFIX_COLS); @@ -806,6 +817,7 @@ pub fn render_overlay(f: &mut ratatui::Frame, area: Rect, overlay: &OverlayState dims, lock, index_age, + symbol_warnings, } => { let title = format!(" {} — Index Info ", alias); let lines = vec![ @@ -855,12 +867,28 @@ pub fn render_overlay(f: &mut ratatui::Frame, area: Rect, overlay: &OverlayState Span::styled(" Index age: ", Style::default().fg(Color::DarkGray)), Span::styled(index_age.clone(), Style::default().fg(Color::White)), ]), - Line::from(""), - Line::from(Span::styled( - " [Esc] close", - Style::default().fg(Color::DarkGray), - )), ]; + let mut lines = lines; + // Durable record of a degraded symbol index: the build problems + // the helper survived. Red, like the table's `C#⚠` indicator. + if !symbol_warnings.is_empty() { + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + " ⚠ Symbol index warnings:", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ))); + for warning in symbol_warnings { + lines.push(Line::from(Span::styled( + format!(" {warning}"), + Style::default().fg(Color::Red), + ))); + } + } + lines.push(Line::from("")); + lines.push(Line::from(Span::styled( + " [Esc] close", + Style::default().fg(Color::DarkGray), + ))); render_centered_modal(f, area, &title, lines); } OverlayState::RemoteInfo { @@ -1272,6 +1300,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Open C#!".to_string(), Color::Red), + "partial" => ("Open C#⚠".to_string(), Color::Yellow), _ => ("Open".to_string(), Color::Green), }, "warm" => match csharp { @@ -1285,6 +1314,7 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { }, ), "error" => ("Warm C#!".to_string(), Color::Red), + "partial" => ("Warm C#⚠".to_string(), Color::Yellow), _ => ("Warm".to_string(), Color::Yellow), }, "readonly" => ("Readonly".to_string(), Color::Cyan), @@ -1313,6 +1343,17 @@ fn detail_status_style(status: &str, csharp: &str) -> (String, Color) { } } +/// Display width of one alias-column indicator suffix (`" C#·"` family). +/// The partial glyph ⚠ is double-width (unicode-width EAW=W), so that +/// variant budgets 6 display cols; the 1-col glyphs budget 4. +fn indicator_cols(status: &str) -> usize { + match status { + "ready" | "error" | "indexing" => 4, + "partial" => 6, + _ => 0, + } +} + // --------------------------------------------------------------------------- // Terminal helpers // --------------------------------------------------------------------------- @@ -1355,6 +1396,69 @@ mod tests { ); } + /// A degraded (partial) C# index renders YELLOW with the ⚠ glyph in the + /// detail panel — a warning, not an error; only a general error is red. + #[test] + fn detail_status_style_maps_partial_to_yellow_warning() { + assert_eq!( + detail_status_style("open", "partial"), + ("Open C#⚠".to_string(), Color::Yellow) + ); + assert_eq!( + detail_status_style("warm", "partial"), + ("Warm C#⚠".to_string(), Color::Yellow) + ); + assert_eq!( + detail_status_style("warm", "error").1, + Color::Red, + "a general error stays fully red" + ); + } + + /// Alias-column indicators: the alias text is styled by the caller and + /// stays neutral; degraded reads yellow, only a general error is red. + #[test] + fn alias_indicators_use_yellow_for_degraded_and_red_only_for_errors() { + let (glyph, style) = csharp_indicator("partial", false).unwrap(); + assert_eq!(glyph, " C#⚠"); + assert_eq!(style, Style::default().fg(Color::Yellow)); + + let (glyph, style) = csharp_indicator("error", false).unwrap(); + assert_eq!(glyph, " C#!"); + assert_eq!( + style, + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD) + ); + + let (glyph, style) = ts_indicator("partial").unwrap(); + assert_eq!(glyph, " TS⚠"); + assert_eq!(style, Style::default().fg(Color::Yellow)); + + // The transient dim/pulse phase: indexing indicators must stay dim — + // an inherited bright style here once turned the whole alias red. + let (_, style) = csharp_indicator("indexing", false).unwrap(); + assert_eq!(style, Style::default().fg(Color::DarkGray)); + let (_, style) = ts_indicator("indexing").unwrap(); + assert_eq!(style, Style::default().fg(Color::DarkGray)); + } + + /// The alias-column width budget must cover the actually rendered + /// suffixes: the partial glyph ⚠ is double-width (EAW=W), so " C#⚠" is + /// 5 display cols — budgeting it at 4 (the 1-col glyph family) would + /// clip the ⚠ exactly when a partial row defines the widest alias. + #[test] + fn indicator_cols_budgets_double_width_partial_glyph() { + assert_eq!(indicator_cols("ready"), 4); + assert_eq!(indicator_cols("error"), 4); + assert_eq!(indicator_cols("indexing"), 4); + assert_eq!( + indicator_cols("partial"), + 6, + "double-width glyph needs more than the 4-col family budget" + ); + assert_eq!(indicator_cols("none"), 0); + } + #[test] fn unknown_status_falls_back_without_idle_treatment() { assert_ne!( diff --git a/src/serve/tui_remote.rs b/src/serve/tui_remote.rs index 6ca26da5..a14335b0 100644 --- a/src/serve/tui_remote.rs +++ b/src/serve/tui_remote.rs @@ -358,6 +358,7 @@ async fn run_remote_tui_loop( dims: info.dims, lock: info.lock, index_age: info.index_age, + symbol_warnings: info.symbol_warnings, }, Err(e) => OverlayState::Doctor { alias, @@ -503,6 +504,10 @@ struct InfoResponse { dims: usize, lock: String, index_age: String, + /// Index-level symbol warnings from the peer's on-disk meta. `default` for + /// the same older-serve compatibility as `path`. + #[serde(default)] + symbol_warnings: Vec, } #[derive(Debug, Deserialize)] diff --git a/src/symbols/csharp.rs b/src/symbols/csharp.rs index 8ad33fa2..1c58ff70 100644 --- a/src/symbols/csharp.rs +++ b/src/symbols/csharp.rs @@ -61,6 +61,70 @@ pub(crate) fn is_helper_warning_line(line: &str) -> bool { line.contains("[WARN]") || line.contains("[Failure]") } +/// Normalize one warning-severity helper stderr line into a concise, +/// index-level completeness warning entry. MSBuild project-load failures +/// (the class that silently strips cross-project references from the index) +/// collapse to `: `; anything else passes through +/// trimmed. `None` for non-warning lines. +pub(crate) fn normalize_index_warning_line(line: &str) -> Option { + if !is_helper_warning_line(line) { + return None; + } + // "[WARN] Workspace error: [Failure] Msbuild failed when processing the + // file '' with message: " → ": " + if let Some((_, rest)) = line.split_once("Msbuild failed when processing the file '") { + if let Some((path, tail)) = rest.split_once('\'') { + let file = path.rsplit(['/', '\\']).next().unwrap_or(path); + let msg = tail + .split_once("with message: ") + .map(|(_, m)| m) + .unwrap_or(tail) + .trim(); + let msg: String = msg.chars().take(160).collect(); + return Some(format!("{file}: {msg}")); + } + } + let trimmed = line.trim(); + if trimmed.is_empty() { + return None; + } + Some(trimmed.chars().take(220).collect()) +} + +/// Collapse the raw warning lines of one helper run into the capped, +/// deduplicated list persisted as `index_warnings` meta. Order-preserving +/// dedup because MSBuild repeats the same failure per project and per +/// pass; capped because the list rides on every find_impact answer. +pub(crate) fn summarize_index_warnings(lines: &[String], cap: usize) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut out: Vec = lines + .iter() + .filter_map(|l| normalize_index_warning_line(l)) + .filter(|w| seen.insert(w.clone())) + .collect(); + if out.len() > cap { + let dropped = out.len() - cap; + out.truncate(cap); + out.push(format!("… and {dropped} more distinct warning(s)")); + } + out +} + +/// The stored form of the summarized warnings: a stand-alone summary entry +/// leads, because the list rides on every find_impact answer AND the TUI +/// info panel and must be understandable alone. +pub(crate) fn index_warnings_stored(entries: &[String]) -> Vec { + if entries.is_empty() { + return Vec::new(); + } + let mut stored = vec![format!( + "The symbol index was built while the C# workspace reported {} distinct failure(s) — cross-project references may be missing from every answer. Fix the underlying build problem (often a dotnet restore) and reindex.", + entries.len() + )]; + stored.extend(entries.iter().cloned()); + stored +} + /// Route one scip-csharp stderr line into tracing at the right severity. /// This is the ONLY sanctioned path for helper stderr: spawn helpers with /// `Stdio::piped()` and drain through here — never `Stdio::inherit()`, @@ -132,6 +196,16 @@ const META_HEAD_SHA: &str = crate::constants::SCIP_HEAD_SHA_KEY; /// built with (see [`crate::constants::SCIP_KEY_FORMAT`]). const META_KEY_FORMAT: &str = crate::constants::SCIP_KEY_FORMAT_KEY; +/// Key in the meta database holding the JSON array of index-level +/// completeness warnings from the last rebuild (see +/// [`summarize_index_warnings`]). +const META_INDEX_WARNINGS: &str = crate::constants::SCIP_INDEX_WARNINGS_KEY; +const META_BUILDER_VERSION: &str = crate::constants::SCIP_INDEX_BUILDER_VERSION_KEY; + +/// How many distinct index-level warnings are persisted (and thus surfaced +/// per find_impact answer) before the list is capped with an overflow note. +const INDEX_WARNINGS_CAP: usize = 10; + /// Key in the meta database storing the count of indexed symbols. #[allow(dead_code)] const META_SYMBOL_COUNT: &str = "symbol_count"; @@ -297,23 +371,36 @@ fn read_ref_warnings(env: &TrackedEnv, rtxn: &heed::RoTxn<'_>, canonical: &str) /// - `"csharp App . FieldDefinition#Validate()."` → `"Validate"` /// - `"csharp SmallSolution.Library . Calculator#Add(int, int)."` → `"Add"` /// - `"csharp . . . Namespace.TopLevel"` → `"TopLevel"` -fn extract_simple_name(scip_symbol: &str) -> String { - // Strip trailing suffix chars: "Validate()." → "Validate", "MyService#" → "MyService" - let cleaned = scip_symbol - .trim_end_matches('.') - .trim_end_matches("()") - .trim_end_matches('#'); - // Take last non-empty segment after '#' or '.' - let last_segment = cleaned - .rsplit(['#', '.']) - .find(|s| !s.trim().is_empty()) - .unwrap_or(cleaned) - .trim(); - // Strip method parameters (e.g. "Add(int, int)" → "Add") - last_segment +/// +/// Parameter lists are stripped BEFORE segmenting: key format 2.0 writes +/// fully-qualified parameter types (`SaveBatchAsync(int, System.Collections +/// .Generic.IReadOnlyList)`), and splitting on `.` first would +/// land inside the parameters — the "Activity, int)" fragments that left +/// every parameterized method unresolvable by fuzzy name. +pub(crate) fn extract_simple_name(scip_symbol: &str) -> String { + // Method/field segment: after the last '#' (types without '#' keep the + // whole string — their name is the last '.'-segment below; a bare + // trailing '#' falls back to the pre-# text). + let after_hash = scip_symbol.rsplit('#').next().unwrap_or(scip_symbol); + let base = if after_hash.trim().is_empty() { + scip_symbol + } else { + after_hash + }; + // Strip parameters FIRST, then the generic-arity marker: keys group all + // overloads and arities under one queryable name. + let name = base .split('(') .next() - .unwrap_or(last_segment) + .unwrap_or(base) + .split('`') + .next() + .unwrap_or(base); + let cleaned = name.trim().trim_end_matches('.').trim_end_matches('#'); + cleaned + .rsplit('.') + .find(|s| !s.trim().is_empty()) + .unwrap_or(cleaned) .trim() .to_string() } @@ -365,6 +452,41 @@ impl CSharpSymbolIndexer { resolved } + /// Writes the rebuild bookkeeping shared by every scope (full and + /// incremental): key-format stamp, index warnings (written or cleared) + /// and the builder-version stamp. Called from `rebuild`'s meta txn, so + /// meta always describes the newest build. `pub(crate)` so the producer + /// test drives the exact write the pipeline runs — a fake helper cannot + /// execute on Windows, where the validated filename must be a real PE. + pub(crate) fn write_rebuild_meta( + meta_db: &Database, + wtxn: &mut heed::RwTxn, + index_warnings: &[String], + ) -> Result<()> { + // Unconditional: has_index refuses an index whose key-format stamp is + // absent or stale, so a key-format change forces exactly one rebuild. + meta_db.put(wtxn, META_KEY_FORMAT, crate::constants::SCIP_KEY_FORMAT)?; + // Index-level completeness warnings from this run — written or + // cleared on EVERY rebuild (full or incremental) so meta always + // describes the newest build. + let stored_warnings = index_warnings_stored(index_warnings); + if stored_warnings.is_empty() { + meta_db.delete(wtxn, META_INDEX_WARNINGS)?; + } else { + let json = serde_json::to_string(&stored_warnings).unwrap_or_else(|_| "[]".to_string()); + meta_db.put(wtxn, META_INDEX_WARNINGS, json.as_str())?; + } + // Stamp the producing build: the rebuild gate treats an index from + // another (or an unknown) build as stale, so a deployed binary + // rebuilds the indexes it inherited itself at startup. + meta_db.put( + wtxn, + META_BUILDER_VERSION, + crate::constants::INDEX_BUILDER_VERSION, + )?; + Ok(()) + } + fn resolve_helper_path(&self) -> Option { // 1. Environment variable override if let Ok(path) = std::env::var(HELPER_ENV_VAR) { @@ -518,13 +640,20 @@ impl CSharpSymbolIndexer { // ── Helper invocation ────────────────────────────────────────── /// Invoke `scip-csharp index` and stream stderr to tracing. + /// + /// Returns the raw warning-severity stderr lines of the run — the input + /// to [`summarize_index_warnings`]. A helper that survives MSBuild + /// project-load failures exits 0 with a syntactically valid but + /// semantically partial index (definitions without cross-project + /// references); those failures land HERE so the rebuild can persist them + /// as index-level warnings instead of them living only in the log. fn invoke_index_helper( &self, helper: &Path, solution: &Path, output_path: &Path, project_filter: Option<&Path>, - ) -> Result<()> { + ) -> Result> { let solution_short = solution .file_name() .map(|s| s.to_string_lossy().into_owned()) @@ -549,11 +678,17 @@ impl CSharpSymbolIndexer { .spawn() .with_context(|| format!("Failed to execute scip-csharp at {}", helper.display()))?; + let captured_warnings: std::sync::Arc>> = + std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); let stderr_handle = child.stderr.take().map(|stderr| { let label = solution_short.clone(); + let sink = std::sync::Arc::clone(&captured_warnings); thread::spawn(move || { drain_pipe_to_tracing(stderr, |line| { if !line.is_empty() { + if is_helper_warning_line(line) { + sink.lock().map(|mut v| v.push(line.to_string())).ok(); + } emit_helper_stderr_line("scip-csharp", &label, line); } }); @@ -591,7 +726,13 @@ impl CSharpSymbolIndexer { // Don't bail — partial output is acceptable per AGENTS.md spec } - Ok(()) + // A poisoned lock still holds every line pushed before the panic — + // recover rather than drop honesty signals on the floor. + let raw = match captured_warnings.lock() { + Ok(mut v) => std::mem::take(&mut *v), + Err(poisoned) => std::mem::take(&mut *poisoned.into_inner()), + }; + Ok(raw) } /// Invoke `scip-csharp find-refs` for a single symbol and return its @@ -1350,8 +1491,11 @@ impl SymbolIndexer for CSharpSymbolIndexer { )); let _output_guard = TempFileGuard(output_path.clone()); - // Invoke helper with stderr streaming - self.invoke_index_helper(&helper, &solution, &output_path, project.as_deref())?; + // Invoke helper with stderr streaming; warning-severity lines become + // the index-level completeness warnings persisted below. + let raw_index_warnings = + self.invoke_index_helper(&helper, &solution, &output_path, project.as_deref())?; + let index_warnings = summarize_index_warnings(&raw_index_warnings, INDEX_WARNINGS_CAP); // Parse the JSON output let index_data = std::fs::read(&output_path) @@ -1689,13 +1833,7 @@ impl SymbolIndexer for CSharpSymbolIndexer { if let Some(sha) = super::current_git_head(repo_path) { meta_db.put(&mut wtxn, META_HEAD_SHA, sha.as_str())?; } - // Unconditional: has_index refuses an index whose key-format stamp is - // absent or stale, so a key-format change forces exactly one rebuild. - meta_db.put( - &mut wtxn, - META_KEY_FORMAT, - crate::constants::SCIP_KEY_FORMAT, - )?; + Self::write_rebuild_meta(&meta_db, &mut wtxn, &index_warnings)?; wtxn.commit()?; @@ -1722,6 +1860,7 @@ impl SymbolIndexer for CSharpSymbolIndexer { symbols_indexed: total_symbols, references_stored: total_defs, // definitions only; refs resolved lazily duration_ms, + index_warnings: index_warnings_stored(&index_warnings), }) } @@ -1846,6 +1985,57 @@ impl SymbolIndexer for CSharpSymbolIndexer { (!sha.is_empty()).then_some(sha) } + /// Index-level completeness warnings from the last rebuild — the + /// workspace/MSBuild failures the helper survived. Non-empty means + /// reference lists from this index may be incomplete even when a + /// lookup succeeds (observed live: a worktree whose NuGet restore was + /// broken indexed every definition but zero cross-project references, + /// and every find_impact answer passed as clean). Plain LMDB read; + /// unreadable or absent metadata reads as clean (never blocks an answer). + fn index_warnings(&self, db_path: &Path) -> Vec { + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return Vec::new(), + }; + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return Vec::new(), + }; + let meta_db: Database = match env.open_database(&rtxn, Some(SCIP_META_DB_NAME)) { + Ok(Some(db)) => db, + _ => return Vec::new(), + }; + let json = match meta_db.get(&rtxn, META_INDEX_WARNINGS) { + Ok(Some(s)) => s, + _ => return Vec::new(), + }; + serde_json::from_str(json).unwrap_or_default() + } + + /// Which codesearch build produced this index — `None` when the stamp is + /// absent (pre-stamp index) or the meta is unreadable. The rebuild gate + /// compares against [`crate::constants::INDEX_BUILDER_VERSION`] and + /// rebuilds on mismatch: this binary never produced that index. + fn index_builder_version(&self, db_path: &Path) -> Option { + let env = match self.open_scip_env(db_path) { + Ok(e) => e, + Err(_) => return None, + }; + let rtxn = match env.read_txn() { + Ok(t) => t, + Err(_) => return None, + }; + let meta_db: Database = match env.open_database(&rtxn, Some(SCIP_META_DB_NAME)) { + Ok(Some(db)) => db, + _ => return None, + }; + meta_db + .get(&rtxn, META_BUILDER_VERSION) + .ok() + .flatten() + .map(|s| s.to_string()) + } + /// Whether a SCIP index exists AND was built with the current key format. /// An index that is fresh by timestamp but stamped with another (or no) /// key-format generation would serve old-shaped canonical keys as truth, @@ -1893,6 +2083,10 @@ impl SymbolIndexer for CSharpSymbolIndexer { Self::find_solution(repo_path).is_some() } + fn applicability_hint(&self) -> &'static str { + "the scip-csharp adapter requires a .sln file in the repository root" + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/src/symbols/csharp_tests.rs b/src/symbols/csharp_tests.rs index 31991aa0..26259db3 100644 --- a/src/symbols/csharp_tests.rs +++ b/src/symbols/csharp_tests.rs @@ -124,3 +124,254 @@ fn helper_warning_classification_table() { ); } } + +// ── Index-level completeness warnings (features/always-on-symbol-indexes) ── + +/// The live MSBuild project-load failure format observed on a customer +/// worktree (NuGet restore broken: analyzer package version mismatch) — +/// the exact line class whose warnings silently disappeared into the log +/// while every find_impact answer read clean. Identifiers anonymized; the +/// shape is what the parser must match. +const LIVE_MSBUILD_FAILURE: &str = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\demo-worktree\\src\\App.Dam\\App.Dam.csproj' with message: App.Dam depends on Analyzer.X (>= 1.2.3) but Analyzer.X 1.2.3 was not found. Analyzer.X 1.3.0 was resolved instead."; + +/// Normalization collapses the project-load failure to `: `; +/// duplicates (MSBuild repeats per project and per pass) dedupe; plain +/// warning lines pass through trimmed; non-warning lines are dropped. +#[test] +fn summarize_index_warnings_normalizes_dedupes_and_skips_noise() { + let msbuild_second = "[WARN] Workspace error: [Failure] Msbuild failed when processing the file 'C:\\repos\\other\\src\\App.Web\\App.Web.csproj' with message: App.Web depends on Analyzer.X (>= 1.2.3) but Analyzer.X 1.2.3 was not found. Analyzer.X 1.3.0 was resolved instead."; + let lines = vec![ + "MSBuild: registering '.NET Core SDK' v10.0.401".to_string(), // info, dropped + LIVE_MSBUILD_FAILURE.to_string(), + LIVE_MSBUILD_FAILURE.to_string(), // duplicate, deduped + msbuild_second.to_string(), + "[WARN] Grpc.Net.ClientFactory 2.63.0 or earlier could cause issues".to_string(), + "".to_string(), + ]; + + let out = super::csharp::summarize_index_warnings(&lines, 10); + + assert_eq!(out.len(), 3, "expected 3 distinct warnings, got: {out:?}"); + assert!( + out[0].starts_with("App.Dam.csproj: App.Dam depends on Analyzer.X"), + "first entry must be the normalized project failure, got: {}", + out[0] + ); + assert!( + out[1].starts_with("App.Web.csproj: "), + "second entry must be the second project, got: {}", + out[1] + ); + assert!( + out[2].starts_with("[WARN] Grpc"), + "unrecognized warning format passes through verbatim, got: {}", + out[2] + ); + // The failure message is longer than the 160-char cap — pinned so a + // persisted entry can never blow up the warnings array on answers. + let msg = out[0].split_once(": ").expect("normalized shape").1; + assert!(msg.chars().count() <= 160, "message must be capped"); +} + +/// The cap kicks in with an explicit overflow entry — the list rides on +/// every find_impact answer and must stay bounded. +#[test] +fn summarize_index_warnings_caps_with_overflow_note() { + let lines: Vec = (0..5) + .map(|i| format!("[WARN] distinct failure number {i}")) + .collect(); + let out = super::csharp::summarize_index_warnings(&lines, 3); + assert_eq!(out.len(), 4, "3 capped entries + overflow note: {out:?}"); + assert_eq!(out[3], "… and 2 more distinct warning(s)"); +} + +/// The meta roundtrip: what rebuild persists is what `index_warnings` +/// returns; absent key and corrupt JSON both read as clean (never block an +/// answer), and the TypeScript adapter inherits the empty default. +#[test] +fn index_warnings_roundtrip_and_clean_absence() { + use crate::symbols::SymbolIndexer as _; + + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("codesearch.db"); + + let csharp = super::csharp::CSharpSymbolIndexer::new(); + assert!( + csharp.index_warnings(&db_path).is_empty(), + "absent meta must read as clean" + ); + + let env = crate::symbols::get_shared_scip_env(&db_path).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + let stored = vec![ + "summary line".to_string(), + "App.Dam.csproj: Analyzer.X not found".to_string(), + ]; + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + serde_json::to_string(&stored).unwrap().as_str(), + ) + .unwrap(); + wtxn.commit().unwrap(); + + assert_eq!( + csharp.index_warnings(&db_path), + stored, + "persisted warnings must round-trip verbatim" + ); + + // Corrupt JSON reads as clean, not as an error. + let mut wtxn = env.write_txn().unwrap(); + meta.put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + "not json [", + ) + .unwrap(); + wtxn.commit().unwrap(); + assert!( + csharp.index_warnings(&db_path).is_empty(), + "corrupt meta must read as clean, never fail the answer" + ); + + let ts = super::typescript::TypeScriptSymbolIndexer::new(); + assert!( + ts.index_warnings(&db_path).is_empty(), + "non-tracking adapters inherit the empty default" + ); +} + +// ── Builder-version stamp: the rebuild's meta-write site ───────────────── +// +// The gate tests hand-seed meta (consumer half); this one drives the +// producer. The write lives in `write_rebuild_meta` — the exact fn the +// rebuild pipeline calls from its meta txn — because a fake helper cannot +// execute on Windows: the validated filename must be a real PE, not a +// script. Deleting the stamp write must fail this test, or every serve +// start churns all inherited C# indexes (absent stamp reads as stale). + +#[test] +fn rebuild_meta_write_stamps_version_and_writes_or_clears_warnings() { + use crate::constants::{INDEX_BUILDER_VERSION, SCIP_INDEX_BUILDER_VERSION_KEY}; + + let dir = tempfile::TempDir::new().unwrap(); + let db = dir.path().join("db"); + let indexer = super::csharp::CSharpSymbolIndexer::new(); + + // Degraded previous build: stale warnings + an old binary's stamp. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .create_database(&mut wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap(); + meta_db + .put( + &mut wtxn, + crate::constants::SCIP_INDEX_WARNINGS_KEY, + r#"["Old.csproj: Msbuild failed when processing the file"]"#, + ) + .unwrap(); + meta_db + .put(&mut wtxn, SCIP_INDEX_BUILDER_VERSION_KEY, "v0.0.0+1") + .unwrap(); + wtxn.commit().unwrap(); + + // A clean rebuild's meta write: stamp this build, clear the warnings. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + super::csharp::CSharpSymbolIndexer::write_rebuild_meta(&meta_db, &mut wtxn, &[]) + .expect("meta write must succeed"); + wtxn.commit().unwrap(); + + use crate::symbols::SymbolIndexer as _; + assert_eq!( + indexer.index_builder_version(&db).as_deref(), + Some(INDEX_BUILDER_VERSION), + "the rebuild meta write must stamp the producing build" + ); + assert!( + indexer.index_warnings(&db).is_empty(), + "a clean rebuild must clear the previous run's warnings" + ); + + // A degraded rebuild's meta write persists its warnings instead. + let env = crate::symbols::get_shared_scip_env(&db).unwrap(); + let mut wtxn = env.write_txn().unwrap(); + let meta_db: heed::Database = env + .open_database(&wtxn, Some(crate::constants::SCIP_META_DB_NAME)) + .unwrap() + .unwrap(); + super::csharp::CSharpSymbolIndexer::write_rebuild_meta( + &meta_db, + &mut wtxn, + &["Broken.csproj: Msbuild failed when processing the file".to_string()], + ) + .expect("meta write must succeed"); + wtxn.commit().unwrap(); + + // index_warnings_stored prepends the summary entry (summary first). + let warnings = indexer.index_warnings(&db); + assert_eq!( + warnings.len(), + 2, + "summary entry + the raw failed-project line, got: {warnings:?}" + ); + assert!( + warnings[0].contains("1 distinct failure(s)"), + "the summary entry must lead, got: {warnings:?}" + ); + assert_eq!( + warnings[1], + "Broken.csproj: Msbuild failed when processing the file".to_string(), + "a degraded rebuild's warnings must be persisted for the answer path" + ); +} + +// ── Simple-name extraction (fuzzy-lookup keys) ──────────────────────────── + +/// The live defect this pins: key format 2.0 writes fully-qualified +/// parameter types, and the old extractor split on '.' BEFORE stripping the +/// parameter list — landing inside the parameters. Every parameterized +/// method then keyed `scip_simple_names` under a fragment like +/// "Activity, int)" and fuzzy find_impact resolved NOTHING repository-wide +/// after the version-gate rebuilt all indexes to format 2.0. +#[test] +fn extract_simple_name_strips_fqn_parameters_before_segmenting() { + let cases = [ + // (canonical key, expected simple name) + ("csharp App . FieldDefinition#Validate().", "Validate"), + ( + "csharp SmallSolution.Library . Calculator#Add(int, int).", + "Add", + ), + ( + "csharp Acme.Catalog.Azure.TableStorage.DataStore . ActivityVersionStore#SaveBatchAsync(int, System.Collections.Generic.IReadOnlyList).", + "SaveBatchAsync", + ), + ( + "csharp Acme.Catalog.Import.CmdInfra . Arguments#GetVariableValue`1(string, Acme.Catalog.Import.CmdInfra.ArgumentValidator).", + "GetVariableValue", + ), + ("csharp . . . Namespace.TopLevel", "TopLevel"), + ("csharp App . MyService#", "MyService"), + ("csharp Ns.Sub . Class#_field", "_field"), + // Generic TYPE arity lives on the type segment — stripped too, not + // just the method-side backtick. + ("csharp Ns . List`1#", "List"), + ]; + for (key, expected) in cases { + assert_eq!( + super::csharp::extract_simple_name(key), + expected, + "key: {key}" + ); + } +} diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index c3542077..e19b8982 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -69,11 +69,17 @@ pub struct FindImpactResult { /// reference list may be INCOMPLETE — a helper failure was survived /// rather than fatal (a project that would not compile, an exception /// during reference resolution, a non-zero scip-typescript exit) and - /// the partial result was kept. Entries name what failed. Omitted when - /// empty: absent means the answer is as complete as the index knows. + /// the partial result was kept, OR the index itself was built out of a + /// partially loaded solution (index-level rebuild warnings, e.g. + /// MSBuild project-load failures; see `SymbolIndexer::index_warnings`). + /// Entries name what failed. Omitted when empty: absent means the + /// answer is as complete as the index knows. #[serde(skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, - /// Seconds since the symbol index was last rebuilt. + /// Seconds since the symbol index was last rebuilt. `u64::MAX` is the + /// internal "age unknown" sentinel `index_age` returns when the index + /// cannot be opened or read; over the wire it serializes as null. + #[serde(serialize_with = "serialize_index_age")] pub index_age_seconds: u64, /// Language that produced this result. pub language: String, @@ -92,6 +98,20 @@ pub struct FindImpactResult { pub current_head_sha: Option, } +/// Serialize `index_age_seconds`, mapping the internal unknown-age sentinel +/// (`u64::MAX`) to JSON null — a consumer must not see +/// `18446744073709551615` and mistake it for an age. Real ages stay numbers. +fn serialize_index_age(age: &u64, serializer: S) -> Result +where + S: serde::Serializer, +{ + if *age == u64::MAX { + serializer.serialize_none() + } else { + serializer.serialize_u64(*age) + } +} + /// A `find_impact` query in the form the adapters resolve it. /// /// The three input variants of `FindImpactRequest` map onto this one-to-one; @@ -377,6 +397,10 @@ pub struct RebuildSummary { pub references_stored: usize, /// Wall-clock duration in milliseconds. pub duration_ms: u64, + /// Index-level completeness warnings from this build (build-environment + /// failures the helper survived; see [`SymbolIndexer::index_warnings`]). + /// Empty = built clean. Drives the TUI's degraded indicator. + pub index_warnings: Vec, } /// Summary returned after a Phase 3 pre-warm completes. @@ -448,6 +472,26 @@ pub trait SymbolIndexer: Send + Sync { /// How old is the current symbol index (seconds since last rebuild)? fn index_age(&self, db_path: &Path) -> u64; + /// Index-level completeness warnings from the last rebuild — build + /// environment failures the language helper survived while still + /// producing an index (e.g. MSBuild project-load failures for C#). + /// Non-empty means reference lists from this index may be incomplete + /// EVEN WHEN a lookup succeeds, and callers should attach these + /// warnings to answers served from this index. Plain read; empty is + /// the default for adapters that do not track build-environment state. + fn index_warnings(&self, _db_path: &Path) -> Vec { + Vec::new() + } + + /// Which codesearch build produced the current symbol index + /// (`scip_meta[index_builder_version]`). `None` means absent or + /// unreadable — the C# rebuild gate treats that as stale, so a newly + /// deployed binary rebuilds the indexes it inherited itself at startup + /// instead of serving an index it never produced. + fn index_builder_version(&self, _db_path: &Path) -> Option { + None + } + /// The git HEAD sha the current symbol index was built for, when /// recorded. `None` means unknown (pre-fingerprint index, or git was /// unreadable at build time). Compare with the repository's current @@ -479,6 +523,14 @@ pub trait SymbolIndexer: Send + Sync { true } + /// Why `applies_to` would refuse a repository, in one clause. Surfaced + /// by find_impact's missing-index warning so a repo that can never get + /// a symbol index says so — instead of promising a rebuild that will + /// not happen. Adapters override next to their `applies_to`. + fn applicability_hint(&self) -> &'static str { + "the repository has no entrypoint this symbol indexer recognizes" + } + /// Downcast to `Any` for concrete-type method access (e.g. `prewarm_ref_cache`). /// /// This is needed because some adapter-specific methods (like Phase 3 pre-warm) diff --git a/src/symbols/typescript.rs b/src/symbols/typescript.rs index 1cc11b24..2925dc35 100644 --- a/src/symbols/typescript.rs +++ b/src/symbols/typescript.rs @@ -611,6 +611,7 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { symbols_indexed: total_symbols, references_stored: total_refs, duration_ms, + index_warnings: Vec::new(), }) } @@ -779,6 +780,10 @@ impl SymbolIndexer for TypeScriptSymbolIndexer { Self::find_tsconfig(repo_path).is_some() } + fn applicability_hint(&self) -> &'static str { + "the scip-typescript adapter requires a top-level tsconfig.json (monorepo layouts are not resolved yet)" + } + fn as_any(&self) -> &dyn std::any::Any { self } diff --git a/src/watch/mod.rs b/src/watch/mod.rs index 42ddf268..3b39e4de 100644 --- a/src/watch/mod.rs +++ b/src/watch/mod.rs @@ -633,6 +633,8 @@ impl GitHeadWatcher { #[cfg(test)] mod tests { use super::*; + use crate::testing::EnvRestore; + use serial_test::serial; use std::fs; use std::process::Command; use tempfile::tempdir; @@ -672,9 +674,21 @@ mod tests { Ok(()) } + /// Build a `FileWatcher` with the home directory pointed at an empty + /// temporary directory, so the developer's real + /// `~/.codesearch/.codesearchignore` cannot leak into the assertions. + /// CI has no such file, which is why the ambient dependency went unnoticed. + fn watcher_with_isolated_home(root: PathBuf) -> FileWatcher { + let home = tempdir().unwrap(); + let home = home.path().to_string_lossy().into_owned(); + let _guard = EnvRestore::set(&[("HOME", &home), ("USERPROFILE", &home)]); + FileWatcher::new(root) + } + #[test] + #[serial] fn test_is_watchable() { - let watcher = FileWatcher::new(PathBuf::from("/tmp")); + let watcher = watcher_with_isolated_home(PathBuf::from("/tmp")); // Should NOT watch (ignored dirs) assert!(!watcher.is_watchable(Path::new("/tmp/.git/config"))); @@ -688,6 +702,13 @@ mod tests { assert!(!watcher.is_watchable(Path::new("/tmp/image.png"))); assert!(!watcher.is_watchable(Path::new("/tmp/data.bin"))); + // Should NOT watch (dependency lock files — name-based, covers names + // whose extension alone is indexable like .yaml/.json/.lockb) + assert!(!watcher.is_watchable(Path::new("/tmp/pnpm-lock.yaml"))); + assert!(!watcher.is_watchable(Path::new("/tmp/package-lock.json"))); + assert!(!watcher.is_watchable(Path::new("/tmp/npm-shrinkwrap.json"))); + assert!(!watcher.is_watchable(Path::new("/tmp/bun.lockb"))); + // SHOULD watch (code files) assert!(watcher.is_watchable(Path::new("/tmp/src/main.rs"))); assert!(watcher.is_watchable(Path::new("/tmp/src/lib.ts"))); @@ -706,6 +727,7 @@ mod tests { } #[test] + #[serial] fn test_gitignore_rules_respected() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -717,7 +739,7 @@ mod tests { ) .unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!(watcher.gitignore.is_some(), "Should have loaded .gitignore"); // Should NOT watch (gitignored patterns) @@ -751,9 +773,10 @@ mod tests { #[test] #[ignore] // Requires actual filesystem events + #[serial] fn test_file_watcher() { let dir = tempdir().unwrap(); - let mut watcher = FileWatcher::new(dir.path().to_path_buf()); + let mut watcher = watcher_with_isolated_home(dir.path().to_path_buf()); watcher.start(100).unwrap(); @@ -769,6 +792,7 @@ mod tests { } #[test] + #[serial] fn test_codesearchignore_loaded_and_respected() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -776,7 +800,7 @@ mod tests { // Create repo-local .codesearchignore excluding tests/ fs::write(root.join(".codesearchignore"), "tests/\n").unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!( watcher.gitignore.is_some(), "Should have loaded .codesearchignore" @@ -793,6 +817,7 @@ mod tests { } #[test] + #[serial] fn test_codesearchignore_overrides_gitignore() { let dir = tempdir().unwrap(); let root = dir.path(); @@ -802,7 +827,7 @@ mod tests { fs::write(root.join(".gitignore"), "logs/\n").unwrap(); fs::write(root.join(".codesearchignore"), "src/generated/\n").unwrap(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!(watcher.gitignore.is_some()); // .gitignore pattern takes effect @@ -823,11 +848,12 @@ mod tests { } #[test] + #[serial] fn test_no_ignore_files_returns_none() { let dir = tempdir().unwrap(); let root = dir.path(); - let watcher = FileWatcher::new(root.to_path_buf()); + let watcher = watcher_with_isolated_home(root.to_path_buf()); assert!( watcher.gitignore.is_none(), "Should have no gitignore matcher when no ignore files exist"