diff --git a/src/commands/scan.rs b/src/commands/scan.rs index 538707e..1a46c63 100644 --- a/src/commands/scan.rs +++ b/src/commands/scan.rs @@ -143,7 +143,20 @@ pub async fn execute_scan( "batched files" ); - // 4. Process batches and collect issues + // 4. Build brain context once for the entire scan (if use_brain enabled). + // Extracts defined symbols from the file list and queries the symbol + // index for impact analysis, affected tests, and related patterns. + let brain_ctx = if config.context_chain.use_brain { + crate::engine::review::build_scan_brain_context( + &files, + config.context_chain.impact_depth, + &root_abs, + ) + } else { + None + }; + + // 5. Process batches and collect issues let mut all_issues = Vec::new(); let mut total_tokens: Option = None; let mut skipped_batches: Vec<(usize, Vec, String)> = Vec::new(); @@ -165,6 +178,7 @@ pub async fn execute_scan( &config.rules, &config.response_format, None, + brain_ctx.as_deref(), ) .await { diff --git a/src/embed/mod.rs b/src/embed/mod.rs index 2307a2c..e1b35d3 100644 --- a/src/embed/mod.rs +++ b/src/embed/mod.rs @@ -10,10 +10,8 @@ //! compiled into the binary via `include_bytes!` / `include_str!`. //! Higher quality at the cost of ~30 MB binary size. //! -//! Re-exports will be added to this module in Phase 3 when `cora brain` -//! commands are wired up. - -// Embedding engine — consumed by brain module (Phase 3). +//! The [`embed_code_dispatch`] function selects the best available backend at +//! compile time: pretrained-embed (768d) → hashing trick (256d) → FTS5-only. pub mod tokens; @@ -21,3 +19,71 @@ pub mod tokens; // exists. Excluded from crates.io package (too large for 10 MB upload limit). #[cfg(feature = "pretrained-embed")] pub mod token_vocab; + +// Re-export tokenizer — used by both backends. +pub use tokens::EMBEDDING_DIM; + +// Re-export pretrained constants when feature is enabled. +#[cfg(feature = "pretrained-embed")] +pub use token_vocab::{PRETRAINED_DIM, embed_code_pretrained}; + +/// Returns the embedding dimensionality used by the active backend. +/// +/// - `pretrained-embed` feature → 768 +/// - default (hashing trick) → 256 +pub const fn active_dims() -> usize { + #[cfg(feature = "pretrained-embed")] + { + PRETRAINED_DIM + } + #[cfg(not(feature = "pretrained-embed"))] + { + EMBEDDING_DIM + } +} + +/// Returns a human-readable label for the active embedding provider. +pub const fn active_provider_name() -> &'static str { + #[cfg(feature = "pretrained-embed")] + { + "nomic-embed-code (768d, pretrained)" + } + #[cfg(not(feature = "pretrained-embed"))] + { + "hashing-trick (256d, static)" + } +} + +/// Embed a code snippet using the best available backend. +/// +/// Returns an f32 vector that can be passed directly to usearch. +/// +/// - **Pretrained path** (`pretrained-embed` feature): tokenises → looks up +/// each token in the nomic vocabulary → accumulates int8 vectors → L2-normalises. +/// Returns 768-dim vector. +/// +/// - **Hashing-trick fallback**: tokenises → hashes each token into a +/// pseudo-random 256-dim vector → accumulates → L2-normalises. +/// Returns 256-dim vector. +/// +/// Both paths share the same [`tokenize_code`] tokenizer. +pub fn embed_code_dispatch(code: &str) -> Vec { + #[cfg(feature = "pretrained-embed")] + { + embed_code_pretrained(code) + } + #[cfg(not(feature = "pretrained-embed"))] + { + let embedding = tokens::embed_code(code); + embedding.as_slice().iter().map(|&v| v as f32).collect() + } +} + +/// Whether the pretrained embedding backend is available (compile-time). +#[expect( + dead_code, + reason = "used by Phase 3+ features; embed module not yet wired at call sites" +)] +pub const fn has_pretrained() -> bool { + cfg!(feature = "pretrained-embed") +} diff --git a/src/embed/token_vocab.rs b/src/embed/token_vocab.rs index 4080275..c24317d 100644 --- a/src/embed/token_vocab.rs +++ b/src/embed/token_vocab.rs @@ -1,6 +1,3 @@ -//! Pre-trained token vocabulary — reserved for Phase 5 (Voyage-4-Nano ONNX). -#![allow(dead_code)] - //! Pre-trained token vocabulary from nomic-embed-code. //! //! Provides real 768-dimensional code embeddings distilled from @@ -148,6 +145,16 @@ pub fn embed_pretrained(tokens: &HashMap) -> PretrainedEmbedding { PretrainedEmbedding { vec } } +/// Convenience function: tokenize → embed using pretrained vocabulary in one step. +/// +/// Returns an f32 vector (768 dimensions, L2-normalised) ready for usearch. +/// This is the pretrained counterpart of [`crate::embed::tokens::embed_code`]. +pub fn embed_code_pretrained(code: &str) -> Vec { + let tokens = crate::embed::tokens::tokenize_code(code); + let emb = embed_pretrained(&tokens); + emb.into_vec() +} + /// Compute cosine similarity between two [`PretrainedEmbedding`]s. /// /// Because vectors are pre-normalised, this is just the dot product. diff --git a/src/engine/context/resolver.rs b/src/engine/context/resolver.rs index d141d42..ca6ef1f 100644 --- a/src/engine/context/resolver.rs +++ b/src/engine/context/resolver.rs @@ -75,6 +75,31 @@ pub fn build_context_chain( config: &ContextConfig, project_root: &Path, ignore_patterns: &[String], +) -> ContextChain { + // Open the index bridge for the project root. + let bridge = crate::engine::index_bridge::IndexBridge::open(project_root); + build_context_chain_with_bridge( + symbols, + defs, + config, + project_root, + ignore_patterns, + &bridge, + ) +} + +/// Build the full context chain with an explicit [`IndexBridge`]. +/// +/// This is the internal implementation that both the public `build_context_chain` +/// (which opens a bridge automatically) and callers that already hold a bridge +/// can use. +pub fn build_context_chain_with_bridge( + symbols: &[ExtractedSymbol], + defs: &[DefinedSymbol], + config: &ContextConfig, + project_root: &Path, + ignore_patterns: &[String], + bridge: &crate::engine::index_bridge::IndexBridge, ) -> ContextChain { if !config.enabled || (symbols.is_empty() && defs.is_empty()) { return ContextChain::default(); @@ -84,9 +109,41 @@ pub fn build_context_chain( symbols_extracted: symbols.len(), ..Default::default() }; - // Phase 1: Resolve outbound symbols to file locations (what changed code calls) - let mut entries = resolve_symbols(symbols, config, project_root, ignore_patterns, &mut stats); + // When prefer_index is true, try index first, then fall back to regex + // for any symbols not found in the index (new files, stale index, etc.). + let mut entries = if config.prefer_index && bridge.is_available() { + let (index_entries, unresolved) = + resolve_via_index(symbols, bridge, project_root, ignore_patterns, &mut stats); + debug!( + index_resolved = index_entries.len(), + unresolved = unresolved.len(), + source = "index", + "resolved symbols via index" + ); + // Fall back to regex for symbols the index couldn't resolve + if !unresolved.is_empty() { + debug!( + count = unresolved.len(), + "falling back to regex for unresolved symbols" + ); + let mut fallback = resolve_symbols( + &unresolved, + config, + project_root, + ignore_patterns, + &mut stats, + ); + index_entries + .into_iter() + .chain(fallback.drain(..)) + .collect() + } else { + index_entries + } + } else { + resolve_symbols(symbols, config, project_root, ignore_patterns, &mut stats) + }; // Phase 2: Add test file mappings if config.include_tests { @@ -94,7 +151,13 @@ pub fn build_context_chain( } // Phase 3: Resolve inbound callers (blast radius — who calls changed code) - entries.extend(resolve_callers(defs, config, project_root, ignore_patterns)); + entries.extend(resolve_callers_with_bridge( + defs, + config, + project_root, + ignore_patterns, + bridge, + )); // Sort by priority (FunctionDef first, CallerSite last) entries.sort_by_key(|e| e.priority); @@ -165,6 +228,131 @@ pub fn build_context_chain( ContextChain { text, stats } } +/// Resolve extracted symbols using the symbol index (FTS5 search). +/// +/// Resolve extracted symbols to file locations using the symbol index. +/// +/// Returns `(resolved_entries, unresolved_symbols)`. Symbols not found in +/// the index (new files, unsupported languages, stale index) are returned as +/// `unresolved_symbols` so the caller can fall back to regex-based resolution. +fn resolve_via_index( + symbols: &[ExtractedSymbol], + bridge: &crate::engine::index_bridge::IndexBridge, + project_root: &Path, + ignore_patterns: &[String], + stats: &mut ContextStats, +) -> (Vec, Vec) { + let mut entries = Vec::new(); + let mut seen_files: HashMap> = HashMap::new(); + let mut resolved_indices = Vec::new(); // track which input symbols got resolved + + for (sym_idx, sym) in symbols.iter().enumerate() { + let query_text = match &sym.kind { + SymbolKind::FunctionCall(name) => name.clone(), + SymbolKind::TypeRef(name) => name.clone(), + SymbolKind::Import(path) => { + // For imports, try to match the last segment as a module name + path.rsplit("::").next().unwrap_or(path).to_string() + } + }; + + if query_text.len() < 2 { + continue; + } + + let results = bridge.search_symbols(&query_text, 5); + let mut sym_resolved = false; + for result in results { + let sym_file = &result.symbol.file; + + // Skip entries for the same file the symbol came from + if sym_file == &sym.file { + continue; + } + + // Check ignore patterns + if is_ignored(sym_file, ignore_patterns) { + continue; + } + + // Verify the resolved file exists + let full_path = match safe_join(project_root, sym_file) { + Some(p) => p, + None => continue, + }; + if !full_path.exists() { + continue; + } + + sym_resolved = true; + + // Determine priority from the symbol kind + let kind_str = result.symbol.kind.as_str(); + let (priority, label) = match &sym.kind { + SymbolKind::FunctionCall(_) => { + let prio = if kind_str == "function" || kind_str == "method" { + ContextPriority::FunctionDef + } else { + ContextPriority::TypeDef + }; + (prio, format!("fn {}", result.symbol.name)) + } + SymbolKind::TypeRef(_) => ( + ContextPriority::TypeDef, + format!("type {}", result.symbol.name), + ), + SymbolKind::Import(_) => ( + ContextPriority::TypeDef, + format!("module {}", result.symbol.name), + ), + }; + + let line_start = result.symbol.line; + let line_end = (line_start + MAX_FN_LINES as u32).min( + if let Ok(content) = std::fs::read_to_string(&full_path) { + content.lines().count() as u32 + } else { + line_start + }, + ); + + // Merge overlapping ranges for the same file + let file_ranges = seen_files.entry(sym_file.clone()).or_default(); + let overlaps = file_ranges + .iter() + .any(|(s, e)| line_start <= *e && line_end >= *s); + + if !overlaps { + file_ranges.push((line_start, line_end)); + entries.push(ContextEntry { + file: sym_file.clone(), + line_start, + line_end, + label, + priority, + }); + } + } + + if sym_resolved { + resolved_indices.push(sym_idx); + } + } + + stats.symbols_resolved = entries.len(); + + // Collect unresolved symbols (those not found in the index) + let resolved_set: std::collections::HashSet = resolved_indices.into_iter().collect(); + let unresolved: Vec = symbols + .iter() + .enumerate() + .filter(|(i, _)| !resolved_set.contains(i)) + .map(|(_, s)| s.clone()) + .collect(); + + (entries, unresolved) +} + /// Resolve extracted symbols to concrete file locations and line ranges. fn resolve_symbols( symbols: &[ExtractedSymbol], @@ -719,68 +907,78 @@ fn find_go_package_file(dir: &Path, import_path: &str) -> Option { /// never scanned, and is bounded by [`MAX_CALLER_FILES_SCAN`] files and /// [`MAX_CALLERS_PER_SYMBOL`] call-sites per symbol. Caller slices are tiny /// (the call line + 1 line of context) to stay token-economical. +#[allow(dead_code)] fn resolve_callers( defs: &[DefinedSymbol], config: &ContextConfig, project_root: &Path, ignore_patterns: &[String], +) -> Vec { + // Open a fresh bridge for backward compatibility with callers that don't + // pass one explicitly. + let bridge = crate::engine::index_bridge::IndexBridge::open(project_root); + resolve_callers_with_bridge(defs, config, project_root, ignore_patterns, &bridge) +} + +/// Resolve callers using an existing [`IndexBridge`]. +/// +/// When the bridge is available, uses the call-graph index for precise +/// caller resolution. Falls back to regex-based file scanning otherwise. +fn resolve_callers_with_bridge( + defs: &[DefinedSymbol], + config: &ContextConfig, + project_root: &Path, + ignore_patterns: &[String], + bridge: &crate::engine::index_bridge::IndexBridge, ) -> Vec { if !config.include_callers || defs.is_empty() { return Vec::new(); } // ── Index-based caller resolution (preferred) ─────────────────────── - if let Ok(conn) = crate::index::open_global_index() { - if let Ok(project_id) = crate::index::ensure_project(&conn, project_root) { - let mut entries = Vec::new(); - for def in defs { - if def.name.len() < 2 { + if bridge.is_available() { + let mut entries = Vec::new(); + for def in defs { + if def.name.len() < 2 { + continue; + } + let callers = bridge.find_callers(&def.name, 20); + for caller in callers { + // Skip callers in the defining file itself + if caller.file == def.file { continue; } - match crate::index::graph::find_callers(&conn, project_id, &def.name, 20) { - Ok(callers) => { - for caller in callers { - // Skip callers in the defining file itself - if caller.file == def.file { - continue; - } - let rel = caller.file.clone(); - if is_ignored(&rel, ignore_patterns) { - continue; - } - let label = match def.kind { - DefinitionKind::Function => { - format!("caller of fn {}", def.name) - } - DefinitionKind::Type => { - format!("usage of {}", def.name) - } - }; - entries.push(ContextEntry { - file: rel, - line_start: caller.line.saturating_sub(1).max(1), - line_end: caller.line + 1, - label, - priority: ContextPriority::CallerSite, - }); - } + let rel = caller.file.clone(); + if is_ignored(&rel, ignore_patterns) { + continue; + } + let label = match def.kind { + DefinitionKind::Function => { + format!("caller of fn {}", def.name) } - Err(e) => { - debug!(symbol = %def.name, error = %e, "index caller lookup failed"); + DefinitionKind::Type => { + format!("usage of {}", def.name) } - } - } - if !entries.is_empty() { - debug!( - callers_found = entries.len(), - source = "index", - "resolved callers" - ); - return entries; + }; + entries.push(ContextEntry { + file: rel, + line_start: caller.line.saturating_sub(1).max(1), + line_end: caller.line + 1, + label, + priority: ContextPriority::CallerSite, + }); } - // Index found but no callers — fall through to regex scan - debug!("index caller lookup returned empty, falling back to regex"); } + if !entries.is_empty() { + debug!( + callers_found = entries.len(), + source = "index", + "resolved callers via bridge" + ); + return entries; + } + // Index found but no callers — fall through to regex scan + debug!("index caller lookup returned empty, falling back to regex"); } // ── Regex-based fallback (original behavior) ─────────────────────── @@ -1221,6 +1419,7 @@ mod tests { include_callers: false, use_brain: false, impact_depth: 2, + prefer_index: true, }; let symbols = vec![ExtractedSymbol { @@ -1247,6 +1446,7 @@ mod tests { include_callers: false, use_brain: false, impact_depth: 2, + prefer_index: true, }; let symbols = vec![ExtractedSymbol { diff --git a/src/engine/context/types.rs b/src/engine/context/types.rs index 4a7ff56..cb14d3a 100644 --- a/src/engine/context/types.rs +++ b/src/engine/context/types.rs @@ -57,6 +57,13 @@ pub struct ContextConfig { /// Default: 2. #[serde(default = "default_impact_depth")] pub impact_depth: u32, + + /// When `true` (default), prefer the symbol index (FTS5 + call graph) + /// over regex-based file scanning for outbound symbol resolution. + /// This produces more accurate results but requires `cora index` to have + /// been run. Falls back to regex automatically when the index is unavailable. + #[serde(default = "default_true")] + pub prefer_index: bool, } fn default_true() -> bool { @@ -85,6 +92,7 @@ impl Default for ContextConfig { include_callers: true, use_brain: true, impact_depth: 2, + prefer_index: true, } } } diff --git a/src/engine/index_bridge.rs b/src/engine/index_bridge.rs new file mode 100644 index 0000000..b7e004f --- /dev/null +++ b/src/engine/index_bridge.rs @@ -0,0 +1,204 @@ +//! IndexBridge — lightweight connection between the engine and the symbol index. +//! +//! Provides a single struct that wraps an optional `rusqlite::Connection` to the +//! global `cora.db` and the resolved `project_id`. When the index database does +//! not exist or cannot be opened, the bridge reports `is_available() == false` +//! and all query methods return empty results — **zero caller impact**. +//! +//! The bridge is constructed once at the start of a review/scan run and passed +//! through the context chain pipeline, replacing the ad-hoc +//! `crate::index::open_global_index()` calls scattered throughout resolver.rs. + +use std::path::Path; + +use rusqlite::Connection; +use tracing::debug; + +// ── Public API ──────────────────────────────────────────────────────── + +/// Bridge to the cora symbol index. +/// +/// Holds an optional SQLite connection + project_id pair. If the index is +/// unavailable (no `cora.db`, migration failure, etc.) the bridge is *unavailable* +/// but still safe to query — all lookups return `None` / empty `Vec`. +#[allow(dead_code)] +pub struct IndexBridge { + conn: Option, + project_id: Option, +} + +impl IndexBridge { + /// Open the global index and resolve the project id for `project_root`. + /// + /// Returns an `IndexBridge` regardless of whether the index exists. + /// Call `is_available()` to check. + pub fn open(project_root: &Path) -> Self { + let conn = match crate::index::open_global_index() { + Ok(c) => c, + Err(e) => { + debug!(error = %e, "index bridge: global index unavailable"); + return Self::unavailable(); + } + }; + + let project_id = match crate::index::ensure_project(&conn, project_root) { + Ok(id) => Some(id), + Err(e) => { + debug!(error = %e, "index bridge: failed to resolve project_id"); + return Self::unavailable(); + } + }; + + debug!(project_id, "index bridge: opened successfully"); + Self { + conn: Some(conn), + project_id, + } + } + + /// Create an explicitly unavailable bridge (no index / cannot open). + pub fn unavailable() -> Self { + Self { + conn: None, + project_id: None, + } + } + + /// Whether the bridge has an active index connection and a resolved project. + #[inline] + pub fn is_available(&self) -> bool { + self.conn.is_some() && self.project_id.is_some() + } + + /// The resolved project id (if available). + #[allow(dead_code)] + #[inline] + pub fn project_id(&self) -> Option { + self.project_id + } + + // ── Query helpers ──────────────────────────────────────────────────── + + /// Search the symbols table via FTS5 for the given query text. + /// + /// Returns up to `limit` [`crate::index::symbols::SearchResult`] entries. + /// Returns an empty vec if the bridge is unavailable. + pub fn search_symbols(&self, query: &str, limit: usize) -> Vec { + let conn = match self.conn.as_ref() { + Some(c) => c, + None => return Vec::new(), + }; + let pid = match self.project_id { + Some(id) => id, + None => return Vec::new(), + }; + + let sq = crate::index::SymbolQuery::text(query); + let mut sq = sq; + sq.limit = limit; + + crate::index::search(conn, pid, &sq).unwrap_or_default() + } + + /// Find callers of a symbol using the call-graph index. + /// + /// Returns up to `limit` [`crate::index::graph::CallerResult`] entries. + /// Returns an empty vec if the bridge is unavailable. + pub fn find_callers( + &self, + symbol_name: &str, + limit: usize, + ) -> Vec { + let conn = match self.conn.as_ref() { + Some(c) => c, + None => return Vec::new(), + }; + let pid = match self.project_id { + Some(id) => id, + None => return Vec::new(), + }; + + crate::index::graph::find_callers(conn, pid, symbol_name, limit).unwrap_or_default() + } + + /// Run a brain search (FTS5 + vector + graph RRF fusion). + /// + /// Returns up to `limit` [`crate::index::brain::BrainResult`] entries. + /// Returns an empty vec if the bridge is unavailable. + #[allow(dead_code)] + pub fn brain_search(&self, query: &str, limit: usize) -> Vec { + let conn = match self.conn.as_ref() { + Some(c) => c, + None => return Vec::new(), + }; + let pid = match self.project_id { + Some(id) => id, + None => return Vec::new(), + }; + + crate::index::brain::brain_search(conn, pid, query, limit).unwrap_or_default() + } + + /// Run an impact analysis (blast radius) for the given symbol. + /// + /// Returns [`crate::index::graph::ImpactNode`] entries up to `depth`. + /// Returns an empty vec if the bridge is unavailable. + #[allow(dead_code)] + pub fn impact_analysis( + &self, + symbol_name: &str, + depth: u32, + ) -> Vec { + let conn = match self.conn.as_ref() { + Some(c) => c, + None => return Vec::new(), + }; + let pid = match self.project_id { + Some(id) => id, + None => return Vec::new(), + }; + + crate::index::graph::impact_analysis(conn, pid, symbol_name, depth).unwrap_or_default() + } + + /// Direct access to the underlying connection (if available). + /// + /// Used by callers that need raw SQL beyond the typed helpers above. + /// Returns `None` if the bridge is unavailable. + #[allow(dead_code)] + #[inline] + pub fn connection(&self) -> Option<&Connection> { + self.conn.as_ref() + } +} + +// ── Tests ───────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unavailable_bridge_reports_false() { + let bridge = IndexBridge::unavailable(); + assert!(!bridge.is_available()); + assert_eq!(bridge.project_id(), None); + assert!(bridge.search_symbols("foo", 10).is_empty()); + assert!(bridge.find_callers("foo", 10).is_empty()); + assert!(bridge.brain_search("foo", 10).is_empty()); + assert!(bridge.connection().is_none()); + } + + #[test] + fn open_nonexistent_project_returns_unavailable() { + // Opening with a nonexistent project root should still succeed + // (it creates the project row), but we can verify it opens. + let dir = tempfile::tempdir().unwrap(); + // The bridge opens the global index — if it doesn't exist, + // the data_dir crate will create it. + let bridge = IndexBridge::open(dir.path()); + // Either available (index was created) or unavailable — both are valid. + // The key invariant: no panic, no crash. + let _ = bridge.is_available(); + } +} diff --git a/src/engine/llm.rs b/src/engine/llm.rs index c341e62..74af42b 100644 --- a/src/engine/llm.rs +++ b/src/engine/llm.rs @@ -691,6 +691,7 @@ pub async fn scan_files( rules: &[String], response_format: &str, system_prompt_override: Option<&str>, + brain_context: Option<&str>, ) -> std::result::Result<(Vec, Option, Option), CoraError> { let spinner = create_spinner("Scanning files…"); @@ -710,6 +711,15 @@ pub async fn scan_files( .join("\n") )); } + // Inject brain/code-intel context when available (impact analysis, + // related patterns, affected tests from the symbol index). + if let Some(ctx) = brain_context { + if !ctx.is_empty() { + user_prompt.push_str("## Code Intelligence (Brain)\n"); + user_prompt.push_str(ctx); + user_prompt.push_str("\n\n"); + } + } user_prompt.push_str("Files to review:\n\n"); user_prompt.push_str(files_content); diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 468b30c..2252fab 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -5,6 +5,7 @@ pub mod context; pub mod db_writer; pub mod debt_tracker; pub mod diff_parser; +pub mod index_bridge; pub mod index_scanner; pub mod language_analyzer; pub mod llm; diff --git a/src/engine/review.rs b/src/engine/review.rs index 94a24c2..bf732ec 100644 --- a/src/engine/review.rs +++ b/src/engine/review.rs @@ -666,7 +666,7 @@ fn is_valid_file_path(issue_file: &str, valid_files: &[String]) -> bool { /// 3. **Brain search** — semantically related patterns across the codebase /// /// Returns `None` if no index is available or no results found. -fn build_brain_context( +pub(crate) fn build_brain_context( diff_chunks: &[crate::engine::diff_parser::FileChunk], impact_depth: u32, project_root: &std::path::Path, @@ -803,6 +803,102 @@ fn build_brain_context( } } +/// Build brain context for `cora scan` from a list of files. +/// +/// Unlike `build_brain_context` (which works on diff chunks), this variant +/// extracts symbols from the scanned file list and queries the index for +/// impact analysis, affected tests, and related patterns. +/// +/// Returns `None` if no index is available or no results found. +pub(crate) fn build_scan_brain_context( + files: &[crate::engine::scanner::FileEntry], + impact_depth: u32, + project_root: &std::path::Path, +) -> Option { + let conn = crate::index::open_global_index().ok()?; + let project_id = crate::index::ensure_project(&conn, project_root).ok()?; + + // Extract function/type names from each file using simple heuristics. + // For scan we don't have tree-sitter AST — we use the index's FTS5 + // to find symbols defined in these files. + let mut sections = Vec::new(); + let file_paths: Vec<&str> = files.iter().map(|f| f.path.as_str()).collect(); + + // ── Single-pass: collect all symbols from scanned files ──────── + // Query brain_search once per file (capped at 10), then reuse the + // collected symbols for both impact analysis and affected-tests lookup. + let mut all_symbols: Vec = Vec::new(); + for file_path in file_paths.iter().take(10) { + let query = format!("file:\"{file_path}\""); + if let Ok(results) = crate::index::brain::brain_search(&conn, project_id, &query, 5) { + all_symbols.extend(results.into_iter().filter(|r| r.name.len() >= 2)); + } + } + + // Deduplicate by name to avoid redundant impact_analysis calls + let mut seen_names: std::collections::HashSet = std::collections::HashSet::new(); + let unique_symbols: Vec<_> = all_symbols + .into_iter() + .filter(|r| seen_names.insert(r.name.clone())) + .collect(); + + // ── 1. Impact Analysis ─────────────────────────────────────────── + let mut impact_lines: Vec = Vec::new(); + for r in &unique_symbols { + if let Ok(nodes) = + crate::index::graph::impact_analysis(&conn, project_id, &r.name, impact_depth) + { + if nodes.len() > 2 { + impact_lines.push(format!( + "- `{}` ({}:{}): {} downstream caller(s)", + r.name, + r.file, + r.line, + nodes.len() + )); + } + } + } + if !impact_lines.is_empty() { + sections.push(format!( + "### High-Impact Symbols\n{}\n Consider extra scrutiny for these high-call-count symbols.", + impact_lines.join("\n") + )); + } + + // ── 2. Affected Tests ──────────────────────────────────────────── + // Reuse the same symbols — no additional brain_search calls needed. + let mut test_files: std::collections::HashSet = std::collections::HashSet::new(); + for r in &unique_symbols { + if let Ok(nodes) = crate::index::graph::impact_analysis(&conn, project_id, &r.name, 1) { + for node in &nodes { + let lower = node.file.to_lowercase(); + if lower.contains("test") || lower.contains("spec") || lower.contains("_test") { + test_files.insert(node.file.clone()); + } + } + } + } + if !test_files.is_empty() { + let mut test_list: Vec<_> = test_files.into_iter().collect(); + test_list.sort(); + sections.push(format!( + "### Potentially Affected Tests\n{}", + test_list + .iter() + .map(|f| format!("- `{f}`")) + .collect::>() + .join("\n") + )); + } + + if sections.is_empty() { + None + } else { + Some(sections.join("\n\n")) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/index/brain.rs b/src/index/brain.rs index db9d3b2..d917235 100644 --- a/src/index/brain.rs +++ b/src/index/brain.rs @@ -2,10 +2,17 @@ //! //! RRF fusion (k=60) merges 3 signal sources into ranked results. //! Pattern adopted from uteke `doc_search_hybrid()`. +//! +//! Embedding backend is selected at compile time: +//! - `pretrained-embed` feature → nomic-embed-code 768-dim vectors +//! - default → hashing-trick 256-dim vectors +//! +//! A dimension mismatch between an existing on-disk index and the current +//! compile-time backend triggers a warning and advises re-indexing. -use crate::embed::tokens::embed_code; +use crate::embed::{active_dims, active_provider_name, embed_code_dispatch}; use crate::index::symbols::SymbolQuery; -use crate::index::vector::{CodeVectorIndex, DEFAULT_DIMS, cosine_distance_to_similarity}; +use crate::index::vector::{CodeVectorIndex, cosine_distance_to_similarity}; use anyhow::{Context, Result}; use rayon::prelude::*; use rusqlite::Connection; @@ -47,13 +54,89 @@ fn vector_index_path() -> std::path::PathBuf { crate::data_dir::cora_data_dir().join("cora_index.usearch") } +/// Check if an existing on-disk vector index has compatible dimensions. +/// +/// If the index was created with different dimensions (e.g. 256d hashing-trick +/// but now running with 768d pretrained), logs a warning advising re-indexing. +/// +/// The index file contains usearch binary data — we can't easily read the +/// dimensionality without loading it. Instead, we check the `embedding_dims` +/// column in the projects table, which is written during embedding. +fn check_dimension_compat(vi_path: &std::path::Path, expected_dims: usize) { + // Attempt to load the index just to check its dimensions. + // If it fails (empty, corrupt, etc.), we'll create fresh — no warning needed. + let Ok(file) = std::fs::File::open(vi_path) else { + return; + }; + let Ok(metadata) = file.metadata() else { + return; + }; + if metadata.len() == 0 { + return; // Empty file — will create fresh + } + + // Try to load and check dims + let result = std::panic::catch_unwind(|| { + use std::io::{Read, Seek, SeekFrom}; + let mut file = file; + let _ = file.seek(SeekFrom::Start(0)); + let mut buffer = Vec::new(); + let _ = file.read_to_end(&mut buffer); + if buffer.is_empty() { + return None; + } + let index = usearch::Index::restore_from_buffer(&buffer).ok()?; + Some(index.dimensions()) + }); + + match result { + Ok(Some(disk_dims)) if disk_dims != expected_dims => { + tracing::warn!( + "⚠ Vector index dimension mismatch: on-disk={disk_dims}, current={expected_dims} ({})", + active_provider_name() + ); + tracing::warn!( + " The vector index was built with a different embedding backend. \ + Run `cora index` to re-index with the current backend." + ); + // Delete the stale index so embed_project creates a fresh one + if let Err(e) = std::fs::remove_file(vi_path) { + tracing::warn!(" Failed to remove stale index: {e}"); + } else { + let keys_path = vi_path.with_extension("keys"); + let _ = std::fs::remove_file(&keys_path); + tracing::info!(" Removed stale vector index — will create fresh on next index"); + } + } + Ok(Some(_)) => { + // Dimensions match — all good + } + _ => { + // Couldn't read dimensions (corrupt, unsupported, etc.) + // embed_project will handle creation + } + } +} + /// Embed all symbols for a project into the vector index. /// -/// Reads symbols from SQLite, embeds via static token method, -/// stores in usearch. Updates the in-memory cache. +/// Uses the best available embedding backend (selected at compile time): +/// - `pretrained-embed` → nomic-embed-code 768-dim vectors +/// - default → hashing-trick 256-dim vectors +/// +/// Detects dimension mismatch between existing on-disk index and current +/// backend, warning the user to re-index if dimensions changed. /// Call after `index_project`. pub fn embed_project(conn: &Connection, project_id: i64) -> Result { let vi_path = vector_index_path(); + let active = active_dims(); + + // Check for existing index with mismatched dimensions (e.g. user rebuilt + // with/without pretrained-embed feature). Warn but continue — the index + // will be corrupted for searches until re-indexed. + if vi_path.exists() { + check_dimension_compat(&vi_path, active); + } // Acquire write lock — block searches while embedding. // This is fine because embedding only happens during `cora index`, @@ -64,8 +147,7 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { let vi = if let Some(ref mut cached) = *cache { cached } else { - let vi = - CodeVectorIndex::load_or_create(&vi_path, DEFAULT_DIMS).context("load vector index")?; + let vi = CodeVectorIndex::load_or_create(&vi_path, active).context("load vector index")?; *cache = Some(vi); cache.as_mut().unwrap() }; @@ -80,7 +162,7 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { .collect(); // ── Parallel embedding computation (Rayon) ───────────────────────── - // embed_code is pure + CPU-bound. usearch insert is serial. + // embed_code_dispatch is pure + CPU-bound. usearch insert is serial. let t_compute = std::time::Instant::now(); let embedded: Vec<(i64, Vec)> = rows .par_iter() @@ -90,8 +172,7 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { } else { format!("{name} {signature}") }; - let embedding = embed_code(&text); - let vec: Vec = embedding.as_slice().iter().map(|&v| v as f32).collect(); + let vec = embed_code_dispatch(&text); (*sym_id, vec) }) .collect(); @@ -109,10 +190,12 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { let insert_ms = t_insert.elapsed().as_millis(); tracing::debug!( - "embed_compute={}ms, usearch_insert={}ms, symbols={}", + "embed_compute={}ms, usearch_insert={}ms, symbols={}, dims={}, provider={}", compute_ms, insert_ms, - count + count, + active, + active_provider_name() ); if vi.is_dirty() { @@ -125,13 +208,21 @@ pub fn embed_project(conn: &Connection, project_id: i64) -> Result { .unwrap() .insert(project_id, new_ids); + let tier = if cfg!(feature = "pretrained-embed") { + "pretrained" + } else { + "static" + }; conn.execute( - "UPDATE projects SET embedding_tier = 'static', embedding_dims = ?1, \ + "UPDATE projects SET embedding_tier = ?3, embedding_dims = ?1, \ last_embedded_at = datetime('now') WHERE id = ?2", - rusqlite::params![DEFAULT_DIMS, project_id], + rusqlite::params![active, project_id, tier], )?; - tracing::info!("Embedded {count} symbols for project {project_id}"); + tracing::info!( + "Embedded {count} symbols for project {project_id} (provider={}, dims={active})", + active_provider_name() + ); Ok(count) } @@ -215,8 +306,7 @@ fn vector_search(conn: &Connection, project_id: i64, query: &str, limit: usize) _ => return Vec::new(), }; - let embedding = embed_code(query); - let vec: Vec = embedding.as_slice().iter().map(|&v| v as f32).collect(); + let vec = embed_code_dispatch(query); // Over-fetch to compensate for post-filter by project_id. let over_fetch = (limit * 5).max(50); diff --git a/src/index/vector.rs b/src/index/vector.rs index 122ff67..24448cb 100644 --- a/src/index/vector.rs +++ b/src/index/vector.rs @@ -16,8 +16,18 @@ use usearch::{Index, IndexOptions, MetricKind, ScalarKind}; const USEARCH_EXT: &str = "usearch"; -/// Default embedding dimensions (nomic-embed-code static, 768d). -pub const DEFAULT_DIMS: usize = 256; +/// Hashing-trick embedding dimensions (zero-dependency fallback). +pub const FALLBACK_DIMS: usize = 256; + +/// Default embedding dimensions for the vector index. +/// +/// When compiled with `pretrained-embed`, uses nomic-embed-code 768-dim vectors. +/// Otherwise falls back to the hashing-trick 256-dim vectors. +#[cfg(feature = "pretrained-embed")] +pub const DEFAULT_DIMS: usize = 768; + +#[cfg(not(feature = "pretrained-embed"))] +pub const DEFAULT_DIMS: usize = FALLBACK_DIMS; /// Persistent HNSW vector index for code symbol embeddings. ///