From 6cd669f41977a3aa69422f9998db40fc0919a549 Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 08:03:48 +0300 Subject: [PATCH 01/18] fix: defer usearch import to avoid segfault on Python 3.13 + macOS ARM64 On Python 3.13 with macOS ARM64 (Apple Silicon), importing usearch's native extension before calling SentenceTransformer.encode() causes a segmentation fault (signal 11). This is a shared-library symbol conflict between usearch's C++ bindings and the tokenizers/torch native extensions. Root cause: usearch.index was imported at module level in usearch_backend.py, which meant it was always loaded before the embedding model. When store_chunks_batch() later called encode(), the process crashed with SIGSEGV. Fix: - Move usearch import from module-level to a lazy helper (_lazy_usearch) - In create_index() and open_index(), pre-initialize the embedding model BEFORE importing usearch, ensuring torch/tokenizers native libs load first - Use TYPE_CHECKING guard for type annotations Tested: sia-code index --clean on a test project and ~/.dotfiles now successfully indexes files (previously produced 0 chunks with exit 0). --- sia_code/storage/usearch_backend.py | 38 ++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/sia_code/storage/usearch_backend.py b/sia_code/storage/usearch_backend.py index cffb4f2..f1a7912 100644 --- a/sia_code/storage/usearch_backend.py +++ b/sia_code/storage/usearch_backend.py @@ -4,10 +4,17 @@ import sqlite3 from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, TYPE_CHECKING import numpy as np -from usearch.index import Index, MetricKind + +# IMPORTANT: usearch must NOT be imported at module level. +# On Python 3.13 + macOS ARM64, importing usearch before +# sentence_transformers.encode() causes a segfault due to +# a shared-library symbol conflict. We lazy-import usearch +# only when needed and ensure the embedder is loaded first. +if TYPE_CHECKING: + from usearch.index import Index from ..core.models import ( ChangelogEntry, @@ -24,6 +31,17 @@ from .sqlite_runtime import connect_sqlite +def _lazy_usearch(): + """Lazy-import usearch to avoid segfault on macOS ARM64 + Python 3.13. + + The usearch native extension conflicts with tokenizers/torch when imported + before SentenceTransformer.encode(). By deferring the import, we ensure + the embedding model initializes safely first. + """ + from usearch.index import Index, MetricKind + return Index, MetricKind + + class _MemoryAdapter: """Compatibility adapter for legacy mem interface.""" @@ -116,7 +134,7 @@ def __init__( self.db_path = self.path / "index.db" # Will be initialized in create_index() or open_index() - self.vector_index: Index | None = None + self.vector_index: "Index | None" = None self.conn: sqlite3.Connection | None = None self._embedder = None # Lazy-loaded embedding model @@ -436,7 +454,13 @@ def create_index(self) -> None: """Create a new index (vectors + SQLite).""" self.path.mkdir(parents=True, exist_ok=True) - # Create usearch vector index + # Pre-initialize embedder BEFORE importing usearch to avoid segfault + # on Python 3.13 + macOS ARM64 (shared-library symbol conflict). + if self.embedding_enabled: + self._get_embedder() + + # Create usearch vector index (lazy import to avoid segfault) + Index, MetricKind = _lazy_usearch() self.vector_index = Index( ndim=self.ndim, metric=MetricKind.Cos if self.metric == "cos" else MetricKind.L2sq, @@ -462,7 +486,13 @@ def open_index(self, writable: bool = False) -> None: if not self.db_path.exists(): raise FileNotFoundError(f"Database not found: {self.db_path}") + # Pre-initialize embedder BEFORE importing usearch to avoid segfault + # on Python 3.13 + macOS ARM64 (shared-library symbol conflict). + if self.embedding_enabled: + self._get_embedder() + # Load usearch index (memory-mapped for fast access when read-only) + Index, MetricKind = _lazy_usearch() self.vector_index = Index(ndim=self.ndim, metric=MetricKind.Cos, dtype=self.dtype) # Only view if the file is not empty if self.vector_path.stat().st_size > 0: From 1f7ca00765e052421de075fe9fc75f63d33d55e1 Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 11:06:11 +0300 Subject: [PATCH 02/18] feat: add dynamic git memory system with on-demand file history, blast radius, and revert detection - New modules: recency.py, git_dynamic.py, blast_radius.py, revert_detector.py, intent_classifier.py, diff_analyzer.py, semantic_grouper.py - Consolidated CLI: 'memory git-context' command combining all git context features - Consolidated MCP: 'git_context' tool + auto-enrichment in research/bootstrap/trace - Auto-escalating flan-t5 model (large on 16GB+, base otherwise) as fact-rewriter - Exponential recency decay with configurable halflife (30d default) - Branch/worktree awareness with cross-branch search and relevance scoring - Squash-merge dilution guard for co-change coupling - Revert detection via message patterns + fuzzy Jaccard matching - Zero new dependencies (uses existing GitPython + sentence-transformers) - 23 tests passing (8 unit + 15 integration) --- ...-019f173a-fbb0-7341-9b39-2eb5ca609684.json | 4 + sia_code/cli.py | 180 ++++++ sia_code/config.py | 16 + sia_code/mcp.py | 166 ++++++ sia_code/memory/__init__.py | 47 ++ sia_code/memory/blast_radius.py | 340 +++++++++++ sia_code/memory/diff_analyzer.py | 299 ++++++++++ sia_code/memory/git_dynamic.py | 553 ++++++++++++++++++ sia_code/memory/intent_classifier.py | 103 ++++ sia_code/memory/recency.py | 100 ++++ sia_code/memory/revert_detector.py | 188 ++++++ sia_code/memory/semantic_grouper.py | 185 ++++++ sia_code/memory/summarizer.py | 40 +- .../test_git_dynamic_real_repos.py | 154 +++++ tests/unit/test_recency.py | 70 +++ uv.lock | 2 +- 16 files changed, 2445 insertions(+), 2 deletions(-) create mode 100644 .pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json create mode 100644 sia_code/memory/blast_radius.py create mode 100644 sia_code/memory/diff_analyzer.py create mode 100644 sia_code/memory/git_dynamic.py create mode 100644 sia_code/memory/intent_classifier.py create mode 100644 sia_code/memory/recency.py create mode 100644 sia_code/memory/revert_detector.py create mode 100644 sia_code/memory/semantic_grouper.py create mode 100644 tests/integration/test_git_dynamic_real_repos.py create mode 100644 tests/unit/test_recency.py diff --git a/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json new file mode 100644 index 0000000..d9b1714 --- /dev/null +++ b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json @@ -0,0 +1,4 @@ +{ + "nextId": 13, + "tasks": [] +} \ No newline at end of file diff --git a/sia_code/cli.py b/sia_code/cli.py index 6fffb12..d5b5840 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -2398,5 +2398,185 @@ def embed_status(verbose): console.print("\n[dim]Start with: sia-code embed start[/dim]") +# --------------------------------------------------------------------------- +# Dynamic Git Memory CLI Commands (consolidated) +# --------------------------------------------------------------------------- + + +@memory.command(name="git-context") +@click.argument("file_paths", nargs=-1, required=True) +@click.option("--no-blast-radius", is_flag=True, help="Skip blast radius analysis") +@click.option("--no-narrative", is_flag=True, help="Skip evolution narrative") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", +) +def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format): + """Show git context for files: history, blast radius, and evolution narrative. + + Combines file history (revert-aware, cross-branch, recency-scored), + co-change blast radius, and model-generated evolution narrative. + Auto-uses local flan-t5 model for narrative when available. + + Examples: + sia-code memory git-context src/api/datasample.py + sia-code memory git-context src/api.py src/crud.py --format json + sia-code memory git-context src/api.py --no-narrative + """ + from .config import Config + from .memory.blast_radius import BlastRadiusAnalyzer + from .memory.diff_analyzer import DiffSemanticAnalyzer + from .memory.git_dynamic import GitDynamicMemory + from .memory.intent_classifier import IntentClassifier + from .memory.recency import RecencyConfig + + project_dir = Path.cwd() + config = Config.load(project_dir / ".sia-code" / "config.json") + gc = config.git_dynamic + + recency_cfg = RecencyConfig( + halflife_days=gc.recency_halflife_days, + working_window_days=gc.working_window_days, + ) + mem = GitDynamicMemory(project_dir, recency_config=recency_cfg) + classifier = IntentClassifier() + + all_results = {} + for fp in file_paths: + hist = mem.file_history(fp, cross_branch=gc.cross_branch_enabled, limit=15) + if not hist.effective_commits: + console.print(f"[yellow]No history found for {fp}[/yellow]") + continue + + # Classify intents + for c in hist.effective_commits: + intent = classifier.classify(c.message, len(c.files_changed), c.insertions + c.deletions) + c.intent = intent.intent + + entry = {"hist": hist, "radius": None, "narrative": None} + + # Blast radius + if not no_blast_radius: + analyzer = BlastRadiusAnalyzer( + project_dir, + lookback=gc.lookback_commits, + min_coupling=gc.coupling_threshold, + max_files_per_commit=gc.max_files_per_commit, + recency_config=recency_cfg, + ) + entry["radius"] = analyzer.co_changed_files(fp) + + # Narrative + if not no_narrative: + try: + diff_analyzer = DiffSemanticAnalyzer(project_dir, model_name=gc.narrative_model) + entry["narrative"] = diff_analyzer.summarize_evolution(hist) + except Exception: + pass + + all_results[fp] = entry + + if not all_results: + console.print("[red]No results found.[/red]") + return + + if output_format == "json": + import json + + data = {} + for fp, entry in all_results.items(): + hist = entry["hist"] + d = { + "branch_context": { + "current": hist.branch_context.current_branch if hist.branch_context else None, + "base": hist.branch_context.base_branch if hist.branch_context else None, + }, + "owners": hist.owners[:3], + "reverts": [ + {"reverted": r.reverted_hash[:7], "by": r.reverting_hash[:7]} + for r in hist.reverts + ], + "commits": [ + { + "hash": c.hash[:7], + "message": c.message, + "intent": c.intent, + "recency": round(c.recency_score, 3), + "author": c.author, + } + for c in hist.effective_commits[:10] + ], + } + if entry["radius"]: + d["blast_radius"] = [ + {"path": cf.path, "coupling": round(cf.coupling_score, 3)} + for cf in entry["radius"].coupled_files[:10] + ] + if entry["narrative"]: + d["narrative"] = entry["narrative"].narrative + d["phases"] = entry["narrative"].key_phases + d["model_used"] = entry["narrative"].model_used + data[fp] = d + console.print(json.dumps(data, indent=2)) + else: + # Table format + for fp, entry in all_results.items(): + hist = entry["hist"] + console.print(f"\n[bold]{'='*60}[/bold]") + console.print(f"[bold]File:[/bold] {fp}") + if hist.branch_context: + ctx = hist.branch_context + console.print( + f" Branch: {ctx.current_branch} | Base: {ctx.base_branch}" + ) + if hist.owners: + owners_str = ", ".join(f"{a} ({n})" for a, n in hist.owners[:3]) + console.print(f" Owners: {owners_str}") + if hist.reverts: + console.print(f" [yellow]Reverts: {len(hist.reverts)}[/yellow]") + for r in hist.reverts: + console.print(f" {r.reverting_hash[:7]} reverts {r.reverted_hash[:7]}") + + # Narrative + if entry["narrative"]: + n = entry["narrative"] + model_tag = f" [dim](via {n.model_used})[/dim]" if n.model_used else " [dim](heuristic)[/dim]" + console.print(f"\n [bold]Evolution:[/bold]{model_tag}") + console.print(f" {n.narrative}") + if n.key_phases: + console.print(f" Phases: {', '.join(n.key_phases)}") + + # Commits + console.print(f"\n [bold]History[/bold] ({len(hist.effective_commits)} effective):") + for c in hist.effective_commits[:8]: + intent_tag = f"[{c.intent}]" if c.intent else "" + console.print( + f" [{c.recency_score:.2f}] {c.hash[:7]} {c.message[:55]} " + f"[dim]{c.author} {intent_tag}[/dim]" + ) + + # Blast radius + if entry["radius"] and entry["radius"].coupled_files: + radius = entry["radius"] + console.print( + f"\n [bold]Blast Radius[/bold] " + f"({radius.total_commits_analyzed} commits, " + f"{radius.commits_excluded_squash} squash-excluded):" + ) + for cf in radius.coupled_files[:8]: + bar = "█" * int(cf.coupling_score * 20) + console.print( + f" [{cf.coupling_score:.2f}] {bar:20s} {cf.path}" + ) + if radius.change_clusters: + for cl in radius.change_clusters: + console.print( + f" [dim]Cluster (cohesion {cl.cohesion_score:.2f}): " + f"{', '.join(cl.files)}[/dim]" + ) + + if __name__ == "__main__": main() diff --git a/sia_code/config.py b/sia_code/config.py index cc0da42..a45185b 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -183,6 +183,21 @@ class SummarizationConfig(BaseModel): max_commits: int = 20 # Max commits to include in summary +class GitDynamicConfig(BaseModel): + """Configuration for dynamic git memory system.""" + + enabled: bool = True + lookback_commits: int = 200 + coupling_threshold: float = 0.3 + max_files_per_commit: int = 20 # squash dilution guard + working_window_days: int = 14 + recency_halflife_days: float = 30.0 + base_branch: str = "main" + cross_branch_enabled: bool = True + semantic_weight: float = 0.3 # in combined score: git=0.7, semantic=0.3 + narrative_model: str | None = None # None = auto-select (large on 16GB+, base otherwise) + + class StorageConfig(BaseModel): """Storage backend selection configuration.""" @@ -204,6 +219,7 @@ class Config(BaseModel): adaptive: AdaptiveConfig = Field(default_factory=AdaptiveConfig) summarization: SummarizationConfig = Field(default_factory=SummarizationConfig) storage: StorageConfig = Field(default_factory=StorageConfig) + git_dynamic: GitDynamicConfig = Field(default_factory=GitDynamicConfig) @classmethod def load(cls, path: Path) -> "Config": diff --git a/sia_code/mcp.py b/sia_code/mcp.py index ade245e..bef212a 100644 --- a/sia_code/mcp.py +++ b/sia_code/mcp.py @@ -827,6 +827,18 @@ def engineering_bootstrap( else: recommended_next_step = "Start with the top search hits, then escalate to research only if cross-file tracing is needed." + # Auto-enrich with git context for files found in search + git_context_payload = {} + try: + if search_hits and search_hits.get("matches"): + hit_files = list({m["file_path"] for m in search_hits["matches"] if "file_path" in m})[:3] + if hit_files: + git_context_payload = _compute_git_context( + context.workspace_root, hit_files, limit=3 + ) + except Exception: + pass # Graceful — git context is supplementary + return _ok( context=context, result={ @@ -839,6 +851,7 @@ def engineering_bootstrap( "memory_hits": memory_hits, "working_memory": working_memory, "research": research_payload, + "git_context": git_context_payload, "recommended_next_step": recommended_next_step, "fallback_guidance": health["fallback_guidance"], }, @@ -1090,6 +1103,19 @@ def memory_trace( for item in result.events ], } + + # Supplement with dynamic git history for related files + dynamic_git = {} + try: + if result.related_files: + dynamic_git = _compute_git_context( + context.workspace_root, result.related_files[:3], + limit=3, include_blast_radius=False, + ) + except Exception: + pass + payload["dynamic_git"] = dynamic_git + return _ok(context=context, result=payload, embedding_runtime=_embedding_runtime(config)) @mcp.tool() @@ -1252,6 +1278,146 @@ def embed_stop(): return _ok(scope="machine", result={"stopped": stop_daemon()}) + # ------------------------------------------------------------------ + # Dynamic Git Memory (consolidated) + # ------------------------------------------------------------------ + + def _compute_git_context( + workspace_root: Path, + file_paths: list[str], + limit: int = 5, + include_blast_radius: bool = True, + include_narrative: bool = True, + ) -> dict: + """Shared helper — computes git context for files. + + Used by git_context tool, research, engineering_bootstrap, memory_trace. + """ + from .config import Config + from .memory.blast_radius import BlastRadiusAnalyzer + from .memory.diff_analyzer import DiffSemanticAnalyzer + from .memory.git_dynamic import GitDynamicMemory + from .memory.intent_classifier import IntentClassifier + from .memory.recency import RecencyConfig + + config = Config.load(workspace_root / ".sia-code" / "config.json") + gc = config.git_dynamic + + if not gc.enabled: + return {} + + recency_cfg = RecencyConfig( + halflife_days=gc.recency_halflife_days, + working_window_days=gc.working_window_days, + ) + mem = GitDynamicMemory(workspace_root, recency_config=recency_cfg) + classifier = IntentClassifier() + + results = {} + for fp in file_paths[:limit]: + # File history + revert detection + branch context + hist = mem.file_history(fp, cross_branch=gc.cross_branch_enabled, limit=10) + if not hist.effective_commits: + continue + + # Classify intents + for c in hist.effective_commits: + intent = classifier.classify( + c.message, len(c.files_changed), c.insertions + c.deletions + ) + c.intent = intent.intent + + entry: dict = { + "effective_commits": [ + { + "hash": c.hash[:7], + "message": c.message, + "author": c.author, + "date": c.date.isoformat(), + "recency_score": round(c.recency_score, 3), + "branch": c.branch, + "intent": c.intent, + } + for c in hist.effective_commits[:8] + ], + "owners": hist.owners[:3], + "reverts": [ + { + "reverted": r.reverted_hash[:7], + "by": r.reverting_hash[:7], + "method": r.matched_by, + } + for r in hist.reverts + ], + "branch_context": { + "current": hist.branch_context.current_branch if hist.branch_context else None, + "base": hist.branch_context.base_branch if hist.branch_context else None, + "merge_base": hist.branch_context.merge_base if hist.branch_context else None, + }, + } + + # Blast radius + if include_blast_radius: + analyzer = BlastRadiusAnalyzer( + workspace_root, + lookback=gc.lookback_commits, + min_coupling=gc.coupling_threshold, + max_files_per_commit=gc.max_files_per_commit, + recency_config=recency_cfg, + ) + radius = analyzer.co_changed_files(fp) + entry["blast_radius"] = [ + { + "path": cf.path, + "coupling": round(cf.coupling_score, 3), + "co_changes": cf.co_change_count, + } + for cf in radius.coupled_files[:10] + ] + if radius.change_clusters: + entry["clusters"] = [ + {"files": cl.files, "cohesion": round(cl.cohesion_score, 3)} + for cl in radius.change_clusters + ] + + # Evolution narrative (auto-uses local model if available) + if include_narrative: + try: + diff_analyzer = DiffSemanticAnalyzer( + workspace_root, model_name=gc.narrative_model + ) + narrative = diff_analyzer.summarize_evolution(hist) + entry["narrative"] = narrative.narrative + entry["phases"] = narrative.key_phases + entry["model_used"] = narrative.model_used + except Exception: + pass # Graceful degradation + + results[fp] = entry + + return results + + @mcp.tool() + def git_context( + workspace_root: str, + file_paths: list[str], + include_blast_radius: bool = True, + include_narrative: bool = True, + ) -> dict: + """Git-aware context for files: history, blast radius, evolution narrative. + + Combines file history (revert-aware, cross-branch, recency-scored), + co-change blast radius, and model-generated evolution narrative. + Auto-uses local flan-t5 model for narrative when available. + """ + result = _compute_git_context( + Path(workspace_root), + file_paths, + include_blast_radius=include_blast_radius, + include_narrative=include_narrative, + ) + return _ok(scope="project", result=result) + return mcp diff --git a/sia_code/memory/__init__.py b/sia_code/memory/__init__.py index e69de29..2956bfa 100644 --- a/sia_code/memory/__init__.py +++ b/sia_code/memory/__init__.py @@ -0,0 +1,47 @@ +"""Dynamic git memory system for sia-code. + +Provides on-demand file history, blast radius, revert detection, +recency scoring, branch/worktree awareness, and semantic file grouping. +""" + +from .blast_radius import BlastRadius, BlastRadiusAnalyzer, CoupledFile +from .diff_analyzer import ChangeMeaning, DiffSemanticAnalyzer, EvolutionNarrative +from .git_dynamic import ( + BranchContext, + BranchInfo, + BranchResolver, + FileHistory, + GitDynamicMemory, + HistoricalCommit, + WorktreeInfo, +) +from .intent_classifier import CommitIntent, IntentClassifier +from .recency import RecencyConfig, RecencyScorer +from .revert_detector import CommitInfo, RevertDetector, RevertPair +from .semantic_grouper import EnrichedRelation, SemanticFileGrouper, SemanticRelation + +__all__ = [ + "BlastRadius", + "BlastRadiusAnalyzer", + "BranchContext", + "BranchInfo", + "BranchResolver", + "ChangeMeaning", + "CommitInfo", + "CommitIntent", + "CoupledFile", + "DiffSemanticAnalyzer", + "EnrichedRelation", + "EvolutionNarrative", + "FileHistory", + "GitDynamicMemory", + "HistoricalCommit", + "IntentClassifier", + "RecencyConfig", + "RecencyScorer", + "RevertDetector", + "RevertPair", + "SemanticFileGrouper", + "SemanticRelation", + "WorktreeInfo", +] diff --git a/sia_code/memory/blast_radius.py b/sia_code/memory/blast_radius.py new file mode 100644 index 0000000..81298ff --- /dev/null +++ b/sia_code/memory/blast_radius.py @@ -0,0 +1,340 @@ +"""Blast radius analysis — co-change coupling from git history. + +Identifies files that frequently change together with a target file, +with squash-merge dilution guard and recency weighting. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from .recency import RecencyConfig, RecencyScorer + + +@dataclass +class CoupledFile: + """A file that co-changes with the target.""" + + path: str + coupling_score: float # 0.0-1.0 (recency-weighted) + raw_coupling: float # 0.0-1.0 (unweighted) + co_change_count: int + total_changes_target: int + total_changes_self: int + recent_co_change: datetime | None = None + + +@dataclass +class ChangeCluster: + """A group of files that form a logical change unit.""" + + files: list[str] + cohesion_score: float # How tightly coupled the group is + common_commits: int # Commits where all files appear + + +@dataclass +class BlastRadius: + """Complete blast radius analysis for a file.""" + + target_file: str + coupled_files: list[CoupledFile] = field(default_factory=list) + change_clusters: list[ChangeCluster] = field(default_factory=list) + total_commits_analyzed: int = 0 + commits_excluded_squash: int = 0 + + +class BlastRadiusAnalyzer: + """Analyze co-change coupling between files in git history. + + Features: + - Frequency-based coupling score + - Squash-merge dilution guard (skip commits with too many files) + - Recency weighting (recent co-changes score higher) + - Change cluster detection + """ + + def __init__( + self, + repo_path: Path, + lookback: int = 200, + min_coupling: float = 0.3, + max_files_per_commit: int = 20, + recency_config: RecencyConfig | None = None, + ): + self.repo_path = Path(repo_path).resolve() + self.lookback = lookback + self.min_coupling = min_coupling + self.max_files_per_commit = max_files_per_commit + self.recency = RecencyScorer(recency_config) + + def _git(self, *args: str) -> str | None: + """Run git command, return stdout or None.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except (subprocess.CalledProcessError, OSError): + return None + + def co_changed_files( + self, file_path: str, branch: str | None = None + ) -> BlastRadius: + """Find files that co-change with target file. + + Algorithm: + 1. Get all commits touching target (up to lookback) + 2. Apply squash dilution guard (skip commits with > max_files) + 3. Count co-occurrence of other files + 4. Weight by recency (recent co-changes count more) + 5. Compute coupling = weighted_co_occurrences / max(commits_a, commits_b) + + Args: + file_path: Target file path (relative to repo root). + branch: Branch to analyze (default: current HEAD). + + Returns: + BlastRadius with coupled files sorted by score. + """ + # Get commits touching target file with files changed per commit + commits = self._get_commits_with_files(file_path, branch) + + total_analyzed = len(commits) + excluded_squash = 0 + + # Co-occurrence tracking + co_occurrences: dict[str, float] = {} # file → weighted count + co_dates: dict[str, datetime] = {} # file → most recent co-change + raw_co_counts: dict[str, int] = {} # file → raw count + target_commit_count = 0 + + for commit_hash, files, date in commits: + # Squash dilution guard + if len(files) > self.max_files_per_commit: + excluded_squash += 1 + continue + + target_commit_count += 1 + recency_weight = self.recency.score(date) + + for f in files: + if f == file_path: + continue + co_occurrences[f] = co_occurrences.get(f, 0.0) + recency_weight + raw_co_counts[f] = raw_co_counts.get(f, 0) + 1 + if f not in co_dates or date > co_dates[f]: + co_dates[f] = date + + if target_commit_count == 0: + return BlastRadius( + target_file=file_path, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=excluded_squash, + ) + + # Compute coupling scores + coupled: list[CoupledFile] = [] + for f, weighted_count in co_occurrences.items(): + # Get total commits for this file (approximated by co-change context) + other_total = self._count_file_commits(f, branch) + if other_total == 0: + other_total = raw_co_counts[f] + + # Coupling = weighted_co_occ / max(target_commits, other_commits) + denominator = max(target_commit_count, other_total) + coupling = weighted_count / denominator if denominator > 0 else 0.0 + raw_coupling = raw_co_counts[f] / denominator if denominator > 0 else 0.0 + + if raw_coupling >= self.min_coupling: + coupled.append( + CoupledFile( + path=f, + coupling_score=min(coupling, 1.0), + raw_coupling=min(raw_coupling, 1.0), + co_change_count=raw_co_counts[f], + total_changes_target=target_commit_count, + total_changes_self=other_total, + recent_co_change=co_dates.get(f), + ) + ) + + coupled.sort(key=lambda c: c.coupling_score, reverse=True) + + # Detect change clusters + clusters = self._detect_clusters(coupled, commits, file_path) + + return BlastRadius( + target_file=file_path, + coupled_files=coupled, + change_clusters=clusters, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=excluded_squash, + ) + + def change_cluster(self, file_paths: list[str], branch: str | None = None) -> BlastRadius: + """Find blast radius for multiple files (union of their coupled files). + + Useful when a query spans multiple related files. + """ + all_coupled: dict[str, CoupledFile] = {} + total_analyzed = 0 + total_excluded = 0 + + for fp in file_paths: + radius = self.co_changed_files(fp, branch) + total_analyzed += radius.total_commits_analyzed + total_excluded += radius.commits_excluded_squash + for cf in radius.coupled_files: + if cf.path in file_paths: + continue # Don't include query files themselves + if cf.path in all_coupled: + # Take max coupling + if cf.coupling_score > all_coupled[cf.path].coupling_score: + all_coupled[cf.path] = cf + else: + all_coupled[cf.path] = cf + + coupled = sorted(all_coupled.values(), key=lambda c: c.coupling_score, reverse=True) + + return BlastRadius( + target_file=",".join(file_paths), + coupled_files=coupled, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=total_excluded, + ) + + def _get_commits_with_files( + self, file_path: str, branch: str | None + ) -> list[tuple[str, list[str], datetime]]: + """Get commits touching file with ALL files changed per commit. + + Two-step approach: + 1. Get commit hashes touching target (with --follow for renames) + 2. For each commit, get all files changed + + Returns: list of (hash, [all_files_in_commit], date) + """ + branch_arg = branch or "HEAD" + + # Step 1: Get commit hashes that touch this file (with rename tracking) + output = self._git( + "log", + branch_arg, + f"--max-count={self.lookback}", + "--follow", + "--format=%H|%aI", + "--", + file_path, + ) + if not output: + return [] + + # Parse commit hashes + dates + commit_refs: list[tuple[str, datetime]] = [] + for line in output.splitlines(): + line = line.strip() + if not line or "|" not in line: + continue + parts = line.split("|", 1) + if len(parts[0]) >= 7 and all(c in "0123456789abcdef" for c in parts[0][:7]): + try: + date = datetime.fromisoformat(parts[1].strip()) + except (ValueError, IndexError): + date = datetime.now(timezone.utc) + commit_refs.append((parts[0].strip(), date)) + + if not commit_refs: + return [] + + # Step 2: For each commit, get all files changed + commits: list[tuple[str, list[str], datetime]] = [] + for commit_hash, date in commit_refs: + files_output = self._git( + "diff-tree", "--no-commit-id", "-r", "--name-only", commit_hash + ) + if files_output: + files = [f.strip() for f in files_output.splitlines() if f.strip()] + else: + files = [] + commits.append((commit_hash, files, date)) + + return commits + + def _count_file_commits(self, file_path: str, branch: str | None) -> int: + """Quick count of commits touching a file.""" + branch_arg = branch or "HEAD" + output = self._git( + "rev-list", + "--count", + branch_arg, + "--", + file_path, + ) + if output: + try: + return int(output.strip()) + except ValueError: + pass + return 0 + + def _detect_clusters( + self, + coupled: list[CoupledFile], + commits: list[tuple[str, list[str], datetime]], + target: str, + ) -> list[ChangeCluster]: + """Detect groups of files that consistently change together. + + Simple approach: files that appear together in >= 60% of the + target file's commits form a cluster. + """ + if not coupled or not commits: + return [] + + # Only consider top coupled files + top_files = [c.path for c in coupled[:10]] + if not top_files: + return [] + + # Count how often pairs appear together in target's commits + valid_commits = [ + (h, files, d) + for h, files, d in commits + if len(files) <= self.max_files_per_commit + ] + if len(valid_commits) < 3: + return [] + + # Find files that appear in >= 60% of target's commits + file_in_commits: dict[str, int] = {} + for _, files, _ in valid_commits: + for f in files: + if f in top_files: + file_in_commits[f] = file_in_commits.get(f, 0) + 1 + + cluster_members = [ + f + for f, count in file_in_commits.items() + if count / len(valid_commits) >= 0.6 + ] + + if len(cluster_members) >= 2: + cohesion = sum( + file_in_commits[f] / len(valid_commits) for f in cluster_members + ) / len(cluster_members) + return [ + ChangeCluster( + files=[target] + cluster_members, + cohesion_score=cohesion, + common_commits=min(file_in_commits[f] for f in cluster_members), + ) + ] + return [] diff --git a/sia_code/memory/diff_analyzer.py b/sia_code/memory/diff_analyzer.py new file mode 100644 index 0000000..50db41c --- /dev/null +++ b/sia_code/memory/diff_analyzer.py @@ -0,0 +1,299 @@ +"""Semantic diff analysis with auto-escalating local model. + +Derives evolution narratives from file history using: +1. Heuristic analysis (always runs, instant, grounded) +2. Local model rewrite (auto-opt-in when transformers importable) + +Model auto-selects best flan-t5 variant for the machine: +- flan-t5-large (780M) on 16+ GB with MPS/CUDA +- flan-t5-base (250M) otherwise +- Heuristic-only if no transformers + +The model is a FACT REWRITER — it never invents intent. +It rewrites structured heuristic output into fluent prose. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .git_dynamic import FileHistory, HistoricalCommit + +logger = logging.getLogger(__name__) + + +@dataclass +class ChangeMeaning: + """Semantic meaning for a single commit (heuristic-derived).""" + + intent: str + summary: str # first line of commit message (grounded) + impact: str + affected_concepts: list[str] + + +@dataclass +class EvolutionNarrative: + """Model-generated or heuristic narrative of file evolution.""" + + file_path: str + narrative: str + key_phases: list[str] + change_meanings: list[ChangeMeaning] + model_used: str | None = None # which model produced the narrative (None = heuristic) + + +def _auto_select_model() -> str: + """Auto-select best flan-t5 variant for this machine. + + - 16+ GB RAM + MPS/CUDA → flan-t5-large (780M, better quality) + - Otherwise → flan-t5-base (250M, lighter) + """ + try: + import torch + import psutil + + ram_gb = psutil.virtual_memory().total / (1024**3) + has_accel = ( + (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) + or torch.cuda.is_available() + ) + if has_accel and ram_gb >= 16: + return "google/flan-t5-large" + return "google/flan-t5-base" + except ImportError: + return "google/flan-t5-base" + + +class DiffSemanticAnalyzer: + """Analyze file evolution semantically. + + Auto-opts-in to local model when transformers is importable. + Model only rewrites heuristic facts — never processes raw diffs. + """ + + def __init__(self, repo_path: Path, model_name: str | None = None): + """ + Args: + repo_path: Path to git repository. + model_name: Override model (default: auto-select based on hardware). + """ + self.repo_path = Path(repo_path).resolve() + self._model_name = model_name + self._model_checked = False + self._can_model = False + self._summarizer = None + + def _can_use_model(self) -> bool: + """Cheap import check — auto-opt-in.""" + if not self._model_checked: + try: + import transformers # noqa: F401 + + self._can_model = True + except ImportError: + self._can_model = False + self._model_checked = True + return self._can_model + + def _get_model_name(self) -> str: + """Resolve model name: explicit override or auto-select.""" + if self._model_name: + return self._model_name + return _auto_select_model() + + def _get_summarizer(self): + """Lazy-load summarizer with auto-selected model.""" + if self._summarizer is None: + from .summarizer import CommitSummarizer + + model = self._get_model_name() + logger.info(f"Auto-selected narrative model: {model}") + self._summarizer = CommitSummarizer(model) + return self._summarizer + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def analyze_change(self, commit: "HistoricalCommit") -> ChangeMeaning: + """Derive per-commit meaning (heuristic only — fast, grounded).""" + from .intent_classifier import IntentClassifier + + classifier = IntentClassifier() + intent_result = classifier.classify( + commit.message, len(commit.files_changed), commit.insertions + commit.deletions + ) + concepts = self._extract_concepts(commit) + + return ChangeMeaning( + intent=intent_result.intent, + summary=commit.message.split("\n")[0].strip(), + impact=intent_result.impact, + affected_concepts=concepts, + ) + + def summarize_evolution(self, history: "FileHistory") -> EvolutionNarrative: + """Generate evolution narrative for a file. + + Always computes heuristic facts first. Then auto-rewrites with model + if transformers is importable (no config needed). + """ + commits = history.effective_commits + if not commits: + return EvolutionNarrative( + file_path=history.file_path, + narrative="No history available.", + key_phases=[], + change_meanings=[], + ) + + # 1. Heuristic analysis (always, instant) + meanings = [self.analyze_change(c) for c in commits[:15]] + phases = self._detect_phases(meanings) + concepts = self._collect_concepts(meanings) + heuristic_narrative = self._template_narrative( + history.file_path, meanings, phases, concepts + ) + + # 2. Model rewrite (auto if importable) + model_used = None + narrative = heuristic_narrative + if self._can_use_model(): + model_result = self._rewrite_with_model(heuristic_narrative, phases, concepts) + if model_result: + narrative = model_result + model_used = self._get_model_name() + + return EvolutionNarrative( + file_path=history.file_path, + narrative=narrative, + key_phases=phases, + change_meanings=meanings, + model_used=model_used, + ) + + # ------------------------------------------------------------------ + # Model rewriting (single call, fact-based input) + # ------------------------------------------------------------------ + + def _rewrite_with_model( + self, facts: str, phases: list[str], concepts: list[str] + ) -> str | None: + """Single generate() — model rewrites structured facts into fluent prose. + + The model NEVER sees raw diffs. Input is heuristic-derived text only. + This prevents hallucinated intent. + """ + try: + summarizer = self._get_summarizer() + prompt = ( + "Rewrite this file evolution summary as a fluent developer-facing paragraph:\n\n" + f"{facts}\n" + ) + if phases: + prompt += f"Development phases: {', '.join(phases)}\n" + if concepts: + prompt += f"Key concepts: {', '.join(concepts[:6])}\n" + prompt += "\nEvolution paragraph:" + + # Single call, greedy decode (fast), max 150 tokens + result = summarizer.generate(prompt, max_length=150, num_beams=1) + if result and len(result) > 20: + return result + return None + except Exception as e: + logger.debug(f"Model rewrite failed: {e}") + return None + + # ------------------------------------------------------------------ + # Heuristic internals + # ------------------------------------------------------------------ + + def _template_narrative( + self, + file_path: str, + meanings: list[ChangeMeaning], + phases: list[str], + concepts: list[str], + ) -> str: + """Deterministic template narrative from heuristic facts.""" + total = len(meanings) + intent_counts: dict[str, int] = {} + for m in meanings: + if m.intent != "unknown": + intent_counts[m.intent] = intent_counts.get(m.intent, 0) + 1 + + # Build parts + parts = [] + for intent, count in sorted(intent_counts.items(), key=lambda x: -x[1]): + parts.append(f"{count} {intent}{'s' if count > 1 else ''}") + + intent_str = ", ".join(parts) if parts else "mixed changes" + concept_str = ", ".join(concepts[:5]) if concepts else "general" + phase_str = "; ".join(phases) if phases else "single phase" + + return ( + f"{file_path}: {total} changes ({intent_str}). " + f"Phases: {phase_str}. " + f"Concepts: {concept_str}." + ) + + def _detect_phases(self, meanings: list[ChangeMeaning]) -> list[str]: + """Detect development phases from sequential intent patterns.""" + if not meanings: + return [] + + phases: list[str] = [] + current_intent = meanings[0].intent + current_count = 1 + + for m in meanings[1:]: + if m.intent == current_intent: + current_count += 1 + else: + if current_count >= 2: + phases.append(f"{current_intent} ({current_count})") + current_intent = m.intent + current_count = 1 + + if current_count >= 2: + phases.append(f"{current_intent} ({current_count})") + + return phases + + def _collect_concepts(self, meanings: list[ChangeMeaning]) -> list[str]: + """Merge concepts from all commit meanings.""" + all_concepts: dict[str, int] = {} + for m in meanings: + for c in m.affected_concepts: + all_concepts[c] = all_concepts.get(c, 0) + 1 + # Sort by frequency + return [c for c, _ in sorted(all_concepts.items(), key=lambda x: -x[1])][:8] + + def _extract_concepts(self, commit: "HistoricalCommit") -> list[str]: + """Extract domain concepts from a single commit.""" + concepts: set[str] = set() + + # From file paths + for fp in commit.files_changed[:5]: + parts = Path(fp).parts + for part in parts: + if part in ("src", "lib", "app", "tests", "test", "__init__.py", "v1"): + continue + stem = Path(part).stem + if len(stem) > 3 and stem != "__init__": + words = re.findall(r"[a-z]+", stem.lower()) + concepts.update(w for w in words if len(w) > 3) + + # From message + msg_words = re.findall(r"[a-z]+", commit.message.lower()) + noise = {"the", "for", "and", "this", "that", "with", "from", "into", "have", "been"} + concepts.update(w for w in msg_words if len(w) > 4 and w not in noise) + + return sorted(concepts)[:8] diff --git a/sia_code/memory/git_dynamic.py b/sia_code/memory/git_dynamic.py new file mode 100644 index 0000000..2bd14a4 --- /dev/null +++ b/sia_code/memory/git_dynamic.py @@ -0,0 +1,553 @@ +"""Dynamic git memory — on-demand file history with branch/worktree awareness. + +Computes git context dynamically per query rather than relying on pre-indexed batch data. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from .recency import RecencyConfig, RecencyScorer +from .revert_detector import CommitInfo, RevertDetector, RevertPair + + +# --------------------------------------------------------------------------- +# Data models +# --------------------------------------------------------------------------- + + +@dataclass +class HistoricalCommit: + """A commit in a file's history.""" + + hash: str + message: str + author: str + date: datetime + files_changed: list[str] = field(default_factory=list) + insertions: int = 0 + deletions: int = 0 + is_reverted: bool = False + reverted_by: str | None = None + branch: str | None = None + branch_relevance: float = 1.0 + recency_score: float = 1.0 + intent: str | None = None # set by IntentClassifier + + +@dataclass +class BranchInfo: + """Information about a git branch.""" + + name: str + last_commit_date: datetime | None = None + is_merged: bool = False + is_current: bool = False + relevance: float = 0.0 + + +@dataclass +class WorktreeInfo: + """Information about a git worktree.""" + + path: Path + branch: str + head_commit: str + is_current: bool = False + + +@dataclass +class BranchContext: + """Context about the current branch state.""" + + current_branch: str + base_branch: str + merge_base: str | None = None + divergence_commits: int = 0 # commits since divergence on current + worktrees: list[WorktreeInfo] = field(default_factory=list) + + +@dataclass +class FileHistory: + """Complete history for a file with metadata.""" + + file_path: str + all_commits: list[HistoricalCommit] = field(default_factory=list) + effective_commits: list[HistoricalCommit] = field(default_factory=list) + reverts: list[RevertPair] = field(default_factory=list) + branch_context: BranchContext | None = None + owners: list[tuple[str, int]] = field(default_factory=list) # (author, count) + + +# --------------------------------------------------------------------------- +# BranchResolver +# --------------------------------------------------------------------------- + + +@dataclass +class BranchResolverConfig: + """Configuration for branch resolution.""" + + base_branch: str = "main" + base_branch_fallbacks: list[str] = field( + default_factory=lambda: ["master", "develop"] + ) + branch_relevance: dict[str, float] = field( + default_factory=lambda: { + "current": 1.0, + "base": 0.8, + "recent": 0.5, + "merged": 0.3, + "stale": 0.1, + } + ) + stale_days: int = 90 + + +class BranchResolver: + """Resolves branch context, worktrees, and relevance scoring.""" + + def __init__(self, repo_path: Path, config: BranchResolverConfig | None = None): + self.repo_path = repo_path + self.config = config or BranchResolverConfig() + + def _git(self, *args: str, check: bool = True) -> str | None: + """Run a git command, return stdout or None on failure.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=check, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, OSError): + return None + + def current_branch(self) -> str: + """Get current branch name (handles detached HEAD).""" + branch = self._git("rev-parse", "--abbrev-ref", "HEAD") + if branch and branch != "HEAD": + return branch + # Detached HEAD — use short hash + short = self._git("rev-parse", "--short", "HEAD") + return short or "unknown" + + def resolve_base_branch(self) -> str: + """Find the actual base branch (main/master/develop).""" + # Check configured base first + candidates = [self.config.base_branch] + self.config.base_branch_fallbacks + for branch in candidates: + check = self._git("rev-parse", "--verify", f"refs/heads/{branch}", check=False) + if check: + return branch + # Fallback: first branch that isn't current + current = self.current_branch() + branches = self.all_branch_names() + for b in branches: + if b != current: + return b + return current + + def merge_base(self, branch_a: str, branch_b: str) -> str | None: + """Find merge base (common ancestor). Returns None on shallow clone.""" + return self._git("merge-base", branch_a, branch_b, check=False) + + def all_branch_names(self) -> list[str]: + """List all local branch names.""" + output = self._git("branch", "--format=%(refname:short)") + if not output: + return [] + return [b.strip() for b in output.splitlines() if b.strip()] + + def all_branches(self) -> list[BranchInfo]: + """All local branches with metadata and relevance scoring.""" + current = self.current_branch() + base = self.resolve_base_branch() + now = datetime.now(timezone.utc) + + output = self._git( + "branch", + "--format=%(refname:short)|%(committerdate:iso-strict)|%(upstream:track)", + ) + if not output: + return [] + + branches: list[BranchInfo] = [] + for line in output.splitlines(): + parts = line.strip().split("|") + if len(parts) < 2: + continue + name = parts[0].strip() + date_str = parts[1].strip() if len(parts) > 1 else "" + + last_date = None + if date_str: + try: + last_date = datetime.fromisoformat(date_str) + except ValueError: + pass + + # Determine if merged into base + is_merged = False + if name != base: + merge_check = self._git( + "branch", "--merged", base, "--format=%(refname:short)", check=False + ) + if merge_check: + is_merged = name in merge_check.splitlines() + + # Compute relevance + relevance = self._compute_relevance( + name, current, base, last_date, is_merged, now + ) + + branches.append( + BranchInfo( + name=name, + last_commit_date=last_date, + is_merged=is_merged, + is_current=(name == current), + relevance=relevance, + ) + ) + + return sorted(branches, key=lambda b: b.relevance, reverse=True) + + def _compute_relevance( + self, + name: str, + current: str, + base: str, + last_date: datetime | None, + is_merged: bool, + now: datetime, + ) -> float: + """Compute branch relevance score.""" + rel = self.config.branch_relevance + if name == current: + return rel.get("current", 1.0) + if name == base: + return rel.get("base", 0.8) + if is_merged: + return rel.get("merged", 0.3) + if last_date: + days_ago = (now - last_date).total_seconds() / 86400 + if days_ago > self.config.stale_days: + return rel.get("stale", 0.1) + return rel.get("recent", 0.5) + return rel.get("stale", 0.1) + + def branches_touching_file(self, file_path: str) -> list[str]: + """Which branches have commits touching this file.""" + output = self._git( + "log", "--all", "--format=%D", "--", file_path, check=False + ) + if not output: + return [] + branches: set[str] = set() + for line in output.splitlines(): + for ref in line.split(","): + ref = ref.strip() + if ref and "HEAD" not in ref and "->" not in ref: + branches.add(ref.split("/")[-1]) # strip origin/ prefix + return sorted(branches) + + def worktree_list(self) -> list[WorktreeInfo]: + """List all git worktrees.""" + output = self._git("worktree", "list", "--porcelain") + if not output: + return [] + + worktrees: list[WorktreeInfo] = [] + current_wt: dict[str, str] = {} + + for line in output.splitlines(): + if not line.strip(): + if current_wt.get("worktree"): + worktrees.append( + WorktreeInfo( + path=Path(current_wt["worktree"]), + branch=current_wt.get("branch", "").replace( + "refs/heads/", "" + ), + head_commit=current_wt.get("HEAD", ""), + is_current=( + Path(current_wt["worktree"]).resolve() + == self.repo_path.resolve() + ), + ) + ) + current_wt = {} + elif line.startswith("worktree "): + current_wt["worktree"] = line[9:] + elif line.startswith("HEAD "): + current_wt["HEAD"] = line[5:] + elif line.startswith("branch "): + current_wt["branch"] = line[7:] + + # Last entry + if current_wt.get("worktree"): + worktrees.append( + WorktreeInfo( + path=Path(current_wt["worktree"]), + branch=current_wt.get("branch", "").replace("refs/heads/", ""), + head_commit=current_wt.get("HEAD", ""), + is_current=( + Path(current_wt["worktree"]).resolve() + == self.repo_path.resolve() + ), + ) + ) + return worktrees + + def get_branch_context(self) -> BranchContext: + """Get full branch context for current state.""" + current = self.current_branch() + base = self.resolve_base_branch() + mb = self.merge_base(current, base) if current != base else None + + divergence = 0 + if mb: + count = self._git("rev-list", "--count", f"{mb}..HEAD") + if count: + try: + divergence = int(count) + except ValueError: + pass + + return BranchContext( + current_branch=current, + base_branch=base, + merge_base=mb, + divergence_commits=divergence, + worktrees=self.worktree_list(), + ) + + +# --------------------------------------------------------------------------- +# GitDynamicMemory +# --------------------------------------------------------------------------- + + +class GitDynamicMemory: + """On-demand git memory — computes file history dynamically. + + Features: + - Per-file history with rename tracking (--follow) + - Cross-branch context (current + base + other branches) + - Revert detection and filtering + - Recency-weighted scoring + - Worktree awareness + """ + + def __init__( + self, + repo_path: Path, + recency_config: RecencyConfig | None = None, + branch_config: BranchResolverConfig | None = None, + ): + self.repo_path = Path(repo_path).resolve() + self.recency = RecencyScorer(recency_config) + self.branch_resolver = BranchResolver(self.repo_path, branch_config) + self.revert_detector = RevertDetector() + + def _git(self, *args: str) -> str | None: + """Run git command, return stdout or None.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except (subprocess.CalledProcessError, OSError): + return None + + def file_history( + self, + file_path: str, + branch: str | None = None, + cross_branch: bool = True, + limit: int = 20, + ) -> FileHistory: + """Get comprehensive history for a file. + + Args: + file_path: Path relative to repo root. + branch: Specific branch (default: current HEAD). + cross_branch: Include commits from base branch since divergence. + limit: Max commits to return. + + Returns: + FileHistory with scored, revert-filtered commits. + """ + branch_ctx = self.branch_resolver.get_branch_context() + + # Get commits on target branch (with rename following) + target_branch = branch or branch_ctx.current_branch + commits = self._get_file_commits(file_path, target_branch, limit * 2) + + # Cross-branch: also get base branch commits since divergence + cross_commits: list[HistoricalCommit] = [] + if cross_branch and branch_ctx.merge_base and target_branch != branch_ctx.base_branch: + cross_commits = self._get_file_commits_since( + file_path, branch_ctx.base_branch, branch_ctx.merge_base, limit + ) + for c in cross_commits: + c.branch = branch_ctx.base_branch + c.branch_relevance = 0.8 + + # Tag current branch commits + for c in commits: + c.branch = target_branch + c.branch_relevance = 1.0 + + # Merge and deduplicate + all_commits = self._merge_commits(commits + cross_commits) + + # Detect reverts + commit_infos = [ + CommitInfo(hash=c.hash, message=c.message) for c in all_commits + ] + reverts = self.revert_detector.detect_reverts(commit_infos) + reverted_hashes = {p.reverted_hash for p in reverts} + reverting_hashes = {p.reverting_hash for p in reverts} + + for c in all_commits: + if c.hash in reverted_hashes: + c.is_reverted = True + # Find which commit reverted it + for p in reverts: + if p.reverted_hash == c.hash: + c.reverted_by = p.reverting_hash + break + + # Score by recency + for c in all_commits: + c.recency_score = self.recency.weighted_score( + 1.0, c.date, c.branch_relevance + ) + + # Sort by recency score (highest first) + all_commits.sort(key=lambda c: c.recency_score, reverse=True) + + # Effective history: remove reverted + reverting commits + excluded = reverted_hashes | reverting_hashes + effective = [c for c in all_commits if c.hash not in excluded] + + # Compute owners + owners = self._compute_owners(all_commits) + + return FileHistory( + file_path=file_path, + all_commits=all_commits[:limit], + effective_commits=effective[:limit], + reverts=reverts, + branch_context=branch_ctx, + owners=owners, + ) + + def _get_file_commits( + self, file_path: str, branch: str, limit: int + ) -> list[HistoricalCommit]: + """Get commits touching a file on a specific branch using --follow.""" + output = self._git( + "log", + branch, + f"--max-count={limit}", + "--follow", + "--format=%H|%s|%an|%aI", + "--numstat", + "--", + file_path, + ) + if not output: + return [] + return self._parse_log_output(output) + + def _get_file_commits_since( + self, file_path: str, branch: str, since_commit: str, limit: int + ) -> list[HistoricalCommit]: + """Get commits touching a file on branch since a specific commit.""" + output = self._git( + "log", + f"{since_commit}..{branch}", + f"--max-count={limit}", + "--follow", + "--format=%H|%s|%an|%aI", + "--numstat", + "--", + file_path, + ) + if not output: + return [] + return self._parse_log_output(output) + + def _parse_log_output(self, output: str) -> list[HistoricalCommit]: + """Parse git log output with --format=%H|%s|%an|%aI and --numstat.""" + commits: list[HistoricalCommit] = [] + current: HistoricalCommit | None = None + + for line in output.splitlines(): + if "|" in line and len(line.split("|")) >= 4: + # Looks like a commit line + parts = line.split("|", 3) + if len(parts[0]) >= 7 and all(c in "0123456789abcdef" for c in parts[0][:7]): + if current: + commits.append(current) + try: + date = datetime.fromisoformat(parts[3].strip()) + except (ValueError, IndexError): + date = datetime.now(timezone.utc) + current = HistoricalCommit( + hash=parts[0].strip(), + message=parts[1].strip(), + author=parts[2].strip(), + date=date, + ) + continue + + # Numstat line: additions\tdeletions\tfilename + if current and "\t" in line: + parts = line.split("\t") + if len(parts) >= 3: + try: + ins = int(parts[0]) if parts[0] != "-" else 0 + dels = int(parts[1]) if parts[1] != "-" else 0 + current.insertions += ins + current.deletions += dels + current.files_changed.append(parts[2]) + except ValueError: + pass + + if current: + commits.append(current) + return commits + + def _merge_commits( + self, commits: list[HistoricalCommit] + ) -> list[HistoricalCommit]: + """Deduplicate commits by hash, keeping first occurrence.""" + seen: set[str] = set() + result: list[HistoricalCommit] = [] + for c in commits: + if c.hash not in seen: + seen.add(c.hash) + result.append(c) + return result + + def _compute_owners( + self, commits: list[HistoricalCommit] + ) -> list[tuple[str, int]]: + """Compute top authors by commit count.""" + author_counts: dict[str, int] = {} + for c in commits: + author_counts[c.author] = author_counts.get(c.author, 0) + 1 + return sorted(author_counts.items(), key=lambda x: x[1], reverse=True) diff --git a/sia_code/memory/intent_classifier.py b/sia_code/memory/intent_classifier.py new file mode 100644 index 0000000..0a4929a --- /dev/null +++ b/sia_code/memory/intent_classifier.py @@ -0,0 +1,103 @@ +"""Heuristic commit intent classification. + +Classifies commit intent from conventional commit prefixes +and estimates impact from diff statistics. No model required. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +# Conventional commit prefix → intent category +PREFIX_MAP: dict[str, str] = { + "feat": "feature", + "fix": "bugfix", + "refactor": "refactor", + "perf": "performance", + "docs": "documentation", + "test": "testing", + "tests": "testing", + "chore": "maintenance", + "ci": "ci", + "build": "build", + "revert": "revert", + "style": "style", + "release": "release", +} + +# Pattern to extract conventional commit prefix +_PREFIX_RE = re.compile(r"^(\w+)(?:\(.+?\))?[!:]") + + +@dataclass +class CommitIntent: + """Classified intent and impact for a commit.""" + + intent: str # "feature", "bugfix", "refactor", etc. + impact: str # "high", "medium", "low" + confidence: float # 0.0-1.0 + + +class IntentClassifier: + """Classify commit intent from message prefix + diff stats. + + Pure heuristic — no model needed. Fast and deterministic. + """ + + def classify( + self, message: str, files_changed: int = 0, lines_changed: int = 0 + ) -> CommitIntent: + """Classify a commit's intent and impact. + + Args: + message: Commit message (first line). + files_changed: Number of files in commit. + lines_changed: Total insertions + deletions. + + Returns: + CommitIntent with category, impact level, and confidence. + """ + intent = self._classify_intent(message) + impact = self._estimate_impact(files_changed, lines_changed) + confidence = 0.9 if intent != "unknown" else 0.3 + return CommitIntent(intent=intent, impact=impact, confidence=confidence) + + def _classify_intent(self, message: str) -> str: + """Extract intent from conventional commit prefix.""" + first_line = message.split("\n")[0].strip().lower() + + # Try conventional commit pattern: type(scope): description + m = _PREFIX_RE.match(first_line) + if m: + prefix = m.group(1) + if prefix in PREFIX_MAP: + return PREFIX_MAP[prefix] + + # Fallback: keyword detection in message + if any(w in first_line for w in ("fix", "bug", "patch", "hotfix")): + return "bugfix" + if any(w in first_line for w in ("add", "implement", "feature", "new")): + return "feature" + if any(w in first_line for w in ("refactor", "restructure", "clean")): + return "refactor" + if any(w in first_line for w in ("perf", "optim", "speed", "fast")): + return "performance" + if any(w in first_line for w in ("revert", "undo", "rollback")): + return "revert" + if any(w in first_line for w in ("release", "bump", "version")): + return "release" + if any(w in first_line for w in ("doc", "readme", "comment")): + return "documentation" + if any(w in first_line for w in ("test", "spec", "coverage")): + return "testing" + + return "unknown" + + def _estimate_impact(self, files_changed: int, lines_changed: int) -> str: + """Estimate impact from diff size.""" + if lines_changed > 100 or files_changed > 5: + return "high" + if lines_changed > 30 or files_changed > 2: + return "medium" + return "low" diff --git a/sia_code/memory/recency.py b/sia_code/memory/recency.py new file mode 100644 index 0000000..3ccd64a --- /dev/null +++ b/sia_code/memory/recency.py @@ -0,0 +1,100 @@ +"""Recency scoring for git commits. + +Exponential time decay with a configurable working window. +Commits within the working window get full weight (1.0). +Beyond the window, score decays exponentially with configurable halflife. +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from dataclasses import dataclass + + +@dataclass +class RecencyConfig: + """Configuration for recency scoring.""" + + halflife_days: float = 30.0 + """Time in days for score to decay to 50% beyond working window.""" + + working_window_days: int = 14 + """Commits within this many days get full weight (no decay).""" + + +class RecencyScorer: + """Score commits by recency using exponential decay. + + Behaviour: + - Commits within working_window_days → weight = 1.0 + - Beyond window: weight = exp(-lambda * (days_ago - window)) + - lambda = ln(2) / halflife_days + + Examples (default config: window=14d, halflife=30d): + - 7 days old → 1.0 + - 14 days old → 1.0 (still in window) + - 44 days old → 0.5 (30 days beyond window = one halflife) + - 74 days old → 0.25 (60 days beyond) + - 134 days old → 0.0625 (120 days beyond) + """ + + def __init__(self, config: RecencyConfig | None = None): + self.config = config or RecencyConfig() + self._decay_lambda = math.log(2) / self.config.halflife_days + + @property + def halflife_days(self) -> float: + return self.config.halflife_days + + @property + def working_window_days(self) -> int: + return self.config.working_window_days + + def score(self, commit_date: datetime, now: datetime | None = None) -> float: + """Compute recency weight for a commit. + + Args: + commit_date: When the commit was authored (timezone-aware preferred). + now: Reference time (default: utcnow). Pass for deterministic tests. + + Returns: + Weight in (0.0, 1.0]. + """ + if now is None: + now = datetime.now(timezone.utc) + + # Normalize to UTC for comparison + if commit_date.tzinfo is None: + commit_date = commit_date.replace(tzinfo=timezone.utc) + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) + + days_ago = (now - commit_date).total_seconds() / 86400.0 + if days_ago <= 0: + return 1.0 + if days_ago <= self.config.working_window_days: + return 1.0 + + effective_days = days_ago - self.config.working_window_days + return math.exp(-self._decay_lambda * effective_days) + + def weighted_score( + self, + base_score: float, + commit_date: datetime, + branch_relevance: float = 1.0, + now: datetime | None = None, + ) -> float: + """Combined scoring: base * recency * branch_relevance. + + Args: + base_score: Raw score (e.g., coupling, importance). + commit_date: When the commit was authored. + branch_relevance: Branch relevance weight (1.0 = current, 0.8 = base, etc.). + now: Reference time for deterministic tests. + + Returns: + Weighted score incorporating all factors. + """ + return base_score * self.score(commit_date, now=now) * branch_relevance diff --git a/sia_code/memory/revert_detector.py b/sia_code/memory/revert_detector.py new file mode 100644 index 0000000..a55d40f --- /dev/null +++ b/sia_code/memory/revert_detector.py @@ -0,0 +1,188 @@ +"""Revert detection for git commits. + +Detects revert commits via message patterns and marks original commits as reverted. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + + +# Patterns that identify a revert commit +REVERT_PATTERNS: list[re.Pattern] = [ + re.compile(r'^[Rr]evert "(.+)"'), # Git default: Revert "original message" + re.compile(r"^[Rr]evert:\s*(.+)"), # Conventional commit: revert: message + re.compile(r"[Tt]his reverts commit ([a-f0-9]{7,40})"), # Body reference +] + + +@dataclass +class RevertPair: + """A pair of commits: the original and the commit that reverts it.""" + + reverted_hash: str + """Hash of the commit that was reverted.""" + + reverting_hash: str + """Hash of the commit that performs the revert.""" + + matched_by: str + """Which pattern matched (for debugging).""" + + +@dataclass +class CommitInfo: + """Minimal commit info for revert detection.""" + + hash: str + message: str + body: str = "" + + +class RevertDetector: + """Detects revert relationships between commits. + + Strategy (message-based, fast): + 1. Match revert patterns in commit message/body + 2. For message-match reverts: find original by matching quoted message + 3. For hash-reference reverts: direct hash lookup + """ + + def detect_reverts(self, commits: list[CommitInfo]) -> list[RevertPair]: + """Find all revert relationships in a list of commits. + + Args: + commits: List of commits to analyze (newer first). + + Returns: + List of RevertPair identifying original→reverting relationships. + """ + pairs: list[RevertPair] = [] + # Build message→hash index for message-matching + msg_to_hash: dict[str, str] = {} + for c in commits: + # Use first line of message for matching + first_line = c.message.split("\n")[0].strip() + msg_to_hash[first_line] = c.hash + + for c in commits: + full_text = f"{c.message}\n{c.body}" + pair = self._check_commit(c, full_text, msg_to_hash) + if pair: + pairs.append(pair) + + return pairs + + def _check_commit( + self, commit: CommitInfo, full_text: str, msg_to_hash: dict[str, str] + ) -> RevertPair | None: + """Check if a single commit is a revert.""" + first_line = commit.message.split("\n")[0].strip() + + # Pattern 1: Revert "original message" (exact quote match) + m = REVERT_PATTERNS[0].match(first_line) + if m: + original_msg = m.group(1).strip() + original_hash = msg_to_hash.get(original_msg) + if original_hash and original_hash != commit.hash: + return RevertPair( + reverted_hash=original_hash, + reverting_hash=commit.hash, + matched_by="message_quote", + ) + + # Pattern 2: revert: message (conventional commit) + m = REVERT_PATTERNS[1].match(first_line) + if m: + revert_desc = m.group(1).strip() + # Try exact match first + original_hash = msg_to_hash.get(revert_desc) + if original_hash and original_hash != commit.hash: + return RevertPair( + reverted_hash=original_hash, + reverting_hash=commit.hash, + matched_by="conventional_prefix", + ) + # Fuzzy: find recent commit with highest keyword overlap + best = self._fuzzy_match(revert_desc, msg_to_hash, commit.hash) + if best: + return RevertPair( + reverted_hash=best, + reverting_hash=commit.hash, + matched_by="conventional_fuzzy", + ) + + # Pattern 3: "This reverts commit " in body + m = REVERT_PATTERNS[2].search(full_text) + if m: + reverted_hash = m.group(1) + return RevertPair( + reverted_hash=reverted_hash, + reverting_hash=commit.hash, + matched_by="hash_reference", + ) + + return None + + def _fuzzy_match( + self, revert_desc: str, msg_to_hash: dict[str, str], exclude_hash: str + ) -> str | None: + """Find the most likely original commit via keyword overlap. + + Uses Jaccard similarity on normalized word sets. + Threshold: >= 0.3 overlap to consider a match. + """ + revert_words = self._normalize_words(revert_desc) + if len(revert_words) < 2: + return None + + best_score = 0.0 + best_hash: str | None = None + + for msg, h in msg_to_hash.items(): + if h == exclude_hash: + continue + msg_words = self._normalize_words(msg) + if not msg_words: + continue + # Jaccard similarity + intersection = revert_words & msg_words + union = revert_words | msg_words + score = len(intersection) / len(union) if union else 0 + if score > best_score and score >= 0.3: + best_score = score + best_hash = h + + return best_hash + + @staticmethod + def _normalize_words(text: str) -> set[str]: + """Extract significant words (lowercase, strip punctuation, drop short).""" + # Remove common prefixes + for prefix in ("feat:", "fix:", "chore:", "refactor:", "revert:", "docs:", "ci:"): + if text.lower().startswith(prefix): + text = text[len(prefix):] + break + words = set(re.findall(r'[a-z0-9]+', text.lower())) + # Drop very short words and common noise + noise = {"the", "to", "for", "in", "of", "a", "an", "and", "or", "is", "was"} + return {w for w in words if len(w) > 2 and w not in noise} + + def filter_reverted(self, commits: list[CommitInfo]) -> list[CommitInfo]: + """Return commits with reverted ones removed. + + Removes BOTH the reverted commit AND the reverting commit from the + effective history (neither adds meaningful signal). + """ + pairs = self.detect_reverts(commits) + excluded: set[str] = set() + for pair in pairs: + excluded.add(pair.reverted_hash) + excluded.add(pair.reverting_hash) + return [c for c in commits if c.hash not in excluded] + + def reverted_set(self, commits: list[CommitInfo]) -> set[str]: + """Return set of commit hashes that have been reverted.""" + pairs = self.detect_reverts(commits) + return {p.reverted_hash for p in pairs} diff --git a/sia_code/memory/semantic_grouper.py b/sia_code/memory/semantic_grouper.py new file mode 100644 index 0000000..128642c --- /dev/null +++ b/sia_code/memory/semantic_grouper.py @@ -0,0 +1,185 @@ +"""Semantic file grouping using existing code-chunk embeddings. + +Finds files semantically related to a target by querying the +existing sia-code index (no new vector store needed). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..storage.sqlite_vec_backend import SqliteVecBackend + +from .blast_radius import CoupledFile + + +@dataclass +class SemanticRelation: + """A file semantically related to the target.""" + + file_path: str + similarity_score: float # 0.0-1.0 + relation_type: str = "semantic" # "semantic", "co-change", "both" + + +@dataclass +class EnrichedRelation: + """Combined git + semantic relation.""" + + file_path: str + git_coupling: float # 0.0-1.0 from co-change + semantic_similarity: float # 0.0-1.0 from embeddings + combined_score: float + relation_type: str # "co-change", "semantic", "both" + + +class SemanticFileGrouper: + """Find semantically related files using existing indexed embeddings. + + Leverages the persistent sqlite-vec index that sia-code already maintains. + No new embedding work — just queries existing chunks. + + Strategy: + 1. Get chunks belonging to target file from index + 2. For each chunk, find nearest neighbors in vector space + 3. Aggregate neighbor chunks by file_path + 4. Score by sum of similarity scores per file + """ + + def __init__(self, backend: "SqliteVecBackend"): + self.backend = backend + + def is_available(self) -> bool: + """Check if semantic search is available (index exists + embeddings enabled).""" + return ( + self.backend is not None + and self.backend.conn is not None + and self.backend.embedding_enabled + ) + + def related_files(self, file_path: str, k: int = 10) -> list[SemanticRelation]: + """Find files semantically related to target using existing embeddings. + + Args: + file_path: Target file path (relative to repo root). + k: Max related files to return. + + Returns: + List of SemanticRelation sorted by similarity score. + """ + if not self.is_available(): + return [] + + # Use search_files with file content as implicit query + # Strategy: get the target file's content/chunks, find similar files + try: + # Read chunks for target file from DB + cursor = self.backend.conn.cursor() + cursor.execute( + "SELECT content FROM chunks WHERE file_path = ? LIMIT 5", + (file_path,), + ) + rows = cursor.fetchall() + + if not rows: + # File not in index — try with variations + cursor.execute( + "SELECT content FROM chunks WHERE file_path LIKE ? LIMIT 5", + (f"%{file_path}",), + ) + rows = cursor.fetchall() + + if not rows: + return [] + + # Use first few chunks as query text + query_text = "\n".join(row[0][:500] for row in rows[:3]) + + # Search for similar files + results = self.backend.search_files( + query=query_text, k=k + 5, vector_weight=0.9 + ) + + # Filter out self and convert to SemanticRelation + relations: list[SemanticRelation] = [] + max_score = results[0][1] if results else 1.0 + + for result_path, score in results: + # Skip self (match by suffix to handle path variations) + if result_path.endswith(file_path) or file_path.endswith(result_path): + continue + # Normalize score to 0-1 + normalized = score / max_score if max_score > 0 else 0 + relations.append( + SemanticRelation( + file_path=result_path, + similarity_score=min(normalized, 1.0), + ) + ) + if len(relations) >= k: + break + + return relations + + except Exception: + # Graceful degradation — index might be corrupt or unavailable + return [] + + def semantic_blast_radius( + self, + file_path: str, + git_coupled: list[CoupledFile], + k: int = 15, + git_weight: float = 0.7, + ) -> list[EnrichedRelation]: + """Combine git co-change with semantic similarity. + + Args: + file_path: Target file. + git_coupled: Co-change results from BlastRadiusAnalyzer. + k: Max results. + git_weight: Weight for git signal (1 - git_weight = semantic weight). + + Returns: + Merged + ranked EnrichedRelation list. + """ + semantic_weight = 1.0 - git_weight + + # Get semantic relations + semantic = self.related_files(file_path, k=k * 2) + semantic_map: dict[str, float] = {r.file_path: r.similarity_score for r in semantic} + + # Build git map + git_map: dict[str, float] = {c.path: c.coupling_score for c in git_coupled} + + # Merge all files + all_files = set(semantic_map.keys()) | set(git_map.keys()) + enriched: list[EnrichedRelation] = [] + + for fp in all_files: + git_score = git_map.get(fp, 0.0) + sem_score = semantic_map.get(fp, 0.0) + combined = git_weight * git_score + semantic_weight * sem_score + + # Determine relation type + if git_score > 0 and sem_score > 0: + rel_type = "both" + elif git_score > 0: + rel_type = "co-change" + else: + rel_type = "semantic" + + enriched.append( + EnrichedRelation( + file_path=fp, + git_coupling=git_score, + semantic_similarity=sem_score, + combined_score=combined, + relation_type=rel_type, + ) + ) + + enriched.sort(key=lambda e: e.combined_score, reverse=True) + return enriched[:k] diff --git a/sia_code/memory/summarizer.py b/sia_code/memory/summarizer.py index 835e2a6..3d66fcd 100644 --- a/sia_code/memory/summarizer.py +++ b/sia_code/memory/summarizer.py @@ -100,7 +100,45 @@ def summarize_commits(self, commits: list[str], max_length: int = 100) -> Option logger.warning(f"Error during summarization: {e}") return None - def enhance_changelog(self, tag: str, original_summary: str, commits: list[str]) -> str: + def generate(self, prompt: str, max_length: int = 150, num_beams: int = 1) -> Optional[str]: + """General-purpose generation with custom prompt. + + Used by DiffSemanticAnalyzer for narrative rewriting. + Supports greedy (num_beams=1) for speed or beam search for quality. + + Args: + prompt: Input prompt text. + max_length: Maximum output tokens. + num_beams: Beam width (1 = greedy, fast; 4 = beam search, better quality). + + Returns: + Generated text, or None if model unavailable. + """ + self._load_model() + if self._model is None or self._tokenizer is None: + return None + + try: + inputs = self._tokenizer( + prompt, return_tensors="pt", max_length=512, truncation=True + ) + device = next(self._model.parameters()).device + inputs = {k: v.to(device) for k, v in inputs.items()} + + outputs = self._model.generate( + **inputs, max_length=max_length, num_beams=num_beams, + early_stopping=(num_beams > 1), + ) + result = self._tokenizer.decode(outputs[0], skip_special_tokens=True) + return result.strip() if result.strip() else None + + except Exception as e: + logger.debug(f"Generation failed: {e}") + return None + + def enhance_changelog( + self, tag: str, original_summary: str, commits: list[str] + ) -> str: """Enhance a sparse changelog entry with commit details. Args: diff --git a/tests/integration/test_git_dynamic_real_repos.py b/tests/integration/test_git_dynamic_real_repos.py new file mode 100644 index 0000000..9ef98fb --- /dev/null +++ b/tests/integration/test_git_dynamic_real_repos.py @@ -0,0 +1,154 @@ +"""Integration tests for dynamic git memory against real repos. + +Run with: + pytest tests/integration/test_git_dynamic_real_repos.py -v +""" + +from __future__ import annotations + +import pytest +from pathlib import Path + +from sia_code.memory.git_dynamic import GitDynamicMemory +from sia_code.memory.blast_radius import BlastRadiusAnalyzer +from sia_code.memory.recency import RecencyConfig, RecencyScorer +from sia_code.memory.revert_detector import RevertDetector, CommitInfo +from sia_code.memory.intent_classifier import IntentClassifier + + +# Test repo paths +MLDB_REPO = Path.home() / "dev/ai.platform/ai.platform.mldb" +PIPELINES_REPO = Path.home() / "dev/ai.platform/ai.platform.pipelines" + + +def _skip_if_missing(repo: Path): + if not (repo / ".git").exists(): + pytest.skip(f"Repo not available: {repo}") + + +class TestGitDynamicMemoryMLDB: + """Tests against ai.platform.mldb — rich co-change patterns.""" + + @pytest.fixture(autouse=True) + def setup(self): + _skip_if_missing(MLDB_REPO) + self.config = RecencyConfig(halflife_days=30.0, working_window_days=14) + self.mem = GitDynamicMemory(MLDB_REPO, recency_config=self.config) + + def test_file_history_returns_commits(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert len(hist.effective_commits) > 0 + assert hist.file_path == "mldb/app/api/v1/datasample_metadata.py" + + def test_recency_scoring_decreases_with_age(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + scores = [c.recency_score for c in hist.effective_commits] + # Should be generally decreasing (most recent first) + assert scores[0] >= scores[-1] + + def test_owners_detected(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert len(hist.owners) > 0 + # Top owner should have multiple commits + assert hist.owners[0][1] > 1 + + def test_branch_context_available(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert hist.branch_context is not None + assert hist.branch_context.current_branch + + def test_blast_radius_finds_coupled_files(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.2, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + assert len(result.coupled_files) > 0 + # Expect CRUD layer to be coupled + paths = [cf.path for cf in result.coupled_files] + assert any("crud" in p for p in paths) + + def test_blast_radius_squash_guard(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.1, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + # Should have excluded some squash commits + assert result.commits_excluded_squash >= 0 + + def test_change_cluster_detected(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.2, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + # Expect a cluster of related files + if result.change_clusters: + assert len(result.change_clusters[0].files) >= 2 + + def test_intent_classification(self): + classifier = IntentClassifier() + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + intents = [ + classifier.classify(c.message, len(c.files_changed), c.insertions + c.deletions) + for c in hist.effective_commits + ] + # At least some should be classified + classified = [i for i in intents if i.intent != "unknown"] + assert len(classified) > 0 + + +class TestGitDynamicMemoryPipelines: + """Tests against ai.platform.pipelines — revert detection.""" + + @pytest.fixture(autouse=True) + def setup(self): + _skip_if_missing(PIPELINES_REPO) + self.config = RecencyConfig(halflife_days=30.0, working_window_days=14) + self.mem = GitDynamicMemory(PIPELINES_REPO, recency_config=self.config) + + def test_revert_detected(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + assert len(hist.reverts) >= 1 + # Known revert: e534b21 reverts 75591dd + revert_hashes = {r.reverting_hash[:7] for r in hist.reverts} + assert "e534b21" in revert_hashes + + def test_effective_history_excludes_reverts(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + effective_hashes = {c.hash[:7] for c in hist.effective_commits} + # Both revert and reverted should be excluded + assert "e534b21" not in effective_hashes + assert "75591dd" not in effective_hashes + + def test_all_commits_contains_reverted(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + all_hashes = {c.hash[:7] for c in hist.all_commits} + # All commits should include both + assert "e534b21" in all_hashes + assert "75591dd" in all_hashes + + def test_branch_resolver_lists_branches(self): + branches = self.mem.branch_resolver.all_branch_names() + assert len(branches) >= 1 + assert "main" in branches + + def test_worktree_list(self): + worktrees = self.mem.branch_resolver.worktree_list() + assert len(worktrees) >= 1 + # Current worktree should be marked + current = [w for w in worktrees if w.is_current] + assert len(current) == 1 + + +class TestRecencyScorer: + """Unit-level tests for recency math.""" + + def test_within_window_full_weight(self): + from datetime import datetime, timedelta, timezone + + scorer = RecencyScorer(RecencyConfig(halflife_days=30, working_window_days=14)) + now = datetime.now(timezone.utc) + assert scorer.score(now - timedelta(days=7), now=now) == 1.0 + + def test_halflife_gives_half_weight(self): + from datetime import datetime, timedelta, timezone + import math + + scorer = RecencyScorer(RecencyConfig(halflife_days=30, working_window_days=14)) + now = datetime.now(timezone.utc) + # 44 days ago = 30 days beyond window = one halflife + score = scorer.score(now - timedelta(days=44), now=now) + assert abs(score - 0.5) < 0.01 diff --git a/tests/unit/test_recency.py b/tests/unit/test_recency.py new file mode 100644 index 0000000..35e4817 --- /dev/null +++ b/tests/unit/test_recency.py @@ -0,0 +1,70 @@ +"""Tests for RecencyScorer.""" + +import math +from datetime import datetime, timedelta, timezone + +from sia_code.memory.recency import RecencyConfig, RecencyScorer + + +def _days_ago(days: float) -> datetime: + """Create a commit date exactly N days ago (truncated to seconds to avoid fp edge).""" + dt = datetime.now(timezone.utc) - timedelta(days=days) + return dt.replace(microsecond=0) + + +class TestRecencyScorer: + def test_within_working_window_full_weight(self): + scorer = RecencyScorer() + assert scorer.score(_days_ago(0)) == 1.0 + assert scorer.score(_days_ago(7)) == 1.0 + assert scorer.score(_days_ago(13.9)) == 1.0 # just inside window + + def test_one_halflife_beyond_window(self): + scorer = RecencyScorer() # halflife=30, window=14 + # 44 days old = 30 days beyond window = one halflife → 0.5 + score = scorer.score(_days_ago(44)) + assert abs(score - 0.5) < 0.01 + + def test_two_halflives_beyond_window(self): + scorer = RecencyScorer() + # 74 days old = 60 days beyond window = two halflives → 0.25 + score = scorer.score(_days_ago(74)) + assert abs(score - 0.25) < 0.01 + + def test_future_commit_full_weight(self): + scorer = RecencyScorer() + future = datetime.now(timezone.utc) + timedelta(hours=1) + assert scorer.score(future) == 1.0 + + def test_custom_config(self): + config = RecencyConfig(halflife_days=7.0, working_window_days=3) + scorer = RecencyScorer(config) + # 2.9 days old → still in window + assert scorer.score(_days_ago(2.9)) == 1.0 + # 10 days old = 7 days beyond window = one halflife → 0.5 + score = scorer.score(_days_ago(10)) + assert abs(score - 0.5) < 0.01 + + def test_weighted_score(self): + scorer = RecencyScorer() + date = _days_ago(44) # one halflife beyond window → recency=0.5 + # base=0.8, recency=0.5, branch=0.8 + result = scorer.weighted_score(0.8, date, branch_relevance=0.8) + expected = 0.8 * 0.5 * 0.8 + assert abs(result - expected) < 0.01 + + def test_deterministic_with_now_param(self): + scorer = RecencyScorer() + now = datetime(2024, 6, 1, tzinfo=timezone.utc) + commit = datetime(2024, 5, 1, tzinfo=timezone.utc) # 31 days ago + # 31 - 14 = 17 days beyond window + expected = math.exp(-math.log(2) / 30.0 * 17) + score = scorer.score(commit, now=now) + assert abs(score - expected) < 0.001 + + def test_naive_datetime_treated_as_utc(self): + scorer = RecencyScorer() + naive_commit = datetime.now() - timedelta(days=7) + # Should not crash, treated as UTC + score = scorer.score(naive_commit) + assert score == 1.0 diff --git a/uv.lock b/uv.lock index 016e90f..802d693 100644 --- a/uv.lock +++ b/uv.lock @@ -2955,7 +2955,7 @@ wheels = [ [[package]] name = "sia-code" -version = "0.7.1" +version = "0.7.2" source = { editable = "." } dependencies = [ { name = "click" }, From 27ffa141e4b26c9eff6bfb727250b7eee8a94074 Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 13:26:28 +0300 Subject: [PATCH 03/18] feat: multi-repo fan-out indexing + embed daemon auto-start - Auto-detect multi-repo workspace (folders with git sub-directories) - Fan-out indexing: each repo indexed independently with own .sia-code/ - Multi-repo registry (.sia-code/multi-repo.json) for aggregated search - Aggregated search: queries all registered sub-repo indexes, merges results - Embed daemon auto-start: _try_auto_start_daemon() in _get_embedder() Uses subprocess.Popen (not in-process fork which causes sys.exit) - MPS device detection in both daemon and fallback embedder - HF_HUB_OFFLINE=1 set after first model load (prevents network calls) - Model stays warm via daemon (1h idle timeout) --- ...-019f173a-fbb0-7341-9b39-2eb5ca609684.json | 2 +- sia_code/cli.py | 177 ++++++++++++++++-- sia_code/embed_server/daemon.py | 12 +- sia_code/storage/multi_repo.py | 134 +++++++++++++ sia_code/storage/sqlite_vec_backend.py | 53 ++++++ 5 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 sia_code/storage/multi_repo.py diff --git a/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json index d9b1714..b1cc5ec 100644 --- a/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json +++ b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json @@ -1,4 +1,4 @@ { - "nextId": 13, + "nextId": 18, "tasks": [] } \ No newline at end of file diff --git a/sia_code/cli.py b/sia_code/cli.py index d5b5840..2ad1259 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -4,7 +4,7 @@ import sys import logging import subprocess -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path import click @@ -360,6 +360,34 @@ def require_initialized() -> tuple[Path, Config]: return sia_dir, config +def get_multi_repo_backends(cwd: Path | None = None): + """Check if in a multi-repo workspace and return all backends. + + Returns: + list of (repo_name, backend) tuples, or empty list if not multi-repo + """ + from .storage.multi_repo import get_registry_path, MultiRepoRegistry + + workspace = cwd or Path.cwd() + registry_path = get_registry_path(workspace) + registry = MultiRepoRegistry.load(registry_path) + if not registry or not registry.repos: + return [] + + backends = [] + for entry in registry.repos: + repo_sia_dir = workspace / entry.index_dir + if (repo_sia_dir / "index.db").exists(): + try: + config = Config.load(repo_sia_dir / "config.json") + backend = create_backend(repo_sia_dir, config, suppress_stdout_notices=True) + backend.open_index() + backends.append((entry.name, backend)) + except Exception: + pass + return backends + + @click.group() @click.version_option(version=__version__) @click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging") @@ -515,6 +543,89 @@ def index( # Index directory directory = Path(path).resolve() + # Auto-detect multi-repo workspace (multiple git sub-repos) + from .storage.multi_repo import ( + detect_sub_repos, + is_multi_repo_workspace, + build_registry, + get_registry_path, + ) + + if is_multi_repo_workspace(directory): + sub_repos = detect_sub_repos(directory) + console.print( + f"[cyan]Detected multi-repo workspace with {len(sub_repos)} repos[/cyan]" + ) + for repo in sub_repos: + console.print(f" [dim]• {repo.name}[/dim]") + console.print() + + # Fan-out: index each repo independently + registry = build_registry(directory, sub_repos) + all_stats = [] + + for i, repo_path in enumerate(sub_repos, 1): + console.print( + f"[cyan][{i}/{len(sub_repos)}] Indexing {repo_path.name}...[/cyan]" + ) + repo_sia_dir = repo_path / ".sia-code" + repo_sia_dir.mkdir(parents=True, exist_ok=True) + + # Write minimal config if none exists + repo_config_path = repo_sia_dir / "config.json" + if not repo_config_path.exists(): + config.save(repo_config_path) + + # Create backend for this repo + repo_backend = create_backend(repo_sia_dir, config) + if clean: + index_path = repo_sia_dir / "index.db" + if index_path.exists(): + index_path.unlink() + repo_backend.create_index() + else: + try: + repo_backend.open_index() + except Exception: + repo_backend.create_index() + + repo_coord = IndexingCoordinator(config, repo_backend) + try: + stats = repo_coord.index_directory(repo_path) + all_stats.append(stats) + # Update registry entry + for entry in registry.repos: + if entry.name == repo_path.name: + entry.indexed_at = ( + datetime.now(timezone.utc).isoformat() + ) + entry.file_count = stats.get("indexed_files", 0) + break + console.print( + f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " + f"{stats.get('total_chunks', 0)} chunks" + ) + except Exception as e: + console.print(f" [red]✗ Failed: {e}[/red]") + + # Close this repo's backend + try: + repo_backend.close() + except Exception: + pass + + # Save registry + registry_path = get_registry_path(directory) + registry.save(registry_path) + console.print("\n[green]✓ Multi-repo indexing complete[/green]") + console.print(f" Registry: {registry_path}") + total_files = sum(s.get("indexed_files", 0) for s in all_stats) + total_chunks = sum(s.get("total_chunks", 0) for s in all_stats) + console.print(f" Total: {total_files} files, {total_chunks} chunks") + return + + # --- Single-repo indexing (original flow) --- + if update: console.print(f"[cyan]Incremental indexing {directory}...[/cyan]") console.print("[dim]Checking for changes...[/dim]") @@ -799,8 +910,18 @@ def search( console.print("[red]Error: --no-deps and --deps-only are mutually exclusive[/red]") sys.exit(1) - backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) - backend.open_index() + # Multi-repo: if in workspace with registry, search all sub-repos + multi_backends = get_multi_repo_backends() + _multi_repo_mode = bool(multi_backends) + if _multi_repo_mode: + console.print( + f"[dim]Multi-repo: searching across {len(multi_backends)} repos[/dim]" + ) + # Use primary backend for config-based settings; actual search aggregates all + backend = multi_backends[0][1] + else: + backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) + backend.open_index() # Determine dependency filtering # Default: include deps (from config or True) @@ -825,23 +946,41 @@ def search( console.print(f"[dim]Searching ({mode}{filter_status}{deps_status})...[/dim]") # Execute search based on mode - if regex: - results = backend.search_lexical( - query, k=limit, include_deps=include_deps, tier_boost=tier_boost - ) - elif semantic_only: - results = backend.search_semantic( - query, k=limit, include_deps=include_deps, tier_boost=tier_boost - ) + def _search_one_backend(be): + if regex: + return be.search_lexical( + query, k=limit, include_deps=include_deps, tier_boost=tier_boost + ) + elif semantic_only: + return be.search_semantic( + query, k=limit, include_deps=include_deps, tier_boost=tier_boost + ) + else: + return be.search_hybrid( + query, + k=limit, + vector_weight=config.search.vector_weight, + include_deps=include_deps, + tier_boost=tier_boost, + ) + + if _multi_repo_mode: + # Aggregate results from all repos + all_results = [] + for repo_name, be in multi_backends: + try: + repo_results = _search_one_backend(be) + # Prefix file paths with repo name for disambiguation + for r in repo_results: + if hasattr(r, "chunk") and hasattr(r.chunk, "file_path"): + r.chunk.file_path = f"{repo_name}/{r.chunk.file_path}" + all_results.extend(repo_results) + except Exception: + pass + # Sort by score descending, take top `limit` + results = sorted(all_results, key=lambda r: r.score, reverse=True)[:limit] else: - # NEW: Hybrid search (BM25 + semantic) for best performance - results = backend.search_hybrid( - query, - k=limit, - vector_weight=config.search.vector_weight, - include_deps=include_deps, - tier_boost=tier_boost, - ) + results = _search_one_backend(backend) # Filter for --deps-only after search if deps_only and results: diff --git a/sia_code/embed_server/daemon.py b/sia_code/embed_server/daemon.py index 7d0398c..bdfafba 100644 --- a/sia_code/embed_server/daemon.py +++ b/sia_code/embed_server/daemon.py @@ -137,13 +137,23 @@ def _load_model(self, model_name: str) -> Any: # Auto-detect device on first load if not self.models: # First model - self.device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + self.device = "cuda" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + self.device = "mps" + else: + self.device = "cpu" logger.info(f"Using device: {self.device}") # Load model model = SentenceTransformer(model_name, device=self.device) self.models[model_name] = model + # After successful load, prevent future network calls + # Model is now cached locally — no need to hit HF Hub again + import os + os.environ["HF_HUB_OFFLINE"] = "1" + logger.info(f"Model loaded: {model_name} ({len(self.models)} total)") return self.models[model_name] diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py new file mode 100644 index 0000000..bb74b17 --- /dev/null +++ b/sia_code/storage/multi_repo.py @@ -0,0 +1,134 @@ +"""Multi-repo detection and fan-out indexing support. + +Detects git repositories as immediate sub-directories and indexes +each independently, then registers them for aggregated search. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass +class RepoEntry: + """A registered sub-repo in a multi-repo workspace.""" + + name: str + path: str # relative to workspace root + index_dir: str # relative path to .sia-code/ dir + indexed_at: str | None = None + file_count: int = 0 + + +@dataclass +class MultiRepoRegistry: + """Registry of sub-repos in a multi-repo workspace.""" + + repos: list[RepoEntry] = field(default_factory=list) + created_at: str = "" + workspace_root: str = "" + + def save(self, registry_path: Path) -> None: + """Save registry to .sia-code/multi-repo.json.""" + registry_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "workspace_root": self.workspace_root, + "created_at": self.created_at, + "repos": [ + { + "name": r.name, + "path": r.path, + "index_dir": r.index_dir, + "indexed_at": r.indexed_at, + "file_count": r.file_count, + } + for r in self.repos + ], + } + registry_path.write_text(json.dumps(data, indent=2)) + + @classmethod + def load(cls, registry_path: Path) -> MultiRepoRegistry | None: + """Load registry from file. Returns None if not found.""" + if not registry_path.exists(): + return None + try: + data = json.loads(registry_path.read_text()) + return cls( + workspace_root=data.get("workspace_root", ""), + created_at=data.get("created_at", ""), + repos=[ + RepoEntry( + name=r["name"], + path=r["path"], + index_dir=r["index_dir"], + indexed_at=r.get("indexed_at"), + file_count=r.get("file_count", 0), + ) + for r in data.get("repos", []) + ], + ) + except (json.JSONDecodeError, KeyError) as e: + logger.warning(f"Failed to load multi-repo registry: {e}") + return None + + +def detect_sub_repos(directory: Path) -> list[Path]: + """Detect git repositories as immediate sub-directories. + + Args: + directory: Parent directory to scan + + Returns: + Sorted list of paths to sub-directories that are git repos + """ + repos = [] + try: + for child in sorted(directory.iterdir()): + if child.is_dir() and not child.name.startswith("."): + # Check for .git/ directory (is a git repo) + if (child / ".git").exists(): + repos.append(child) + except PermissionError: + pass + return repos + + +def is_multi_repo_workspace(directory: Path) -> bool: + """Check if directory is a multi-repo workspace (has git sub-repos but is not itself a repo).""" + # If directory IS a git repo, it's not a multi-repo workspace + if (directory / ".git").exists(): + return False + # Check if it has at least 2 sub-repos + sub_repos = detect_sub_repos(directory) + return len(sub_repos) >= 2 + + +def get_registry_path(workspace_root: Path) -> Path: + """Get path to multi-repo registry file.""" + return workspace_root / ".sia-code" / "multi-repo.json" + + +def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry: + """Build a fresh registry from detected repos.""" + entries = [] + for repo_path in repos: + rel_path = str(repo_path.relative_to(workspace_root)) + entries.append( + RepoEntry( + name=repo_path.name, + path=rel_path, + index_dir=f"{rel_path}/.sia-code", + ) + ) + return MultiRepoRegistry( + workspace_root=str(workspace_root), + created_at=datetime.now(timezone.utc).isoformat(), + repos=entries, + ) diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index 35d9165..082db95 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -359,12 +359,27 @@ def _get_embedder(self): except Exception as e: logger.debug(f"Embedding daemon not available: {e}") + # Auto-start daemon if not running (keeps model warm across repos) + if self._try_auto_start_daemon(): + try: + from ..embed_server.client import EmbedClient + + self._embedder = EmbedClient(model_name=self.embedding_model) + logger.info( + f"Auto-started embedding daemon for {self.embedding_model}" + ) + return self._embedder + except Exception as e: + logger.debug(f"Auto-started daemon not usable: {e}") + # Fallback to local model (current behavior) from sentence_transformers import SentenceTransformer import torch # Auto-detect device (GPU if available, CPU fallback) device = "cuda" if torch.cuda.is_available() else "cpu" + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + device = "mps" self._embedder = SentenceTransformer(self.embedding_model, device=device) @@ -373,6 +388,44 @@ def _get_embedder(self): return self._embedder + def _try_auto_start_daemon(self) -> bool: + """Auto-start embedding daemon in background if not running. + + Returns True if daemon started successfully and is reachable. + """ + import logging + + logger = logging.getLogger(__name__) + try: + from ..embed_server.client import EmbedClient + import subprocess + import sys + import time + + logger.info("Auto-starting embedding daemon...") + # Start daemon as separate process (NOT in-process fork which calls sys.exit) + python_exe = sys.executable + subprocess.Popen( + [python_exe, "-c", + "from sia_code.embed_server.daemon import start_daemon; " + "start_daemon(foreground=True)"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + # Wait for socket to become available (up to 10s for model load) + for _ in range(100): + if EmbedClient.is_available(): + return True + time.sleep(0.1) + + logger.debug("Daemon started but socket not reachable within timeout") + return False + except Exception as e: + logger.debug(f"Failed to auto-start daemon: {e}") + return False + def _get_thread_conn(self) -> sqlite3.Connection: """Get thread-local SQLite connection for parallel operations. From 0f95afa8bc57bc4a6df7cead73c47a212c25588d Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 14:40:28 +0300 Subject: [PATCH 04/18] fix: numpy shape mismatch in brute-force vector search + indexing speed - Fix ValueError: shapes (768,) and (1,768) not aligned in _vector_search by flattening query vector with .flatten() - Pre-warm embed daemon before multi-repo fan-out loop - Add per-repo timing to multi-repo indexing output --- sia_code/cli.py | 17 ++++++++++++++++- sia_code/storage/sqlite_vec_backend.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 2ad1259..8cc44b3 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -564,10 +564,23 @@ def index( registry = build_registry(directory, sub_repos) all_stats = [] + # Pre-warm embed daemon before the loop (shared across all repos) + try: + backend_tmp = create_backend( + directory / ".sia-code", config, suppress_stdout_notices=True + ) + backend_tmp._get_embedder() # triggers auto-start daemon + console.print("[dim]Embedding daemon ready[/dim]") + except Exception: + pass + + import time as _time + for i, repo_path in enumerate(sub_repos, 1): console.print( f"[cyan][{i}/{len(sub_repos)}] Indexing {repo_path.name}...[/cyan]" ) + _t0 = _time.monotonic() repo_sia_dir = repo_path / ".sia-code" repo_sia_dir.mkdir(parents=True, exist_ok=True) @@ -601,9 +614,11 @@ def index( ) entry.file_count = stats.get("indexed_files", 0) break + _elapsed = _time.monotonic() - _t0 console.print( f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " - f"{stats.get('total_chunks', 0)} chunks" + f"{stats.get('total_chunks', 0)} chunks " + f"[dim]({_elapsed:.1f}s)[/dim]" ) except Exception as e: console.print(f" [red]✗ Failed: {e}[/red]") diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index 082db95..b1d920e 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -323,7 +323,7 @@ def _vector_search(self, query_vector: np.ndarray, k: int) -> list[tuple[str, fl if not rows: return [] - query = np.asarray(query_vector, dtype=np.float32) + query = np.asarray(query_vector, dtype=np.float32).flatten() query_norm = np.linalg.norm(query) or 1.0 scored = [] for row in rows: From b7d43aa7357fb99498574cef66b69dc9355ea973 Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 15:01:00 +0300 Subject: [PATCH 05/18] fix: store multi-repo indexes under workspace .sia-code/repos/ + crash isolation - Indexes stored at .sia-code/repos// instead of /.sia-code/ No pollution of sub-repo directories - Each repo indexed in isolated subprocess (5 min timeout per repo) One repo crashing doesn't kill the entire fan-out - sqlite-vec warning fires once per session (class-level dedup) - Subprocess env sets HF_HUB_OFFLINE=1 to prevent network calls --- sia_code/cli.py | 123 ++++++++++++++++--------- sia_code/storage/multi_repo.py | 2 +- sia_code/storage/sqlite_vec_backend.py | 8 +- 3 files changed, 88 insertions(+), 45 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 8cc44b3..5ecc56d 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -561,74 +561,115 @@ def index( console.print() # Fan-out: index each repo independently + # All indexes stored under workspace .sia-code/repos// (no pollution in sub-repos) registry = build_registry(directory, sub_repos) all_stats = [] + workspace_sia = directory / ".sia-code" - # Pre-warm embed daemon before the loop (shared across all repos) + # Pre-warm embed daemon (shared across all repos) try: backend_tmp = create_backend( - directory / ".sia-code", config, suppress_stdout_notices=True + workspace_sia, config, suppress_stdout_notices=True ) - backend_tmp._get_embedder() # triggers auto-start daemon + backend_tmp._get_embedder() console.print("[dim]Embedding daemon ready[/dim]") except Exception: pass import time as _time + import subprocess as _sp + import json as _json for i, repo_path in enumerate(sub_repos, 1): console.print( f"[cyan][{i}/{len(sub_repos)}] Indexing {repo_path.name}...[/cyan]" ) _t0 = _time.monotonic() - repo_sia_dir = repo_path / ".sia-code" - repo_sia_dir.mkdir(parents=True, exist_ok=True) - # Write minimal config if none exists - repo_config_path = repo_sia_dir / "config.json" + # Store index under workspace: .sia-code/repos// + repo_index_dir = workspace_sia / "repos" / repo_path.name + repo_index_dir.mkdir(parents=True, exist_ok=True) + + # Write config for this sub-index + repo_config_path = repo_index_dir / "config.json" if not repo_config_path.exists(): config.save(repo_config_path) - # Create backend for this repo - repo_backend = create_backend(repo_sia_dir, config) - if clean: - index_path = repo_sia_dir / "index.db" - if index_path.exists(): - index_path.unlink() - repo_backend.create_index() - else: - try: - repo_backend.open_index() - except Exception: - repo_backend.create_index() - - repo_coord = IndexingCoordinator(config, repo_backend) + # Update registry to point to workspace-level index dir + for entry in registry.repos: + if entry.name == repo_path.name: + entry.index_dir = str( + (workspace_sia / "repos" / repo_path.name).relative_to(directory) + ) + break + + # Index in subprocess (crash-isolated, 5 min timeout per repo) + clean_flag = "True" if clean else "False" + index_script = ( + "import sys, json, os\n" + "os.environ.setdefault('HF_HUB_OFFLINE', '1')\n" + "from pathlib import Path\n" + "from sia_code.config import Config\n" + "from sia_code.cli import create_backend\n" + "from sia_code.indexer.coordinator import IndexingCoordinator\n" + f"sia_dir = Path({str(repo_index_dir)!r})\n" + f"repo_dir = Path({str(repo_path)!r})\n" + f"do_clean = {clean_flag}\n" + "config = Config.load(sia_dir / 'config.json')\n" + "backend = create_backend(sia_dir, config, suppress_stdout_notices=True)\n" + "if do_clean:\n" + " idx = sia_dir / 'index.db'\n" + " if idx.exists(): idx.unlink()\n" + " backend.create_index()\n" + "else:\n" + " try:\n" + " backend.open_index()\n" + " except Exception:\n" + " backend.create_index()\n" + "coord = IndexingCoordinator(config, backend)\n" + "stats = coord.index_directory(repo_dir)\n" + "backend.close()\n" + "print(json.dumps(stats))\n" + ) try: - stats = repo_coord.index_directory(repo_path) - all_stats.append(stats) - # Update registry entry - for entry in registry.repos: - if entry.name == repo_path.name: - entry.indexed_at = ( - datetime.now(timezone.utc).isoformat() - ) - entry.file_count = stats.get("indexed_files", 0) - break + result = _sp.run( + [sys.executable, "-c", index_script], + capture_output=True, text=True, + timeout=300, + ) + _elapsed = _time.monotonic() - _t0 + if result.returncode == 0: + stdout_lines = result.stdout.strip().splitlines() + stats_line = stdout_lines[-1] if stdout_lines else '{}' + try: + stats = _json.loads(stats_line) + except _json.JSONDecodeError: + stats = {"indexed_files": 0, "total_chunks": 0} + all_stats.append(stats) + for entry in registry.repos: + if entry.name == repo_path.name: + entry.indexed_at = datetime.now(timezone.utc).isoformat() + entry.file_count = stats.get("indexed_files", 0) + break + console.print( + f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " + f"{stats.get('total_chunks', 0)} chunks " + f"[dim]({_elapsed:.1f}s)[/dim]" + ) + else: + err_lines = result.stderr.strip().splitlines() + err_msg = err_lines[-1][:80] if err_lines else "unknown" + console.print( + f" [red]✗ Failed ({_elapsed:.1f}s): {err_msg}[/red]" + ) + except _sp.TimeoutExpired: _elapsed = _time.monotonic() - _t0 console.print( - f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " - f"{stats.get('total_chunks', 0)} chunks " - f"[dim]({_elapsed:.1f}s)[/dim]" + f" [yellow]⚠ Timeout ({_elapsed:.0f}s) — skipped[/yellow]" ) except Exception as e: console.print(f" [red]✗ Failed: {e}[/red]") - # Close this repo's backend - try: - repo_backend.close() - except Exception: - pass - # Save registry registry_path = get_registry_path(directory) registry.save(registry_path) @@ -636,7 +677,7 @@ def index( console.print(f" Registry: {registry_path}") total_files = sum(s.get("indexed_files", 0) for s in all_stats) total_chunks = sum(s.get("total_chunks", 0) for s in all_stats) - console.print(f" Total: {total_files} files, {total_chunks} chunks") + console.print(f" Total: {total_files} files, {total_chunks} chunks across {len(all_stats)} repos") return # --- Single-repo indexing (original flow) --- diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index bb74b17..937486a 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -124,7 +124,7 @@ def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry RepoEntry( name=repo_path.name, path=rel_path, - index_dir=f"{rel_path}/.sia-code", + index_dir=f".sia-code/repos/{repo_path.name}", ) ) return MultiRepoRegistry( diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index b1d920e..75e2ad6 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -246,9 +246,11 @@ def _ensure_vector_table(self) -> None: import logging logger = logging.getLogger(__name__) - logger.warning( - "sqlite-vec extension not available; falling back to brute-force vector search." - ) + if not getattr(SqliteVecBackend, "_sqlite_vec_warned", False): + logger.warning( + "sqlite-vec extension not available; falling back to brute-force vector search." + ) + SqliteVecBackend._sqlite_vec_warned = True cursor.execute( """ CREATE TABLE IF NOT EXISTS vectors ( From 34d5f8b2da9b4499874b952003c4550ead60ce2c Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 16:02:19 +0300 Subject: [PATCH 06/18] fix: dynamic multi-repo timeouts + aggregate status - Scale per-repo timeout from estimated file count (5m..30m) - Save multi-repo registry before and during fan-out so partial runs are visible - Aggregate Error: Sia Code not initialized. Run 'sia-code init' first. across workspace .sia-code/repos even if run was interrupted - Increase embedding batch size on higher-memory machines (96/128) - Fallback multi-repo status when registry missing but repos dir exists --- sia_code/cli.py | 83 ++++++++++++++++++++++---- sia_code/storage/multi_repo.py | 44 ++++++++++++++ sia_code/storage/sqlite_vec_backend.py | 6 +- 3 files changed, 118 insertions(+), 15 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 5ecc56d..1b949b1 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -366,23 +366,35 @@ def get_multi_repo_backends(cwd: Path | None = None): Returns: list of (repo_name, backend) tuples, or empty list if not multi-repo """ - from .storage.multi_repo import get_registry_path, MultiRepoRegistry + from .storage.multi_repo import MultiRepoRegistry, get_registry_path workspace = cwd or Path.cwd() registry_path = get_registry_path(workspace) registry = MultiRepoRegistry.load(registry_path) - if not registry or not registry.repos: - return [] + + entries = [] + if registry and registry.repos: + entries = [(entry.name, workspace / entry.index_dir) for entry in registry.repos] + else: + # Fallback: infer from workspace-level repos dir even if registry missing/interrupted + repos_root = workspace / ".sia-code" / "repos" + if repos_root.exists(): + entries = [ + (child.name, child) + for child in sorted(repos_root.iterdir()) + if child.is_dir() and (child / "index.db").exists() + ] + if not entries: + return [] backends = [] - for entry in registry.repos: - repo_sia_dir = workspace / entry.index_dir + for repo_name, repo_sia_dir in entries: if (repo_sia_dir / "index.db").exists(): try: config = Config.load(repo_sia_dir / "config.json") backend = create_backend(repo_sia_dir, config, suppress_stdout_notices=True) backend.open_index() - backends.append((entry.name, backend)) + backends.append((repo_name, backend)) except Exception: pass return backends @@ -545,10 +557,12 @@ def index( # Auto-detect multi-repo workspace (multiple git sub-repos) from .storage.multi_repo import ( - detect_sub_repos, - is_multi_repo_workspace, build_registry, + detect_sub_repos, + estimate_indexable_files, get_registry_path, + is_multi_repo_workspace, + recommend_repo_timeout_seconds, ) if is_multi_repo_workspace(directory): @@ -565,6 +579,8 @@ def index( registry = build_registry(directory, sub_repos) all_stats = [] workspace_sia = directory / ".sia-code" + registry_path = get_registry_path(directory) + registry.save(registry_path) # Pre-warm embed daemon (shared across all repos) try: @@ -603,7 +619,14 @@ def index( ) break - # Index in subprocess (crash-isolated, 5 min timeout per repo) + # Estimate size for timeout sizing and visibility + estimated_files = estimate_indexable_files(repo_path, config) + repo_timeout = recommend_repo_timeout_seconds(estimated_files) + console.print( + f" [dim]~{estimated_files} files, timeout {repo_timeout}s[/dim]" + ) + + # Index in subprocess (crash-isolated, dynamic timeout per repo) clean_flag = "True" if clean else "False" index_script = ( "import sys, json, os\n" @@ -634,8 +657,9 @@ def index( try: result = _sp.run( [sys.executable, "-c", index_script], - capture_output=True, text=True, - timeout=300, + capture_output=True, + text=True, + timeout=repo_timeout, ) _elapsed = _time.monotonic() - _t0 if result.returncode == 0: @@ -651,6 +675,7 @@ def index( entry.indexed_at = datetime.now(timezone.utc).isoformat() entry.file_count = stats.get("indexed_files", 0) break + registry.save(registry_path) console.print( f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " f"{stats.get('total_chunks', 0)} chunks " @@ -665,13 +690,12 @@ def index( except _sp.TimeoutExpired: _elapsed = _time.monotonic() - _t0 console.print( - f" [yellow]⚠ Timeout ({_elapsed:.0f}s) — skipped[/yellow]" + f" [yellow]⚠ Timeout ({_elapsed:.0f}s/{repo_timeout}s) — skipped[/yellow]" ) except Exception as e: console.print(f" [red]✗ Failed: {e}[/red]") # Save registry - registry_path = get_registry_path(directory) registry.save(registry_path) console.print("\n[green]✓ Multi-repo indexing complete[/green]") console.print(f" Registry: {registry_path}") @@ -1389,6 +1413,39 @@ def status(): sia_dir, config = require_initialized() + # Multi-repo aggregate status + multi_backends = get_multi_repo_backends() + if multi_backends: + total_files = 0 + total_chunks = 0 + repo_rows = [] + for repo_name, backend in multi_backends: + try: + stats = backend.get_stats() + total_files += stats.total_files + total_chunks += stats.total_chunks + repo_rows.append((repo_name, stats.total_files, stats.total_chunks)) + except Exception: + pass + + table = Table(title="Sia Code Index Status (Multi-Repo)") + table.add_column("Property", style="cyan") + table.add_column("Value", style="green") + table.add_row("Workspace Index Path", str(sia_dir)) + table.add_row("Registered Repos", f"{len(multi_backends):,}") + table.add_row("Total Files", f"{total_files:,}") + table.add_row("Total Chunks", f"{total_chunks:,}") + console.print(table) + + repo_table = Table(title="Per-Repo Status") + repo_table.add_column("Repo", style="cyan") + repo_table.add_column("Files", justify="right") + repo_table.add_column("Chunks", justify="right") + for repo_name, files_n, chunks_n in repo_rows: + repo_table.add_row(repo_name, f"{files_n:,}", f"{chunks_n:,}") + console.print(repo_table) + return + backend = create_backend(sia_dir, config) backend.open_index() stats = backend.get_stats() diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 937486a..2d0e490 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -12,6 +12,8 @@ from datetime import datetime, timezone from pathlib import Path +import pathspec + logger = logging.getLogger(__name__) @@ -132,3 +134,45 @@ def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry created_at=datetime.now(timezone.utc).isoformat(), repos=entries, ) + + +def estimate_indexable_files(directory: Path, config) -> int: + """Estimate how many files will be indexed for timeout sizing. + + Mirrors IndexingCoordinator._discover_files() but counts only. + """ + effective_patterns = config.indexing.get_effective_exclude_patterns(directory) + spec = pathspec.PathSpec.from_lines("gitwildmatch", effective_patterns) + + count = 0 + seen: set[Path] = set() + max_bytes = config.indexing.max_file_size_mb * 1024 * 1024 + for pattern in config.indexing.include_patterns: + glob_pattern = pattern if "*" in pattern else f"**/*{pattern}" + for file_path in directory.rglob(glob_pattern): + if not file_path.is_file() or file_path in seen: + continue + rel_path = file_path.relative_to(directory) + if spec.match_file(str(rel_path)): + continue + try: + file_size = file_path.stat().st_size + except OSError: + continue + if file_size == 0 or file_size > max_bytes: + continue + seen.add(file_path) + count += 1 + return count + + +def recommend_repo_timeout_seconds(file_count: int) -> int: + """Compute per-repo timeout from estimated file count. + + Small repos keep 5m floor. Large repos scale up but stay bounded. + """ + if file_count <= 0: + return 300 + # ~0.8s per file plus 60s overhead, bounded 5m..30m + seconds = int(60 + file_count * 0.8) + return max(300, min(1800, seconds)) diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index 75e2ad6..f88c8ed 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -499,13 +499,15 @@ def _get_embed_batch_size(self) -> int: mem_based = 16 elif mem_gb < 24: mem_based = 32 + elif mem_gb < 48: + mem_based = 96 else: - mem_based = 64 + mem_based = 128 cpu_count = os.cpu_count() or 2 max_by_cpu = max(8, cpu_count * 8) size = min(mem_based, max_by_cpu) - size = max(8, min(64, size)) + size = max(8, min(128, size)) self._embed_batch_size = int(size) return self._embed_batch_size From 928f62a3e8fb53e9a084be60bcb328b66cac353f Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 22:38:18 +0300 Subject: [PATCH 07/18] perf: speed up heavy multi-repo indexing with chunk-aware policy - add repo-specific multi-repo override config schema - add built-in ai-toolkit indexing policy (core first-party focus) - estimate chunks for heavy repos and size timeout from chunk count - persist per-repo status, estimated chunks, and last error in registry - improve multi-repo status output with inferred indexed state for legacy runs --- sia_code/cli.py | 118 +++++++++++++++++++++++++++---- sia_code/config.py | 20 ++++++ sia_code/storage/multi_repo.py | 123 +++++++++++++++++++++++++++++++-- 3 files changed, 245 insertions(+), 16 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 1b949b1..5ffd0ef 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -558,7 +558,9 @@ def index( # Auto-detect multi-repo workspace (multiple git sub-repos) from .storage.multi_repo import ( build_registry, + build_repo_config, detect_sub_repos, + estimate_chunks, estimate_indexable_files, get_registry_path, is_multi_repo_workspace, @@ -606,10 +608,10 @@ def index( repo_index_dir = workspace_sia / "repos" / repo_path.name repo_index_dir.mkdir(parents=True, exist_ok=True) - # Write config for this sub-index + # Build per-repo config override and persist for this sub-index + repo_config = build_repo_config(config, repo_path.name) repo_config_path = repo_index_dir / "config.json" - if not repo_config_path.exists(): - config.save(repo_config_path) + repo_config.save(repo_config_path) # Update registry to point to workspace-level index dir for entry in registry.repos: @@ -620,11 +622,21 @@ def index( break # Estimate size for timeout sizing and visibility - estimated_files = estimate_indexable_files(repo_path, config) - repo_timeout = recommend_repo_timeout_seconds(estimated_files) + estimated_files = estimate_indexable_files(repo_path, repo_config) + estimated_chunks = estimate_chunks(repo_path, repo_config) + repo_timeout = recommend_repo_timeout_seconds( + estimated_files, estimated_chunks + ) console.print( - f" [dim]~{estimated_files} files, timeout {repo_timeout}s[/dim]" + f" [dim]~{estimated_files} files, ~{estimated_chunks} chunks, timeout {repo_timeout}s[/dim]" ) + for entry in registry.repos: + if entry.name == repo_path.name: + entry.estimated_chunks = estimated_chunks + entry.status = "indexing" + entry.last_error = None + break + registry.save(registry_path) # Index in subprocess (crash-isolated, dynamic timeout per repo) clean_flag = "True" if clean else "False" @@ -674,6 +686,8 @@ def index( if entry.name == repo_path.name: entry.indexed_at = datetime.now(timezone.utc).isoformat() entry.file_count = stats.get("indexed_files", 0) + entry.status = "full" + entry.last_error = None break registry.save(registry_path) console.print( @@ -683,16 +697,34 @@ def index( ) else: err_lines = result.stderr.strip().splitlines() - err_msg = err_lines[-1][:80] if err_lines else "unknown" + err_msg = err_lines[-1][:120] if err_lines else "unknown" + for entry in registry.repos: + if entry.name == repo_path.name: + entry.status = "failed" + entry.last_error = err_msg + break + registry.save(registry_path) console.print( f" [red]✗ Failed ({_elapsed:.1f}s): {err_msg}[/red]" ) except _sp.TimeoutExpired: _elapsed = _time.monotonic() - _t0 + for entry in registry.repos: + if entry.name == repo_path.name: + entry.status = "timed_out" + entry.last_error = f"timeout after {repo_timeout}s" + break + registry.save(registry_path) console.print( f" [yellow]⚠ Timeout ({_elapsed:.0f}s/{repo_timeout}s) — skipped[/yellow]" ) except Exception as e: + for entry in registry.repos: + if entry.name == repo_path.name: + entry.status = "failed" + entry.last_error = str(e)[:120] + break + registry.save(registry_path) console.print(f" [red]✗ Failed: {e}[/red]") # Save registry @@ -1416,33 +1448,95 @@ def status(): # Multi-repo aggregate status multi_backends = get_multi_repo_backends() if multi_backends: + from .storage.multi_repo import MultiRepoRegistry, get_registry_path + + registry = MultiRepoRegistry.load(get_registry_path(Path.cwd())) total_files = 0 total_chunks = 0 repo_rows = [] + status_counts: dict[str, int] = {} + meta_by_name = {entry.name: entry for entry in (registry.repos if registry else [])} for repo_name, backend in multi_backends: try: stats = backend.get_stats() total_files += stats.total_files total_chunks += stats.total_chunks - repo_rows.append((repo_name, stats.total_files, stats.total_chunks)) + meta = meta_by_name.get(repo_name) + status_value = meta.status if meta else "indexed" + if status_value == "pending" and stats.total_chunks > 0: + status_value = "indexed" + status_counts[status_value] = status_counts.get(status_value, 0) + 1 + repo_rows.append( + ( + repo_name, + status_value, + meta.estimated_chunks if meta else 0, + stats.total_files, + stats.total_chunks, + meta.last_error if meta else None, + ) + ) except Exception: - pass + meta = meta_by_name.get(repo_name) + status_value = meta.status if meta else "error" + status_counts[status_value] = status_counts.get(status_value, 0) + 1 + repo_rows.append( + ( + repo_name, + status_value, + meta.estimated_chunks if meta else 0, + 0, + 0, + meta.last_error if meta else None, + ) + ) + + if registry: + for entry in registry.repos: + if entry.name not in {row[0] for row in repo_rows}: + status_counts[entry.status] = status_counts.get(entry.status, 0) + 1 + repo_rows.append( + ( + entry.name, + entry.status, + entry.estimated_chunks, + 0, + 0, + entry.last_error, + ) + ) table = Table(title="Sia Code Index Status (Multi-Repo)") table.add_column("Property", style="cyan") table.add_column("Value", style="green") table.add_row("Workspace Index Path", str(sia_dir)) - table.add_row("Registered Repos", f"{len(multi_backends):,}") + table.add_row("Registered Repos", f"{len(repo_rows):,}") + table.add_row("Indexed Repos", f"{len(multi_backends):,}") table.add_row("Total Files", f"{total_files:,}") table.add_row("Total Chunks", f"{total_chunks:,}") + if status_counts: + table.add_row( + "Repo States", + ", ".join(f"{k}={v}" for k, v in sorted(status_counts.items())), + ) console.print(table) repo_table = Table(title="Per-Repo Status") repo_table.add_column("Repo", style="cyan") + repo_table.add_column("State") + repo_table.add_column("Est Chunks", justify="right") repo_table.add_column("Files", justify="right") repo_table.add_column("Chunks", justify="right") - for repo_name, files_n, chunks_n in repo_rows: - repo_table.add_row(repo_name, f"{files_n:,}", f"{chunks_n:,}") + repo_table.add_column("Last Error", overflow="fold") + for repo_name, state, est_chunks, files_n, chunks_n, last_error in sorted(repo_rows): + repo_table.add_row( + repo_name, + state, + f"{est_chunks:,}" if est_chunks else "-", + f"{files_n:,}", + f"{chunks_n:,}", + (last_error or "")[:80], + ) console.print(repo_table) return diff --git a/sia_code/config.py b/sia_code/config.py index a45185b..f00dc9d 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -198,6 +198,25 @@ class GitDynamicConfig(BaseModel): narrative_model: str | None = None # None = auto-select (large on 16GB+, base otherwise) +class RepoIndexOverride(BaseModel): + """Repo-specific indexing policy override for multi-repo workspaces.""" + + index_first: list[str] = Field(default_factory=list) + dependency_tier: list[str] = Field(default_factory=list) + lazy_index: list[str] = Field(default_factory=list) + skip: list[str] = Field(default_factory=list) + + +class MultiRepoConfig(BaseModel): + """Workspace-level multi-repo indexing controls.""" + + enabled: bool = True + fanout_concurrency: int = 1 + heavy_repo_chunk_threshold: int = 4000 + heavy_repo_run_dependency_tier: bool = False + repo_overrides: dict[str, RepoIndexOverride] = Field(default_factory=dict) + + class StorageConfig(BaseModel): """Storage backend selection configuration.""" @@ -220,6 +239,7 @@ class Config(BaseModel): summarization: SummarizationConfig = Field(default_factory=SummarizationConfig) storage: StorageConfig = Field(default_factory=StorageConfig) git_dynamic: GitDynamicConfig = Field(default_factory=GitDynamicConfig) + multi_repo: MultiRepoConfig = Field(default_factory=MultiRepoConfig) @classmethod def load(cls, path: Path) -> "Config": diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 2d0e490..c2ad863 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -14,6 +14,8 @@ import pathspec +from ..config import Config, RepoIndexOverride + logger = logging.getLogger(__name__) @@ -26,6 +28,9 @@ class RepoEntry: index_dir: str # relative path to .sia-code/ dir indexed_at: str | None = None file_count: int = 0 + estimated_chunks: int = 0 + status: str = "pending" + last_error: str | None = None @dataclass @@ -49,6 +54,9 @@ def save(self, registry_path: Path) -> None: "index_dir": r.index_dir, "indexed_at": r.indexed_at, "file_count": r.file_count, + "estimated_chunks": r.estimated_chunks, + "status": r.status, + "last_error": r.last_error, } for r in self.repos ], @@ -72,6 +80,9 @@ def load(cls, registry_path: Path) -> MultiRepoRegistry | None: index_dir=r["index_dir"], indexed_at=r.get("indexed_at"), file_count=r.get("file_count", 0), + estimated_chunks=r.get("estimated_chunks", 0), + status=r.get("status", "pending"), + last_error=r.get("last_error"), ) for r in data.get("repos", []) ], @@ -136,6 +147,68 @@ def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry ) +def get_repo_override(config: Config, repo_name: str) -> RepoIndexOverride | None: + """Return repo-specific indexing override, including built-in defaults.""" + override = config.multi_repo.repo_overrides.get(repo_name) + if override: + return override + + # Built-in performance policy for known heavy mixed repo. + if repo_name == "ai.platform.forks.ai-toolkit": + return RepoIndexOverride( + index_first=[ + "toolkit/**", + "jobs/**", + "ui/src/**", + "ui/cron/**", + "ui/prisma/schema.prisma", + "run.py", + "run_modal.py", + "flux_train_ui.py", + ], + dependency_tier=[ + "extensions_built_in/diffusion_models/**/src/**", + ], + lazy_index=[ + "config/examples/**", + "scripts/**", + "testing/**", + "docker/**", + "extensions/example/**", + ], + skip=[ + "output/**", + "assets/**", + "notebooks/**", + "ui/public/**", + "ui/package-lock.json", + "toolkit/keymaps/**", + ".github/**", + ".vscode/**", + ], + ) + return None + + +def build_repo_config(base_config: Config, repo_name: str) -> Config: + """Clone config and apply repo-specific override for faster indexing.""" + config = base_config.model_copy(deep=True) + override = get_repo_override(config, repo_name) + if not override: + return config + + if override.index_first: + config.indexing.include_patterns = override.index_first + + merged_excludes = list(config.indexing.exclude_patterns) + for group in (override.dependency_tier, override.lazy_index, override.skip): + for pattern in group: + if pattern not in merged_excludes: + merged_excludes.append(pattern) + config.indexing.exclude_patterns = merged_excludes + return config + + def estimate_indexable_files(directory: Path, config) -> int: """Estimate how many files will be indexed for timeout sizing. @@ -166,13 +239,55 @@ def estimate_indexable_files(directory: Path, config) -> int: return count -def recommend_repo_timeout_seconds(file_count: int) -> int: - """Compute per-repo timeout from estimated file count. +def estimate_chunks(directory: Path, config: Config) -> int: + """Estimate chunk count for timeout sizing. + + Uses real chunking against discovered files. This is cheap enough in practice + and much more accurate than file-count heuristics for monolithic repos. + """ + from ..core.types import Language + from ..parser.chunker import CASTChunker + + effective_patterns = config.indexing.get_effective_exclude_patterns(directory) + spec = pathspec.PathSpec.from_lines("gitwildmatch", effective_patterns) + max_bytes = config.indexing.max_file_size_mb * 1024 * 1024 + chunker = CASTChunker(config.chunking) + total = 0 + seen: set[Path] = set() + + for pattern in config.indexing.include_patterns: + glob_pattern = pattern if "*" in pattern else f"**/*{pattern}" + for file_path in directory.rglob(glob_pattern): + if not file_path.is_file() or file_path in seen: + continue + rel_path = file_path.relative_to(directory) + if spec.match_file(str(rel_path)): + continue + try: + file_size = file_path.stat().st_size + except OSError: + continue + if file_size == 0 or file_size > max_bytes: + continue + seen.add(file_path) + try: + language = Language.from_extension(file_path.suffix) + total += len(chunker.chunk_file(file_path, language)) + except Exception: + continue + return total + + +def recommend_repo_timeout_seconds(file_count: int, estimated_chunks: int = 0) -> int: + """Compute per-repo timeout. - Small repos keep 5m floor. Large repos scale up but stay bounded. + Prefer chunk-based sizing because embedding/storage dominates runtime for + large monolithic repos. Bounded to 5m..45m. """ + if estimated_chunks > 0: + seconds = int(120 + (estimated_chunks / 20.0)) + return max(300, min(2700, seconds)) if file_count <= 0: return 300 - # ~0.8s per file plus 60s overhead, bounded 5m..30m seconds = int(60 + file_count * 0.8) return max(300, min(1800, seconds)) From 7fc74a9fbcf617117248b2913c7713c6939b7612 Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 22:42:18 +0300 Subject: [PATCH 08/18] perf: add bounded fan-out concurrency for multi-repo indexing - run light repos concurrently using multi_repo.fanout_concurrency (default 2) - keep heavy repos serialized based on chunk threshold - preserve subprocess crash isolation and per-repo registry updates - default multi-repo concurrency to 2 for faster workspace indexing --- sia_code/cli.py | 170 ++++++++++++++++++++++++++++----------------- sia_code/config.py | 2 +- 2 files changed, 109 insertions(+), 63 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 5ffd0ef..53067a1 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -597,48 +597,49 @@ def index( import time as _time import subprocess as _sp import json as _json + from concurrent.futures import ThreadPoolExecutor, as_completed + plans = [] for i, repo_path in enumerate(sub_repos, 1): - console.print( - f"[cyan][{i}/{len(sub_repos)}] Indexing {repo_path.name}...[/cyan]" - ) - _t0 = _time.monotonic() - - # Store index under workspace: .sia-code/repos// repo_index_dir = workspace_sia / "repos" / repo_path.name repo_index_dir.mkdir(parents=True, exist_ok=True) - # Build per-repo config override and persist for this sub-index repo_config = build_repo_config(config, repo_path.name) repo_config_path = repo_index_dir / "config.json" repo_config.save(repo_config_path) - # Update registry to point to workspace-level index dir - for entry in registry.repos: - if entry.name == repo_path.name: - entry.index_dir = str( - (workspace_sia / "repos" / repo_path.name).relative_to(directory) - ) - break - - # Estimate size for timeout sizing and visibility estimated_files = estimate_indexable_files(repo_path, repo_config) estimated_chunks = estimate_chunks(repo_path, repo_config) repo_timeout = recommend_repo_timeout_seconds( estimated_files, estimated_chunks ) - console.print( - f" [dim]~{estimated_files} files, ~{estimated_chunks} chunks, timeout {repo_timeout}s[/dim]" - ) + is_heavy = estimated_chunks >= config.multi_repo.heavy_repo_chunk_threshold + for entry in registry.repos: if entry.name == repo_path.name: + entry.index_dir = str( + (workspace_sia / "repos" / repo_path.name).relative_to(directory) + ) entry.estimated_chunks = estimated_chunks - entry.status = "indexing" + entry.status = "pending" entry.last_error = None break - registry.save(registry_path) - # Index in subprocess (crash-isolated, dynamic timeout per repo) + plans.append( + { + "seq": i, + "repo_name": repo_path.name, + "repo_path": repo_path, + "repo_index_dir": repo_index_dir, + "estimated_files": estimated_files, + "estimated_chunks": estimated_chunks, + "repo_timeout": repo_timeout, + "heavy": is_heavy, + } + ) + registry.save(registry_path) + + def _run_repo_plan(plan: dict) -> dict: clean_flag = "True" if clean else "False" index_script = ( "import sys, json, os\n" @@ -647,8 +648,8 @@ def index( "from sia_code.config import Config\n" "from sia_code.cli import create_backend\n" "from sia_code.indexer.coordinator import IndexingCoordinator\n" - f"sia_dir = Path({str(repo_index_dir)!r})\n" - f"repo_dir = Path({str(repo_path)!r})\n" + f"sia_dir = Path({str(plan['repo_index_dir'])!r})\n" + f"repo_dir = Path({str(plan['repo_path'])!r})\n" f"do_clean = {clean_flag}\n" "config = Config.load(sia_dir / 'config.json')\n" "backend = create_backend(sia_dir, config, suppress_stdout_notices=True)\n" @@ -666,14 +667,15 @@ def index( "backend.close()\n" "print(json.dumps(stats))\n" ) + t0 = _time.monotonic() try: result = _sp.run( [sys.executable, "-c", index_script], capture_output=True, text=True, - timeout=repo_timeout, + timeout=plan["repo_timeout"], ) - _elapsed = _time.monotonic() - _t0 + elapsed = _time.monotonic() - t0 if result.returncode == 0: stdout_lines = result.stdout.strip().splitlines() stats_line = stdout_lines[-1] if stdout_lines else '{}' @@ -681,51 +683,95 @@ def index( stats = _json.loads(stats_line) except _json.JSONDecodeError: stats = {"indexed_files": 0, "total_chunks": 0} - all_stats.append(stats) - for entry in registry.repos: - if entry.name == repo_path.name: - entry.indexed_at = datetime.now(timezone.utc).isoformat() - entry.file_count = stats.get("indexed_files", 0) - entry.status = "full" - entry.last_error = None - break - registry.save(registry_path) - console.print( - f" [green]✓[/green] {stats.get('indexed_files', 0)} files, " - f"{stats.get('total_chunks', 0)} chunks " - f"[dim]({_elapsed:.1f}s)[/dim]" - ) - else: - err_lines = result.stderr.strip().splitlines() - err_msg = err_lines[-1][:120] if err_lines else "unknown" - for entry in registry.repos: - if entry.name == repo_path.name: - entry.status = "failed" - entry.last_error = err_msg - break - registry.save(registry_path) - console.print( - f" [red]✗ Failed ({_elapsed:.1f}s): {err_msg}[/red]" - ) + return {"kind": "ok", "plan": plan, "elapsed": elapsed, "stats": stats} + err_lines = result.stderr.strip().splitlines() + err_msg = err_lines[-1][:120] if err_lines else "unknown" + return {"kind": "failed", "plan": plan, "elapsed": elapsed, "error": err_msg} except _sp.TimeoutExpired: - _elapsed = _time.monotonic() - _t0 + elapsed = _time.monotonic() - t0 + return {"kind": "timed_out", "plan": plan, "elapsed": elapsed, "error": f"timeout after {plan['repo_timeout']}s"} + except Exception as e: + elapsed = _time.monotonic() - t0 + return {"kind": "failed", "plan": plan, "elapsed": elapsed, "error": str(e)[:120]} + + def _mark_started(plan: dict): + console.print( + f"[cyan][{plan['seq']}/{len(sub_repos)}] Indexing {plan['repo_name']}...[/cyan]" + ) + console.print( + f" [dim]~{plan['estimated_files']} files, ~{plan['estimated_chunks']} chunks, timeout {plan['repo_timeout']}s{' [heavy]' if plan['heavy'] else ''}[/dim]" + ) + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.status = 'indexing' + entry.last_error = None + break + registry.save(registry_path) + + def _record_result(result: dict): + plan = result['plan'] + if result['kind'] == 'ok': + stats = result['stats'] + all_stats.append(stats) for entry in registry.repos: - if entry.name == repo_path.name: - entry.status = "timed_out" - entry.last_error = f"timeout after {repo_timeout}s" + if entry.name == plan['repo_name']: + entry.indexed_at = datetime.now(timezone.utc).isoformat() + entry.file_count = stats.get('indexed_files', 0) + entry.status = 'full' + entry.last_error = None break registry.save(registry_path) console.print( - f" [yellow]⚠ Timeout ({_elapsed:.0f}s/{repo_timeout}s) — skipped[/yellow]" + f" [green]✓[/green] {stats.get('indexed_files', 0)} files, {stats.get('total_chunks', 0)} chunks [dim]({result['elapsed']:.1f}s)[/dim]" ) - except Exception as e: + elif result['kind'] == 'timed_out': for entry in registry.repos: - if entry.name == repo_path.name: - entry.status = "failed" - entry.last_error = str(e)[:120] + if entry.name == plan['repo_name']: + entry.status = 'timed_out' + entry.last_error = result['error'] break registry.save(registry_path) - console.print(f" [red]✗ Failed: {e}[/red]") + console.print( + f" [yellow]⚠ Timeout ({result['elapsed']:.0f}s/{plan['repo_timeout']}s) — skipped[/yellow]" + ) + else: + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.status = 'failed' + entry.last_error = result['error'] + break + registry.save(registry_path) + console.print( + f" [red]✗ Failed ({result['elapsed']:.1f}s): {result['error']}[/red]" + ) + + concurrency = max(1, int(config.multi_repo.fanout_concurrency)) + batch: list[dict] = [] + def _flush_batch(batch_plans: list[dict]): + if not batch_plans: + return + if len(batch_plans) == 1: + _mark_started(batch_plans[0]) + _record_result(_run_repo_plan(batch_plans[0])) + return + for p in batch_plans: + _mark_started(p) + with ThreadPoolExecutor(max_workers=min(concurrency, len(batch_plans))) as ex: + futs = [ex.submit(_run_repo_plan, p) for p in batch_plans] + for fut in as_completed(futs): + _record_result(fut.result()) + + for plan in plans: + if plan['heavy']: + _flush_batch(batch) + batch = [] + _flush_batch([plan]) + else: + batch.append(plan) + if len(batch) >= concurrency: + _flush_batch(batch) + batch = [] + _flush_batch(batch) # Save registry registry.save(registry_path) diff --git a/sia_code/config.py b/sia_code/config.py index f00dc9d..1017680 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -211,7 +211,7 @@ class MultiRepoConfig(BaseModel): """Workspace-level multi-repo indexing controls.""" enabled: bool = True - fanout_concurrency: int = 1 + fanout_concurrency: int = 2 heavy_repo_chunk_threshold: int = 4000 heavy_repo_run_dependency_tier: bool = False repo_overrides: dict[str, RepoIndexOverride] = Field(default_factory=dict) From 5c52813d4d0c67b364104c576773e852dd58fefd Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 22:55:35 +0300 Subject: [PATCH 09/18] perf: add CVAT-specific indexing policy and repo profiles - add lightweight repo profile awareness (general/data_science/annotation_platform) - add built-in CVAT indexing override for annotation-platform repos - prioritize cvat apps/core/ui/sdk first-party source - defer serverless/utils/cvat-cli and lazy-index tests/docs/helm/ai-models - exclude nested test trees with **/tests/** in CVAT first pass - surface repo profile in multi-repo status output --- sia_code/cli.py | 7 ++++- sia_code/storage/multi_repo.py | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 53067a1..8011f74 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -1515,6 +1515,7 @@ def status(): repo_rows.append( ( repo_name, + meta.profile if meta else "general", status_value, meta.estimated_chunks if meta else 0, stats.total_files, @@ -1529,6 +1530,7 @@ def status(): repo_rows.append( ( repo_name, + meta.profile if meta else "general", status_value, meta.estimated_chunks if meta else 0, 0, @@ -1544,6 +1546,7 @@ def status(): repo_rows.append( ( entry.name, + entry.profile, entry.status, entry.estimated_chunks, 0, @@ -1569,14 +1572,16 @@ def status(): repo_table = Table(title="Per-Repo Status") repo_table.add_column("Repo", style="cyan") + repo_table.add_column("Profile") repo_table.add_column("State") repo_table.add_column("Est Chunks", justify="right") repo_table.add_column("Files", justify="right") repo_table.add_column("Chunks", justify="right") repo_table.add_column("Last Error", overflow="fold") - for repo_name, state, est_chunks, files_n, chunks_n, last_error in sorted(repo_rows): + for repo_name, profile, state, est_chunks, files_n, chunks_n, last_error in sorted(repo_rows): repo_table.add_row( repo_name, + profile, state, f"{est_chunks:,}" if est_chunks else "-", f"{files_n:,}", diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index c2ad863..1494bfc 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -26,6 +26,7 @@ class RepoEntry: name: str path: str # relative to workspace root index_dir: str # relative path to .sia-code/ dir + profile: str = "general" indexed_at: str | None = None file_count: int = 0 estimated_chunks: int = 0 @@ -52,6 +53,7 @@ def save(self, registry_path: Path) -> None: "name": r.name, "path": r.path, "index_dir": r.index_dir, + "profile": r.profile, "indexed_at": r.indexed_at, "file_count": r.file_count, "estimated_chunks": r.estimated_chunks, @@ -78,6 +80,7 @@ def load(cls, registry_path: Path) -> MultiRepoRegistry | None: name=r["name"], path=r["path"], index_dir=r["index_dir"], + profile=r.get("profile", "general"), indexed_at=r.get("indexed_at"), file_count=r.get("file_count", 0), estimated_chunks=r.get("estimated_chunks", 0), @@ -128,6 +131,15 @@ def get_registry_path(workspace_root: Path) -> Path: return workspace_root / ".sia-code" / "multi-repo.json" +def get_repo_profile(repo_name: str) -> str: + """Classify known repo families for indexing-aware policy decisions.""" + if repo_name == "ai.platform.forks.ai-toolkit": + return "data_science" + if repo_name == "ai.platform.annotation-suite.cvat": + return "annotation_platform" + return "general" + + def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry: """Build a fresh registry from detected repos.""" entries = [] @@ -138,6 +150,7 @@ def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry name=repo_path.name, path=rel_path, index_dir=f".sia-code/repos/{repo_path.name}", + profile=get_repo_profile(repo_path.name), ) ) return MultiRepoRegistry( @@ -187,6 +200,42 @@ def get_repo_override(config: Config, repo_name: str) -> RepoIndexOverride | Non ".vscode/**", ], ) + + if repo_name == "ai.platform.annotation-suite.cvat": + return RepoIndexOverride( + index_first=[ + "cvat/apps/**", + "cvat/settings/**", + "cvat/utils/**", + "cvat-core/src/**", + "cvat-canvas/src/**", + "cvat-canvas3d/src/**", + "cvat-data/src/**", + "cvat-ui/src/**", + "cvat-sdk/cvat_sdk/**", + ], + dependency_tier=[ + "serverless/**", + "utils/**", + "cvat-cli/**", + ], + lazy_index=[ + "tests/**", + "**/tests/**", + "site/**", + "helm-chart/**", + "ai-models/**", + "backend_entrypoint.d/**", + "changelog.d/**", + ], + skip=[ + "cvat-ui/dist/**", + "cvat-sdk/gen/**", + ".github/**", + ".vscode/**", + ".regal/**", + ], + ) return None From 0658c362057626b5c644c21e0b86b94f5a4ac3cd Mon Sep 17 00:00:00 2001 From: dxta Date: Tue, 30 Jun 2026 23:26:33 +0300 Subject: [PATCH 10/18] perf: add profile-specific chunk sizing for heavy repos - use larger chunks for data_science repos - use even larger chunks for annotation_platform repos - reduce first-pass chunk volume for ai-toolkit and CVAT - keep repo-specific path policies while lowering embedding/storage cost --- sia_code/storage/multi_repo.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 1494bfc..3877fdd 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -242,19 +242,29 @@ def get_repo_override(config: Config, repo_name: str) -> RepoIndexOverride | Non def build_repo_config(base_config: Config, repo_name: str) -> Config: """Clone config and apply repo-specific override for faster indexing.""" config = base_config.model_copy(deep=True) + profile = get_repo_profile(repo_name) override = get_repo_override(config, repo_name) - if not override: - return config - - if override.index_first: - config.indexing.include_patterns = override.index_first - - merged_excludes = list(config.indexing.exclude_patterns) - for group in (override.dependency_tier, override.lazy_index, override.skip): - for pattern in group: - if pattern not in merged_excludes: - merged_excludes.append(pattern) - config.indexing.exclude_patterns = merged_excludes + if override: + if override.index_first: + config.indexing.include_patterns = override.index_first + + merged_excludes = list(config.indexing.exclude_patterns) + for group in (override.dependency_tier, override.lazy_index, override.skip): + for pattern in group: + if pattern not in merged_excludes: + merged_excludes.append(pattern) + config.indexing.exclude_patterns = merged_excludes + + # Profile-specific chunking for faster first-pass indexing. + # Heavy data-science / annotation repos benefit from fewer, larger chunks. + if profile == "data_science": + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1800) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 120) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.9) + elif profile == "annotation_platform": + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 2200) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 140) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.92) return config From bc00706659b81a3b4f850f166207955fb138d5e0 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 00:13:58 +0300 Subject: [PATCH 11/18] perf: accelerate heavy-repo indexing with selective semantic budget - add embedding granularity/budget config fields - use smaller bge-small model for heavy repo profiles when cached - add persistent content-addressed embedding cache across runs - embed only selected chunks per file for heavy profiles - pass batch size through daemon protocol/client/server - add safer daemon retry fallback in embedding path - keep vector writes stable by deduping selected chunk ids - add local heavy-repo benchmark script --- scripts/bench_heavy_repo_indexing.py | 65 +++++++ sia_code/cli.py | 17 +- sia_code/config.py | 6 + sia_code/embed_server/client.py | 2 +- sia_code/embed_server/daemon.py | 8 +- sia_code/embed_server/protocol.py | 4 +- sia_code/storage/multi_repo.py | 39 ++++- sia_code/storage/sqlite_vec_backend.py | 225 +++++++++++++++++++++---- 8 files changed, 313 insertions(+), 53 deletions(-) create mode 100644 scripts/bench_heavy_repo_indexing.py diff --git a/scripts/bench_heavy_repo_indexing.py b/scripts/bench_heavy_repo_indexing.py new file mode 100644 index 0000000..a7fc26d --- /dev/null +++ b/scripts/bench_heavy_repo_indexing.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import shutil +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from sia_code.cli import create_backend +from sia_code.config import Config +from sia_code.indexer.coordinator import IndexingCoordinator +from sia_code.storage.multi_repo import ( + build_repo_config, + estimate_chunks, + estimate_indexable_files, + estimate_semantic_vectors, + recommend_repo_timeout_seconds, +) + + +def bench_repo(repo_name: str) -> dict: + repo = Path.home() / 'dev' / 'ai.platform' / repo_name + out = Path.home() / 'dev' / 'ai.platform' / '.sia-code' / 'bench' / repo_name + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True, exist_ok=True) + + cfg = build_repo_config(Config(), repo.name) + cfg.save(out / 'config.json') + files = estimate_indexable_files(repo, cfg) + chunks = estimate_chunks(repo, cfg) + vectors = estimate_semantic_vectors(repo, cfg) + timeout = recommend_repo_timeout_seconds(files, vectors) + + backend = create_backend(out, cfg, suppress_stdout_notices=True) + backend.create_index() + coord = IndexingCoordinator(cfg, backend) + t0 = time.time() + stats = coord.index_directory(repo) + elapsed = time.time() - t0 + backend.close() + return { + 'repo': repo_name, + 'model': cfg.embedding.model, + 'dims': cfg.embedding.dimensions, + 'granularity': cfg.embedding.granularity, + 'max_vectors_per_file': cfg.embedding.max_vectors_per_file, + 'files': files, + 'chunks': chunks, + 'vectors': vectors, + 'timeout': timeout, + 'elapsed_s': round(elapsed, 2), + 'stats': stats, + } + + +if __name__ == '__main__': + repos = sys.argv[1:] or [ + 'ai.platform.forks.ai-toolkit', + 'ai.platform.annotation-suite.cvat', + ] + for repo_name in repos: + print(json.dumps(bench_repo(repo_name)), flush=True) diff --git a/sia_code/cli.py b/sia_code/cli.py index 8011f74..4ed9889 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -182,6 +182,10 @@ def create_backend( embedding_enabled=config.embedding.enabled, embedding_model=config.embedding.model, ndim=config.embedding.dimensions, + embedding_granularity=config.embedding.granularity, + max_vectors_per_file=config.embedding.max_vectors_per_file, + semantic_chunk_types=config.embedding.semantic_chunk_types, + persistent_embedding_cache=config.embedding.persistent_cache, valid_chunks=valid_chunks, ) @@ -562,6 +566,7 @@ def index( detect_sub_repos, estimate_chunks, estimate_indexable_files, + estimate_semantic_vectors, get_registry_path, is_multi_repo_workspace, recommend_repo_timeout_seconds, @@ -610,17 +615,18 @@ def index( estimated_files = estimate_indexable_files(repo_path, repo_config) estimated_chunks = estimate_chunks(repo_path, repo_config) + estimated_vectors = estimate_semantic_vectors(repo_path, repo_config) repo_timeout = recommend_repo_timeout_seconds( - estimated_files, estimated_chunks + estimated_files, estimated_vectors ) - is_heavy = estimated_chunks >= config.multi_repo.heavy_repo_chunk_threshold + is_heavy = estimated_vectors >= config.multi_repo.heavy_repo_chunk_threshold for entry in registry.repos: if entry.name == repo_path.name: entry.index_dir = str( (workspace_sia / "repos" / repo_path.name).relative_to(directory) ) - entry.estimated_chunks = estimated_chunks + entry.estimated_chunks = estimated_vectors entry.status = "pending" entry.last_error = None break @@ -632,7 +638,8 @@ def index( "repo_path": repo_path, "repo_index_dir": repo_index_dir, "estimated_files": estimated_files, - "estimated_chunks": estimated_chunks, + "estimated_chunks": estimated_vectors, + "raw_chunks": estimated_chunks, "repo_timeout": repo_timeout, "heavy": is_heavy, } @@ -699,7 +706,7 @@ def _mark_started(plan: dict): f"[cyan][{plan['seq']}/{len(sub_repos)}] Indexing {plan['repo_name']}...[/cyan]" ) console.print( - f" [dim]~{plan['estimated_files']} files, ~{plan['estimated_chunks']} chunks, timeout {plan['repo_timeout']}s{' [heavy]' if plan['heavy'] else ''}[/dim]" + f" [dim]~{plan['estimated_files']} files, ~{plan['raw_chunks']} chunks, ~{plan['estimated_chunks']} vectors, timeout {plan['repo_timeout']}s{' [heavy]' if plan['heavy'] else ''}[/dim]" ) for entry in registry.repos: if entry.name == plan['repo_name']: diff --git a/sia_code/config.py b/sia_code/config.py index 1017680..1d7e153 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -73,6 +73,12 @@ class EmbeddingConfig(BaseModel): model: str = "BAAI/bge-base-en-v1.5" # Model name (see supported models above) api_key_env: str = "" # Deprecated legacy field; ignored by local-only runtime dimensions: int = 768 # Embedding dimensions (auto-detected for most models) + granularity: Literal["chunk", "budget"] = "chunk" + max_vectors_per_file: int = 0 # 0 = unlimited + semantic_chunk_types: list[str] = Field( + default_factory=lambda: ["class", "function", "method", "definition"] + ) + persistent_cache: bool = True class IndexingConfig(BaseModel): diff --git a/sia_code/embed_server/client.py b/sia_code/embed_server/client.py index d3e68d6..c0435a2 100644 --- a/sia_code/embed_server/client.py +++ b/sia_code/embed_server/client.py @@ -151,7 +151,7 @@ def encode( # Create request request_id = str(uuid.uuid4()) - request = EmbedRequest.create(request_id, self.model_name, sentences) + request = EmbedRequest.create(request_id, self.model_name, sentences, batch_size=batch_size) # Send request response = self._send_request(request) diff --git a/sia_code/embed_server/daemon.py b/sia_code/embed_server/daemon.py index bdfafba..3bba02b 100644 --- a/sia_code/embed_server/daemon.py +++ b/sia_code/embed_server/daemon.py @@ -158,19 +158,20 @@ def _load_model(self, model_name: str) -> Any: return self.models[model_name] - def _handle_embed(self, model: str, texts: list[str]) -> dict: + def _handle_embed(self, model: str, texts: list[str], batch_size: int = 32) -> dict: """Handle embedding request. Args: model: Model name texts: List of texts to embed + batch_size: Preferred batch size from client Returns: Response dict with embeddings """ try: embedder = self._load_model(model) - vectors = embedder.encode(texts, convert_to_numpy=True, batch_size=32) + vectors = embedder.encode(texts, convert_to_numpy=True, batch_size=batch_size) return { "embeddings": vectors.tolist(), @@ -234,13 +235,14 @@ def _handle_connection(self, conn: socket.socket): params = request.get("params", {}) model = params.get("model") texts = params.get("texts", []) + batch_size = int(params.get("batch_size", 32) or 32) if not model or not texts: response = ErrorResponse.create( request_id, "Missing model or texts", "InvalidRequest" ) else: - result = self._handle_embed(model, texts) + result = self._handle_embed(model, texts, batch_size=batch_size) response = EmbedResponse.create( request_id, result["embeddings"], diff --git a/sia_code/embed_server/protocol.py b/sia_code/embed_server/protocol.py index 227aac0..327fa84 100644 --- a/sia_code/embed_server/protocol.py +++ b/sia_code/embed_server/protocol.py @@ -73,12 +73,12 @@ class EmbedRequest: """Embedding request message.""" @staticmethod - def create(request_id: str, model: str, texts: list[str]) -> dict: + def create(request_id: str, model: str, texts: list[str], batch_size: int = 32) -> dict: """Create embedding request.""" return { "id": request_id, "method": "embed", - "params": {"model": model, "texts": texts}, + "params": {"model": model, "texts": texts, "batch_size": batch_size}, } diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 3877fdd..c0ec4aa 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -19,6 +19,12 @@ logger = logging.getLogger(__name__) +def is_model_cached(model_name: str) -> bool: + """Return True if HuggingFace model appears cached locally.""" + hub_name = model_name.replace('/', '--') + return (Path.home() / '.cache' / 'huggingface' / 'hub' / f'models--{hub_name}').exists() + + @dataclass class RepoEntry: """A registered sub-repo in a multi-repo workspace.""" @@ -255,16 +261,27 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: merged_excludes.append(pattern) config.indexing.exclude_patterns = merged_excludes - # Profile-specific chunking for faster first-pass indexing. - # Heavy data-science / annotation repos benefit from fewer, larger chunks. + # Profile-specific chunking + embedding for faster first-pass indexing. + # Heavy data-science / annotation repos benefit from fewer, larger chunks + # plus a semantic budget and smaller embedding model when cached locally. if profile == "data_science": config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1800) config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 120) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.9) + config.embedding.granularity = "budget" + config.embedding.max_vectors_per_file = 16 + if is_model_cached("BAAI/bge-small-en-v1.5"): + config.embedding.model = "BAAI/bge-small-en-v1.5" + config.embedding.dimensions = 384 elif profile == "annotation_platform": config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 2200) config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 140) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.92) + config.embedding.granularity = "budget" + config.embedding.max_vectors_per_file = 12 + if is_model_cached("BAAI/bge-small-en-v1.5"): + config.embedding.model = "BAAI/bge-small-en-v1.5" + config.embedding.dimensions = 384 return config @@ -299,10 +316,16 @@ def estimate_indexable_files(directory: Path, config) -> int: def estimate_chunks(directory: Path, config: Config) -> int: - """Estimate chunk count for timeout sizing. + """Estimate raw chunk count for visibility / status.""" + return estimate_semantic_vectors(directory, config, raw_chunks_only=True) + + +def estimate_semantic_vectors( + directory: Path, config: Config, raw_chunks_only: bool = False +) -> int: + """Estimate semantic vectors to be embedded for timeout sizing. - Uses real chunking against discovered files. This is cheap enough in practice - and much more accurate than file-count heuristics for monolithic repos. + In budget mode, this counts vectors after per-file cap is applied. """ from ..core.types import Language from ..parser.chunker import CASTChunker @@ -331,7 +354,11 @@ def estimate_chunks(directory: Path, config: Config) -> int: seen.add(file_path) try: language = Language.from_extension(file_path.suffix) - total += len(chunker.chunk_file(file_path, language)) + count = len(chunker.chunk_file(file_path, language)) + if raw_chunks_only or config.embedding.granularity != "budget" or config.embedding.max_vectors_per_file <= 0: + total += count + else: + total += min(count, config.embedding.max_vectors_per_file) except Exception: continue return total diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index f88c8ed..aa7d4cf 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -102,6 +102,10 @@ def __init__( self.embedding_enabled = embedding_enabled self.embedding_model = embedding_model self.ndim = ndim + self.embedding_granularity = kwargs.pop("embedding_granularity", "chunk") + self.max_vectors_per_file = int(kwargs.pop("max_vectors_per_file", 0) or 0) + self.semantic_chunk_types = set(kwargs.pop("semantic_chunk_types", []) or []) + self.persistent_embedding_cache = bool(kwargs.pop("persistent_embedding_cache", True)) # Paths self.db_path = self.path / "index.db" @@ -122,6 +126,9 @@ def __init__( self._search_cache: dict[str, list] | None = None self._search_cache_enabled = False + # Persistent embedding cache (global across repos/runs) + self._embedding_cache_conn: sqlite3.Connection | None = None + self.mem = _MemoryAdapter(self) # Vector key prefixes for unified index @@ -278,22 +285,29 @@ def _serialize_vector(self, vector: np.ndarray) -> bytes: def _vector_insert(self, vector_id: int, vector: np.ndarray) -> None: """Insert or replace a vector embedding.""" + self._vector_insert_many([(vector_id, vector)]) + + def _vector_insert_many(self, items: list[tuple[int, np.ndarray]]) -> None: + """Bulk insert or replace vector embeddings.""" if self.conn is None: raise RuntimeError("Database connection not initialized") - if not self.embedding_enabled: + if not self.embedding_enabled or not items: return self._ensure_vector_table() - payload = self._serialize_vector(vector) cursor = self.conn.cursor() + rows = [(vector_id, self._serialize_vector(vector)) for vector_id, vector in items] if self._using_vec_extension: - cursor.execute( - "INSERT OR REPLACE INTO vectors(rowid, embedding) VALUES (?, ?)", - (vector_id, payload), - ) + # sqlite-vec virtual table path is safer with row-wise inserts. + # executemany + OR REPLACE can hit primary-key issues on some builds. + for row in rows: + cursor.execute( + "INSERT OR REPLACE INTO vectors(rowid, embedding) VALUES (?, ?)", + row, + ) else: - cursor.execute( + cursor.executemany( "INSERT OR REPLACE INTO vectors(id, embedding) VALUES (?, ?)", - (vector_id, payload), + rows, ) def _vector_search(self, query_vector: np.ndarray, k: int) -> list[tuple[str, float]]: @@ -440,6 +454,88 @@ def _get_thread_conn(self) -> sqlite3.Connection: self._local.conn = conn return self._local.conn + def _get_embedding_cache_conn(self) -> sqlite3.Connection | None: + """Open persistent embedding cache DB shared across repos/runs.""" + if not self.persistent_embedding_cache: + return None + if self._embedding_cache_conn is not None: + return self._embedding_cache_conn + try: + cache_dir = Path.home() / ".cache" / "sia-code" + cache_dir.mkdir(parents=True, exist_ok=True) + cache_db = cache_dir / "embedding-cache.sqlite3" + conn = sqlite3.connect(cache_db) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS embeddings_cache ( + cache_key TEXT PRIMARY KEY, + model TEXT NOT NULL, + embedding BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_embeddings_cache_model ON embeddings_cache(model)" + ) + conn.commit() + self._embedding_cache_conn = conn + return conn + except Exception: + return None + + def _embedding_cache_key(self, text: str) -> str: + """Stable key for persistent embedding cache.""" + import hashlib + + return hashlib.sha256(f"{self.embedding_model}\0{text}".encode("utf-8")).hexdigest() + + def _get_cached_embeddings(self, texts: list[str]) -> tuple[dict[int, np.ndarray], list[int], list[str]]: + """Return cached embeddings plus list of misses preserving order.""" + conn = self._get_embedding_cache_conn() + if conn is None or not texts: + return {}, list(range(len(texts))), list(texts) + + keys = [self._embedding_cache_key(t) for t in texts] + placeholders = ",".join("?" for _ in keys) + rows = conn.execute( + f"SELECT cache_key, embedding FROM embeddings_cache WHERE cache_key IN ({placeholders})", + keys, + ).fetchall() + cached = { + row[0]: np.frombuffer(row[1], dtype=np.float32).copy() for row in rows + } + hit_map: dict[int, np.ndarray] = {} + miss_idx: list[int] = [] + miss_texts: list[str] = [] + for i, key in enumerate(keys): + vec = cached.get(key) + if vec is None: + miss_idx.append(i) + miss_texts.append(texts[i]) + else: + hit_map[i] = vec + return hit_map, miss_idx, miss_texts + + def _put_cached_embeddings(self, texts: list[str], vectors: np.ndarray) -> None: + """Persist newly computed embeddings.""" + conn = self._get_embedding_cache_conn() + if conn is None or not texts: + return + rows = [ + ( + self._embedding_cache_key(text), + self.embedding_model, + np.asarray(vec, dtype=np.float32).tobytes(), + ) + for text, vec in zip(texts, vectors, strict=False) + ] + conn.executemany( + "INSERT OR REPLACE INTO embeddings_cache(cache_key, model, embedding) VALUES (?, ?, ?)", + rows, + ) + conn.commit() + def _embed(self, text: str) -> np.ndarray | None: """Embed text to vector with caching. @@ -526,26 +622,42 @@ def _embed_batch(self, texts: list[str]) -> np.ndarray | None: if not texts: return np.empty((0, self.ndim), dtype=np.float32) - embedder = self._get_embedder() - batch_size = self._get_embed_batch_size() - encoded = [] - - # Process in batches to avoid memory spikes - for idx in range(0, len(texts), batch_size): - batch = texts[idx : idx + batch_size] - vectors = embedder.encode( - batch, - batch_size=batch_size, - show_progress_bar=False, - convert_to_numpy=True, - ) - encoded.append(np.asarray(vectors, dtype=np.float32)) - - # Combine all batches - if len(encoded) == 1: - return encoded[0] - else: - return np.vstack(encoded) + # Persistent cache across runs/rebuilds + hit_map, miss_idx, miss_texts = self._get_cached_embeddings(texts) + out = np.empty((len(texts), self.ndim), dtype=np.float32) + for i, vec in hit_map.items(): + out[i] = vec + + if miss_texts: + batch_size = self._get_embed_batch_size() + encoded = [] + for idx in range(0, len(miss_texts), batch_size): + batch = miss_texts[idx : idx + batch_size] + last_error = None + for attempt in range(2): + embedder = self._get_embedder() + try: + vectors = embedder.encode( + batch, + batch_size=batch_size, + show_progress_bar=False, + convert_to_numpy=True, + ) + encoded.append(np.asarray(vectors, dtype=np.float32)) + last_error = None + break + except Exception as e: + last_error = e + # Daemon may have died after availability check; force re-resolve once. + self._embedder = None + if last_error is not None: + raise last_error + new_vectors = encoded[0] if len(encoded) == 1 else np.vstack(encoded) + self._put_cached_embeddings(miss_texts, new_vectors) + for pos, vec in zip(miss_idx, new_vectors, strict=False): + out[pos] = vec + + return out def _make_chunk_key(self, chunk_id: int) -> str: """Create vector index key for chunk.""" @@ -658,6 +770,10 @@ def close(self) -> None: self.conn.commit() self.conn.close() self.conn = None + if self._embedding_cache_conn is not None: + self._embedding_cache_conn.commit() + self._embedding_cache_conn.close() + self._embedding_cache_conn = None def seal(self) -> None: """Seal the index to finalize WAL and reduce storage. @@ -892,6 +1008,36 @@ def ensure_column(table: str, column: str, column_type: str) -> None: # Code Operations # =================================================================== + def _select_semantic_embedding_indices(self, chunks: list[Chunk]) -> list[int]: + """Select which chunks should receive semantic vectors. + + All chunks are still stored lexically/FTS. This only reduces vector count + for heavy repo profiles. + """ + if self.embedding_granularity != "budget" or self.max_vectors_per_file <= 0: + return list(range(len(chunks))) + + allowed_types = self.semantic_chunk_types or {"class", "function", "method", "definition"} + by_file: dict[str, list[tuple[int, Chunk]]] = {} + for i, chunk in enumerate(chunks): + by_file.setdefault(str(chunk.file_path), []).append((i, chunk)) + + selected: list[int] = [] + for items in by_file.values(): + preferred = [ + idx for idx, ch in items if getattr(ch.chunk_type, "value", str(ch.chunk_type)) in allowed_types + ] + fallback = [idx for idx, _ in items] + chosen = preferred[: self.max_vectors_per_file] + if len(chosen) < self.max_vectors_per_file: + for idx in fallback: + if idx not in chosen: + chosen.append(idx) + if len(chosen) >= self.max_vectors_per_file: + break + selected.extend(chosen) + return sorted(selected) + def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: """Store multiple code chunks. @@ -906,7 +1052,6 @@ def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: cursor = self.conn.cursor() chunk_ids: list[int] = [] - embed_texts: list[str] = [] # Phase 1: preserve stable IDs on conflict without REPLACE row churn for chunk in chunks: @@ -958,16 +1103,24 @@ def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: ) chunk_ids.append(chunk_id) - embed_texts.append(f"{chunk.symbol}\n\n{chunk.code}") - # Phase 2: Batch-embed all chunks (inserted or updated) + # Phase 2: Batch-embed selected chunks only (all chunks remain lexically searchable) if self.embedding_enabled and chunk_ids: try: - vectors = self._embed_batch(embed_texts) - - if vectors is not None: - for j, chunk_id in enumerate(chunk_ids): - self._vector_insert(int(chunk_id), vectors[j]) + selected_indices = self._select_semantic_embedding_indices(chunks) + if selected_indices: + embed_texts = [ + f"{chunks[idx].symbol}\n\n{chunks[idx].code}" for idx in selected_indices + ] + vectors = self._embed_batch(embed_texts) + if vectors is not None: + # Dedupe by chunk_id in case multiple chunks in the batch + # resolve to the same stable URI/id. + vector_map = { + int(chunk_ids[idx]): vectors[pos] + for pos, idx in enumerate(selected_indices) + } + self._vector_insert_many(list(vector_map.items())) except Exception: # Rollback SQLite inserts to avoid chunks without embeddings self.conn.rollback() From 254bc3875b3cf362c53e2b9ec92608031e7def87 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 00:24:48 +0300 Subject: [PATCH 12/18] tune: raise heavy-profile semantic vector budgets - data_science repos: 24 vectors/file - annotation_platform repos: 16 vectors/file - improves research recall while keeping timeout at 300s floor for current heavy profiles --- sia_code/storage/multi_repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index c0ec4aa..7c3ea67 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -269,7 +269,7 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 120) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.9) config.embedding.granularity = "budget" - config.embedding.max_vectors_per_file = 16 + config.embedding.max_vectors_per_file = 24 if is_model_cached("BAAI/bge-small-en-v1.5"): config.embedding.model = "BAAI/bge-small-en-v1.5" config.embedding.dimensions = 384 @@ -278,7 +278,7 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 140) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.92) config.embedding.granularity = "budget" - config.embedding.max_vectors_per_file = 12 + config.embedding.max_vectors_per_file = 16 if is_model_cached("BAAI/bge-small-en-v1.5"): config.embedding.model = "BAAI/bge-small-en-v1.5" config.embedding.dimensions = 384 From 89534b4910066e95fc905a63359b5bbd8ed1ddc3 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 00:47:29 +0300 Subject: [PATCH 13/18] fix: restore multi-repo search and research results - fix multi-repo search dropping all results due to immutable chunk mutation - make CLI research multi-repo aware by aggregating repo-local research results - use hybrid seeding for budgeted indexes in MultiHopSearchStrategy - normalize displayed paths to repo-relative multi-repo paths --- sia_code/cli.py | 87 +++++++++++++++++++++++++++++------- sia_code/search/multi_hop.py | 7 ++- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 4ed9889..3c9161b 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -1132,14 +1132,26 @@ def _search_one_backend(be): if _multi_repo_mode: # Aggregate results from all repos all_results = [] + from dataclasses import replace + from pathlib import Path as _Path + for repo_name, be in multi_backends: try: repo_results = _search_one_backend(be) - # Prefix file paths with repo name for disambiguation + rewritten = [] + repo_root = Path.cwd() / repo_name for r in repo_results: - if hasattr(r, "chunk") and hasattr(r.chunk, "file_path"): - r.chunk.file_path = f"{repo_name}/{r.chunk.file_path}" - all_results.extend(repo_results) + try: + orig_path = Path(r.chunk.file_path) + rel_path = orig_path.relative_to(repo_root) if orig_path.is_absolute() else orig_path + except Exception: + rel_path = Path(r.chunk.file_path).name + new_chunk = replace( + r.chunk, + file_path=_Path(repo_name) / rel_path, + ) + rewritten.append(replace(r, chunk=new_chunk)) + all_results.extend(rewritten) except Exception: pass # Sort by score descending, take top `limit` @@ -1429,20 +1441,65 @@ def research(question: str, hops: int, graph: bool, limit: int, no_filter: bool) except Exception: pass # Silently fall back to no filtering - backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) - backend.open_index() - - strategy = MultiHopSearchStrategy(backend, max_hops=hops) - + multi_backends = get_multi_repo_backends() console.print(f"[dim]Researching: {question}[/dim]") console.print(f"[dim]Max hops: {hops}, Results per hop: {limit}[/dim]\n") - with Progress( - SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console - ) as progress: - task = progress.add_task("Analyzing code relationships...", total=None) - result = strategy.research(question, max_results_per_hop=limit) - progress.update(task, completed=True) + if multi_backends: + from dataclasses import replace + from pathlib import Path as _Path + + combined_chunks = [] + combined_relationships = [] + max_hops_executed = 0 + total_entities_found = 0 + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console + ) as progress: + task = progress.add_task("Analyzing code relationships across repos...", total=None) + for repo_name, backend in multi_backends: + try: + strategy = MultiHopSearchStrategy(backend, max_hops=hops) + repo_result = strategy.research(question, max_results_per_hop=limit) + if repo_result.chunks: + repo_root = Path.cwd() / repo_name + rewritten_chunks = [] + for chunk in repo_result.chunks: + try: + orig_path = Path(chunk.file_path) + rel_path = orig_path.relative_to(repo_root) if orig_path.is_absolute() else orig_path + except Exception: + rel_path = Path(chunk.file_path).name + rewritten_chunks.append( + replace(chunk, file_path=_Path(repo_name) / rel_path) + ) + combined_chunks.extend(rewritten_chunks) + combined_relationships.extend(repo_result.relationships) + max_hops_executed = max(max_hops_executed, repo_result.hops_executed) + total_entities_found += repo_result.total_entities_found + except Exception: + pass + progress.update(task, completed=True) + + from .search.multi_hop import ResearchResult + result = ResearchResult( + question=question, + chunks=combined_chunks[: max(10, limit * 4)], + relationships=combined_relationships, + hops_executed=max_hops_executed, + total_entities_found=total_entities_found, + ) + else: + backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) + backend.open_index() + strategy = MultiHopSearchStrategy(backend, max_hops=hops) + + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console + ) as progress: + task = progress.add_task("Analyzing code relationships...", total=None) + result = strategy.research(question, max_results_per_hop=limit) + progress.update(task, completed=True) # Display results summary console.print("\n[bold green]✓ Research Complete[/bold green]") diff --git a/sia_code/search/multi_hop.py b/sia_code/search/multi_hop.py index 09dab14..4d08424 100644 --- a/sia_code/search/multi_hop.py +++ b/sia_code/search/multi_hop.py @@ -62,11 +62,16 @@ def _initial_search(self, question: str, k: int) -> list: """ if self.backend.embedding_enabled: try: + granularity = getattr(self.backend, "embedding_granularity", "chunk") + if granularity == "budget": + logger.info(f"Using hybrid search for budgeted index query: {question[:100]}") + return self.backend.search_hybrid(question, k=k, vector_weight=0.35) + logger.info(f"Using semantic search for query: {question[:100]}") return self.backend.search_semantic(question, k=k) except Exception as e: logger.warning( - f"Semantic search failed ({e.__class__.__name__}: {str(e)}), " + f"Semantic/hybrid search failed ({e.__class__.__name__}: {str(e)}), " "falling back to lexical search" ) # Fall through to lexical search From b1c1a0f3537a544a8935b5d7ea1357dcc6832133 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 01:04:35 +0300 Subject: [PATCH 14/18] feat: improve budgeted semantic selection and research query rewrites - rank budgeted semantic vectors by chunk type and role hints per file - prefer class/method/function/definition chunks over low-value comments/blocks - add query variant expansion for research seeding - synthesize code-like identifiers (CamelCase/snake_case/API forms) - aggregate 2-3 rewritten seed queries in MultiHopSearchStrategy - use hybrid seeding for budgeted indexes --- sia_code/search/multi_hop.py | 78 ++++++++++++++------- sia_code/search/query_preprocessor.py | 97 ++++++++++++++++++++++++++ sia_code/storage/sqlite_vec_backend.py | 40 +++++++---- tests/unit/test_multi_hop.py | 21 ++++++ tests/unit/test_query_preprocessor.py | 23 ++++++ 5 files changed, 223 insertions(+), 36 deletions(-) diff --git a/sia_code/search/multi_hop.py b/sia_code/search/multi_hop.py index 4d08424..25d2b25 100644 --- a/sia_code/search/multi_hop.py +++ b/sia_code/search/multi_hop.py @@ -1,10 +1,10 @@ """Multi-hop code research for discovering code relationships.""" import logging -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Set -from ..core.models import Chunk, CodeRelationshipRecord +from ..core.models import Chunk, CodeRelationshipRecord, SearchResult from ..core.types import ChunkId from ..storage.base import StorageBackend from .entity_extractor import EntityExtractor, Entity @@ -50,39 +50,69 @@ def __init__(self, backend: StorageBackend, max_hops: int = 2): self.extractor = EntityExtractor() self._preprocessor = QueryPreprocessor() # Cache instance to avoid recreation - def _initial_search(self, question: str, k: int) -> list: - """Perform initial search with adaptive mode selection. - - Args: - question: Natural language question - k: Number of results to return + def _aggregate_seed_results(self, result_sets: list[list[SearchResult]], k: int) -> list[SearchResult]: + """Merge result sets from multiple query variants. - Returns: - List of search results + Uses chunk identity when available, otherwise file/symbol/line tuple. + Rewards repeated hits modestly without overwhelming score scales. """ + merged: dict[str, tuple[SearchResult, float, int]] = {} + for results in result_sets: + for r in results: + key = str(r.chunk.id) if r.chunk.id else f"{r.chunk.file_path}:{r.chunk.symbol}:{r.chunk.start_line}:{r.chunk.end_line}" + if key in merged: + base, best_score, hits = merged[key] + merged[key] = (base, max(best_score, r.score), hits + 1) + else: + merged[key] = (r, r.score, 1) + + ranked: list[SearchResult] = [] + for base, best_score, hits in merged.values(): + combined = best_score + (0.03 * (hits - 1)) + ranked.append(replace(base, score=combined)) + ranked.sort(key=lambda x: x.score, reverse=True) + return ranked[:k] + + def _initial_search(self, question: str, k: int) -> list: + """Perform initial search with adaptive mode selection and query rewrites.""" + variants = self._preprocessor.expand_variants(question) + if not variants: + variants = [question] + if self.backend.embedding_enabled: try: granularity = getattr(self.backend, "embedding_granularity", "chunk") + result_sets: list[list[SearchResult]] = [] + if granularity == "budget": - logger.info(f"Using hybrid search for budgeted index query: {question[:100]}") - return self.backend.search_hybrid(question, k=k, vector_weight=0.35) + logger.info( + f"Using hybrid multi-variant search for budgeted index query: {question[:100]}" + ) + for i, variant in enumerate(variants): + vw = 0.4 if i == 0 else 0.3 + result_sets.append( + self.backend.search_hybrid(variant, k=max(k, 8), vector_weight=vw) + ) + return self._aggregate_seed_results(result_sets, k) - logger.info(f"Using semantic search for query: {question[:100]}") - return self.backend.search_semantic(question, k=k) + logger.info(f"Using semantic+hybrid variant search for query: {question[:100]}") + result_sets.append(self.backend.search_semantic(variants[0], k=max(k, 8))) + if len(variants) > 1: + result_sets.append( + self.backend.search_hybrid(variants[1], k=max(k, 8), vector_weight=0.5) + ) + if len(variants) > 2: + result_sets.append(self.backend.search_lexical(variants[2], k=max(k, 6))) + return self._aggregate_seed_results(result_sets, k) except Exception as e: logger.warning( - f"Semantic/hybrid search failed ({e.__class__.__name__}: {str(e)}), " + f"Semantic/hybrid variant search failed ({e.__class__.__name__}: {str(e)}), " "falling back to lexical search" ) - # Fall through to lexical search - - # Lexical search path - logger.info(f"Using lexical search for query: {question[:100]}") - processed_query = self._preprocessor.preprocess(question) - search_query = processed_query or question - if not processed_query: - logger.debug(f"Preprocessing returned empty for query: {question[:100]}") - return self.backend.search_lexical(search_query, k=k) + + logger.info(f"Using lexical variant search for query: {question[:100]}") + result_sets = [self.backend.search_lexical(v, k=max(k, 8)) for v in variants] + return self._aggregate_seed_results(result_sets, k) def _expand_from_persisted_graph( self, diff --git a/sia_code/search/query_preprocessor.py b/sia_code/search/query_preprocessor.py index 720cca8..abe8a97 100644 --- a/sia_code/search/query_preprocessor.py +++ b/sia_code/search/query_preprocessor.py @@ -106,6 +106,103 @@ def preprocess(self, question: str) -> str: # Rejoin with spaces return " ".join(keywords) + def expand_variants(self, question: str) -> list[str]: + """Produce 2-3 query variants for research seeding. + + Variants aim to improve recall for code search by combining: + - raw natural language + - cleaned keyword query + - synthesized code-like identifiers / framework-role forms + """ + if not question or not question.strip(): + return [] + + variants: list[str] = [] + raw = question.strip() + variants.append(raw) + + keywords = self.extract_keywords(question) + keyword_query = " ".join(keywords) + if keyword_query and keyword_query.lower() != raw.lower(): + variants.append(keyword_query) + + focused_parts: list[str] = [] + identifiers = [t for t in keywords if self._is_code_identifier(t)] + plain = [t for t in keywords if not self._is_code_identifier(t)] + focused_parts.extend(identifiers[:4]) + focused_parts.extend(self._synthesize_code_forms(keywords)) + focused_parts.extend([t for t in plain if len(t) >= 4 and t.lower() not in {'work', 'works'}][:4]) + focused = " ".join(dict.fromkeys(p for p in focused_parts if p)).strip() + if focused and focused.lower() not in {v.lower() for v in variants}: + variants.append(focused) + + deduped: list[str] = [] + seen: set[str] = set() + for v in variants: + key = v.lower().strip() + if key and key not in seen: + deduped.append(v) + seen.add(key) + return deduped[:3] + + def _synthesize_code_forms(self, tokens: list[str]) -> list[str]: + """Generate code-like identifier variants from plain-language tokens. + + Examples: + - stable diffusion trainer -> StableDiffusionTrainer, stable_diffusion_trainer + - task rest api -> TaskViewSet, TaskSerializer, TaskAPIView + """ + if not tokens: + return [] + + lowered = [t.lower() for t in tokens] + generic_noise = {'work', 'works', 'working'} + results: list[str] = [] + + role_terms = { + "trainer": "Trainer", + "config": "Config", + "service": "Service", + "manager": "Manager", + "controller": "Controller", + "model": "Model", + "serializer": "Serializer", + "viewset": "ViewSet", + "handler": "Handler", + "pipeline": "Pipeline", + } + + role_idx = next((i for i, t in enumerate(lowered) if t in role_terms), None) + if role_idx is not None and role_idx > 0: + base_tokens = lowered[: role_idx + 1] + camel = "".join(t.capitalize() for t in base_tokens) + snake = "_".join(base_tokens) + results.extend([camel, snake]) + + # API-ish expansion: task REST API -> TaskViewSet / TaskSerializer / TaskAPIView + api_tokens = {"api", "rest", "endpoint"} + if any(t in api_tokens for t in lowered): + base = next((t for t in lowered if t not in api_tokens and t not in generic_noise and len(t) > 2), None) + if base: + title = base.capitalize() + results.extend([f"{title}ViewSet", f"{title}Serializer", f"{title}APIView"]) + + # Generic multi-token camel/snake for first 2-3 meaningful words + content = [t for t in lowered if len(t) > 2 and t not in api_tokens and t not in generic_noise][:3] + if len(content) >= 2: + results.append("".join(t.capitalize() for t in content)) + results.append("_".join(content)) + + # Keep small and unique + out: list[str] = [] + seen: set[str] = set() + for r in results: + k = r.lower() + if r and k not in seen: + out.append(r) + seen.add(k) + return out[:6] + def extract_keywords(self, question: str) -> list[str]: """Extract meaningful keywords from a question. diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index aa7d4cf..c199994 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -1012,29 +1012,45 @@ def _select_semantic_embedding_indices(self, chunks: list[Chunk]) -> list[int]: """Select which chunks should receive semantic vectors. All chunks are still stored lexically/FTS. This only reduces vector count - for heavy repo profiles. + for heavy repo profiles. Selection is type-prioritized within each file. """ if self.embedding_granularity != "budget" or self.max_vectors_per_file <= 0: return list(range(len(chunks))) allowed_types = self.semantic_chunk_types or {"class", "function", "method", "definition"} + type_priority = { + "class": 100, + "method": 95, + "function": 90, + "definition": 88, + "structure": 60, + "import": 45, + "block": 35, + "call": 25, + "docstring": 18, + "comment": 10, + "unknown": 5, + } + + def _chunk_score(ch: Chunk) -> tuple[int, int, int, int]: + t = getattr(ch.chunk_type, "value", str(ch.chunk_type)) + base = type_priority.get(t, 0) + if t in allowed_types: + base += 1000 + symbol = (ch.symbol or "").lower() + if any(k in symbol for k in ("view", "viewset", "api", "handler", "endpoint", "trainer", "process", "service", "serializer", "model")): + base += 20 + span = max(0, int(ch.end_line) - int(ch.start_line)) + return (base, min(span, 400), 1 if ch.parent_header else 0, -int(ch.start_line)) + by_file: dict[str, list[tuple[int, Chunk]]] = {} for i, chunk in enumerate(chunks): by_file.setdefault(str(chunk.file_path), []).append((i, chunk)) selected: list[int] = [] for items in by_file.values(): - preferred = [ - idx for idx, ch in items if getattr(ch.chunk_type, "value", str(ch.chunk_type)) in allowed_types - ] - fallback = [idx for idx, _ in items] - chosen = preferred[: self.max_vectors_per_file] - if len(chosen) < self.max_vectors_per_file: - for idx in fallback: - if idx not in chosen: - chosen.append(idx) - if len(chosen) >= self.max_vectors_per_file: - break + ranked = sorted(items, key=lambda it: _chunk_score(it[1]), reverse=True) + chosen = [idx for idx, _ in ranked[: self.max_vectors_per_file]] selected.extend(chosen) return sorted(selected) diff --git a/tests/unit/test_multi_hop.py b/tests/unit/test_multi_hop.py index f12bccb..f2a953e 100644 --- a/tests/unit/test_multi_hop.py +++ b/tests/unit/test_multi_hop.py @@ -500,6 +500,27 @@ def mock_search_lexical(query, *args, **kwargs): first_query = calls[0] assert "how" not in first_query.lower() or "main" in first_query.lower() + def test_uses_hybrid_variants_when_budgeted_embeddings_enabled(self, backend, sample_chunks): + """Budgeted indexes should seed research via multiple hybrid query variants.""" + backend.store_chunks_batch(sample_chunks) + backend.embedding_enabled = True + backend.embedding_granularity = "budget" + + original_search_hybrid = backend.search_hybrid + calls = [] + + def mock_search_hybrid(query, *args, **kwargs): + calls.append(query) + return original_search_hybrid(query, *args, **kwargs) + + backend.search_hybrid = mock_search_hybrid + + strategy = MultiHopSearchStrategy(backend, max_hops=1) + strategy.research("How does load_config work in main flow?", max_results_per_hop=5) + + assert len(calls) >= 2 + assert any("load_config" in q for q in calls) + class TestNaturalLanguageQueries: """Test that research handles natural language questions.""" diff --git a/tests/unit/test_query_preprocessor.py b/tests/unit/test_query_preprocessor.py index d8aa71e..8725692 100644 --- a/tests/unit/test_query_preprocessor.py +++ b/tests/unit/test_query_preprocessor.py @@ -89,6 +89,29 @@ def test_multiple_code_identifiers(self, preprocessor): assert "load_config" in result assert "use" in result + def test_expand_variants_returns_2_to_3_unique_forms(self, preprocessor): + """Expanded variants should keep raw + focused forms without duplicates.""" + variants = preprocessor.expand_variants( + "How does ChipCountingService use load_config in authentication flow?" + ) + assert 2 <= len(variants) <= 3 + assert variants[0] == "How does ChipCountingService use load_config in authentication flow?" + assert any("ChipCountingService" in v for v in variants) + assert any("load_config" in v for v in variants) + assert len({v.lower() for v in variants}) == len(variants) + + def test_expand_variants_synthesizes_code_forms(self, preprocessor): + variants = preprocessor.expand_variants( + "How does stable diffusion trainer config work?" + ) + combined = " ".join(variants) + assert "StableDiffusionTrainer" in combined or "stable_diffusion_trainer" in combined + + def test_expand_variants_synthesizes_api_forms(self, preprocessor): + variants = preprocessor.expand_variants("How does task REST API work?") + combined = " ".join(variants) + assert any(x in combined for x in ["TaskViewSet", "TaskSerializer", "TaskAPIView"]) + class TestExtractKeywords: """Test keyword extraction specifically.""" From 144c63f9406af3eb47355c68a8e788fc134a17d9 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 01:17:38 +0300 Subject: [PATCH 15/18] perf: apply fast indexing baseline to all repos by default - baseline fast policy for any repo: larger chunks, higher merge threshold, budgeted semantic indexing - default general repos to 32 vectors/file under budget mode - keep stronger profile-specific tuning for data_science and annotation_platform repos --- sia_code/storage/multi_repo.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 7c3ea67..cd5c770 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -246,10 +246,25 @@ def get_repo_override(config: Config, repo_name: str) -> RepoIndexOverride | Non def build_repo_config(base_config: Config, repo_name: str) -> Config: - """Clone config and apply repo-specific override for faster indexing.""" + """Clone config and apply fast indexing policy by default. + + Baseline fast policy applies to any repo, then known profiles add stronger + tuning and repo-specific include/exclude overrides. + """ config = base_config.model_copy(deep=True) profile = get_repo_profile(repo_name) override = get_repo_override(config, repo_name) + + # Baseline fast policy for ALL repos. + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1400) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 80) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.85) + config.embedding.granularity = "budget" + if config.embedding.max_vectors_per_file <= 0: + config.embedding.max_vectors_per_file = 32 + else: + config.embedding.max_vectors_per_file = min(config.embedding.max_vectors_per_file, 32) + if override: if override.index_first: config.indexing.include_patterns = override.index_first @@ -268,7 +283,6 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1800) config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 120) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.9) - config.embedding.granularity = "budget" config.embedding.max_vectors_per_file = 24 if is_model_cached("BAAI/bge-small-en-v1.5"): config.embedding.model = "BAAI/bge-small-en-v1.5" @@ -277,7 +291,6 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 2200) config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 140) config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.92) - config.embedding.granularity = "budget" config.embedding.max_vectors_per_file = 16 if is_model_cached("BAAI/bge-small-en-v1.5"): config.embedding.model = "BAAI/bge-small-en-v1.5" From ecef62f691b696e78af786acaccfb08d2b9a79fd Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 09:02:23 +0300 Subject: [PATCH 16/18] fix: improve research rewrite flow and address PR feedback - make FLAN query rewrite opt-in via config and cached-only - cache FLAN rewrite summarizer in-process when enabled - keep heuristic+BGE-assisted rewrite as default path - gate multi-repo fan-out on multi_repo.enabled - honor heavy_repo_run_dependency_tier in repo config - use incremental indexing for multi-repo update subprocesses - clean stale backend artifacts before multi-repo clean rebuilds - keep git-context JSON output parseable with structured warnings - resolve remote base branches like origin/main - gate FLAN narrative use on local model cache --- sia_code/cli.py | 68 +++++++++++++++++++++----- sia_code/config.py | 1 + sia_code/memory/diff_analyzer.py | 7 +-- sia_code/memory/git_dynamic.py | 11 +++-- sia_code/search/multi_hop.py | 34 ++++++++----- sia_code/search/query_preprocessor.py | 51 +++++++++++++++++-- sia_code/storage/multi_repo.py | 5 +- sia_code/storage/sqlite_vec_backend.py | 1 + 8 files changed, 140 insertions(+), 38 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 3c9161b..dc878d3 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -186,6 +186,7 @@ def create_backend( max_vectors_per_file=config.embedding.max_vectors_per_file, semantic_chunk_types=config.embedding.semantic_chunk_types, persistent_embedding_cache=config.embedding.persistent_cache, + flan_query_rewrite=config.search.flan_query_rewrite, valid_chunks=valid_chunks, ) @@ -572,7 +573,7 @@ def index( recommend_repo_timeout_seconds, ) - if is_multi_repo_workspace(directory): + if config.multi_repo.enabled and is_multi_repo_workspace(directory): sub_repos = detect_sub_repos(directory) console.print( f"[cyan]Detected multi-repo workspace with {len(sub_repos)} repos[/cyan]" @@ -648,6 +649,7 @@ def index( def _run_repo_plan(plan: dict) -> dict: clean_flag = "True" if clean else "False" + update_flag = "True" if update else "False" index_script = ( "import sys, json, os\n" "os.environ.setdefault('HF_HUB_OFFLINE', '1')\n" @@ -658,11 +660,14 @@ def _run_repo_plan(plan: dict) -> dict: f"sia_dir = Path({str(plan['repo_index_dir'])!r})\n" f"repo_dir = Path({str(plan['repo_path'])!r})\n" f"do_clean = {clean_flag}\n" + f"do_update = {update_flag}\n" "config = Config.load(sia_dir / 'config.json')\n" + "if do_clean:\n" + " for stale in ('index.db', 'vectors.usearch'):\n" + " fp = sia_dir / stale\n" + " if fp.exists(): fp.unlink()\n" "backend = create_backend(sia_dir, config, suppress_stdout_notices=True)\n" "if do_clean:\n" - " idx = sia_dir / 'index.db'\n" - " if idx.exists(): idx.unlink()\n" " backend.create_index()\n" "else:\n" " try:\n" @@ -670,7 +675,14 @@ def _run_repo_plan(plan: dict) -> dict: " except Exception:\n" " backend.create_index()\n" "coord = IndexingCoordinator(config, backend)\n" - "stats = coord.index_directory(repo_dir)\n" + "if do_update:\n" + " from sia_code.indexer.hash_cache import HashCache\n" + " from sia_code.indexer.chunk_index import ChunkIndex\n" + " cache = HashCache(sia_dir / 'cache' / 'file_hashes.json')\n" + " chunk_index = ChunkIndex(sia_dir / 'chunk_index.json')\n" + " stats = coord.index_directory_incremental_v2(repo_dir, cache, chunk_index)\n" + "else:\n" + " stats = coord.index_directory(repo_dir)\n" "backend.close()\n" "print(json.dumps(stats))\n" ) @@ -789,6 +801,32 @@ def _flush_batch(batch_plans: list[dict]): console.print(f" Total: {total_files} files, {total_chunks} chunks across {len(all_stats)} repos") return + # --- Single-repo indexing with repo-aware profile overrides --- + repo_config = build_repo_config(config, directory.name) + if repo_config != config: + config = repo_config + # Persist effective repo config for transparency and consistent subsequent commands + try: + config.save(sia_dir / "config.json") + except Exception: + pass + try: + backend.close() + except Exception: + pass + backend = create_backend(sia_dir, config) + index_path = sia_dir / "index.db" + if clean: + if index_path.exists(): + index_path.unlink() + backend.create_index() + else: + try: + backend.open_index() + except Exception: + backend.create_index() + coordinator = IndexingCoordinator(config, backend) + # --- Single-repo indexing (original flow) --- if update: @@ -2905,10 +2943,13 @@ def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format) classifier = IntentClassifier() all_results = {} + warnings = [] for fp in file_paths: hist = mem.file_history(fp, cross_branch=gc.cross_branch_enabled, limit=15) if not hist.effective_commits: - console.print(f"[yellow]No history found for {fp}[/yellow]") + warnings.append(f"No history found for {fp}") + if output_format != "json": + console.print(f"[yellow]No history found for {fp}[/yellow]") continue # Classify intents @@ -2939,14 +2980,10 @@ def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format) all_results[fp] = entry - if not all_results: - console.print("[red]No results found.[/red]") - return - if output_format == "json": import json - data = {} + data = {"files": {}, "warnings": warnings} for fp, entry in all_results.items(): hist = entry["hist"] d = { @@ -2979,10 +3016,15 @@ def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format) d["narrative"] = entry["narrative"].narrative d["phases"] = entry["narrative"].key_phases d["model_used"] = entry["narrative"].model_used - data[fp] = d + data["files"][fp] = d console.print(json.dumps(data, indent=2)) - else: - # Table format + return + + if not all_results: + console.print("[red]No results found.[/red]") + return + + # Table format for fp, entry in all_results.items(): hist = entry["hist"] console.print(f"\n[bold]{'='*60}[/bold]") diff --git a/sia_code/config.py b/sia_code/config.py index 1d7e153..32692f6 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -141,6 +141,7 @@ class SearchConfig(BaseModel): default_limit: int = 10 multi_hop_enabled: bool = True max_hops: int = 2 + flan_query_rewrite: bool = False # optional, cached-only FLAN extra rewrite candidate vector_weight: float = ( 0.7 # Weight for vector search in hybrid (0.0=lexical only, 1.0=semantic only) ) diff --git a/sia_code/memory/diff_analyzer.py b/sia_code/memory/diff_analyzer.py index 50db41c..5e688d0 100644 --- a/sia_code/memory/diff_analyzer.py +++ b/sia_code/memory/diff_analyzer.py @@ -21,6 +21,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from ..storage.multi_repo import is_model_cached + if TYPE_CHECKING: from .git_dynamic import FileHistory, HistoricalCommit @@ -90,12 +92,11 @@ def __init__(self, repo_path: Path, model_name: str | None = None): self._summarizer = None def _can_use_model(self) -> bool: - """Cheap import check — auto-opt-in.""" + """Only use model when transformers is importable AND weights are cached.""" if not self._model_checked: try: import transformers # noqa: F401 - - self._can_model = True + self._can_model = is_model_cached(self._get_model_name()) except ImportError: self._can_model = False self._model_checked = True diff --git a/sia_code/memory/git_dynamic.py b/sia_code/memory/git_dynamic.py index 2bd14a4..7bb85ad 100644 --- a/sia_code/memory/git_dynamic.py +++ b/sia_code/memory/git_dynamic.py @@ -138,14 +138,15 @@ def current_branch(self) -> str: return short or "unknown" def resolve_base_branch(self) -> str: - """Find the actual base branch (main/master/develop).""" - # Check configured base first + """Find the actual base branch (main/master/develop), local or remote.""" candidates = [self.config.base_branch] + self.config.base_branch_fallbacks for branch in candidates: - check = self._git("rev-parse", "--verify", f"refs/heads/{branch}", check=False) - if check: + local_check = self._git("rev-parse", "--verify", f"refs/heads/{branch}", check=False) + if local_check: return branch - # Fallback: first branch that isn't current + remote_check = self._git("rev-parse", "--verify", f"refs/remotes/origin/{branch}", check=False) + if remote_check: + return f"origin/{branch}" current = self.current_branch() branches = self.all_branch_names() for b in branches: diff --git a/sia_code/search/multi_hop.py b/sia_code/search/multi_hop.py index 25d2b25..6796c9e 100644 --- a/sia_code/search/multi_hop.py +++ b/sia_code/search/multi_hop.py @@ -50,32 +50,42 @@ def __init__(self, backend: StorageBackend, max_hops: int = 2): self.extractor = EntityExtractor() self._preprocessor = QueryPreprocessor() # Cache instance to avoid recreation - def _aggregate_seed_results(self, result_sets: list[list[SearchResult]], k: int) -> list[SearchResult]: + def _aggregate_seed_results( + self, result_sets: list[list[SearchResult]], k: int + ) -> list[SearchResult]: """Merge result sets from multiple query variants. - Uses chunk identity when available, otherwise file/symbol/line tuple. - Rewards repeated hits modestly without overwhelming score scales. + Uses retrieval evidence to rank variants implicitly: + - better scores win + - repeated hits across variants get a modest boost + - earlier ranks contribute slightly more than later ranks """ - merged: dict[str, tuple[SearchResult, float, int]] = {} + merged: dict[str, tuple[SearchResult, float, int, float]] = {} for results in result_sets: - for r in results: - key = str(r.chunk.id) if r.chunk.id else f"{r.chunk.file_path}:{r.chunk.symbol}:{r.chunk.start_line}:{r.chunk.end_line}" + for rank, r in enumerate(results, start=1): + key = ( + str(r.chunk.id) + if r.chunk.id + else f"{r.chunk.file_path}:{r.chunk.symbol}:{r.chunk.start_line}:{r.chunk.end_line}" + ) + rank_bonus = 0.05 / rank if key in merged: - base, best_score, hits = merged[key] - merged[key] = (base, max(best_score, r.score), hits + 1) + base, best_score, hits, bonus = merged[key] + merged[key] = (base, max(best_score, r.score), hits + 1, bonus + rank_bonus) else: - merged[key] = (r, r.score, 1) + merged[key] = (r, r.score, 1, rank_bonus) ranked: list[SearchResult] = [] - for base, best_score, hits in merged.values(): - combined = best_score + (0.03 * (hits - 1)) + for base, best_score, hits, bonus in merged.values(): + combined = best_score + (0.03 * (hits - 1)) + bonus ranked.append(replace(base, score=combined)) ranked.sort(key=lambda x: x.score, reverse=True) return ranked[:k] def _initial_search(self, question: str, k: int) -> list: """Perform initial search with adaptive mode selection and query rewrites.""" - variants = self._preprocessor.expand_variants(question) + allow_model = bool(getattr(self.backend, "flan_query_rewrite", False)) + variants = self._preprocessor.expand_variants(question, allow_model=allow_model) if not variants: variants = [question] diff --git a/sia_code/search/query_preprocessor.py b/sia_code/search/query_preprocessor.py index abe8a97..b004be4 100644 --- a/sia_code/search/query_preprocessor.py +++ b/sia_code/search/query_preprocessor.py @@ -3,6 +3,10 @@ import re from typing import Set +from ..storage.multi_repo import is_model_cached + +_FLAN_REWRITE_SUMMARIZER = None + class QueryPreprocessor: """Preprocess natural language queries for lexical search. @@ -106,13 +110,14 @@ def preprocess(self, question: str) -> str: # Rejoin with spaces return " ".join(keywords) - def expand_variants(self, question: str) -> list[str]: - """Produce 2-3 query variants for research seeding. + def expand_variants(self, question: str, allow_model: bool = False) -> list[str]: + """Produce 2-4 query variants for research seeding. - Variants aim to improve recall for code search by combining: + Variants combine: - raw natural language - cleaned keyword query - synthesized code-like identifiers / framework-role forms + - optional cached-FLAN rewrite candidate """ if not question or not question.strip(): return [] @@ -136,6 +141,11 @@ def expand_variants(self, question: str) -> list[str]: if focused and focused.lower() not in {v.lower() for v in variants}: variants.append(focused) + if allow_model: + flan_variant = self._generate_flan_variant(question, keywords) + if flan_variant and flan_variant.lower() not in {v.lower() for v in variants}: + variants.append(flan_variant) + deduped: list[str] = [] seen: set[str] = set() for v in variants: @@ -143,7 +153,7 @@ def expand_variants(self, question: str) -> list[str]: if key and key not in seen: deduped.append(v) seen.add(key) - return deduped[:3] + return deduped[:4] def _synthesize_code_forms(self, tokens: list[str]) -> list[str]: """Generate code-like identifier variants from plain-language tokens. @@ -203,6 +213,39 @@ def _synthesize_code_forms(self, tokens: list[str]) -> list[str]: seen.add(k) return out[:6] + def _generate_flan_variant(self, question: str, keywords: list[str]) -> str | None: + """Optionally generate one concise code-search rewrite using cached FLAN. + + Never downloads models. Returns None if FLAN base not cached or transformers unavailable. + """ + model_name = 'google/flan-t5-base' + if not is_model_cached(model_name): + return None + # Only worth it for natural-language questions, not direct symbol lookups + if sum(1 for t in keywords if not self._is_code_identifier(t)) < 2: + return None + try: + from ..memory.summarizer import CommitSummarizer + + prompt = ( + 'Rewrite this software engineering question into a concise code search query. ' + 'Keep likely identifiers, components, API names, and configuration terms. ' + 'Do not answer the question.\n\n' + f'Question: {question}\n' + 'Query:' + ) + global _FLAN_REWRITE_SUMMARIZER + if _FLAN_REWRITE_SUMMARIZER is None: + _FLAN_REWRITE_SUMMARIZER = CommitSummarizer(model_name) + summarizer = _FLAN_REWRITE_SUMMARIZER + result = summarizer.generate(prompt, max_length=48, num_beams=1) + if not result: + return None + result = result.strip().strip('"\'') + return result if result and len(result.split()) <= 14 else None + except Exception: + return None + def extract_keywords(self, question: str) -> list[str]: """Extract meaningful keywords from a question. diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index cd5c770..05d3815 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -270,7 +270,10 @@ def build_repo_config(base_config: Config, repo_name: str) -> Config: config.indexing.include_patterns = override.index_first merged_excludes = list(config.indexing.exclude_patterns) - for group in (override.dependency_tier, override.lazy_index, override.skip): + exclude_groups = [override.lazy_index, override.skip] + if not config.multi_repo.heavy_repo_run_dependency_tier: + exclude_groups.insert(0, override.dependency_tier) + for group in exclude_groups: for pattern in group: if pattern not in merged_excludes: merged_excludes.append(pattern) diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index c199994..01e12b5 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -106,6 +106,7 @@ def __init__( self.max_vectors_per_file = int(kwargs.pop("max_vectors_per_file", 0) or 0) self.semantic_chunk_types = set(kwargs.pop("semantic_chunk_types", []) or []) self.persistent_embedding_cache = bool(kwargs.pop("persistent_embedding_cache", True)) + self.flan_query_rewrite = bool(kwargs.pop("flan_query_rewrite", False)) # Paths self.db_path = self.path / "index.db" From 45737dd726cf9e63d1cc255c50171c0bed2d38ec Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 09:52:23 +0300 Subject: [PATCH 17/18] feat: repo-aware ranking for workspace search/research - Add RepoRankingContext and build_ranking_context helper - Score repos by: cwd boost + query name match + profile hint match - Skip zero-relevance repos in research when positive-score repos exist - Apply path bonus for first-party code, penalty for tests/docs/vendor - Set flan_query_rewrite=True by default (cached-only remains) - Improves multi-repo workspace research precision dramatically --- sia_code/cli.py | 51 +++++++++++- sia_code/config.py | 2 +- sia_code/storage/multi_repo.py | 142 +++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 4 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index dc878d3..93f758d 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -1192,8 +1192,22 @@ def _search_one_backend(be): all_results.extend(rewritten) except Exception: pass - # Sort by score descending, take top `limit` - results = sorted(all_results, key=lambda r: r.score, reverse=True)[:limit] + # Repo-aware re-ranking before slice + from .storage.multi_repo import build_ranking_context, get_repo_profile + _rctx = build_ranking_context(Path.cwd(), query) + reranked = [] + for r in all_results: + _repo = str(r.chunk.metadata.get("_repo_name", "")) + if not _repo: + # infer from rewritten file_path first component + _repo = str(r.chunk.file_path).split("/")[0] + _profile = get_repo_profile(_repo) + _adj = _rctx.adjusted_score( + r.score, _repo, _profile, str(r.chunk.file_path) + ) + reranked.append(replace(r, score=_adj)) + # Sort by adjusted score descending, take top `limit` + results = sorted(reranked, key=lambda r: r.score, reverse=True)[:limit] else: results = _search_one_backend(backend) @@ -1486,17 +1500,35 @@ def research(question: str, hops: int, graph: bool, limit: int, no_filter: bool) if multi_backends: from dataclasses import replace from pathlib import Path as _Path + from .storage.multi_repo import build_ranking_context, get_repo_profile combined_chunks = [] combined_relationships = [] max_hops_executed = 0 total_entities_found = 0 + + # Build ranking context once for this question to score each repo + _rctx_pre = build_ranking_context(Path.cwd(), question) + _repo_scores = { + rname: _rctx_pre.repo_score(rname, get_repo_profile(rname)) + for rname, _ in multi_backends + } + # Determine include threshold: take repos with top-N scores or score > 0 + _sorted_scores = sorted(_repo_scores.values(), reverse=True) + _top_threshold = _sorted_scores[min(4, len(_sorted_scores) - 1)] if _sorted_scores else 0.0 + # Always include repos with any non-zero score; fallback: include all if none score > 0 + _any_positive = any(v > 0 for v in _repo_scores.values()) + with Progress( SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console ) as progress: task = progress.add_task("Analyzing code relationships across repos...", total=None) for repo_name, backend in multi_backends: try: + repo_score = _repo_scores.get(repo_name, 0.0) + # Skip repos with zero relevance if at least some repos are relevant + if _any_positive and repo_score <= 0: + continue strategy = MultiHopSearchStrategy(backend, max_hops=hops) repo_result = strategy.research(question, max_results_per_hop=limit) if repo_result.chunks: @@ -1520,9 +1552,22 @@ def research(question: str, hops: int, graph: bool, limit: int, no_filter: bool) progress.update(task, completed=True) from .search.multi_hop import ResearchResult + # Repo-aware re-ranking of combined research chunks + from .storage.multi_repo import build_ranking_context, get_repo_profile + _rctx = build_ranking_context(Path.cwd(), question) + reranked_chunks = [] + _n = max(len(combined_chunks), 1) + for _i, chunk in enumerate(combined_chunks): + _repo = str(chunk.file_path).split("/")[0] + _profile = get_repo_profile(_repo) + _base = 1.0 - (_i / _n) * 0.5 + _adj = _rctx.adjusted_score(_base, _repo, _profile, str(chunk.file_path)) + reranked_chunks.append((_adj, chunk)) + reranked_chunks.sort(key=lambda x: x[0], reverse=True) + sorted_chunks = [c for _, c in reranked_chunks] result = ResearchResult( question=question, - chunks=combined_chunks[: max(10, limit * 4)], + chunks=sorted_chunks[: max(10, limit * 4)], relationships=combined_relationships, hops_executed=max_hops_executed, total_entities_found=total_entities_found, diff --git a/sia_code/config.py b/sia_code/config.py index 32692f6..d3780a8 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -141,7 +141,7 @@ class SearchConfig(BaseModel): default_limit: int = 10 multi_hop_enabled: bool = True max_hops: int = 2 - flan_query_rewrite: bool = False # optional, cached-only FLAN extra rewrite candidate + flan_query_rewrite: bool = True # cached-only FLAN extra rewrite candidate (on by default if model cached) vector_weight: float = ( 0.7 # Weight for vector search in hybrid (0.0=lexical only, 1.0=semantic only) ) diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py index 05d3815..348ded4 100644 --- a/sia_code/storage/multi_repo.py +++ b/sia_code/storage/multi_repo.py @@ -8,6 +8,8 @@ import json import logging +import re as _re +from dataclasses import dataclass as _dataclass from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path @@ -393,3 +395,143 @@ def recommend_repo_timeout_seconds(file_count: int, estimated_chunks: int = 0) - return 300 seconds = int(60 + file_count * 0.8) return max(300, min(1800, seconds)) + + +# --------------------------------------------------------------------------- +# Repo-aware ranking helpers +# --------------------------------------------------------------------------- + + +# Words that strongly suggest a specific repo domain when found in a query +_REPO_HINT_PATTERNS: dict[str, list[str]] = { + # annotation / CVAT-like repos + "annotation_platform": [ + "annotation", "cvat", "task", "label", "labeling", "dataset", + "segmentation", "bounding box", "keypoint", "job", "organization", + "member", "project", "review", "webhook", + ], + # data science / trainer repos + "data_science": [ + "stable diffusion", "diffusion", "trainer", "training", "lora", + "dreambooth", "flux", "controlnet", "vae", "unet", "sdxl", + "checkpoint", "finetune", "fine-tune", "fine_tune", "latent", + "dataset loader", "augment", "epoch", + ], + # general ml db / ingestion + "mldb": [ + "mldb", "datasample", "ingestion", "pipeline", "metadata", + "ml platform", "gcputils", + ], +} + +# Path patterns that indicate high-value first-party code (bonus) +_FIRST_PARTY_PATTERNS = ( + r"(apps|src|core|api|views|serializer|service|handler|trainer|pipeline|" + r"processor|model|router|schema|query|manager|controller)", +) +# Path patterns that indicate low-value content (penalty) +_LOW_VALUE_PATTERNS = ( + r"(test|spec|fixture|conftest|mock|__pycache__|node_modules|" + r"vendor|generated|dist|build|docs|site|migration|changelog|" + r"example|sample|assets|static|i18n)", +) + + +@_dataclass +class RepoRankingContext: + """Per-query scoring state for repo-aware multi-repo ranking.""" + + cwd: Path + query: str + workspace_root: Path + registry: "MultiRepoRegistry | None" + + # Computed once from query vocabulary + _profile_hints: dict[str, float] = None # profile -> hint_score + _repo_name_scores: dict[str, float] = None # repo_name -> name_score + + # Compiled path patterns + _first_party_re: object = None + _low_value_re: object = None + + def __post_init__(self) -> None: + q_lower = self.query.lower() + tokens = set(_re.findall(r"[a-z][a-z0-9]*", q_lower)) + + # Score each profile by how many hint terms appear in query + profile_hints: dict[str, float] = {} + for profile, hints in _REPO_HINT_PATTERNS.items(): + score = 0.0 + for hint in hints: + if hint in q_lower: + score += 1.0 + if score > 0: + profile_hints[profile] = min(score / 3.0, 1.0) + self._profile_hints = profile_hints + + # Score repo names: boost repos whose name tokens appear in query + repo_name_scores: dict[str, float] = {} + if self.registry: + for entry in self.registry.repos: + name_tokens = set(_re.findall(r"[a-z][a-z0-9]*", entry.name.lower())) + overlap = name_tokens & tokens + if overlap: + repo_name_scores[entry.name] = min(len(overlap) / 3.0, 1.0) + self._repo_name_scores = repo_name_scores + + self._first_party_re = _re.compile(_FIRST_PARTY_PATTERNS[0], _re.IGNORECASE) + self._low_value_re = _re.compile(_LOW_VALUE_PATTERNS[0], _re.IGNORECASE) + + def cwd_boost(self, repo_name: str) -> float: + """Return 1.0 if cwd is inside this repo, else 0.0.""" + try: + repo_path = self.workspace_root / repo_name + self.cwd.relative_to(repo_path) + return 1.0 + except (ValueError, TypeError): + return 0.0 + + def repo_score(self, repo_name: str, profile: str) -> float: + """Composite repo-level score: cwd + name match + profile match.""" + score = 0.0 + score += self.cwd_boost(repo_name) * 0.60 + score += self._repo_name_scores.get(repo_name, 0.0) * 0.25 + score += self._profile_hints.get(profile, 0.0) * 0.15 + return score + + def path_adjustment(self, file_path: str) -> float: + """Return bonus (+) or penalty (-) based on file path quality.""" + p = str(file_path).lower() + if self._low_value_re.search(p): + return -0.08 + if self._first_party_re.search(p): + return +0.04 + return 0.0 + + def adjusted_score( + self, + base_score: float, + repo_name: str, + profile: str, + file_path: str, + ) -> float: + """Final score = base + repo_boost + path_adjustment.""" + repo_boost = self.repo_score(repo_name, profile) + path_adj = self.path_adjustment(file_path) + return base_score + repo_boost * 0.6 + path_adj + + +def build_ranking_context( + workspace_root: Path, + query: str, + repo_profiles: "dict[str, str] | None" = None, +) -> RepoRankingContext: + """Construct a ranking context from workspace root and query.""" + registry = MultiRepoRegistry.load(get_registry_path(workspace_root)) + cwd = Path.cwd() + return RepoRankingContext( + cwd=cwd, + query=query, + workspace_root=workspace_root, + registry=registry, + ) From e016bcd292402710319191600f0a642a3faa9365 Mon Sep 17 00:00:00 2001 From: dxta Date: Wed, 1 Jul 2026 10:21:34 +0300 Subject: [PATCH 18/18] fix: force Rich console output in non-TTY environments and fix table indent bug --- sia_code/cli.py | 100 ++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/sia_code/cli.py b/sia_code/cli.py index 93f758d..a2b7bb0 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -23,7 +23,7 @@ from .config import Config from .indexer.coordinator import IndexingCoordinator -console = Console() +console = Console(force_terminal=True) def _display_skip_summary( @@ -3070,60 +3070,60 @@ def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format) return # Table format - for fp, entry in all_results.items(): - hist = entry["hist"] - console.print(f"\n[bold]{'='*60}[/bold]") - console.print(f"[bold]File:[/bold] {fp}") - if hist.branch_context: - ctx = hist.branch_context - console.print( - f" Branch: {ctx.current_branch} | Base: {ctx.base_branch}" - ) - if hist.owners: - owners_str = ", ".join(f"{a} ({n})" for a, n in hist.owners[:3]) - console.print(f" Owners: {owners_str}") - if hist.reverts: - console.print(f" [yellow]Reverts: {len(hist.reverts)}[/yellow]") - for r in hist.reverts: - console.print(f" {r.reverting_hash[:7]} reverts {r.reverted_hash[:7]}") - - # Narrative - if entry["narrative"]: - n = entry["narrative"] - model_tag = f" [dim](via {n.model_used})[/dim]" if n.model_used else " [dim](heuristic)[/dim]" - console.print(f"\n [bold]Evolution:[/bold]{model_tag}") - console.print(f" {n.narrative}") - if n.key_phases: - console.print(f" Phases: {', '.join(n.key_phases)}") - - # Commits - console.print(f"\n [bold]History[/bold] ({len(hist.effective_commits)} effective):") - for c in hist.effective_commits[:8]: - intent_tag = f"[{c.intent}]" if c.intent else "" - console.print( - f" [{c.recency_score:.2f}] {c.hash[:7]} {c.message[:55]} " - f"[dim]{c.author} {intent_tag}[/dim]" - ) + for fp, entry in all_results.items(): + hist = entry["hist"] + console.print(f"\n[bold]{'='*60}[/bold]") + console.print(f"[bold]File:[/bold] {fp}") + if hist.branch_context: + ctx = hist.branch_context + console.print( + f" Branch: {ctx.current_branch} | Base: {ctx.base_branch}" + ) + if hist.owners: + owners_str = ", ".join(f"{a} ({n})" for a, n in hist.owners[:3]) + console.print(f" Owners: {owners_str}") + if hist.reverts: + console.print(f" [yellow]Reverts: {len(hist.reverts)}[/yellow]") + for r in hist.reverts: + console.print(f" {r.reverting_hash[:7]} reverts {r.reverted_hash[:7]}") - # Blast radius - if entry["radius"] and entry["radius"].coupled_files: - radius = entry["radius"] + # Narrative + if entry["narrative"]: + n = entry["narrative"] + model_tag = f" [dim](via {n.model_used})[/dim]" if n.model_used else " [dim](heuristic)[/dim]" + console.print(f"\n [bold]Evolution:[/bold]{model_tag}") + console.print(f" {n.narrative}") + if n.key_phases: + console.print(f" Phases: {', '.join(n.key_phases)}") + + # Commits + console.print(f"\n [bold]History[/bold] ({len(hist.effective_commits)} effective):") + for c in hist.effective_commits[:8]: + intent_tag = f"[{c.intent}]" if c.intent else "" + console.print( + f" [{c.recency_score:.2f}] {c.hash[:7]} {c.message[:55]} " + f"[dim]{c.author} {intent_tag}[/dim]" + ) + + # Blast radius + if entry["radius"] and entry["radius"].coupled_files: + radius = entry["radius"] + console.print( + f"\n [bold]Blast Radius[/bold] " + f"({radius.total_commits_analyzed} commits, " + f"{radius.commits_excluded_squash} squash-excluded):" + ) + for cf in radius.coupled_files[:8]: + bar = "█" * int(cf.coupling_score * 20) console.print( - f"\n [bold]Blast Radius[/bold] " - f"({radius.total_commits_analyzed} commits, " - f"{radius.commits_excluded_squash} squash-excluded):" + f" [{cf.coupling_score:.2f}] {bar:20s} {cf.path}" ) - for cf in radius.coupled_files[:8]: - bar = "█" * int(cf.coupling_score * 20) + if radius.change_clusters: + for cl in radius.change_clusters: console.print( - f" [{cf.coupling_score:.2f}] {bar:20s} {cf.path}" + f" [dim]Cluster (cohesion {cl.cohesion_score:.2f}): " + f"{', '.join(cl.files)}[/dim]" ) - if radius.change_clusters: - for cl in radius.change_clusters: - console.print( - f" [dim]Cluster (cohesion {cl.cohesion_score:.2f}): " - f"{', '.join(cl.files)}[/dim]" - ) if __name__ == "__main__":