diff --git a/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json new file mode 100644 index 0000000..b1cc5ec --- /dev/null +++ b/.pi/tasks/tasks-019f173a-fbb0-7341-9b39-2eb5ca609684.json @@ -0,0 +1,4 @@ +{ + "nextId": 18, + "tasks": [] +} \ No newline at end of file diff --git a/scripts/bench_heavy_repo_indexing.py b/scripts/bench_heavy_repo_indexing.py new file mode 100644 index 0000000..a7fc26d --- /dev/null +++ b/scripts/bench_heavy_repo_indexing.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +import shutil +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from sia_code.cli import create_backend +from sia_code.config import Config +from sia_code.indexer.coordinator import IndexingCoordinator +from sia_code.storage.multi_repo import ( + build_repo_config, + estimate_chunks, + estimate_indexable_files, + estimate_semantic_vectors, + recommend_repo_timeout_seconds, +) + + +def bench_repo(repo_name: str) -> dict: + repo = Path.home() / 'dev' / 'ai.platform' / repo_name + out = Path.home() / 'dev' / 'ai.platform' / '.sia-code' / 'bench' / repo_name + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True, exist_ok=True) + + cfg = build_repo_config(Config(), repo.name) + cfg.save(out / 'config.json') + files = estimate_indexable_files(repo, cfg) + chunks = estimate_chunks(repo, cfg) + vectors = estimate_semantic_vectors(repo, cfg) + timeout = recommend_repo_timeout_seconds(files, vectors) + + backend = create_backend(out, cfg, suppress_stdout_notices=True) + backend.create_index() + coord = IndexingCoordinator(cfg, backend) + t0 = time.time() + stats = coord.index_directory(repo) + elapsed = time.time() - t0 + backend.close() + return { + 'repo': repo_name, + 'model': cfg.embedding.model, + 'dims': cfg.embedding.dimensions, + 'granularity': cfg.embedding.granularity, + 'max_vectors_per_file': cfg.embedding.max_vectors_per_file, + 'files': files, + 'chunks': chunks, + 'vectors': vectors, + 'timeout': timeout, + 'elapsed_s': round(elapsed, 2), + 'stats': stats, + } + + +if __name__ == '__main__': + repos = sys.argv[1:] or [ + 'ai.platform.forks.ai-toolkit', + 'ai.platform.annotation-suite.cvat', + ] + for repo_name in repos: + print(json.dumps(bench_repo(repo_name)), flush=True) diff --git a/sia_code/cli.py b/sia_code/cli.py index 6fffb12..a2b7bb0 100644 --- a/sia_code/cli.py +++ b/sia_code/cli.py @@ -4,7 +4,7 @@ import sys import logging import subprocess -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path import click @@ -23,7 +23,7 @@ from .config import Config from .indexer.coordinator import IndexingCoordinator -console = Console() +console = Console(force_terminal=True) def _display_skip_summary( @@ -182,6 +182,11 @@ def create_backend( embedding_enabled=config.embedding.enabled, embedding_model=config.embedding.model, ndim=config.embedding.dimensions, + embedding_granularity=config.embedding.granularity, + max_vectors_per_file=config.embedding.max_vectors_per_file, + semantic_chunk_types=config.embedding.semantic_chunk_types, + persistent_embedding_cache=config.embedding.persistent_cache, + flan_query_rewrite=config.search.flan_query_rewrite, valid_chunks=valid_chunks, ) @@ -360,6 +365,46 @@ def require_initialized() -> tuple[Path, Config]: return sia_dir, config +def get_multi_repo_backends(cwd: Path | None = None): + """Check if in a multi-repo workspace and return all backends. + + Returns: + list of (repo_name, backend) tuples, or empty list if not multi-repo + """ + from .storage.multi_repo import MultiRepoRegistry, get_registry_path + + workspace = cwd or Path.cwd() + registry_path = get_registry_path(workspace) + registry = MultiRepoRegistry.load(registry_path) + + entries = [] + if registry and registry.repos: + entries = [(entry.name, workspace / entry.index_dir) for entry in registry.repos] + else: + # Fallback: infer from workspace-level repos dir even if registry missing/interrupted + repos_root = workspace / ".sia-code" / "repos" + if repos_root.exists(): + entries = [ + (child.name, child) + for child in sorted(repos_root.iterdir()) + if child.is_dir() and (child / "index.db").exists() + ] + if not entries: + return [] + + backends = [] + for repo_name, repo_sia_dir in entries: + if (repo_sia_dir / "index.db").exists(): + try: + config = Config.load(repo_sia_dir / "config.json") + backend = create_backend(repo_sia_dir, config, suppress_stdout_notices=True) + backend.open_index() + backends.append((repo_name, backend)) + except Exception: + pass + return backends + + @click.group() @click.version_option(version=__version__) @click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging") @@ -515,6 +560,275 @@ def index( # Index directory directory = Path(path).resolve() + # Auto-detect multi-repo workspace (multiple git sub-repos) + from .storage.multi_repo import ( + build_registry, + build_repo_config, + detect_sub_repos, + estimate_chunks, + estimate_indexable_files, + estimate_semantic_vectors, + get_registry_path, + is_multi_repo_workspace, + recommend_repo_timeout_seconds, + ) + + if config.multi_repo.enabled and is_multi_repo_workspace(directory): + sub_repos = detect_sub_repos(directory) + console.print( + f"[cyan]Detected multi-repo workspace with {len(sub_repos)} repos[/cyan]" + ) + for repo in sub_repos: + console.print(f" [dim]• {repo.name}[/dim]") + console.print() + + # Fan-out: index each repo independently + # All indexes stored under workspace .sia-code/repos// (no pollution in sub-repos) + registry = build_registry(directory, sub_repos) + all_stats = [] + workspace_sia = directory / ".sia-code" + registry_path = get_registry_path(directory) + registry.save(registry_path) + + # Pre-warm embed daemon (shared across all repos) + try: + backend_tmp = create_backend( + workspace_sia, config, suppress_stdout_notices=True + ) + backend_tmp._get_embedder() + console.print("[dim]Embedding daemon ready[/dim]") + except Exception: + pass + + import time as _time + import subprocess as _sp + import json as _json + from concurrent.futures import ThreadPoolExecutor, as_completed + + plans = [] + for i, repo_path in enumerate(sub_repos, 1): + repo_index_dir = workspace_sia / "repos" / repo_path.name + repo_index_dir.mkdir(parents=True, exist_ok=True) + + repo_config = build_repo_config(config, repo_path.name) + repo_config_path = repo_index_dir / "config.json" + repo_config.save(repo_config_path) + + estimated_files = estimate_indexable_files(repo_path, repo_config) + estimated_chunks = estimate_chunks(repo_path, repo_config) + estimated_vectors = estimate_semantic_vectors(repo_path, repo_config) + repo_timeout = recommend_repo_timeout_seconds( + estimated_files, estimated_vectors + ) + is_heavy = estimated_vectors >= config.multi_repo.heavy_repo_chunk_threshold + + for entry in registry.repos: + if entry.name == repo_path.name: + entry.index_dir = str( + (workspace_sia / "repos" / repo_path.name).relative_to(directory) + ) + entry.estimated_chunks = estimated_vectors + entry.status = "pending" + entry.last_error = None + break + + plans.append( + { + "seq": i, + "repo_name": repo_path.name, + "repo_path": repo_path, + "repo_index_dir": repo_index_dir, + "estimated_files": estimated_files, + "estimated_chunks": estimated_vectors, + "raw_chunks": estimated_chunks, + "repo_timeout": repo_timeout, + "heavy": is_heavy, + } + ) + registry.save(registry_path) + + def _run_repo_plan(plan: dict) -> dict: + clean_flag = "True" if clean else "False" + update_flag = "True" if update else "False" + index_script = ( + "import sys, json, os\n" + "os.environ.setdefault('HF_HUB_OFFLINE', '1')\n" + "from pathlib import Path\n" + "from sia_code.config import Config\n" + "from sia_code.cli import create_backend\n" + "from sia_code.indexer.coordinator import IndexingCoordinator\n" + f"sia_dir = Path({str(plan['repo_index_dir'])!r})\n" + f"repo_dir = Path({str(plan['repo_path'])!r})\n" + f"do_clean = {clean_flag}\n" + f"do_update = {update_flag}\n" + "config = Config.load(sia_dir / 'config.json')\n" + "if do_clean:\n" + " for stale in ('index.db', 'vectors.usearch'):\n" + " fp = sia_dir / stale\n" + " if fp.exists(): fp.unlink()\n" + "backend = create_backend(sia_dir, config, suppress_stdout_notices=True)\n" + "if do_clean:\n" + " backend.create_index()\n" + "else:\n" + " try:\n" + " backend.open_index()\n" + " except Exception:\n" + " backend.create_index()\n" + "coord = IndexingCoordinator(config, backend)\n" + "if do_update:\n" + " from sia_code.indexer.hash_cache import HashCache\n" + " from sia_code.indexer.chunk_index import ChunkIndex\n" + " cache = HashCache(sia_dir / 'cache' / 'file_hashes.json')\n" + " chunk_index = ChunkIndex(sia_dir / 'chunk_index.json')\n" + " stats = coord.index_directory_incremental_v2(repo_dir, cache, chunk_index)\n" + "else:\n" + " stats = coord.index_directory(repo_dir)\n" + "backend.close()\n" + "print(json.dumps(stats))\n" + ) + t0 = _time.monotonic() + try: + result = _sp.run( + [sys.executable, "-c", index_script], + capture_output=True, + text=True, + timeout=plan["repo_timeout"], + ) + elapsed = _time.monotonic() - t0 + if result.returncode == 0: + stdout_lines = result.stdout.strip().splitlines() + stats_line = stdout_lines[-1] if stdout_lines else '{}' + try: + stats = _json.loads(stats_line) + except _json.JSONDecodeError: + stats = {"indexed_files": 0, "total_chunks": 0} + return {"kind": "ok", "plan": plan, "elapsed": elapsed, "stats": stats} + err_lines = result.stderr.strip().splitlines() + err_msg = err_lines[-1][:120] if err_lines else "unknown" + return {"kind": "failed", "plan": plan, "elapsed": elapsed, "error": err_msg} + except _sp.TimeoutExpired: + elapsed = _time.monotonic() - t0 + return {"kind": "timed_out", "plan": plan, "elapsed": elapsed, "error": f"timeout after {plan['repo_timeout']}s"} + except Exception as e: + elapsed = _time.monotonic() - t0 + return {"kind": "failed", "plan": plan, "elapsed": elapsed, "error": str(e)[:120]} + + def _mark_started(plan: dict): + console.print( + f"[cyan][{plan['seq']}/{len(sub_repos)}] Indexing {plan['repo_name']}...[/cyan]" + ) + console.print( + f" [dim]~{plan['estimated_files']} files, ~{plan['raw_chunks']} chunks, ~{plan['estimated_chunks']} vectors, timeout {plan['repo_timeout']}s{' [heavy]' if plan['heavy'] else ''}[/dim]" + ) + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.status = 'indexing' + entry.last_error = None + break + registry.save(registry_path) + + def _record_result(result: dict): + plan = result['plan'] + if result['kind'] == 'ok': + stats = result['stats'] + all_stats.append(stats) + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.indexed_at = datetime.now(timezone.utc).isoformat() + entry.file_count = stats.get('indexed_files', 0) + entry.status = 'full' + entry.last_error = None + break + registry.save(registry_path) + console.print( + f" [green]✓[/green] {stats.get('indexed_files', 0)} files, {stats.get('total_chunks', 0)} chunks [dim]({result['elapsed']:.1f}s)[/dim]" + ) + elif result['kind'] == 'timed_out': + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.status = 'timed_out' + entry.last_error = result['error'] + break + registry.save(registry_path) + console.print( + f" [yellow]⚠ Timeout ({result['elapsed']:.0f}s/{plan['repo_timeout']}s) — skipped[/yellow]" + ) + else: + for entry in registry.repos: + if entry.name == plan['repo_name']: + entry.status = 'failed' + entry.last_error = result['error'] + break + registry.save(registry_path) + console.print( + f" [red]✗ Failed ({result['elapsed']:.1f}s): {result['error']}[/red]" + ) + + concurrency = max(1, int(config.multi_repo.fanout_concurrency)) + batch: list[dict] = [] + def _flush_batch(batch_plans: list[dict]): + if not batch_plans: + return + if len(batch_plans) == 1: + _mark_started(batch_plans[0]) + _record_result(_run_repo_plan(batch_plans[0])) + return + for p in batch_plans: + _mark_started(p) + with ThreadPoolExecutor(max_workers=min(concurrency, len(batch_plans))) as ex: + futs = [ex.submit(_run_repo_plan, p) for p in batch_plans] + for fut in as_completed(futs): + _record_result(fut.result()) + + for plan in plans: + if plan['heavy']: + _flush_batch(batch) + batch = [] + _flush_batch([plan]) + else: + batch.append(plan) + if len(batch) >= concurrency: + _flush_batch(batch) + batch = [] + _flush_batch(batch) + + # Save registry + registry.save(registry_path) + console.print("\n[green]✓ Multi-repo indexing complete[/green]") + console.print(f" Registry: {registry_path}") + total_files = sum(s.get("indexed_files", 0) for s in all_stats) + total_chunks = sum(s.get("total_chunks", 0) for s in all_stats) + console.print(f" Total: {total_files} files, {total_chunks} chunks across {len(all_stats)} repos") + return + + # --- Single-repo indexing with repo-aware profile overrides --- + repo_config = build_repo_config(config, directory.name) + if repo_config != config: + config = repo_config + # Persist effective repo config for transparency and consistent subsequent commands + try: + config.save(sia_dir / "config.json") + except Exception: + pass + try: + backend.close() + except Exception: + pass + backend = create_backend(sia_dir, config) + index_path = sia_dir / "index.db" + if clean: + if index_path.exists(): + index_path.unlink() + backend.create_index() + else: + try: + backend.open_index() + except Exception: + backend.create_index() + coordinator = IndexingCoordinator(config, backend) + + # --- Single-repo indexing (original flow) --- + if update: console.print(f"[cyan]Incremental indexing {directory}...[/cyan]") console.print("[dim]Checking for changes...[/dim]") @@ -799,8 +1113,18 @@ def search( console.print("[red]Error: --no-deps and --deps-only are mutually exclusive[/red]") sys.exit(1) - backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) - backend.open_index() + # Multi-repo: if in workspace with registry, search all sub-repos + multi_backends = get_multi_repo_backends() + _multi_repo_mode = bool(multi_backends) + if _multi_repo_mode: + console.print( + f"[dim]Multi-repo: searching across {len(multi_backends)} repos[/dim]" + ) + # Use primary backend for config-based settings; actual search aggregates all + backend = multi_backends[0][1] + else: + backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) + backend.open_index() # Determine dependency filtering # Default: include deps (from config or True) @@ -825,23 +1149,67 @@ def search( console.print(f"[dim]Searching ({mode}{filter_status}{deps_status})...[/dim]") # Execute search based on mode - if regex: - results = backend.search_lexical( - query, k=limit, include_deps=include_deps, tier_boost=tier_boost - ) - elif semantic_only: - results = backend.search_semantic( - query, k=limit, include_deps=include_deps, tier_boost=tier_boost - ) + def _search_one_backend(be): + if regex: + return be.search_lexical( + query, k=limit, include_deps=include_deps, tier_boost=tier_boost + ) + elif semantic_only: + return be.search_semantic( + query, k=limit, include_deps=include_deps, tier_boost=tier_boost + ) + else: + return be.search_hybrid( + query, + k=limit, + vector_weight=config.search.vector_weight, + include_deps=include_deps, + tier_boost=tier_boost, + ) + + if _multi_repo_mode: + # Aggregate results from all repos + all_results = [] + from dataclasses import replace + from pathlib import Path as _Path + + for repo_name, be in multi_backends: + try: + repo_results = _search_one_backend(be) + rewritten = [] + repo_root = Path.cwd() / repo_name + for r in repo_results: + try: + orig_path = Path(r.chunk.file_path) + rel_path = orig_path.relative_to(repo_root) if orig_path.is_absolute() else orig_path + except Exception: + rel_path = Path(r.chunk.file_path).name + new_chunk = replace( + r.chunk, + file_path=_Path(repo_name) / rel_path, + ) + rewritten.append(replace(r, chunk=new_chunk)) + all_results.extend(rewritten) + except Exception: + pass + # Repo-aware re-ranking before slice + from .storage.multi_repo import build_ranking_context, get_repo_profile + _rctx = build_ranking_context(Path.cwd(), query) + reranked = [] + for r in all_results: + _repo = str(r.chunk.metadata.get("_repo_name", "")) + if not _repo: + # infer from rewritten file_path first component + _repo = str(r.chunk.file_path).split("/")[0] + _profile = get_repo_profile(_repo) + _adj = _rctx.adjusted_score( + r.score, _repo, _profile, str(r.chunk.file_path) + ) + reranked.append(replace(r, score=_adj)) + # Sort by adjusted score descending, take top `limit` + results = sorted(reranked, key=lambda r: r.score, reverse=True)[:limit] else: - # NEW: Hybrid search (BM25 + semantic) for best performance - results = backend.search_hybrid( - query, - k=limit, - vector_weight=config.search.vector_weight, - include_deps=include_deps, - tier_boost=tier_boost, - ) + results = _search_one_backend(backend) # Filter for --deps-only after search if deps_only and results: @@ -1125,20 +1493,96 @@ def research(question: str, hops: int, graph: bool, limit: int, no_filter: bool) except Exception: pass # Silently fall back to no filtering - backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) - backend.open_index() - - strategy = MultiHopSearchStrategy(backend, max_hops=hops) - + multi_backends = get_multi_repo_backends() console.print(f"[dim]Researching: {question}[/dim]") console.print(f"[dim]Max hops: {hops}, Results per hop: {limit}[/dim]\n") - with Progress( - SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console - ) as progress: - task = progress.add_task("Analyzing code relationships...", total=None) - result = strategy.research(question, max_results_per_hop=limit) - progress.update(task, completed=True) + if multi_backends: + from dataclasses import replace + from pathlib import Path as _Path + from .storage.multi_repo import build_ranking_context, get_repo_profile + + combined_chunks = [] + combined_relationships = [] + max_hops_executed = 0 + total_entities_found = 0 + + # Build ranking context once for this question to score each repo + _rctx_pre = build_ranking_context(Path.cwd(), question) + _repo_scores = { + rname: _rctx_pre.repo_score(rname, get_repo_profile(rname)) + for rname, _ in multi_backends + } + # Determine include threshold: take repos with top-N scores or score > 0 + _sorted_scores = sorted(_repo_scores.values(), reverse=True) + _top_threshold = _sorted_scores[min(4, len(_sorted_scores) - 1)] if _sorted_scores else 0.0 + # Always include repos with any non-zero score; fallback: include all if none score > 0 + _any_positive = any(v > 0 for v in _repo_scores.values()) + + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console + ) as progress: + task = progress.add_task("Analyzing code relationships across repos...", total=None) + for repo_name, backend in multi_backends: + try: + repo_score = _repo_scores.get(repo_name, 0.0) + # Skip repos with zero relevance if at least some repos are relevant + if _any_positive and repo_score <= 0: + continue + strategy = MultiHopSearchStrategy(backend, max_hops=hops) + repo_result = strategy.research(question, max_results_per_hop=limit) + if repo_result.chunks: + repo_root = Path.cwd() / repo_name + rewritten_chunks = [] + for chunk in repo_result.chunks: + try: + orig_path = Path(chunk.file_path) + rel_path = orig_path.relative_to(repo_root) if orig_path.is_absolute() else orig_path + except Exception: + rel_path = Path(chunk.file_path).name + rewritten_chunks.append( + replace(chunk, file_path=_Path(repo_name) / rel_path) + ) + combined_chunks.extend(rewritten_chunks) + combined_relationships.extend(repo_result.relationships) + max_hops_executed = max(max_hops_executed, repo_result.hops_executed) + total_entities_found += repo_result.total_entities_found + except Exception: + pass + progress.update(task, completed=True) + + from .search.multi_hop import ResearchResult + # Repo-aware re-ranking of combined research chunks + from .storage.multi_repo import build_ranking_context, get_repo_profile + _rctx = build_ranking_context(Path.cwd(), question) + reranked_chunks = [] + _n = max(len(combined_chunks), 1) + for _i, chunk in enumerate(combined_chunks): + _repo = str(chunk.file_path).split("/")[0] + _profile = get_repo_profile(_repo) + _base = 1.0 - (_i / _n) * 0.5 + _adj = _rctx.adjusted_score(_base, _repo, _profile, str(chunk.file_path)) + reranked_chunks.append((_adj, chunk)) + reranked_chunks.sort(key=lambda x: x[0], reverse=True) + sorted_chunks = [c for _, c in reranked_chunks] + result = ResearchResult( + question=question, + chunks=sorted_chunks[: max(10, limit * 4)], + relationships=combined_relationships, + hops_executed=max_hops_executed, + total_entities_found=total_entities_found, + ) + else: + backend = create_backend(sia_dir, config, valid_chunks=valid_chunks) + backend.open_index() + strategy = MultiHopSearchStrategy(backend, max_hops=hops) + + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), console=console + ) as progress: + task = progress.add_task("Analyzing code relationships...", total=None) + result = strategy.research(question, max_results_per_hop=limit) + progress.update(task, completed=True) # Display results summary console.print("\n[bold green]✓ Research Complete[/bold green]") @@ -1194,6 +1638,106 @@ def status(): sia_dir, config = require_initialized() + # Multi-repo aggregate status + multi_backends = get_multi_repo_backends() + if multi_backends: + from .storage.multi_repo import MultiRepoRegistry, get_registry_path + + registry = MultiRepoRegistry.load(get_registry_path(Path.cwd())) + total_files = 0 + total_chunks = 0 + repo_rows = [] + status_counts: dict[str, int] = {} + meta_by_name = {entry.name: entry for entry in (registry.repos if registry else [])} + for repo_name, backend in multi_backends: + try: + stats = backend.get_stats() + total_files += stats.total_files + total_chunks += stats.total_chunks + meta = meta_by_name.get(repo_name) + status_value = meta.status if meta else "indexed" + if status_value == "pending" and stats.total_chunks > 0: + status_value = "indexed" + status_counts[status_value] = status_counts.get(status_value, 0) + 1 + repo_rows.append( + ( + repo_name, + meta.profile if meta else "general", + status_value, + meta.estimated_chunks if meta else 0, + stats.total_files, + stats.total_chunks, + meta.last_error if meta else None, + ) + ) + except Exception: + meta = meta_by_name.get(repo_name) + status_value = meta.status if meta else "error" + status_counts[status_value] = status_counts.get(status_value, 0) + 1 + repo_rows.append( + ( + repo_name, + meta.profile if meta else "general", + status_value, + meta.estimated_chunks if meta else 0, + 0, + 0, + meta.last_error if meta else None, + ) + ) + + if registry: + for entry in registry.repos: + if entry.name not in {row[0] for row in repo_rows}: + status_counts[entry.status] = status_counts.get(entry.status, 0) + 1 + repo_rows.append( + ( + entry.name, + entry.profile, + entry.status, + entry.estimated_chunks, + 0, + 0, + entry.last_error, + ) + ) + + table = Table(title="Sia Code Index Status (Multi-Repo)") + table.add_column("Property", style="cyan") + table.add_column("Value", style="green") + table.add_row("Workspace Index Path", str(sia_dir)) + table.add_row("Registered Repos", f"{len(repo_rows):,}") + table.add_row("Indexed Repos", f"{len(multi_backends):,}") + table.add_row("Total Files", f"{total_files:,}") + table.add_row("Total Chunks", f"{total_chunks:,}") + if status_counts: + table.add_row( + "Repo States", + ", ".join(f"{k}={v}" for k, v in sorted(status_counts.items())), + ) + console.print(table) + + repo_table = Table(title="Per-Repo Status") + repo_table.add_column("Repo", style="cyan") + repo_table.add_column("Profile") + repo_table.add_column("State") + repo_table.add_column("Est Chunks", justify="right") + repo_table.add_column("Files", justify="right") + repo_table.add_column("Chunks", justify="right") + repo_table.add_column("Last Error", overflow="fold") + for repo_name, profile, state, est_chunks, files_n, chunks_n, last_error in sorted(repo_rows): + repo_table.add_row( + repo_name, + profile, + state, + f"{est_chunks:,}" if est_chunks else "-", + f"{files_n:,}", + f"{chunks_n:,}", + (last_error or "")[:80], + ) + console.print(repo_table) + return + backend = create_backend(sia_dir, config) backend.open_index() stats = backend.get_stats() @@ -2398,5 +2942,189 @@ def embed_status(verbose): console.print("\n[dim]Start with: sia-code embed start[/dim]") +# --------------------------------------------------------------------------- +# Dynamic Git Memory CLI Commands (consolidated) +# --------------------------------------------------------------------------- + + +@memory.command(name="git-context") +@click.argument("file_paths", nargs=-1, required=True) +@click.option("--no-blast-radius", is_flag=True, help="Skip blast radius analysis") +@click.option("--no-narrative", is_flag=True, help="Skip evolution narrative") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", +) +def memory_git_context(file_paths, no_blast_radius, no_narrative, output_format): + """Show git context for files: history, blast radius, and evolution narrative. + + Combines file history (revert-aware, cross-branch, recency-scored), + co-change blast radius, and model-generated evolution narrative. + Auto-uses local flan-t5 model for narrative when available. + + Examples: + sia-code memory git-context src/api/datasample.py + sia-code memory git-context src/api.py src/crud.py --format json + sia-code memory git-context src/api.py --no-narrative + """ + from .config import Config + from .memory.blast_radius import BlastRadiusAnalyzer + from .memory.diff_analyzer import DiffSemanticAnalyzer + from .memory.git_dynamic import GitDynamicMemory + from .memory.intent_classifier import IntentClassifier + from .memory.recency import RecencyConfig + + project_dir = Path.cwd() + config = Config.load(project_dir / ".sia-code" / "config.json") + gc = config.git_dynamic + + recency_cfg = RecencyConfig( + halflife_days=gc.recency_halflife_days, + working_window_days=gc.working_window_days, + ) + mem = GitDynamicMemory(project_dir, recency_config=recency_cfg) + classifier = IntentClassifier() + + all_results = {} + warnings = [] + for fp in file_paths: + hist = mem.file_history(fp, cross_branch=gc.cross_branch_enabled, limit=15) + if not hist.effective_commits: + warnings.append(f"No history found for {fp}") + if output_format != "json": + console.print(f"[yellow]No history found for {fp}[/yellow]") + continue + + # Classify intents + for c in hist.effective_commits: + intent = classifier.classify(c.message, len(c.files_changed), c.insertions + c.deletions) + c.intent = intent.intent + + entry = {"hist": hist, "radius": None, "narrative": None} + + # Blast radius + if not no_blast_radius: + analyzer = BlastRadiusAnalyzer( + project_dir, + lookback=gc.lookback_commits, + min_coupling=gc.coupling_threshold, + max_files_per_commit=gc.max_files_per_commit, + recency_config=recency_cfg, + ) + entry["radius"] = analyzer.co_changed_files(fp) + + # Narrative + if not no_narrative: + try: + diff_analyzer = DiffSemanticAnalyzer(project_dir, model_name=gc.narrative_model) + entry["narrative"] = diff_analyzer.summarize_evolution(hist) + except Exception: + pass + + all_results[fp] = entry + + if output_format == "json": + import json + + data = {"files": {}, "warnings": warnings} + for fp, entry in all_results.items(): + hist = entry["hist"] + d = { + "branch_context": { + "current": hist.branch_context.current_branch if hist.branch_context else None, + "base": hist.branch_context.base_branch if hist.branch_context else None, + }, + "owners": hist.owners[:3], + "reverts": [ + {"reverted": r.reverted_hash[:7], "by": r.reverting_hash[:7]} + for r in hist.reverts + ], + "commits": [ + { + "hash": c.hash[:7], + "message": c.message, + "intent": c.intent, + "recency": round(c.recency_score, 3), + "author": c.author, + } + for c in hist.effective_commits[:10] + ], + } + if entry["radius"]: + d["blast_radius"] = [ + {"path": cf.path, "coupling": round(cf.coupling_score, 3)} + for cf in entry["radius"].coupled_files[:10] + ] + if entry["narrative"]: + d["narrative"] = entry["narrative"].narrative + d["phases"] = entry["narrative"].key_phases + d["model_used"] = entry["narrative"].model_used + data["files"][fp] = d + console.print(json.dumps(data, indent=2)) + return + + if not all_results: + console.print("[red]No results found.[/red]") + return + + # Table format + for fp, entry in all_results.items(): + hist = entry["hist"] + console.print(f"\n[bold]{'='*60}[/bold]") + console.print(f"[bold]File:[/bold] {fp}") + if hist.branch_context: + ctx = hist.branch_context + console.print( + f" Branch: {ctx.current_branch} | Base: {ctx.base_branch}" + ) + if hist.owners: + owners_str = ", ".join(f"{a} ({n})" for a, n in hist.owners[:3]) + console.print(f" Owners: {owners_str}") + if hist.reverts: + console.print(f" [yellow]Reverts: {len(hist.reverts)}[/yellow]") + for r in hist.reverts: + console.print(f" {r.reverting_hash[:7]} reverts {r.reverted_hash[:7]}") + + # Narrative + if entry["narrative"]: + n = entry["narrative"] + model_tag = f" [dim](via {n.model_used})[/dim]" if n.model_used else " [dim](heuristic)[/dim]" + console.print(f"\n [bold]Evolution:[/bold]{model_tag}") + console.print(f" {n.narrative}") + if n.key_phases: + console.print(f" Phases: {', '.join(n.key_phases)}") + + # Commits + console.print(f"\n [bold]History[/bold] ({len(hist.effective_commits)} effective):") + for c in hist.effective_commits[:8]: + intent_tag = f"[{c.intent}]" if c.intent else "" + console.print( + f" [{c.recency_score:.2f}] {c.hash[:7]} {c.message[:55]} " + f"[dim]{c.author} {intent_tag}[/dim]" + ) + + # Blast radius + if entry["radius"] and entry["radius"].coupled_files: + radius = entry["radius"] + console.print( + f"\n [bold]Blast Radius[/bold] " + f"({radius.total_commits_analyzed} commits, " + f"{radius.commits_excluded_squash} squash-excluded):" + ) + for cf in radius.coupled_files[:8]: + bar = "█" * int(cf.coupling_score * 20) + console.print( + f" [{cf.coupling_score:.2f}] {bar:20s} {cf.path}" + ) + if radius.change_clusters: + for cl in radius.change_clusters: + console.print( + f" [dim]Cluster (cohesion {cl.cohesion_score:.2f}): " + f"{', '.join(cl.files)}[/dim]" + ) + + if __name__ == "__main__": main() diff --git a/sia_code/config.py b/sia_code/config.py index cc0da42..d3780a8 100644 --- a/sia_code/config.py +++ b/sia_code/config.py @@ -73,6 +73,12 @@ class EmbeddingConfig(BaseModel): model: str = "BAAI/bge-base-en-v1.5" # Model name (see supported models above) api_key_env: str = "" # Deprecated legacy field; ignored by local-only runtime dimensions: int = 768 # Embedding dimensions (auto-detected for most models) + granularity: Literal["chunk", "budget"] = "chunk" + max_vectors_per_file: int = 0 # 0 = unlimited + semantic_chunk_types: list[str] = Field( + default_factory=lambda: ["class", "function", "method", "definition"] + ) + persistent_cache: bool = True class IndexingConfig(BaseModel): @@ -135,6 +141,7 @@ class SearchConfig(BaseModel): default_limit: int = 10 multi_hop_enabled: bool = True max_hops: int = 2 + flan_query_rewrite: bool = True # cached-only FLAN extra rewrite candidate (on by default if model cached) vector_weight: float = ( 0.7 # Weight for vector search in hybrid (0.0=lexical only, 1.0=semantic only) ) @@ -183,6 +190,40 @@ class SummarizationConfig(BaseModel): max_commits: int = 20 # Max commits to include in summary +class GitDynamicConfig(BaseModel): + """Configuration for dynamic git memory system.""" + + enabled: bool = True + lookback_commits: int = 200 + coupling_threshold: float = 0.3 + max_files_per_commit: int = 20 # squash dilution guard + working_window_days: int = 14 + recency_halflife_days: float = 30.0 + base_branch: str = "main" + cross_branch_enabled: bool = True + semantic_weight: float = 0.3 # in combined score: git=0.7, semantic=0.3 + narrative_model: str | None = None # None = auto-select (large on 16GB+, base otherwise) + + +class RepoIndexOverride(BaseModel): + """Repo-specific indexing policy override for multi-repo workspaces.""" + + index_first: list[str] = Field(default_factory=list) + dependency_tier: list[str] = Field(default_factory=list) + lazy_index: list[str] = Field(default_factory=list) + skip: list[str] = Field(default_factory=list) + + +class MultiRepoConfig(BaseModel): + """Workspace-level multi-repo indexing controls.""" + + enabled: bool = True + fanout_concurrency: int = 2 + heavy_repo_chunk_threshold: int = 4000 + heavy_repo_run_dependency_tier: bool = False + repo_overrides: dict[str, RepoIndexOverride] = Field(default_factory=dict) + + class StorageConfig(BaseModel): """Storage backend selection configuration.""" @@ -204,6 +245,8 @@ class Config(BaseModel): adaptive: AdaptiveConfig = Field(default_factory=AdaptiveConfig) summarization: SummarizationConfig = Field(default_factory=SummarizationConfig) storage: StorageConfig = Field(default_factory=StorageConfig) + git_dynamic: GitDynamicConfig = Field(default_factory=GitDynamicConfig) + multi_repo: MultiRepoConfig = Field(default_factory=MultiRepoConfig) @classmethod def load(cls, path: Path) -> "Config": diff --git a/sia_code/embed_server/client.py b/sia_code/embed_server/client.py index d3e68d6..c0435a2 100644 --- a/sia_code/embed_server/client.py +++ b/sia_code/embed_server/client.py @@ -151,7 +151,7 @@ def encode( # Create request request_id = str(uuid.uuid4()) - request = EmbedRequest.create(request_id, self.model_name, sentences) + request = EmbedRequest.create(request_id, self.model_name, sentences, batch_size=batch_size) # Send request response = self._send_request(request) diff --git a/sia_code/embed_server/daemon.py b/sia_code/embed_server/daemon.py index 7d0398c..3bba02b 100644 --- a/sia_code/embed_server/daemon.py +++ b/sia_code/embed_server/daemon.py @@ -137,30 +137,41 @@ def _load_model(self, model_name: str) -> Any: # Auto-detect device on first load if not self.models: # First model - self.device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + self.device = "cuda" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + self.device = "mps" + else: + self.device = "cpu" logger.info(f"Using device: {self.device}") # Load model model = SentenceTransformer(model_name, device=self.device) self.models[model_name] = model + # After successful load, prevent future network calls + # Model is now cached locally — no need to hit HF Hub again + import os + os.environ["HF_HUB_OFFLINE"] = "1" + logger.info(f"Model loaded: {model_name} ({len(self.models)} total)") return self.models[model_name] - def _handle_embed(self, model: str, texts: list[str]) -> dict: + def _handle_embed(self, model: str, texts: list[str], batch_size: int = 32) -> dict: """Handle embedding request. Args: model: Model name texts: List of texts to embed + batch_size: Preferred batch size from client Returns: Response dict with embeddings """ try: embedder = self._load_model(model) - vectors = embedder.encode(texts, convert_to_numpy=True, batch_size=32) + vectors = embedder.encode(texts, convert_to_numpy=True, batch_size=batch_size) return { "embeddings": vectors.tolist(), @@ -224,13 +235,14 @@ def _handle_connection(self, conn: socket.socket): params = request.get("params", {}) model = params.get("model") texts = params.get("texts", []) + batch_size = int(params.get("batch_size", 32) or 32) if not model or not texts: response = ErrorResponse.create( request_id, "Missing model or texts", "InvalidRequest" ) else: - result = self._handle_embed(model, texts) + result = self._handle_embed(model, texts, batch_size=batch_size) response = EmbedResponse.create( request_id, result["embeddings"], diff --git a/sia_code/embed_server/protocol.py b/sia_code/embed_server/protocol.py index 227aac0..327fa84 100644 --- a/sia_code/embed_server/protocol.py +++ b/sia_code/embed_server/protocol.py @@ -73,12 +73,12 @@ class EmbedRequest: """Embedding request message.""" @staticmethod - def create(request_id: str, model: str, texts: list[str]) -> dict: + def create(request_id: str, model: str, texts: list[str], batch_size: int = 32) -> dict: """Create embedding request.""" return { "id": request_id, "method": "embed", - "params": {"model": model, "texts": texts}, + "params": {"model": model, "texts": texts, "batch_size": batch_size}, } diff --git a/sia_code/mcp.py b/sia_code/mcp.py index ade245e..bef212a 100644 --- a/sia_code/mcp.py +++ b/sia_code/mcp.py @@ -827,6 +827,18 @@ def engineering_bootstrap( else: recommended_next_step = "Start with the top search hits, then escalate to research only if cross-file tracing is needed." + # Auto-enrich with git context for files found in search + git_context_payload = {} + try: + if search_hits and search_hits.get("matches"): + hit_files = list({m["file_path"] for m in search_hits["matches"] if "file_path" in m})[:3] + if hit_files: + git_context_payload = _compute_git_context( + context.workspace_root, hit_files, limit=3 + ) + except Exception: + pass # Graceful — git context is supplementary + return _ok( context=context, result={ @@ -839,6 +851,7 @@ def engineering_bootstrap( "memory_hits": memory_hits, "working_memory": working_memory, "research": research_payload, + "git_context": git_context_payload, "recommended_next_step": recommended_next_step, "fallback_guidance": health["fallback_guidance"], }, @@ -1090,6 +1103,19 @@ def memory_trace( for item in result.events ], } + + # Supplement with dynamic git history for related files + dynamic_git = {} + try: + if result.related_files: + dynamic_git = _compute_git_context( + context.workspace_root, result.related_files[:3], + limit=3, include_blast_radius=False, + ) + except Exception: + pass + payload["dynamic_git"] = dynamic_git + return _ok(context=context, result=payload, embedding_runtime=_embedding_runtime(config)) @mcp.tool() @@ -1252,6 +1278,146 @@ def embed_stop(): return _ok(scope="machine", result={"stopped": stop_daemon()}) + # ------------------------------------------------------------------ + # Dynamic Git Memory (consolidated) + # ------------------------------------------------------------------ + + def _compute_git_context( + workspace_root: Path, + file_paths: list[str], + limit: int = 5, + include_blast_radius: bool = True, + include_narrative: bool = True, + ) -> dict: + """Shared helper — computes git context for files. + + Used by git_context tool, research, engineering_bootstrap, memory_trace. + """ + from .config import Config + from .memory.blast_radius import BlastRadiusAnalyzer + from .memory.diff_analyzer import DiffSemanticAnalyzer + from .memory.git_dynamic import GitDynamicMemory + from .memory.intent_classifier import IntentClassifier + from .memory.recency import RecencyConfig + + config = Config.load(workspace_root / ".sia-code" / "config.json") + gc = config.git_dynamic + + if not gc.enabled: + return {} + + recency_cfg = RecencyConfig( + halflife_days=gc.recency_halflife_days, + working_window_days=gc.working_window_days, + ) + mem = GitDynamicMemory(workspace_root, recency_config=recency_cfg) + classifier = IntentClassifier() + + results = {} + for fp in file_paths[:limit]: + # File history + revert detection + branch context + hist = mem.file_history(fp, cross_branch=gc.cross_branch_enabled, limit=10) + if not hist.effective_commits: + continue + + # Classify intents + for c in hist.effective_commits: + intent = classifier.classify( + c.message, len(c.files_changed), c.insertions + c.deletions + ) + c.intent = intent.intent + + entry: dict = { + "effective_commits": [ + { + "hash": c.hash[:7], + "message": c.message, + "author": c.author, + "date": c.date.isoformat(), + "recency_score": round(c.recency_score, 3), + "branch": c.branch, + "intent": c.intent, + } + for c in hist.effective_commits[:8] + ], + "owners": hist.owners[:3], + "reverts": [ + { + "reverted": r.reverted_hash[:7], + "by": r.reverting_hash[:7], + "method": r.matched_by, + } + for r in hist.reverts + ], + "branch_context": { + "current": hist.branch_context.current_branch if hist.branch_context else None, + "base": hist.branch_context.base_branch if hist.branch_context else None, + "merge_base": hist.branch_context.merge_base if hist.branch_context else None, + }, + } + + # Blast radius + if include_blast_radius: + analyzer = BlastRadiusAnalyzer( + workspace_root, + lookback=gc.lookback_commits, + min_coupling=gc.coupling_threshold, + max_files_per_commit=gc.max_files_per_commit, + recency_config=recency_cfg, + ) + radius = analyzer.co_changed_files(fp) + entry["blast_radius"] = [ + { + "path": cf.path, + "coupling": round(cf.coupling_score, 3), + "co_changes": cf.co_change_count, + } + for cf in radius.coupled_files[:10] + ] + if radius.change_clusters: + entry["clusters"] = [ + {"files": cl.files, "cohesion": round(cl.cohesion_score, 3)} + for cl in radius.change_clusters + ] + + # Evolution narrative (auto-uses local model if available) + if include_narrative: + try: + diff_analyzer = DiffSemanticAnalyzer( + workspace_root, model_name=gc.narrative_model + ) + narrative = diff_analyzer.summarize_evolution(hist) + entry["narrative"] = narrative.narrative + entry["phases"] = narrative.key_phases + entry["model_used"] = narrative.model_used + except Exception: + pass # Graceful degradation + + results[fp] = entry + + return results + + @mcp.tool() + def git_context( + workspace_root: str, + file_paths: list[str], + include_blast_radius: bool = True, + include_narrative: bool = True, + ) -> dict: + """Git-aware context for files: history, blast radius, evolution narrative. + + Combines file history (revert-aware, cross-branch, recency-scored), + co-change blast radius, and model-generated evolution narrative. + Auto-uses local flan-t5 model for narrative when available. + """ + result = _compute_git_context( + Path(workspace_root), + file_paths, + include_blast_radius=include_blast_radius, + include_narrative=include_narrative, + ) + return _ok(scope="project", result=result) + return mcp diff --git a/sia_code/memory/__init__.py b/sia_code/memory/__init__.py index e69de29..2956bfa 100644 --- a/sia_code/memory/__init__.py +++ b/sia_code/memory/__init__.py @@ -0,0 +1,47 @@ +"""Dynamic git memory system for sia-code. + +Provides on-demand file history, blast radius, revert detection, +recency scoring, branch/worktree awareness, and semantic file grouping. +""" + +from .blast_radius import BlastRadius, BlastRadiusAnalyzer, CoupledFile +from .diff_analyzer import ChangeMeaning, DiffSemanticAnalyzer, EvolutionNarrative +from .git_dynamic import ( + BranchContext, + BranchInfo, + BranchResolver, + FileHistory, + GitDynamicMemory, + HistoricalCommit, + WorktreeInfo, +) +from .intent_classifier import CommitIntent, IntentClassifier +from .recency import RecencyConfig, RecencyScorer +from .revert_detector import CommitInfo, RevertDetector, RevertPair +from .semantic_grouper import EnrichedRelation, SemanticFileGrouper, SemanticRelation + +__all__ = [ + "BlastRadius", + "BlastRadiusAnalyzer", + "BranchContext", + "BranchInfo", + "BranchResolver", + "ChangeMeaning", + "CommitInfo", + "CommitIntent", + "CoupledFile", + "DiffSemanticAnalyzer", + "EnrichedRelation", + "EvolutionNarrative", + "FileHistory", + "GitDynamicMemory", + "HistoricalCommit", + "IntentClassifier", + "RecencyConfig", + "RecencyScorer", + "RevertDetector", + "RevertPair", + "SemanticFileGrouper", + "SemanticRelation", + "WorktreeInfo", +] diff --git a/sia_code/memory/blast_radius.py b/sia_code/memory/blast_radius.py new file mode 100644 index 0000000..81298ff --- /dev/null +++ b/sia_code/memory/blast_radius.py @@ -0,0 +1,340 @@ +"""Blast radius analysis — co-change coupling from git history. + +Identifies files that frequently change together with a target file, +with squash-merge dilution guard and recency weighting. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from .recency import RecencyConfig, RecencyScorer + + +@dataclass +class CoupledFile: + """A file that co-changes with the target.""" + + path: str + coupling_score: float # 0.0-1.0 (recency-weighted) + raw_coupling: float # 0.0-1.0 (unweighted) + co_change_count: int + total_changes_target: int + total_changes_self: int + recent_co_change: datetime | None = None + + +@dataclass +class ChangeCluster: + """A group of files that form a logical change unit.""" + + files: list[str] + cohesion_score: float # How tightly coupled the group is + common_commits: int # Commits where all files appear + + +@dataclass +class BlastRadius: + """Complete blast radius analysis for a file.""" + + target_file: str + coupled_files: list[CoupledFile] = field(default_factory=list) + change_clusters: list[ChangeCluster] = field(default_factory=list) + total_commits_analyzed: int = 0 + commits_excluded_squash: int = 0 + + +class BlastRadiusAnalyzer: + """Analyze co-change coupling between files in git history. + + Features: + - Frequency-based coupling score + - Squash-merge dilution guard (skip commits with too many files) + - Recency weighting (recent co-changes score higher) + - Change cluster detection + """ + + def __init__( + self, + repo_path: Path, + lookback: int = 200, + min_coupling: float = 0.3, + max_files_per_commit: int = 20, + recency_config: RecencyConfig | None = None, + ): + self.repo_path = Path(repo_path).resolve() + self.lookback = lookback + self.min_coupling = min_coupling + self.max_files_per_commit = max_files_per_commit + self.recency = RecencyScorer(recency_config) + + def _git(self, *args: str) -> str | None: + """Run git command, return stdout or None.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except (subprocess.CalledProcessError, OSError): + return None + + def co_changed_files( + self, file_path: str, branch: str | None = None + ) -> BlastRadius: + """Find files that co-change with target file. + + Algorithm: + 1. Get all commits touching target (up to lookback) + 2. Apply squash dilution guard (skip commits with > max_files) + 3. Count co-occurrence of other files + 4. Weight by recency (recent co-changes count more) + 5. Compute coupling = weighted_co_occurrences / max(commits_a, commits_b) + + Args: + file_path: Target file path (relative to repo root). + branch: Branch to analyze (default: current HEAD). + + Returns: + BlastRadius with coupled files sorted by score. + """ + # Get commits touching target file with files changed per commit + commits = self._get_commits_with_files(file_path, branch) + + total_analyzed = len(commits) + excluded_squash = 0 + + # Co-occurrence tracking + co_occurrences: dict[str, float] = {} # file → weighted count + co_dates: dict[str, datetime] = {} # file → most recent co-change + raw_co_counts: dict[str, int] = {} # file → raw count + target_commit_count = 0 + + for commit_hash, files, date in commits: + # Squash dilution guard + if len(files) > self.max_files_per_commit: + excluded_squash += 1 + continue + + target_commit_count += 1 + recency_weight = self.recency.score(date) + + for f in files: + if f == file_path: + continue + co_occurrences[f] = co_occurrences.get(f, 0.0) + recency_weight + raw_co_counts[f] = raw_co_counts.get(f, 0) + 1 + if f not in co_dates or date > co_dates[f]: + co_dates[f] = date + + if target_commit_count == 0: + return BlastRadius( + target_file=file_path, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=excluded_squash, + ) + + # Compute coupling scores + coupled: list[CoupledFile] = [] + for f, weighted_count in co_occurrences.items(): + # Get total commits for this file (approximated by co-change context) + other_total = self._count_file_commits(f, branch) + if other_total == 0: + other_total = raw_co_counts[f] + + # Coupling = weighted_co_occ / max(target_commits, other_commits) + denominator = max(target_commit_count, other_total) + coupling = weighted_count / denominator if denominator > 0 else 0.0 + raw_coupling = raw_co_counts[f] / denominator if denominator > 0 else 0.0 + + if raw_coupling >= self.min_coupling: + coupled.append( + CoupledFile( + path=f, + coupling_score=min(coupling, 1.0), + raw_coupling=min(raw_coupling, 1.0), + co_change_count=raw_co_counts[f], + total_changes_target=target_commit_count, + total_changes_self=other_total, + recent_co_change=co_dates.get(f), + ) + ) + + coupled.sort(key=lambda c: c.coupling_score, reverse=True) + + # Detect change clusters + clusters = self._detect_clusters(coupled, commits, file_path) + + return BlastRadius( + target_file=file_path, + coupled_files=coupled, + change_clusters=clusters, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=excluded_squash, + ) + + def change_cluster(self, file_paths: list[str], branch: str | None = None) -> BlastRadius: + """Find blast radius for multiple files (union of their coupled files). + + Useful when a query spans multiple related files. + """ + all_coupled: dict[str, CoupledFile] = {} + total_analyzed = 0 + total_excluded = 0 + + for fp in file_paths: + radius = self.co_changed_files(fp, branch) + total_analyzed += radius.total_commits_analyzed + total_excluded += radius.commits_excluded_squash + for cf in radius.coupled_files: + if cf.path in file_paths: + continue # Don't include query files themselves + if cf.path in all_coupled: + # Take max coupling + if cf.coupling_score > all_coupled[cf.path].coupling_score: + all_coupled[cf.path] = cf + else: + all_coupled[cf.path] = cf + + coupled = sorted(all_coupled.values(), key=lambda c: c.coupling_score, reverse=True) + + return BlastRadius( + target_file=",".join(file_paths), + coupled_files=coupled, + total_commits_analyzed=total_analyzed, + commits_excluded_squash=total_excluded, + ) + + def _get_commits_with_files( + self, file_path: str, branch: str | None + ) -> list[tuple[str, list[str], datetime]]: + """Get commits touching file with ALL files changed per commit. + + Two-step approach: + 1. Get commit hashes touching target (with --follow for renames) + 2. For each commit, get all files changed + + Returns: list of (hash, [all_files_in_commit], date) + """ + branch_arg = branch or "HEAD" + + # Step 1: Get commit hashes that touch this file (with rename tracking) + output = self._git( + "log", + branch_arg, + f"--max-count={self.lookback}", + "--follow", + "--format=%H|%aI", + "--", + file_path, + ) + if not output: + return [] + + # Parse commit hashes + dates + commit_refs: list[tuple[str, datetime]] = [] + for line in output.splitlines(): + line = line.strip() + if not line or "|" not in line: + continue + parts = line.split("|", 1) + if len(parts[0]) >= 7 and all(c in "0123456789abcdef" for c in parts[0][:7]): + try: + date = datetime.fromisoformat(parts[1].strip()) + except (ValueError, IndexError): + date = datetime.now(timezone.utc) + commit_refs.append((parts[0].strip(), date)) + + if not commit_refs: + return [] + + # Step 2: For each commit, get all files changed + commits: list[tuple[str, list[str], datetime]] = [] + for commit_hash, date in commit_refs: + files_output = self._git( + "diff-tree", "--no-commit-id", "-r", "--name-only", commit_hash + ) + if files_output: + files = [f.strip() for f in files_output.splitlines() if f.strip()] + else: + files = [] + commits.append((commit_hash, files, date)) + + return commits + + def _count_file_commits(self, file_path: str, branch: str | None) -> int: + """Quick count of commits touching a file.""" + branch_arg = branch or "HEAD" + output = self._git( + "rev-list", + "--count", + branch_arg, + "--", + file_path, + ) + if output: + try: + return int(output.strip()) + except ValueError: + pass + return 0 + + def _detect_clusters( + self, + coupled: list[CoupledFile], + commits: list[tuple[str, list[str], datetime]], + target: str, + ) -> list[ChangeCluster]: + """Detect groups of files that consistently change together. + + Simple approach: files that appear together in >= 60% of the + target file's commits form a cluster. + """ + if not coupled or not commits: + return [] + + # Only consider top coupled files + top_files = [c.path for c in coupled[:10]] + if not top_files: + return [] + + # Count how often pairs appear together in target's commits + valid_commits = [ + (h, files, d) + for h, files, d in commits + if len(files) <= self.max_files_per_commit + ] + if len(valid_commits) < 3: + return [] + + # Find files that appear in >= 60% of target's commits + file_in_commits: dict[str, int] = {} + for _, files, _ in valid_commits: + for f in files: + if f in top_files: + file_in_commits[f] = file_in_commits.get(f, 0) + 1 + + cluster_members = [ + f + for f, count in file_in_commits.items() + if count / len(valid_commits) >= 0.6 + ] + + if len(cluster_members) >= 2: + cohesion = sum( + file_in_commits[f] / len(valid_commits) for f in cluster_members + ) / len(cluster_members) + return [ + ChangeCluster( + files=[target] + cluster_members, + cohesion_score=cohesion, + common_commits=min(file_in_commits[f] for f in cluster_members), + ) + ] + return [] diff --git a/sia_code/memory/diff_analyzer.py b/sia_code/memory/diff_analyzer.py new file mode 100644 index 0000000..5e688d0 --- /dev/null +++ b/sia_code/memory/diff_analyzer.py @@ -0,0 +1,300 @@ +"""Semantic diff analysis with auto-escalating local model. + +Derives evolution narratives from file history using: +1. Heuristic analysis (always runs, instant, grounded) +2. Local model rewrite (auto-opt-in when transformers importable) + +Model auto-selects best flan-t5 variant for the machine: +- flan-t5-large (780M) on 16+ GB with MPS/CUDA +- flan-t5-base (250M) otherwise +- Heuristic-only if no transformers + +The model is a FACT REWRITER — it never invents intent. +It rewrites structured heuristic output into fluent prose. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from ..storage.multi_repo import is_model_cached + +if TYPE_CHECKING: + from .git_dynamic import FileHistory, HistoricalCommit + +logger = logging.getLogger(__name__) + + +@dataclass +class ChangeMeaning: + """Semantic meaning for a single commit (heuristic-derived).""" + + intent: str + summary: str # first line of commit message (grounded) + impact: str + affected_concepts: list[str] + + +@dataclass +class EvolutionNarrative: + """Model-generated or heuristic narrative of file evolution.""" + + file_path: str + narrative: str + key_phases: list[str] + change_meanings: list[ChangeMeaning] + model_used: str | None = None # which model produced the narrative (None = heuristic) + + +def _auto_select_model() -> str: + """Auto-select best flan-t5 variant for this machine. + + - 16+ GB RAM + MPS/CUDA → flan-t5-large (780M, better quality) + - Otherwise → flan-t5-base (250M, lighter) + """ + try: + import torch + import psutil + + ram_gb = psutil.virtual_memory().total / (1024**3) + has_accel = ( + (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()) + or torch.cuda.is_available() + ) + if has_accel and ram_gb >= 16: + return "google/flan-t5-large" + return "google/flan-t5-base" + except ImportError: + return "google/flan-t5-base" + + +class DiffSemanticAnalyzer: + """Analyze file evolution semantically. + + Auto-opts-in to local model when transformers is importable. + Model only rewrites heuristic facts — never processes raw diffs. + """ + + def __init__(self, repo_path: Path, model_name: str | None = None): + """ + Args: + repo_path: Path to git repository. + model_name: Override model (default: auto-select based on hardware). + """ + self.repo_path = Path(repo_path).resolve() + self._model_name = model_name + self._model_checked = False + self._can_model = False + self._summarizer = None + + def _can_use_model(self) -> bool: + """Only use model when transformers is importable AND weights are cached.""" + if not self._model_checked: + try: + import transformers # noqa: F401 + self._can_model = is_model_cached(self._get_model_name()) + except ImportError: + self._can_model = False + self._model_checked = True + return self._can_model + + def _get_model_name(self) -> str: + """Resolve model name: explicit override or auto-select.""" + if self._model_name: + return self._model_name + return _auto_select_model() + + def _get_summarizer(self): + """Lazy-load summarizer with auto-selected model.""" + if self._summarizer is None: + from .summarizer import CommitSummarizer + + model = self._get_model_name() + logger.info(f"Auto-selected narrative model: {model}") + self._summarizer = CommitSummarizer(model) + return self._summarizer + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def analyze_change(self, commit: "HistoricalCommit") -> ChangeMeaning: + """Derive per-commit meaning (heuristic only — fast, grounded).""" + from .intent_classifier import IntentClassifier + + classifier = IntentClassifier() + intent_result = classifier.classify( + commit.message, len(commit.files_changed), commit.insertions + commit.deletions + ) + concepts = self._extract_concepts(commit) + + return ChangeMeaning( + intent=intent_result.intent, + summary=commit.message.split("\n")[0].strip(), + impact=intent_result.impact, + affected_concepts=concepts, + ) + + def summarize_evolution(self, history: "FileHistory") -> EvolutionNarrative: + """Generate evolution narrative for a file. + + Always computes heuristic facts first. Then auto-rewrites with model + if transformers is importable (no config needed). + """ + commits = history.effective_commits + if not commits: + return EvolutionNarrative( + file_path=history.file_path, + narrative="No history available.", + key_phases=[], + change_meanings=[], + ) + + # 1. Heuristic analysis (always, instant) + meanings = [self.analyze_change(c) for c in commits[:15]] + phases = self._detect_phases(meanings) + concepts = self._collect_concepts(meanings) + heuristic_narrative = self._template_narrative( + history.file_path, meanings, phases, concepts + ) + + # 2. Model rewrite (auto if importable) + model_used = None + narrative = heuristic_narrative + if self._can_use_model(): + model_result = self._rewrite_with_model(heuristic_narrative, phases, concepts) + if model_result: + narrative = model_result + model_used = self._get_model_name() + + return EvolutionNarrative( + file_path=history.file_path, + narrative=narrative, + key_phases=phases, + change_meanings=meanings, + model_used=model_used, + ) + + # ------------------------------------------------------------------ + # Model rewriting (single call, fact-based input) + # ------------------------------------------------------------------ + + def _rewrite_with_model( + self, facts: str, phases: list[str], concepts: list[str] + ) -> str | None: + """Single generate() — model rewrites structured facts into fluent prose. + + The model NEVER sees raw diffs. Input is heuristic-derived text only. + This prevents hallucinated intent. + """ + try: + summarizer = self._get_summarizer() + prompt = ( + "Rewrite this file evolution summary as a fluent developer-facing paragraph:\n\n" + f"{facts}\n" + ) + if phases: + prompt += f"Development phases: {', '.join(phases)}\n" + if concepts: + prompt += f"Key concepts: {', '.join(concepts[:6])}\n" + prompt += "\nEvolution paragraph:" + + # Single call, greedy decode (fast), max 150 tokens + result = summarizer.generate(prompt, max_length=150, num_beams=1) + if result and len(result) > 20: + return result + return None + except Exception as e: + logger.debug(f"Model rewrite failed: {e}") + return None + + # ------------------------------------------------------------------ + # Heuristic internals + # ------------------------------------------------------------------ + + def _template_narrative( + self, + file_path: str, + meanings: list[ChangeMeaning], + phases: list[str], + concepts: list[str], + ) -> str: + """Deterministic template narrative from heuristic facts.""" + total = len(meanings) + intent_counts: dict[str, int] = {} + for m in meanings: + if m.intent != "unknown": + intent_counts[m.intent] = intent_counts.get(m.intent, 0) + 1 + + # Build parts + parts = [] + for intent, count in sorted(intent_counts.items(), key=lambda x: -x[1]): + parts.append(f"{count} {intent}{'s' if count > 1 else ''}") + + intent_str = ", ".join(parts) if parts else "mixed changes" + concept_str = ", ".join(concepts[:5]) if concepts else "general" + phase_str = "; ".join(phases) if phases else "single phase" + + return ( + f"{file_path}: {total} changes ({intent_str}). " + f"Phases: {phase_str}. " + f"Concepts: {concept_str}." + ) + + def _detect_phases(self, meanings: list[ChangeMeaning]) -> list[str]: + """Detect development phases from sequential intent patterns.""" + if not meanings: + return [] + + phases: list[str] = [] + current_intent = meanings[0].intent + current_count = 1 + + for m in meanings[1:]: + if m.intent == current_intent: + current_count += 1 + else: + if current_count >= 2: + phases.append(f"{current_intent} ({current_count})") + current_intent = m.intent + current_count = 1 + + if current_count >= 2: + phases.append(f"{current_intent} ({current_count})") + + return phases + + def _collect_concepts(self, meanings: list[ChangeMeaning]) -> list[str]: + """Merge concepts from all commit meanings.""" + all_concepts: dict[str, int] = {} + for m in meanings: + for c in m.affected_concepts: + all_concepts[c] = all_concepts.get(c, 0) + 1 + # Sort by frequency + return [c for c, _ in sorted(all_concepts.items(), key=lambda x: -x[1])][:8] + + def _extract_concepts(self, commit: "HistoricalCommit") -> list[str]: + """Extract domain concepts from a single commit.""" + concepts: set[str] = set() + + # From file paths + for fp in commit.files_changed[:5]: + parts = Path(fp).parts + for part in parts: + if part in ("src", "lib", "app", "tests", "test", "__init__.py", "v1"): + continue + stem = Path(part).stem + if len(stem) > 3 and stem != "__init__": + words = re.findall(r"[a-z]+", stem.lower()) + concepts.update(w for w in words if len(w) > 3) + + # From message + msg_words = re.findall(r"[a-z]+", commit.message.lower()) + noise = {"the", "for", "and", "this", "that", "with", "from", "into", "have", "been"} + concepts.update(w for w in msg_words if len(w) > 4 and w not in noise) + + return sorted(concepts)[:8] diff --git a/sia_code/memory/git_dynamic.py b/sia_code/memory/git_dynamic.py new file mode 100644 index 0000000..7bb85ad --- /dev/null +++ b/sia_code/memory/git_dynamic.py @@ -0,0 +1,554 @@ +"""Dynamic git memory — on-demand file history with branch/worktree awareness. + +Computes git context dynamically per query rather than relying on pre-indexed batch data. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from .recency import RecencyConfig, RecencyScorer +from .revert_detector import CommitInfo, RevertDetector, RevertPair + + +# --------------------------------------------------------------------------- +# Data models +# --------------------------------------------------------------------------- + + +@dataclass +class HistoricalCommit: + """A commit in a file's history.""" + + hash: str + message: str + author: str + date: datetime + files_changed: list[str] = field(default_factory=list) + insertions: int = 0 + deletions: int = 0 + is_reverted: bool = False + reverted_by: str | None = None + branch: str | None = None + branch_relevance: float = 1.0 + recency_score: float = 1.0 + intent: str | None = None # set by IntentClassifier + + +@dataclass +class BranchInfo: + """Information about a git branch.""" + + name: str + last_commit_date: datetime | None = None + is_merged: bool = False + is_current: bool = False + relevance: float = 0.0 + + +@dataclass +class WorktreeInfo: + """Information about a git worktree.""" + + path: Path + branch: str + head_commit: str + is_current: bool = False + + +@dataclass +class BranchContext: + """Context about the current branch state.""" + + current_branch: str + base_branch: str + merge_base: str | None = None + divergence_commits: int = 0 # commits since divergence on current + worktrees: list[WorktreeInfo] = field(default_factory=list) + + +@dataclass +class FileHistory: + """Complete history for a file with metadata.""" + + file_path: str + all_commits: list[HistoricalCommit] = field(default_factory=list) + effective_commits: list[HistoricalCommit] = field(default_factory=list) + reverts: list[RevertPair] = field(default_factory=list) + branch_context: BranchContext | None = None + owners: list[tuple[str, int]] = field(default_factory=list) # (author, count) + + +# --------------------------------------------------------------------------- +# BranchResolver +# --------------------------------------------------------------------------- + + +@dataclass +class BranchResolverConfig: + """Configuration for branch resolution.""" + + base_branch: str = "main" + base_branch_fallbacks: list[str] = field( + default_factory=lambda: ["master", "develop"] + ) + branch_relevance: dict[str, float] = field( + default_factory=lambda: { + "current": 1.0, + "base": 0.8, + "recent": 0.5, + "merged": 0.3, + "stale": 0.1, + } + ) + stale_days: int = 90 + + +class BranchResolver: + """Resolves branch context, worktrees, and relevance scoring.""" + + def __init__(self, repo_path: Path, config: BranchResolverConfig | None = None): + self.repo_path = repo_path + self.config = config or BranchResolverConfig() + + def _git(self, *args: str, check: bool = True) -> str | None: + """Run a git command, return stdout or None on failure.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=check, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, OSError): + return None + + def current_branch(self) -> str: + """Get current branch name (handles detached HEAD).""" + branch = self._git("rev-parse", "--abbrev-ref", "HEAD") + if branch and branch != "HEAD": + return branch + # Detached HEAD — use short hash + short = self._git("rev-parse", "--short", "HEAD") + return short or "unknown" + + def resolve_base_branch(self) -> str: + """Find the actual base branch (main/master/develop), local or remote.""" + candidates = [self.config.base_branch] + self.config.base_branch_fallbacks + for branch in candidates: + local_check = self._git("rev-parse", "--verify", f"refs/heads/{branch}", check=False) + if local_check: + return branch + remote_check = self._git("rev-parse", "--verify", f"refs/remotes/origin/{branch}", check=False) + if remote_check: + return f"origin/{branch}" + current = self.current_branch() + branches = self.all_branch_names() + for b in branches: + if b != current: + return b + return current + + def merge_base(self, branch_a: str, branch_b: str) -> str | None: + """Find merge base (common ancestor). Returns None on shallow clone.""" + return self._git("merge-base", branch_a, branch_b, check=False) + + def all_branch_names(self) -> list[str]: + """List all local branch names.""" + output = self._git("branch", "--format=%(refname:short)") + if not output: + return [] + return [b.strip() for b in output.splitlines() if b.strip()] + + def all_branches(self) -> list[BranchInfo]: + """All local branches with metadata and relevance scoring.""" + current = self.current_branch() + base = self.resolve_base_branch() + now = datetime.now(timezone.utc) + + output = self._git( + "branch", + "--format=%(refname:short)|%(committerdate:iso-strict)|%(upstream:track)", + ) + if not output: + return [] + + branches: list[BranchInfo] = [] + for line in output.splitlines(): + parts = line.strip().split("|") + if len(parts) < 2: + continue + name = parts[0].strip() + date_str = parts[1].strip() if len(parts) > 1 else "" + + last_date = None + if date_str: + try: + last_date = datetime.fromisoformat(date_str) + except ValueError: + pass + + # Determine if merged into base + is_merged = False + if name != base: + merge_check = self._git( + "branch", "--merged", base, "--format=%(refname:short)", check=False + ) + if merge_check: + is_merged = name in merge_check.splitlines() + + # Compute relevance + relevance = self._compute_relevance( + name, current, base, last_date, is_merged, now + ) + + branches.append( + BranchInfo( + name=name, + last_commit_date=last_date, + is_merged=is_merged, + is_current=(name == current), + relevance=relevance, + ) + ) + + return sorted(branches, key=lambda b: b.relevance, reverse=True) + + def _compute_relevance( + self, + name: str, + current: str, + base: str, + last_date: datetime | None, + is_merged: bool, + now: datetime, + ) -> float: + """Compute branch relevance score.""" + rel = self.config.branch_relevance + if name == current: + return rel.get("current", 1.0) + if name == base: + return rel.get("base", 0.8) + if is_merged: + return rel.get("merged", 0.3) + if last_date: + days_ago = (now - last_date).total_seconds() / 86400 + if days_ago > self.config.stale_days: + return rel.get("stale", 0.1) + return rel.get("recent", 0.5) + return rel.get("stale", 0.1) + + def branches_touching_file(self, file_path: str) -> list[str]: + """Which branches have commits touching this file.""" + output = self._git( + "log", "--all", "--format=%D", "--", file_path, check=False + ) + if not output: + return [] + branches: set[str] = set() + for line in output.splitlines(): + for ref in line.split(","): + ref = ref.strip() + if ref and "HEAD" not in ref and "->" not in ref: + branches.add(ref.split("/")[-1]) # strip origin/ prefix + return sorted(branches) + + def worktree_list(self) -> list[WorktreeInfo]: + """List all git worktrees.""" + output = self._git("worktree", "list", "--porcelain") + if not output: + return [] + + worktrees: list[WorktreeInfo] = [] + current_wt: dict[str, str] = {} + + for line in output.splitlines(): + if not line.strip(): + if current_wt.get("worktree"): + worktrees.append( + WorktreeInfo( + path=Path(current_wt["worktree"]), + branch=current_wt.get("branch", "").replace( + "refs/heads/", "" + ), + head_commit=current_wt.get("HEAD", ""), + is_current=( + Path(current_wt["worktree"]).resolve() + == self.repo_path.resolve() + ), + ) + ) + current_wt = {} + elif line.startswith("worktree "): + current_wt["worktree"] = line[9:] + elif line.startswith("HEAD "): + current_wt["HEAD"] = line[5:] + elif line.startswith("branch "): + current_wt["branch"] = line[7:] + + # Last entry + if current_wt.get("worktree"): + worktrees.append( + WorktreeInfo( + path=Path(current_wt["worktree"]), + branch=current_wt.get("branch", "").replace("refs/heads/", ""), + head_commit=current_wt.get("HEAD", ""), + is_current=( + Path(current_wt["worktree"]).resolve() + == self.repo_path.resolve() + ), + ) + ) + return worktrees + + def get_branch_context(self) -> BranchContext: + """Get full branch context for current state.""" + current = self.current_branch() + base = self.resolve_base_branch() + mb = self.merge_base(current, base) if current != base else None + + divergence = 0 + if mb: + count = self._git("rev-list", "--count", f"{mb}..HEAD") + if count: + try: + divergence = int(count) + except ValueError: + pass + + return BranchContext( + current_branch=current, + base_branch=base, + merge_base=mb, + divergence_commits=divergence, + worktrees=self.worktree_list(), + ) + + +# --------------------------------------------------------------------------- +# GitDynamicMemory +# --------------------------------------------------------------------------- + + +class GitDynamicMemory: + """On-demand git memory — computes file history dynamically. + + Features: + - Per-file history with rename tracking (--follow) + - Cross-branch context (current + base + other branches) + - Revert detection and filtering + - Recency-weighted scoring + - Worktree awareness + """ + + def __init__( + self, + repo_path: Path, + recency_config: RecencyConfig | None = None, + branch_config: BranchResolverConfig | None = None, + ): + self.repo_path = Path(repo_path).resolve() + self.recency = RecencyScorer(recency_config) + self.branch_resolver = BranchResolver(self.repo_path, branch_config) + self.revert_detector = RevertDetector() + + def _git(self, *args: str) -> str | None: + """Run git command, return stdout or None.""" + try: + result = subprocess.run( + ["git", *args], + cwd=self.repo_path, + capture_output=True, + text=True, + check=True, + ) + return result.stdout + except (subprocess.CalledProcessError, OSError): + return None + + def file_history( + self, + file_path: str, + branch: str | None = None, + cross_branch: bool = True, + limit: int = 20, + ) -> FileHistory: + """Get comprehensive history for a file. + + Args: + file_path: Path relative to repo root. + branch: Specific branch (default: current HEAD). + cross_branch: Include commits from base branch since divergence. + limit: Max commits to return. + + Returns: + FileHistory with scored, revert-filtered commits. + """ + branch_ctx = self.branch_resolver.get_branch_context() + + # Get commits on target branch (with rename following) + target_branch = branch or branch_ctx.current_branch + commits = self._get_file_commits(file_path, target_branch, limit * 2) + + # Cross-branch: also get base branch commits since divergence + cross_commits: list[HistoricalCommit] = [] + if cross_branch and branch_ctx.merge_base and target_branch != branch_ctx.base_branch: + cross_commits = self._get_file_commits_since( + file_path, branch_ctx.base_branch, branch_ctx.merge_base, limit + ) + for c in cross_commits: + c.branch = branch_ctx.base_branch + c.branch_relevance = 0.8 + + # Tag current branch commits + for c in commits: + c.branch = target_branch + c.branch_relevance = 1.0 + + # Merge and deduplicate + all_commits = self._merge_commits(commits + cross_commits) + + # Detect reverts + commit_infos = [ + CommitInfo(hash=c.hash, message=c.message) for c in all_commits + ] + reverts = self.revert_detector.detect_reverts(commit_infos) + reverted_hashes = {p.reverted_hash for p in reverts} + reverting_hashes = {p.reverting_hash for p in reverts} + + for c in all_commits: + if c.hash in reverted_hashes: + c.is_reverted = True + # Find which commit reverted it + for p in reverts: + if p.reverted_hash == c.hash: + c.reverted_by = p.reverting_hash + break + + # Score by recency + for c in all_commits: + c.recency_score = self.recency.weighted_score( + 1.0, c.date, c.branch_relevance + ) + + # Sort by recency score (highest first) + all_commits.sort(key=lambda c: c.recency_score, reverse=True) + + # Effective history: remove reverted + reverting commits + excluded = reverted_hashes | reverting_hashes + effective = [c for c in all_commits if c.hash not in excluded] + + # Compute owners + owners = self._compute_owners(all_commits) + + return FileHistory( + file_path=file_path, + all_commits=all_commits[:limit], + effective_commits=effective[:limit], + reverts=reverts, + branch_context=branch_ctx, + owners=owners, + ) + + def _get_file_commits( + self, file_path: str, branch: str, limit: int + ) -> list[HistoricalCommit]: + """Get commits touching a file on a specific branch using --follow.""" + output = self._git( + "log", + branch, + f"--max-count={limit}", + "--follow", + "--format=%H|%s|%an|%aI", + "--numstat", + "--", + file_path, + ) + if not output: + return [] + return self._parse_log_output(output) + + def _get_file_commits_since( + self, file_path: str, branch: str, since_commit: str, limit: int + ) -> list[HistoricalCommit]: + """Get commits touching a file on branch since a specific commit.""" + output = self._git( + "log", + f"{since_commit}..{branch}", + f"--max-count={limit}", + "--follow", + "--format=%H|%s|%an|%aI", + "--numstat", + "--", + file_path, + ) + if not output: + return [] + return self._parse_log_output(output) + + def _parse_log_output(self, output: str) -> list[HistoricalCommit]: + """Parse git log output with --format=%H|%s|%an|%aI and --numstat.""" + commits: list[HistoricalCommit] = [] + current: HistoricalCommit | None = None + + for line in output.splitlines(): + if "|" in line and len(line.split("|")) >= 4: + # Looks like a commit line + parts = line.split("|", 3) + if len(parts[0]) >= 7 and all(c in "0123456789abcdef" for c in parts[0][:7]): + if current: + commits.append(current) + try: + date = datetime.fromisoformat(parts[3].strip()) + except (ValueError, IndexError): + date = datetime.now(timezone.utc) + current = HistoricalCommit( + hash=parts[0].strip(), + message=parts[1].strip(), + author=parts[2].strip(), + date=date, + ) + continue + + # Numstat line: additions\tdeletions\tfilename + if current and "\t" in line: + parts = line.split("\t") + if len(parts) >= 3: + try: + ins = int(parts[0]) if parts[0] != "-" else 0 + dels = int(parts[1]) if parts[1] != "-" else 0 + current.insertions += ins + current.deletions += dels + current.files_changed.append(parts[2]) + except ValueError: + pass + + if current: + commits.append(current) + return commits + + def _merge_commits( + self, commits: list[HistoricalCommit] + ) -> list[HistoricalCommit]: + """Deduplicate commits by hash, keeping first occurrence.""" + seen: set[str] = set() + result: list[HistoricalCommit] = [] + for c in commits: + if c.hash not in seen: + seen.add(c.hash) + result.append(c) + return result + + def _compute_owners( + self, commits: list[HistoricalCommit] + ) -> list[tuple[str, int]]: + """Compute top authors by commit count.""" + author_counts: dict[str, int] = {} + for c in commits: + author_counts[c.author] = author_counts.get(c.author, 0) + 1 + return sorted(author_counts.items(), key=lambda x: x[1], reverse=True) diff --git a/sia_code/memory/intent_classifier.py b/sia_code/memory/intent_classifier.py new file mode 100644 index 0000000..0a4929a --- /dev/null +++ b/sia_code/memory/intent_classifier.py @@ -0,0 +1,103 @@ +"""Heuristic commit intent classification. + +Classifies commit intent from conventional commit prefixes +and estimates impact from diff statistics. No model required. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +# Conventional commit prefix → intent category +PREFIX_MAP: dict[str, str] = { + "feat": "feature", + "fix": "bugfix", + "refactor": "refactor", + "perf": "performance", + "docs": "documentation", + "test": "testing", + "tests": "testing", + "chore": "maintenance", + "ci": "ci", + "build": "build", + "revert": "revert", + "style": "style", + "release": "release", +} + +# Pattern to extract conventional commit prefix +_PREFIX_RE = re.compile(r"^(\w+)(?:\(.+?\))?[!:]") + + +@dataclass +class CommitIntent: + """Classified intent and impact for a commit.""" + + intent: str # "feature", "bugfix", "refactor", etc. + impact: str # "high", "medium", "low" + confidence: float # 0.0-1.0 + + +class IntentClassifier: + """Classify commit intent from message prefix + diff stats. + + Pure heuristic — no model needed. Fast and deterministic. + """ + + def classify( + self, message: str, files_changed: int = 0, lines_changed: int = 0 + ) -> CommitIntent: + """Classify a commit's intent and impact. + + Args: + message: Commit message (first line). + files_changed: Number of files in commit. + lines_changed: Total insertions + deletions. + + Returns: + CommitIntent with category, impact level, and confidence. + """ + intent = self._classify_intent(message) + impact = self._estimate_impact(files_changed, lines_changed) + confidence = 0.9 if intent != "unknown" else 0.3 + return CommitIntent(intent=intent, impact=impact, confidence=confidence) + + def _classify_intent(self, message: str) -> str: + """Extract intent from conventional commit prefix.""" + first_line = message.split("\n")[0].strip().lower() + + # Try conventional commit pattern: type(scope): description + m = _PREFIX_RE.match(first_line) + if m: + prefix = m.group(1) + if prefix in PREFIX_MAP: + return PREFIX_MAP[prefix] + + # Fallback: keyword detection in message + if any(w in first_line for w in ("fix", "bug", "patch", "hotfix")): + return "bugfix" + if any(w in first_line for w in ("add", "implement", "feature", "new")): + return "feature" + if any(w in first_line for w in ("refactor", "restructure", "clean")): + return "refactor" + if any(w in first_line for w in ("perf", "optim", "speed", "fast")): + return "performance" + if any(w in first_line for w in ("revert", "undo", "rollback")): + return "revert" + if any(w in first_line for w in ("release", "bump", "version")): + return "release" + if any(w in first_line for w in ("doc", "readme", "comment")): + return "documentation" + if any(w in first_line for w in ("test", "spec", "coverage")): + return "testing" + + return "unknown" + + def _estimate_impact(self, files_changed: int, lines_changed: int) -> str: + """Estimate impact from diff size.""" + if lines_changed > 100 or files_changed > 5: + return "high" + if lines_changed > 30 or files_changed > 2: + return "medium" + return "low" diff --git a/sia_code/memory/recency.py b/sia_code/memory/recency.py new file mode 100644 index 0000000..3ccd64a --- /dev/null +++ b/sia_code/memory/recency.py @@ -0,0 +1,100 @@ +"""Recency scoring for git commits. + +Exponential time decay with a configurable working window. +Commits within the working window get full weight (1.0). +Beyond the window, score decays exponentially with configurable halflife. +""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from dataclasses import dataclass + + +@dataclass +class RecencyConfig: + """Configuration for recency scoring.""" + + halflife_days: float = 30.0 + """Time in days for score to decay to 50% beyond working window.""" + + working_window_days: int = 14 + """Commits within this many days get full weight (no decay).""" + + +class RecencyScorer: + """Score commits by recency using exponential decay. + + Behaviour: + - Commits within working_window_days → weight = 1.0 + - Beyond window: weight = exp(-lambda * (days_ago - window)) + - lambda = ln(2) / halflife_days + + Examples (default config: window=14d, halflife=30d): + - 7 days old → 1.0 + - 14 days old → 1.0 (still in window) + - 44 days old → 0.5 (30 days beyond window = one halflife) + - 74 days old → 0.25 (60 days beyond) + - 134 days old → 0.0625 (120 days beyond) + """ + + def __init__(self, config: RecencyConfig | None = None): + self.config = config or RecencyConfig() + self._decay_lambda = math.log(2) / self.config.halflife_days + + @property + def halflife_days(self) -> float: + return self.config.halflife_days + + @property + def working_window_days(self) -> int: + return self.config.working_window_days + + def score(self, commit_date: datetime, now: datetime | None = None) -> float: + """Compute recency weight for a commit. + + Args: + commit_date: When the commit was authored (timezone-aware preferred). + now: Reference time (default: utcnow). Pass for deterministic tests. + + Returns: + Weight in (0.0, 1.0]. + """ + if now is None: + now = datetime.now(timezone.utc) + + # Normalize to UTC for comparison + if commit_date.tzinfo is None: + commit_date = commit_date.replace(tzinfo=timezone.utc) + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) + + days_ago = (now - commit_date).total_seconds() / 86400.0 + if days_ago <= 0: + return 1.0 + if days_ago <= self.config.working_window_days: + return 1.0 + + effective_days = days_ago - self.config.working_window_days + return math.exp(-self._decay_lambda * effective_days) + + def weighted_score( + self, + base_score: float, + commit_date: datetime, + branch_relevance: float = 1.0, + now: datetime | None = None, + ) -> float: + """Combined scoring: base * recency * branch_relevance. + + Args: + base_score: Raw score (e.g., coupling, importance). + commit_date: When the commit was authored. + branch_relevance: Branch relevance weight (1.0 = current, 0.8 = base, etc.). + now: Reference time for deterministic tests. + + Returns: + Weighted score incorporating all factors. + """ + return base_score * self.score(commit_date, now=now) * branch_relevance diff --git a/sia_code/memory/revert_detector.py b/sia_code/memory/revert_detector.py new file mode 100644 index 0000000..a55d40f --- /dev/null +++ b/sia_code/memory/revert_detector.py @@ -0,0 +1,188 @@ +"""Revert detection for git commits. + +Detects revert commits via message patterns and marks original commits as reverted. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + + +# Patterns that identify a revert commit +REVERT_PATTERNS: list[re.Pattern] = [ + re.compile(r'^[Rr]evert "(.+)"'), # Git default: Revert "original message" + re.compile(r"^[Rr]evert:\s*(.+)"), # Conventional commit: revert: message + re.compile(r"[Tt]his reverts commit ([a-f0-9]{7,40})"), # Body reference +] + + +@dataclass +class RevertPair: + """A pair of commits: the original and the commit that reverts it.""" + + reverted_hash: str + """Hash of the commit that was reverted.""" + + reverting_hash: str + """Hash of the commit that performs the revert.""" + + matched_by: str + """Which pattern matched (for debugging).""" + + +@dataclass +class CommitInfo: + """Minimal commit info for revert detection.""" + + hash: str + message: str + body: str = "" + + +class RevertDetector: + """Detects revert relationships between commits. + + Strategy (message-based, fast): + 1. Match revert patterns in commit message/body + 2. For message-match reverts: find original by matching quoted message + 3. For hash-reference reverts: direct hash lookup + """ + + def detect_reverts(self, commits: list[CommitInfo]) -> list[RevertPair]: + """Find all revert relationships in a list of commits. + + Args: + commits: List of commits to analyze (newer first). + + Returns: + List of RevertPair identifying original→reverting relationships. + """ + pairs: list[RevertPair] = [] + # Build message→hash index for message-matching + msg_to_hash: dict[str, str] = {} + for c in commits: + # Use first line of message for matching + first_line = c.message.split("\n")[0].strip() + msg_to_hash[first_line] = c.hash + + for c in commits: + full_text = f"{c.message}\n{c.body}" + pair = self._check_commit(c, full_text, msg_to_hash) + if pair: + pairs.append(pair) + + return pairs + + def _check_commit( + self, commit: CommitInfo, full_text: str, msg_to_hash: dict[str, str] + ) -> RevertPair | None: + """Check if a single commit is a revert.""" + first_line = commit.message.split("\n")[0].strip() + + # Pattern 1: Revert "original message" (exact quote match) + m = REVERT_PATTERNS[0].match(first_line) + if m: + original_msg = m.group(1).strip() + original_hash = msg_to_hash.get(original_msg) + if original_hash and original_hash != commit.hash: + return RevertPair( + reverted_hash=original_hash, + reverting_hash=commit.hash, + matched_by="message_quote", + ) + + # Pattern 2: revert: message (conventional commit) + m = REVERT_PATTERNS[1].match(first_line) + if m: + revert_desc = m.group(1).strip() + # Try exact match first + original_hash = msg_to_hash.get(revert_desc) + if original_hash and original_hash != commit.hash: + return RevertPair( + reverted_hash=original_hash, + reverting_hash=commit.hash, + matched_by="conventional_prefix", + ) + # Fuzzy: find recent commit with highest keyword overlap + best = self._fuzzy_match(revert_desc, msg_to_hash, commit.hash) + if best: + return RevertPair( + reverted_hash=best, + reverting_hash=commit.hash, + matched_by="conventional_fuzzy", + ) + + # Pattern 3: "This reverts commit " in body + m = REVERT_PATTERNS[2].search(full_text) + if m: + reverted_hash = m.group(1) + return RevertPair( + reverted_hash=reverted_hash, + reverting_hash=commit.hash, + matched_by="hash_reference", + ) + + return None + + def _fuzzy_match( + self, revert_desc: str, msg_to_hash: dict[str, str], exclude_hash: str + ) -> str | None: + """Find the most likely original commit via keyword overlap. + + Uses Jaccard similarity on normalized word sets. + Threshold: >= 0.3 overlap to consider a match. + """ + revert_words = self._normalize_words(revert_desc) + if len(revert_words) < 2: + return None + + best_score = 0.0 + best_hash: str | None = None + + for msg, h in msg_to_hash.items(): + if h == exclude_hash: + continue + msg_words = self._normalize_words(msg) + if not msg_words: + continue + # Jaccard similarity + intersection = revert_words & msg_words + union = revert_words | msg_words + score = len(intersection) / len(union) if union else 0 + if score > best_score and score >= 0.3: + best_score = score + best_hash = h + + return best_hash + + @staticmethod + def _normalize_words(text: str) -> set[str]: + """Extract significant words (lowercase, strip punctuation, drop short).""" + # Remove common prefixes + for prefix in ("feat:", "fix:", "chore:", "refactor:", "revert:", "docs:", "ci:"): + if text.lower().startswith(prefix): + text = text[len(prefix):] + break + words = set(re.findall(r'[a-z0-9]+', text.lower())) + # Drop very short words and common noise + noise = {"the", "to", "for", "in", "of", "a", "an", "and", "or", "is", "was"} + return {w for w in words if len(w) > 2 and w not in noise} + + def filter_reverted(self, commits: list[CommitInfo]) -> list[CommitInfo]: + """Return commits with reverted ones removed. + + Removes BOTH the reverted commit AND the reverting commit from the + effective history (neither adds meaningful signal). + """ + pairs = self.detect_reverts(commits) + excluded: set[str] = set() + for pair in pairs: + excluded.add(pair.reverted_hash) + excluded.add(pair.reverting_hash) + return [c for c in commits if c.hash not in excluded] + + def reverted_set(self, commits: list[CommitInfo]) -> set[str]: + """Return set of commit hashes that have been reverted.""" + pairs = self.detect_reverts(commits) + return {p.reverted_hash for p in pairs} diff --git a/sia_code/memory/semantic_grouper.py b/sia_code/memory/semantic_grouper.py new file mode 100644 index 0000000..128642c --- /dev/null +++ b/sia_code/memory/semantic_grouper.py @@ -0,0 +1,185 @@ +"""Semantic file grouping using existing code-chunk embeddings. + +Finds files semantically related to a target by querying the +existing sia-code index (no new vector store needed). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..storage.sqlite_vec_backend import SqliteVecBackend + +from .blast_radius import CoupledFile + + +@dataclass +class SemanticRelation: + """A file semantically related to the target.""" + + file_path: str + similarity_score: float # 0.0-1.0 + relation_type: str = "semantic" # "semantic", "co-change", "both" + + +@dataclass +class EnrichedRelation: + """Combined git + semantic relation.""" + + file_path: str + git_coupling: float # 0.0-1.0 from co-change + semantic_similarity: float # 0.0-1.0 from embeddings + combined_score: float + relation_type: str # "co-change", "semantic", "both" + + +class SemanticFileGrouper: + """Find semantically related files using existing indexed embeddings. + + Leverages the persistent sqlite-vec index that sia-code already maintains. + No new embedding work — just queries existing chunks. + + Strategy: + 1. Get chunks belonging to target file from index + 2. For each chunk, find nearest neighbors in vector space + 3. Aggregate neighbor chunks by file_path + 4. Score by sum of similarity scores per file + """ + + def __init__(self, backend: "SqliteVecBackend"): + self.backend = backend + + def is_available(self) -> bool: + """Check if semantic search is available (index exists + embeddings enabled).""" + return ( + self.backend is not None + and self.backend.conn is not None + and self.backend.embedding_enabled + ) + + def related_files(self, file_path: str, k: int = 10) -> list[SemanticRelation]: + """Find files semantically related to target using existing embeddings. + + Args: + file_path: Target file path (relative to repo root). + k: Max related files to return. + + Returns: + List of SemanticRelation sorted by similarity score. + """ + if not self.is_available(): + return [] + + # Use search_files with file content as implicit query + # Strategy: get the target file's content/chunks, find similar files + try: + # Read chunks for target file from DB + cursor = self.backend.conn.cursor() + cursor.execute( + "SELECT content FROM chunks WHERE file_path = ? LIMIT 5", + (file_path,), + ) + rows = cursor.fetchall() + + if not rows: + # File not in index — try with variations + cursor.execute( + "SELECT content FROM chunks WHERE file_path LIKE ? LIMIT 5", + (f"%{file_path}",), + ) + rows = cursor.fetchall() + + if not rows: + return [] + + # Use first few chunks as query text + query_text = "\n".join(row[0][:500] for row in rows[:3]) + + # Search for similar files + results = self.backend.search_files( + query=query_text, k=k + 5, vector_weight=0.9 + ) + + # Filter out self and convert to SemanticRelation + relations: list[SemanticRelation] = [] + max_score = results[0][1] if results else 1.0 + + for result_path, score in results: + # Skip self (match by suffix to handle path variations) + if result_path.endswith(file_path) or file_path.endswith(result_path): + continue + # Normalize score to 0-1 + normalized = score / max_score if max_score > 0 else 0 + relations.append( + SemanticRelation( + file_path=result_path, + similarity_score=min(normalized, 1.0), + ) + ) + if len(relations) >= k: + break + + return relations + + except Exception: + # Graceful degradation — index might be corrupt or unavailable + return [] + + def semantic_blast_radius( + self, + file_path: str, + git_coupled: list[CoupledFile], + k: int = 15, + git_weight: float = 0.7, + ) -> list[EnrichedRelation]: + """Combine git co-change with semantic similarity. + + Args: + file_path: Target file. + git_coupled: Co-change results from BlastRadiusAnalyzer. + k: Max results. + git_weight: Weight for git signal (1 - git_weight = semantic weight). + + Returns: + Merged + ranked EnrichedRelation list. + """ + semantic_weight = 1.0 - git_weight + + # Get semantic relations + semantic = self.related_files(file_path, k=k * 2) + semantic_map: dict[str, float] = {r.file_path: r.similarity_score for r in semantic} + + # Build git map + git_map: dict[str, float] = {c.path: c.coupling_score for c in git_coupled} + + # Merge all files + all_files = set(semantic_map.keys()) | set(git_map.keys()) + enriched: list[EnrichedRelation] = [] + + for fp in all_files: + git_score = git_map.get(fp, 0.0) + sem_score = semantic_map.get(fp, 0.0) + combined = git_weight * git_score + semantic_weight * sem_score + + # Determine relation type + if git_score > 0 and sem_score > 0: + rel_type = "both" + elif git_score > 0: + rel_type = "co-change" + else: + rel_type = "semantic" + + enriched.append( + EnrichedRelation( + file_path=fp, + git_coupling=git_score, + semantic_similarity=sem_score, + combined_score=combined, + relation_type=rel_type, + ) + ) + + enriched.sort(key=lambda e: e.combined_score, reverse=True) + return enriched[:k] diff --git a/sia_code/memory/summarizer.py b/sia_code/memory/summarizer.py index 835e2a6..3d66fcd 100644 --- a/sia_code/memory/summarizer.py +++ b/sia_code/memory/summarizer.py @@ -100,7 +100,45 @@ def summarize_commits(self, commits: list[str], max_length: int = 100) -> Option logger.warning(f"Error during summarization: {e}") return None - def enhance_changelog(self, tag: str, original_summary: str, commits: list[str]) -> str: + def generate(self, prompt: str, max_length: int = 150, num_beams: int = 1) -> Optional[str]: + """General-purpose generation with custom prompt. + + Used by DiffSemanticAnalyzer for narrative rewriting. + Supports greedy (num_beams=1) for speed or beam search for quality. + + Args: + prompt: Input prompt text. + max_length: Maximum output tokens. + num_beams: Beam width (1 = greedy, fast; 4 = beam search, better quality). + + Returns: + Generated text, or None if model unavailable. + """ + self._load_model() + if self._model is None or self._tokenizer is None: + return None + + try: + inputs = self._tokenizer( + prompt, return_tensors="pt", max_length=512, truncation=True + ) + device = next(self._model.parameters()).device + inputs = {k: v.to(device) for k, v in inputs.items()} + + outputs = self._model.generate( + **inputs, max_length=max_length, num_beams=num_beams, + early_stopping=(num_beams > 1), + ) + result = self._tokenizer.decode(outputs[0], skip_special_tokens=True) + return result.strip() if result.strip() else None + + except Exception as e: + logger.debug(f"Generation failed: {e}") + return None + + def enhance_changelog( + self, tag: str, original_summary: str, commits: list[str] + ) -> str: """Enhance a sparse changelog entry with commit details. Args: diff --git a/sia_code/search/multi_hop.py b/sia_code/search/multi_hop.py index 09dab14..6796c9e 100644 --- a/sia_code/search/multi_hop.py +++ b/sia_code/search/multi_hop.py @@ -1,10 +1,10 @@ """Multi-hop code research for discovering code relationships.""" import logging -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Set -from ..core.models import Chunk, CodeRelationshipRecord +from ..core.models import Chunk, CodeRelationshipRecord, SearchResult from ..core.types import ChunkId from ..storage.base import StorageBackend from .entity_extractor import EntityExtractor, Entity @@ -50,34 +50,79 @@ def __init__(self, backend: StorageBackend, max_hops: int = 2): self.extractor = EntityExtractor() self._preprocessor = QueryPreprocessor() # Cache instance to avoid recreation - def _initial_search(self, question: str, k: int) -> list: - """Perform initial search with adaptive mode selection. - - Args: - question: Natural language question - k: Number of results to return + def _aggregate_seed_results( + self, result_sets: list[list[SearchResult]], k: int + ) -> list[SearchResult]: + """Merge result sets from multiple query variants. - Returns: - List of search results + Uses retrieval evidence to rank variants implicitly: + - better scores win + - repeated hits across variants get a modest boost + - earlier ranks contribute slightly more than later ranks """ + merged: dict[str, tuple[SearchResult, float, int, float]] = {} + for results in result_sets: + for rank, r in enumerate(results, start=1): + key = ( + str(r.chunk.id) + if r.chunk.id + else f"{r.chunk.file_path}:{r.chunk.symbol}:{r.chunk.start_line}:{r.chunk.end_line}" + ) + rank_bonus = 0.05 / rank + if key in merged: + base, best_score, hits, bonus = merged[key] + merged[key] = (base, max(best_score, r.score), hits + 1, bonus + rank_bonus) + else: + merged[key] = (r, r.score, 1, rank_bonus) + + ranked: list[SearchResult] = [] + for base, best_score, hits, bonus in merged.values(): + combined = best_score + (0.03 * (hits - 1)) + bonus + ranked.append(replace(base, score=combined)) + ranked.sort(key=lambda x: x.score, reverse=True) + return ranked[:k] + + def _initial_search(self, question: str, k: int) -> list: + """Perform initial search with adaptive mode selection and query rewrites.""" + allow_model = bool(getattr(self.backend, "flan_query_rewrite", False)) + variants = self._preprocessor.expand_variants(question, allow_model=allow_model) + if not variants: + variants = [question] + if self.backend.embedding_enabled: try: - logger.info(f"Using semantic search for query: {question[:100]}") - return self.backend.search_semantic(question, k=k) + granularity = getattr(self.backend, "embedding_granularity", "chunk") + result_sets: list[list[SearchResult]] = [] + + if granularity == "budget": + logger.info( + f"Using hybrid multi-variant search for budgeted index query: {question[:100]}" + ) + for i, variant in enumerate(variants): + vw = 0.4 if i == 0 else 0.3 + result_sets.append( + self.backend.search_hybrid(variant, k=max(k, 8), vector_weight=vw) + ) + return self._aggregate_seed_results(result_sets, k) + + logger.info(f"Using semantic+hybrid variant search for query: {question[:100]}") + result_sets.append(self.backend.search_semantic(variants[0], k=max(k, 8))) + if len(variants) > 1: + result_sets.append( + self.backend.search_hybrid(variants[1], k=max(k, 8), vector_weight=0.5) + ) + if len(variants) > 2: + result_sets.append(self.backend.search_lexical(variants[2], k=max(k, 6))) + return self._aggregate_seed_results(result_sets, k) except Exception as e: logger.warning( - f"Semantic search failed ({e.__class__.__name__}: {str(e)}), " + f"Semantic/hybrid variant search failed ({e.__class__.__name__}: {str(e)}), " "falling back to lexical search" ) - # Fall through to lexical search - - # Lexical search path - logger.info(f"Using lexical search for query: {question[:100]}") - processed_query = self._preprocessor.preprocess(question) - search_query = processed_query or question - if not processed_query: - logger.debug(f"Preprocessing returned empty for query: {question[:100]}") - return self.backend.search_lexical(search_query, k=k) + + logger.info(f"Using lexical variant search for query: {question[:100]}") + result_sets = [self.backend.search_lexical(v, k=max(k, 8)) for v in variants] + return self._aggregate_seed_results(result_sets, k) def _expand_from_persisted_graph( self, diff --git a/sia_code/search/query_preprocessor.py b/sia_code/search/query_preprocessor.py index 720cca8..b004be4 100644 --- a/sia_code/search/query_preprocessor.py +++ b/sia_code/search/query_preprocessor.py @@ -3,6 +3,10 @@ import re from typing import Set +from ..storage.multi_repo import is_model_cached + +_FLAN_REWRITE_SUMMARIZER = None + class QueryPreprocessor: """Preprocess natural language queries for lexical search. @@ -106,6 +110,142 @@ def preprocess(self, question: str) -> str: # Rejoin with spaces return " ".join(keywords) + def expand_variants(self, question: str, allow_model: bool = False) -> list[str]: + """Produce 2-4 query variants for research seeding. + + Variants combine: + - raw natural language + - cleaned keyword query + - synthesized code-like identifiers / framework-role forms + - optional cached-FLAN rewrite candidate + """ + if not question or not question.strip(): + return [] + + variants: list[str] = [] + raw = question.strip() + variants.append(raw) + + keywords = self.extract_keywords(question) + keyword_query = " ".join(keywords) + if keyword_query and keyword_query.lower() != raw.lower(): + variants.append(keyword_query) + + focused_parts: list[str] = [] + identifiers = [t for t in keywords if self._is_code_identifier(t)] + plain = [t for t in keywords if not self._is_code_identifier(t)] + focused_parts.extend(identifiers[:4]) + focused_parts.extend(self._synthesize_code_forms(keywords)) + focused_parts.extend([t for t in plain if len(t) >= 4 and t.lower() not in {'work', 'works'}][:4]) + focused = " ".join(dict.fromkeys(p for p in focused_parts if p)).strip() + if focused and focused.lower() not in {v.lower() for v in variants}: + variants.append(focused) + + if allow_model: + flan_variant = self._generate_flan_variant(question, keywords) + if flan_variant and flan_variant.lower() not in {v.lower() for v in variants}: + variants.append(flan_variant) + + deduped: list[str] = [] + seen: set[str] = set() + for v in variants: + key = v.lower().strip() + if key and key not in seen: + deduped.append(v) + seen.add(key) + return deduped[:4] + + def _synthesize_code_forms(self, tokens: list[str]) -> list[str]: + """Generate code-like identifier variants from plain-language tokens. + + Examples: + - stable diffusion trainer -> StableDiffusionTrainer, stable_diffusion_trainer + - task rest api -> TaskViewSet, TaskSerializer, TaskAPIView + """ + if not tokens: + return [] + + lowered = [t.lower() for t in tokens] + generic_noise = {'work', 'works', 'working'} + results: list[str] = [] + + role_terms = { + "trainer": "Trainer", + "config": "Config", + "service": "Service", + "manager": "Manager", + "controller": "Controller", + "model": "Model", + "serializer": "Serializer", + "viewset": "ViewSet", + "handler": "Handler", + "pipeline": "Pipeline", + } + + role_idx = next((i for i, t in enumerate(lowered) if t in role_terms), None) + if role_idx is not None and role_idx > 0: + base_tokens = lowered[: role_idx + 1] + camel = "".join(t.capitalize() for t in base_tokens) + snake = "_".join(base_tokens) + results.extend([camel, snake]) + + # API-ish expansion: task REST API -> TaskViewSet / TaskSerializer / TaskAPIView + api_tokens = {"api", "rest", "endpoint"} + if any(t in api_tokens for t in lowered): + base = next((t for t in lowered if t not in api_tokens and t not in generic_noise and len(t) > 2), None) + if base: + title = base.capitalize() + results.extend([f"{title}ViewSet", f"{title}Serializer", f"{title}APIView"]) + + # Generic multi-token camel/snake for first 2-3 meaningful words + content = [t for t in lowered if len(t) > 2 and t not in api_tokens and t not in generic_noise][:3] + if len(content) >= 2: + results.append("".join(t.capitalize() for t in content)) + results.append("_".join(content)) + + # Keep small and unique + out: list[str] = [] + seen: set[str] = set() + for r in results: + k = r.lower() + if r and k not in seen: + out.append(r) + seen.add(k) + return out[:6] + + def _generate_flan_variant(self, question: str, keywords: list[str]) -> str | None: + """Optionally generate one concise code-search rewrite using cached FLAN. + + Never downloads models. Returns None if FLAN base not cached or transformers unavailable. + """ + model_name = 'google/flan-t5-base' + if not is_model_cached(model_name): + return None + # Only worth it for natural-language questions, not direct symbol lookups + if sum(1 for t in keywords if not self._is_code_identifier(t)) < 2: + return None + try: + from ..memory.summarizer import CommitSummarizer + + prompt = ( + 'Rewrite this software engineering question into a concise code search query. ' + 'Keep likely identifiers, components, API names, and configuration terms. ' + 'Do not answer the question.\n\n' + f'Question: {question}\n' + 'Query:' + ) + global _FLAN_REWRITE_SUMMARIZER + if _FLAN_REWRITE_SUMMARIZER is None: + _FLAN_REWRITE_SUMMARIZER = CommitSummarizer(model_name) + summarizer = _FLAN_REWRITE_SUMMARIZER + result = summarizer.generate(prompt, max_length=48, num_beams=1) + if not result: + return None + result = result.strip().strip('"\'') + return result if result and len(result.split()) <= 14 else None + except Exception: + return None + def extract_keywords(self, question: str) -> list[str]: """Extract meaningful keywords from a question. diff --git a/sia_code/storage/multi_repo.py b/sia_code/storage/multi_repo.py new file mode 100644 index 0000000..348ded4 --- /dev/null +++ b/sia_code/storage/multi_repo.py @@ -0,0 +1,537 @@ +"""Multi-repo detection and fan-out indexing support. + +Detects git repositories as immediate sub-directories and indexes +each independently, then registers them for aggregated search. +""" + +from __future__ import annotations + +import json +import logging +import re as _re +from dataclasses import dataclass as _dataclass +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +import pathspec + +from ..config import Config, RepoIndexOverride + +logger = logging.getLogger(__name__) + + +def is_model_cached(model_name: str) -> bool: + """Return True if HuggingFace model appears cached locally.""" + hub_name = model_name.replace('/', '--') + return (Path.home() / '.cache' / 'huggingface' / 'hub' / f'models--{hub_name}').exists() + + +@dataclass +class RepoEntry: + """A registered sub-repo in a multi-repo workspace.""" + + name: str + path: str # relative to workspace root + index_dir: str # relative path to .sia-code/ dir + profile: str = "general" + indexed_at: str | None = None + file_count: int = 0 + estimated_chunks: int = 0 + status: str = "pending" + last_error: str | None = None + + +@dataclass +class MultiRepoRegistry: + """Registry of sub-repos in a multi-repo workspace.""" + + repos: list[RepoEntry] = field(default_factory=list) + created_at: str = "" + workspace_root: str = "" + + def save(self, registry_path: Path) -> None: + """Save registry to .sia-code/multi-repo.json.""" + registry_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "workspace_root": self.workspace_root, + "created_at": self.created_at, + "repos": [ + { + "name": r.name, + "path": r.path, + "index_dir": r.index_dir, + "profile": r.profile, + "indexed_at": r.indexed_at, + "file_count": r.file_count, + "estimated_chunks": r.estimated_chunks, + "status": r.status, + "last_error": r.last_error, + } + for r in self.repos + ], + } + registry_path.write_text(json.dumps(data, indent=2)) + + @classmethod + def load(cls, registry_path: Path) -> MultiRepoRegistry | None: + """Load registry from file. Returns None if not found.""" + if not registry_path.exists(): + return None + try: + data = json.loads(registry_path.read_text()) + return cls( + workspace_root=data.get("workspace_root", ""), + created_at=data.get("created_at", ""), + repos=[ + RepoEntry( + name=r["name"], + path=r["path"], + index_dir=r["index_dir"], + profile=r.get("profile", "general"), + indexed_at=r.get("indexed_at"), + file_count=r.get("file_count", 0), + estimated_chunks=r.get("estimated_chunks", 0), + status=r.get("status", "pending"), + last_error=r.get("last_error"), + ) + for r in data.get("repos", []) + ], + ) + except (json.JSONDecodeError, KeyError) as e: + logger.warning(f"Failed to load multi-repo registry: {e}") + return None + + +def detect_sub_repos(directory: Path) -> list[Path]: + """Detect git repositories as immediate sub-directories. + + Args: + directory: Parent directory to scan + + Returns: + Sorted list of paths to sub-directories that are git repos + """ + repos = [] + try: + for child in sorted(directory.iterdir()): + if child.is_dir() and not child.name.startswith("."): + # Check for .git/ directory (is a git repo) + if (child / ".git").exists(): + repos.append(child) + except PermissionError: + pass + return repos + + +def is_multi_repo_workspace(directory: Path) -> bool: + """Check if directory is a multi-repo workspace (has git sub-repos but is not itself a repo).""" + # If directory IS a git repo, it's not a multi-repo workspace + if (directory / ".git").exists(): + return False + # Check if it has at least 2 sub-repos + sub_repos = detect_sub_repos(directory) + return len(sub_repos) >= 2 + + +def get_registry_path(workspace_root: Path) -> Path: + """Get path to multi-repo registry file.""" + return workspace_root / ".sia-code" / "multi-repo.json" + + +def get_repo_profile(repo_name: str) -> str: + """Classify known repo families for indexing-aware policy decisions.""" + if repo_name == "ai.platform.forks.ai-toolkit": + return "data_science" + if repo_name == "ai.platform.annotation-suite.cvat": + return "annotation_platform" + return "general" + + +def build_registry(workspace_root: Path, repos: list[Path]) -> MultiRepoRegistry: + """Build a fresh registry from detected repos.""" + entries = [] + for repo_path in repos: + rel_path = str(repo_path.relative_to(workspace_root)) + entries.append( + RepoEntry( + name=repo_path.name, + path=rel_path, + index_dir=f".sia-code/repos/{repo_path.name}", + profile=get_repo_profile(repo_path.name), + ) + ) + return MultiRepoRegistry( + workspace_root=str(workspace_root), + created_at=datetime.now(timezone.utc).isoformat(), + repos=entries, + ) + + +def get_repo_override(config: Config, repo_name: str) -> RepoIndexOverride | None: + """Return repo-specific indexing override, including built-in defaults.""" + override = config.multi_repo.repo_overrides.get(repo_name) + if override: + return override + + # Built-in performance policy for known heavy mixed repo. + if repo_name == "ai.platform.forks.ai-toolkit": + return RepoIndexOverride( + index_first=[ + "toolkit/**", + "jobs/**", + "ui/src/**", + "ui/cron/**", + "ui/prisma/schema.prisma", + "run.py", + "run_modal.py", + "flux_train_ui.py", + ], + dependency_tier=[ + "extensions_built_in/diffusion_models/**/src/**", + ], + lazy_index=[ + "config/examples/**", + "scripts/**", + "testing/**", + "docker/**", + "extensions/example/**", + ], + skip=[ + "output/**", + "assets/**", + "notebooks/**", + "ui/public/**", + "ui/package-lock.json", + "toolkit/keymaps/**", + ".github/**", + ".vscode/**", + ], + ) + + if repo_name == "ai.platform.annotation-suite.cvat": + return RepoIndexOverride( + index_first=[ + "cvat/apps/**", + "cvat/settings/**", + "cvat/utils/**", + "cvat-core/src/**", + "cvat-canvas/src/**", + "cvat-canvas3d/src/**", + "cvat-data/src/**", + "cvat-ui/src/**", + "cvat-sdk/cvat_sdk/**", + ], + dependency_tier=[ + "serverless/**", + "utils/**", + "cvat-cli/**", + ], + lazy_index=[ + "tests/**", + "**/tests/**", + "site/**", + "helm-chart/**", + "ai-models/**", + "backend_entrypoint.d/**", + "changelog.d/**", + ], + skip=[ + "cvat-ui/dist/**", + "cvat-sdk/gen/**", + ".github/**", + ".vscode/**", + ".regal/**", + ], + ) + return None + + +def build_repo_config(base_config: Config, repo_name: str) -> Config: + """Clone config and apply fast indexing policy by default. + + Baseline fast policy applies to any repo, then known profiles add stronger + tuning and repo-specific include/exclude overrides. + """ + config = base_config.model_copy(deep=True) + profile = get_repo_profile(repo_name) + override = get_repo_override(config, repo_name) + + # Baseline fast policy for ALL repos. + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1400) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 80) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.85) + config.embedding.granularity = "budget" + if config.embedding.max_vectors_per_file <= 0: + config.embedding.max_vectors_per_file = 32 + else: + config.embedding.max_vectors_per_file = min(config.embedding.max_vectors_per_file, 32) + + if override: + if override.index_first: + config.indexing.include_patterns = override.index_first + + merged_excludes = list(config.indexing.exclude_patterns) + exclude_groups = [override.lazy_index, override.skip] + if not config.multi_repo.heavy_repo_run_dependency_tier: + exclude_groups.insert(0, override.dependency_tier) + for group in exclude_groups: + for pattern in group: + if pattern not in merged_excludes: + merged_excludes.append(pattern) + config.indexing.exclude_patterns = merged_excludes + + # Profile-specific chunking + embedding for faster first-pass indexing. + # Heavy data-science / annotation repos benefit from fewer, larger chunks + # plus a semantic budget and smaller embedding model when cached locally. + if profile == "data_science": + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 1800) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 120) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.9) + config.embedding.max_vectors_per_file = 24 + if is_model_cached("BAAI/bge-small-en-v1.5"): + config.embedding.model = "BAAI/bge-small-en-v1.5" + config.embedding.dimensions = 384 + elif profile == "annotation_platform": + config.chunking.max_chunk_size = max(config.chunking.max_chunk_size, 2200) + config.chunking.min_chunk_size = max(config.chunking.min_chunk_size, 140) + config.chunking.merge_threshold = max(config.chunking.merge_threshold, 0.92) + config.embedding.max_vectors_per_file = 16 + if is_model_cached("BAAI/bge-small-en-v1.5"): + config.embedding.model = "BAAI/bge-small-en-v1.5" + config.embedding.dimensions = 384 + return config + + +def estimate_indexable_files(directory: Path, config) -> int: + """Estimate how many files will be indexed for timeout sizing. + + Mirrors IndexingCoordinator._discover_files() but counts only. + """ + effective_patterns = config.indexing.get_effective_exclude_patterns(directory) + spec = pathspec.PathSpec.from_lines("gitwildmatch", effective_patterns) + + count = 0 + seen: set[Path] = set() + max_bytes = config.indexing.max_file_size_mb * 1024 * 1024 + for pattern in config.indexing.include_patterns: + glob_pattern = pattern if "*" in pattern else f"**/*{pattern}" + for file_path in directory.rglob(glob_pattern): + if not file_path.is_file() or file_path in seen: + continue + rel_path = file_path.relative_to(directory) + if spec.match_file(str(rel_path)): + continue + try: + file_size = file_path.stat().st_size + except OSError: + continue + if file_size == 0 or file_size > max_bytes: + continue + seen.add(file_path) + count += 1 + return count + + +def estimate_chunks(directory: Path, config: Config) -> int: + """Estimate raw chunk count for visibility / status.""" + return estimate_semantic_vectors(directory, config, raw_chunks_only=True) + + +def estimate_semantic_vectors( + directory: Path, config: Config, raw_chunks_only: bool = False +) -> int: + """Estimate semantic vectors to be embedded for timeout sizing. + + In budget mode, this counts vectors after per-file cap is applied. + """ + from ..core.types import Language + from ..parser.chunker import CASTChunker + + effective_patterns = config.indexing.get_effective_exclude_patterns(directory) + spec = pathspec.PathSpec.from_lines("gitwildmatch", effective_patterns) + max_bytes = config.indexing.max_file_size_mb * 1024 * 1024 + chunker = CASTChunker(config.chunking) + total = 0 + seen: set[Path] = set() + + for pattern in config.indexing.include_patterns: + glob_pattern = pattern if "*" in pattern else f"**/*{pattern}" + for file_path in directory.rglob(glob_pattern): + if not file_path.is_file() or file_path in seen: + continue + rel_path = file_path.relative_to(directory) + if spec.match_file(str(rel_path)): + continue + try: + file_size = file_path.stat().st_size + except OSError: + continue + if file_size == 0 or file_size > max_bytes: + continue + seen.add(file_path) + try: + language = Language.from_extension(file_path.suffix) + count = len(chunker.chunk_file(file_path, language)) + if raw_chunks_only or config.embedding.granularity != "budget" or config.embedding.max_vectors_per_file <= 0: + total += count + else: + total += min(count, config.embedding.max_vectors_per_file) + except Exception: + continue + return total + + +def recommend_repo_timeout_seconds(file_count: int, estimated_chunks: int = 0) -> int: + """Compute per-repo timeout. + + Prefer chunk-based sizing because embedding/storage dominates runtime for + large monolithic repos. Bounded to 5m..45m. + """ + if estimated_chunks > 0: + seconds = int(120 + (estimated_chunks / 20.0)) + return max(300, min(2700, seconds)) + if file_count <= 0: + return 300 + seconds = int(60 + file_count * 0.8) + return max(300, min(1800, seconds)) + + +# --------------------------------------------------------------------------- +# Repo-aware ranking helpers +# --------------------------------------------------------------------------- + + +# Words that strongly suggest a specific repo domain when found in a query +_REPO_HINT_PATTERNS: dict[str, list[str]] = { + # annotation / CVAT-like repos + "annotation_platform": [ + "annotation", "cvat", "task", "label", "labeling", "dataset", + "segmentation", "bounding box", "keypoint", "job", "organization", + "member", "project", "review", "webhook", + ], + # data science / trainer repos + "data_science": [ + "stable diffusion", "diffusion", "trainer", "training", "lora", + "dreambooth", "flux", "controlnet", "vae", "unet", "sdxl", + "checkpoint", "finetune", "fine-tune", "fine_tune", "latent", + "dataset loader", "augment", "epoch", + ], + # general ml db / ingestion + "mldb": [ + "mldb", "datasample", "ingestion", "pipeline", "metadata", + "ml platform", "gcputils", + ], +} + +# Path patterns that indicate high-value first-party code (bonus) +_FIRST_PARTY_PATTERNS = ( + r"(apps|src|core|api|views|serializer|service|handler|trainer|pipeline|" + r"processor|model|router|schema|query|manager|controller)", +) +# Path patterns that indicate low-value content (penalty) +_LOW_VALUE_PATTERNS = ( + r"(test|spec|fixture|conftest|mock|__pycache__|node_modules|" + r"vendor|generated|dist|build|docs|site|migration|changelog|" + r"example|sample|assets|static|i18n)", +) + + +@_dataclass +class RepoRankingContext: + """Per-query scoring state for repo-aware multi-repo ranking.""" + + cwd: Path + query: str + workspace_root: Path + registry: "MultiRepoRegistry | None" + + # Computed once from query vocabulary + _profile_hints: dict[str, float] = None # profile -> hint_score + _repo_name_scores: dict[str, float] = None # repo_name -> name_score + + # Compiled path patterns + _first_party_re: object = None + _low_value_re: object = None + + def __post_init__(self) -> None: + q_lower = self.query.lower() + tokens = set(_re.findall(r"[a-z][a-z0-9]*", q_lower)) + + # Score each profile by how many hint terms appear in query + profile_hints: dict[str, float] = {} + for profile, hints in _REPO_HINT_PATTERNS.items(): + score = 0.0 + for hint in hints: + if hint in q_lower: + score += 1.0 + if score > 0: + profile_hints[profile] = min(score / 3.0, 1.0) + self._profile_hints = profile_hints + + # Score repo names: boost repos whose name tokens appear in query + repo_name_scores: dict[str, float] = {} + if self.registry: + for entry in self.registry.repos: + name_tokens = set(_re.findall(r"[a-z][a-z0-9]*", entry.name.lower())) + overlap = name_tokens & tokens + if overlap: + repo_name_scores[entry.name] = min(len(overlap) / 3.0, 1.0) + self._repo_name_scores = repo_name_scores + + self._first_party_re = _re.compile(_FIRST_PARTY_PATTERNS[0], _re.IGNORECASE) + self._low_value_re = _re.compile(_LOW_VALUE_PATTERNS[0], _re.IGNORECASE) + + def cwd_boost(self, repo_name: str) -> float: + """Return 1.0 if cwd is inside this repo, else 0.0.""" + try: + repo_path = self.workspace_root / repo_name + self.cwd.relative_to(repo_path) + return 1.0 + except (ValueError, TypeError): + return 0.0 + + def repo_score(self, repo_name: str, profile: str) -> float: + """Composite repo-level score: cwd + name match + profile match.""" + score = 0.0 + score += self.cwd_boost(repo_name) * 0.60 + score += self._repo_name_scores.get(repo_name, 0.0) * 0.25 + score += self._profile_hints.get(profile, 0.0) * 0.15 + return score + + def path_adjustment(self, file_path: str) -> float: + """Return bonus (+) or penalty (-) based on file path quality.""" + p = str(file_path).lower() + if self._low_value_re.search(p): + return -0.08 + if self._first_party_re.search(p): + return +0.04 + return 0.0 + + def adjusted_score( + self, + base_score: float, + repo_name: str, + profile: str, + file_path: str, + ) -> float: + """Final score = base + repo_boost + path_adjustment.""" + repo_boost = self.repo_score(repo_name, profile) + path_adj = self.path_adjustment(file_path) + return base_score + repo_boost * 0.6 + path_adj + + +def build_ranking_context( + workspace_root: Path, + query: str, + repo_profiles: "dict[str, str] | None" = None, +) -> RepoRankingContext: + """Construct a ranking context from workspace root and query.""" + registry = MultiRepoRegistry.load(get_registry_path(workspace_root)) + cwd = Path.cwd() + return RepoRankingContext( + cwd=cwd, + query=query, + workspace_root=workspace_root, + registry=registry, + ) diff --git a/sia_code/storage/sqlite_vec_backend.py b/sia_code/storage/sqlite_vec_backend.py index 35d9165..01e12b5 100644 --- a/sia_code/storage/sqlite_vec_backend.py +++ b/sia_code/storage/sqlite_vec_backend.py @@ -102,6 +102,11 @@ def __init__( self.embedding_enabled = embedding_enabled self.embedding_model = embedding_model self.ndim = ndim + self.embedding_granularity = kwargs.pop("embedding_granularity", "chunk") + self.max_vectors_per_file = int(kwargs.pop("max_vectors_per_file", 0) or 0) + self.semantic_chunk_types = set(kwargs.pop("semantic_chunk_types", []) or []) + self.persistent_embedding_cache = bool(kwargs.pop("persistent_embedding_cache", True)) + self.flan_query_rewrite = bool(kwargs.pop("flan_query_rewrite", False)) # Paths self.db_path = self.path / "index.db" @@ -122,6 +127,9 @@ def __init__( self._search_cache: dict[str, list] | None = None self._search_cache_enabled = False + # Persistent embedding cache (global across repos/runs) + self._embedding_cache_conn: sqlite3.Connection | None = None + self.mem = _MemoryAdapter(self) # Vector key prefixes for unified index @@ -246,9 +254,11 @@ def _ensure_vector_table(self) -> None: import logging logger = logging.getLogger(__name__) - logger.warning( - "sqlite-vec extension not available; falling back to brute-force vector search." - ) + if not getattr(SqliteVecBackend, "_sqlite_vec_warned", False): + logger.warning( + "sqlite-vec extension not available; falling back to brute-force vector search." + ) + SqliteVecBackend._sqlite_vec_warned = True cursor.execute( """ CREATE TABLE IF NOT EXISTS vectors ( @@ -276,22 +286,29 @@ def _serialize_vector(self, vector: np.ndarray) -> bytes: def _vector_insert(self, vector_id: int, vector: np.ndarray) -> None: """Insert or replace a vector embedding.""" + self._vector_insert_many([(vector_id, vector)]) + + def _vector_insert_many(self, items: list[tuple[int, np.ndarray]]) -> None: + """Bulk insert or replace vector embeddings.""" if self.conn is None: raise RuntimeError("Database connection not initialized") - if not self.embedding_enabled: + if not self.embedding_enabled or not items: return self._ensure_vector_table() - payload = self._serialize_vector(vector) cursor = self.conn.cursor() + rows = [(vector_id, self._serialize_vector(vector)) for vector_id, vector in items] if self._using_vec_extension: - cursor.execute( - "INSERT OR REPLACE INTO vectors(rowid, embedding) VALUES (?, ?)", - (vector_id, payload), - ) + # sqlite-vec virtual table path is safer with row-wise inserts. + # executemany + OR REPLACE can hit primary-key issues on some builds. + for row in rows: + cursor.execute( + "INSERT OR REPLACE INTO vectors(rowid, embedding) VALUES (?, ?)", + row, + ) else: - cursor.execute( + cursor.executemany( "INSERT OR REPLACE INTO vectors(id, embedding) VALUES (?, ?)", - (vector_id, payload), + rows, ) def _vector_search(self, query_vector: np.ndarray, k: int) -> list[tuple[str, float]]: @@ -323,7 +340,7 @@ def _vector_search(self, query_vector: np.ndarray, k: int) -> list[tuple[str, fl if not rows: return [] - query = np.asarray(query_vector, dtype=np.float32) + query = np.asarray(query_vector, dtype=np.float32).flatten() query_norm = np.linalg.norm(query) or 1.0 scored = [] for row in rows: @@ -359,12 +376,27 @@ def _get_embedder(self): except Exception as e: logger.debug(f"Embedding daemon not available: {e}") + # Auto-start daemon if not running (keeps model warm across repos) + if self._try_auto_start_daemon(): + try: + from ..embed_server.client import EmbedClient + + self._embedder = EmbedClient(model_name=self.embedding_model) + logger.info( + f"Auto-started embedding daemon for {self.embedding_model}" + ) + return self._embedder + except Exception as e: + logger.debug(f"Auto-started daemon not usable: {e}") + # Fallback to local model (current behavior) from sentence_transformers import SentenceTransformer import torch # Auto-detect device (GPU if available, CPU fallback) device = "cuda" if torch.cuda.is_available() else "cpu" + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + device = "mps" self._embedder = SentenceTransformer(self.embedding_model, device=device) @@ -373,6 +405,44 @@ def _get_embedder(self): return self._embedder + def _try_auto_start_daemon(self) -> bool: + """Auto-start embedding daemon in background if not running. + + Returns True if daemon started successfully and is reachable. + """ + import logging + + logger = logging.getLogger(__name__) + try: + from ..embed_server.client import EmbedClient + import subprocess + import sys + import time + + logger.info("Auto-starting embedding daemon...") + # Start daemon as separate process (NOT in-process fork which calls sys.exit) + python_exe = sys.executable + subprocess.Popen( + [python_exe, "-c", + "from sia_code.embed_server.daemon import start_daemon; " + "start_daemon(foreground=True)"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + # Wait for socket to become available (up to 10s for model load) + for _ in range(100): + if EmbedClient.is_available(): + return True + time.sleep(0.1) + + logger.debug("Daemon started but socket not reachable within timeout") + return False + except Exception as e: + logger.debug(f"Failed to auto-start daemon: {e}") + return False + def _get_thread_conn(self) -> sqlite3.Connection: """Get thread-local SQLite connection for parallel operations. @@ -385,6 +455,88 @@ def _get_thread_conn(self) -> sqlite3.Connection: self._local.conn = conn return self._local.conn + def _get_embedding_cache_conn(self) -> sqlite3.Connection | None: + """Open persistent embedding cache DB shared across repos/runs.""" + if not self.persistent_embedding_cache: + return None + if self._embedding_cache_conn is not None: + return self._embedding_cache_conn + try: + cache_dir = Path.home() / ".cache" / "sia-code" + cache_dir.mkdir(parents=True, exist_ok=True) + cache_db = cache_dir / "embedding-cache.sqlite3" + conn = sqlite3.connect(cache_db) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS embeddings_cache ( + cache_key TEXT PRIMARY KEY, + model TEXT NOT NULL, + embedding BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_embeddings_cache_model ON embeddings_cache(model)" + ) + conn.commit() + self._embedding_cache_conn = conn + return conn + except Exception: + return None + + def _embedding_cache_key(self, text: str) -> str: + """Stable key for persistent embedding cache.""" + import hashlib + + return hashlib.sha256(f"{self.embedding_model}\0{text}".encode("utf-8")).hexdigest() + + def _get_cached_embeddings(self, texts: list[str]) -> tuple[dict[int, np.ndarray], list[int], list[str]]: + """Return cached embeddings plus list of misses preserving order.""" + conn = self._get_embedding_cache_conn() + if conn is None or not texts: + return {}, list(range(len(texts))), list(texts) + + keys = [self._embedding_cache_key(t) for t in texts] + placeholders = ",".join("?" for _ in keys) + rows = conn.execute( + f"SELECT cache_key, embedding FROM embeddings_cache WHERE cache_key IN ({placeholders})", + keys, + ).fetchall() + cached = { + row[0]: np.frombuffer(row[1], dtype=np.float32).copy() for row in rows + } + hit_map: dict[int, np.ndarray] = {} + miss_idx: list[int] = [] + miss_texts: list[str] = [] + for i, key in enumerate(keys): + vec = cached.get(key) + if vec is None: + miss_idx.append(i) + miss_texts.append(texts[i]) + else: + hit_map[i] = vec + return hit_map, miss_idx, miss_texts + + def _put_cached_embeddings(self, texts: list[str], vectors: np.ndarray) -> None: + """Persist newly computed embeddings.""" + conn = self._get_embedding_cache_conn() + if conn is None or not texts: + return + rows = [ + ( + self._embedding_cache_key(text), + self.embedding_model, + np.asarray(vec, dtype=np.float32).tobytes(), + ) + for text, vec in zip(texts, vectors, strict=False) + ] + conn.executemany( + "INSERT OR REPLACE INTO embeddings_cache(cache_key, model, embedding) VALUES (?, ?, ?)", + rows, + ) + conn.commit() + def _embed(self, text: str) -> np.ndarray | None: """Embed text to vector with caching. @@ -444,13 +596,15 @@ def _get_embed_batch_size(self) -> int: mem_based = 16 elif mem_gb < 24: mem_based = 32 + elif mem_gb < 48: + mem_based = 96 else: - mem_based = 64 + mem_based = 128 cpu_count = os.cpu_count() or 2 max_by_cpu = max(8, cpu_count * 8) size = min(mem_based, max_by_cpu) - size = max(8, min(64, size)) + size = max(8, min(128, size)) self._embed_batch_size = int(size) return self._embed_batch_size @@ -469,26 +623,42 @@ def _embed_batch(self, texts: list[str]) -> np.ndarray | None: if not texts: return np.empty((0, self.ndim), dtype=np.float32) - embedder = self._get_embedder() - batch_size = self._get_embed_batch_size() - encoded = [] - - # Process in batches to avoid memory spikes - for idx in range(0, len(texts), batch_size): - batch = texts[idx : idx + batch_size] - vectors = embedder.encode( - batch, - batch_size=batch_size, - show_progress_bar=False, - convert_to_numpy=True, - ) - encoded.append(np.asarray(vectors, dtype=np.float32)) - - # Combine all batches - if len(encoded) == 1: - return encoded[0] - else: - return np.vstack(encoded) + # Persistent cache across runs/rebuilds + hit_map, miss_idx, miss_texts = self._get_cached_embeddings(texts) + out = np.empty((len(texts), self.ndim), dtype=np.float32) + for i, vec in hit_map.items(): + out[i] = vec + + if miss_texts: + batch_size = self._get_embed_batch_size() + encoded = [] + for idx in range(0, len(miss_texts), batch_size): + batch = miss_texts[idx : idx + batch_size] + last_error = None + for attempt in range(2): + embedder = self._get_embedder() + try: + vectors = embedder.encode( + batch, + batch_size=batch_size, + show_progress_bar=False, + convert_to_numpy=True, + ) + encoded.append(np.asarray(vectors, dtype=np.float32)) + last_error = None + break + except Exception as e: + last_error = e + # Daemon may have died after availability check; force re-resolve once. + self._embedder = None + if last_error is not None: + raise last_error + new_vectors = encoded[0] if len(encoded) == 1 else np.vstack(encoded) + self._put_cached_embeddings(miss_texts, new_vectors) + for pos, vec in zip(miss_idx, new_vectors, strict=False): + out[pos] = vec + + return out def _make_chunk_key(self, chunk_id: int) -> str: """Create vector index key for chunk.""" @@ -601,6 +771,10 @@ def close(self) -> None: self.conn.commit() self.conn.close() self.conn = None + if self._embedding_cache_conn is not None: + self._embedding_cache_conn.commit() + self._embedding_cache_conn.close() + self._embedding_cache_conn = None def seal(self) -> None: """Seal the index to finalize WAL and reduce storage. @@ -835,6 +1009,52 @@ def ensure_column(table: str, column: str, column_type: str) -> None: # Code Operations # =================================================================== + def _select_semantic_embedding_indices(self, chunks: list[Chunk]) -> list[int]: + """Select which chunks should receive semantic vectors. + + All chunks are still stored lexically/FTS. This only reduces vector count + for heavy repo profiles. Selection is type-prioritized within each file. + """ + if self.embedding_granularity != "budget" or self.max_vectors_per_file <= 0: + return list(range(len(chunks))) + + allowed_types = self.semantic_chunk_types or {"class", "function", "method", "definition"} + type_priority = { + "class": 100, + "method": 95, + "function": 90, + "definition": 88, + "structure": 60, + "import": 45, + "block": 35, + "call": 25, + "docstring": 18, + "comment": 10, + "unknown": 5, + } + + def _chunk_score(ch: Chunk) -> tuple[int, int, int, int]: + t = getattr(ch.chunk_type, "value", str(ch.chunk_type)) + base = type_priority.get(t, 0) + if t in allowed_types: + base += 1000 + symbol = (ch.symbol or "").lower() + if any(k in symbol for k in ("view", "viewset", "api", "handler", "endpoint", "trainer", "process", "service", "serializer", "model")): + base += 20 + span = max(0, int(ch.end_line) - int(ch.start_line)) + return (base, min(span, 400), 1 if ch.parent_header else 0, -int(ch.start_line)) + + by_file: dict[str, list[tuple[int, Chunk]]] = {} + for i, chunk in enumerate(chunks): + by_file.setdefault(str(chunk.file_path), []).append((i, chunk)) + + selected: list[int] = [] + for items in by_file.values(): + ranked = sorted(items, key=lambda it: _chunk_score(it[1]), reverse=True) + chosen = [idx for idx, _ in ranked[: self.max_vectors_per_file]] + selected.extend(chosen) + return sorted(selected) + def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: """Store multiple code chunks. @@ -849,7 +1069,6 @@ def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: cursor = self.conn.cursor() chunk_ids: list[int] = [] - embed_texts: list[str] = [] # Phase 1: preserve stable IDs on conflict without REPLACE row churn for chunk in chunks: @@ -901,16 +1120,24 @@ def store_chunks_batch(self, chunks: list[Chunk]) -> list[str]: ) chunk_ids.append(chunk_id) - embed_texts.append(f"{chunk.symbol}\n\n{chunk.code}") - # Phase 2: Batch-embed all chunks (inserted or updated) + # Phase 2: Batch-embed selected chunks only (all chunks remain lexically searchable) if self.embedding_enabled and chunk_ids: try: - vectors = self._embed_batch(embed_texts) - - if vectors is not None: - for j, chunk_id in enumerate(chunk_ids): - self._vector_insert(int(chunk_id), vectors[j]) + selected_indices = self._select_semantic_embedding_indices(chunks) + if selected_indices: + embed_texts = [ + f"{chunks[idx].symbol}\n\n{chunks[idx].code}" for idx in selected_indices + ] + vectors = self._embed_batch(embed_texts) + if vectors is not None: + # Dedupe by chunk_id in case multiple chunks in the batch + # resolve to the same stable URI/id. + vector_map = { + int(chunk_ids[idx]): vectors[pos] + for pos, idx in enumerate(selected_indices) + } + self._vector_insert_many(list(vector_map.items())) except Exception: # Rollback SQLite inserts to avoid chunks without embeddings self.conn.rollback() diff --git a/sia_code/storage/usearch_backend.py b/sia_code/storage/usearch_backend.py index cffb4f2..f1a7912 100644 --- a/sia_code/storage/usearch_backend.py +++ b/sia_code/storage/usearch_backend.py @@ -4,10 +4,17 @@ import sqlite3 from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, TYPE_CHECKING import numpy as np -from usearch.index import Index, MetricKind + +# IMPORTANT: usearch must NOT be imported at module level. +# On Python 3.13 + macOS ARM64, importing usearch before +# sentence_transformers.encode() causes a segfault due to +# a shared-library symbol conflict. We lazy-import usearch +# only when needed and ensure the embedder is loaded first. +if TYPE_CHECKING: + from usearch.index import Index from ..core.models import ( ChangelogEntry, @@ -24,6 +31,17 @@ from .sqlite_runtime import connect_sqlite +def _lazy_usearch(): + """Lazy-import usearch to avoid segfault on macOS ARM64 + Python 3.13. + + The usearch native extension conflicts with tokenizers/torch when imported + before SentenceTransformer.encode(). By deferring the import, we ensure + the embedding model initializes safely first. + """ + from usearch.index import Index, MetricKind + return Index, MetricKind + + class _MemoryAdapter: """Compatibility adapter for legacy mem interface.""" @@ -116,7 +134,7 @@ def __init__( self.db_path = self.path / "index.db" # Will be initialized in create_index() or open_index() - self.vector_index: Index | None = None + self.vector_index: "Index | None" = None self.conn: sqlite3.Connection | None = None self._embedder = None # Lazy-loaded embedding model @@ -436,7 +454,13 @@ def create_index(self) -> None: """Create a new index (vectors + SQLite).""" self.path.mkdir(parents=True, exist_ok=True) - # Create usearch vector index + # Pre-initialize embedder BEFORE importing usearch to avoid segfault + # on Python 3.13 + macOS ARM64 (shared-library symbol conflict). + if self.embedding_enabled: + self._get_embedder() + + # Create usearch vector index (lazy import to avoid segfault) + Index, MetricKind = _lazy_usearch() self.vector_index = Index( ndim=self.ndim, metric=MetricKind.Cos if self.metric == "cos" else MetricKind.L2sq, @@ -462,7 +486,13 @@ def open_index(self, writable: bool = False) -> None: if not self.db_path.exists(): raise FileNotFoundError(f"Database not found: {self.db_path}") + # Pre-initialize embedder BEFORE importing usearch to avoid segfault + # on Python 3.13 + macOS ARM64 (shared-library symbol conflict). + if self.embedding_enabled: + self._get_embedder() + # Load usearch index (memory-mapped for fast access when read-only) + Index, MetricKind = _lazy_usearch() self.vector_index = Index(ndim=self.ndim, metric=MetricKind.Cos, dtype=self.dtype) # Only view if the file is not empty if self.vector_path.stat().st_size > 0: diff --git a/tests/integration/test_git_dynamic_real_repos.py b/tests/integration/test_git_dynamic_real_repos.py new file mode 100644 index 0000000..9ef98fb --- /dev/null +++ b/tests/integration/test_git_dynamic_real_repos.py @@ -0,0 +1,154 @@ +"""Integration tests for dynamic git memory against real repos. + +Run with: + pytest tests/integration/test_git_dynamic_real_repos.py -v +""" + +from __future__ import annotations + +import pytest +from pathlib import Path + +from sia_code.memory.git_dynamic import GitDynamicMemory +from sia_code.memory.blast_radius import BlastRadiusAnalyzer +from sia_code.memory.recency import RecencyConfig, RecencyScorer +from sia_code.memory.revert_detector import RevertDetector, CommitInfo +from sia_code.memory.intent_classifier import IntentClassifier + + +# Test repo paths +MLDB_REPO = Path.home() / "dev/ai.platform/ai.platform.mldb" +PIPELINES_REPO = Path.home() / "dev/ai.platform/ai.platform.pipelines" + + +def _skip_if_missing(repo: Path): + if not (repo / ".git").exists(): + pytest.skip(f"Repo not available: {repo}") + + +class TestGitDynamicMemoryMLDB: + """Tests against ai.platform.mldb — rich co-change patterns.""" + + @pytest.fixture(autouse=True) + def setup(self): + _skip_if_missing(MLDB_REPO) + self.config = RecencyConfig(halflife_days=30.0, working_window_days=14) + self.mem = GitDynamicMemory(MLDB_REPO, recency_config=self.config) + + def test_file_history_returns_commits(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert len(hist.effective_commits) > 0 + assert hist.file_path == "mldb/app/api/v1/datasample_metadata.py" + + def test_recency_scoring_decreases_with_age(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + scores = [c.recency_score for c in hist.effective_commits] + # Should be generally decreasing (most recent first) + assert scores[0] >= scores[-1] + + def test_owners_detected(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert len(hist.owners) > 0 + # Top owner should have multiple commits + assert hist.owners[0][1] > 1 + + def test_branch_context_available(self): + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + assert hist.branch_context is not None + assert hist.branch_context.current_branch + + def test_blast_radius_finds_coupled_files(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.2, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + assert len(result.coupled_files) > 0 + # Expect CRUD layer to be coupled + paths = [cf.path for cf in result.coupled_files] + assert any("crud" in p for p in paths) + + def test_blast_radius_squash_guard(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.1, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + # Should have excluded some squash commits + assert result.commits_excluded_squash >= 0 + + def test_change_cluster_detected(self): + analyzer = BlastRadiusAnalyzer(MLDB_REPO, min_coupling=0.2, recency_config=self.config) + result = analyzer.co_changed_files("mldb/app/api/v1/datasample_metadata.py") + # Expect a cluster of related files + if result.change_clusters: + assert len(result.change_clusters[0].files) >= 2 + + def test_intent_classification(self): + classifier = IntentClassifier() + hist = self.mem.file_history("mldb/app/api/v1/datasample_metadata.py") + intents = [ + classifier.classify(c.message, len(c.files_changed), c.insertions + c.deletions) + for c in hist.effective_commits + ] + # At least some should be classified + classified = [i for i in intents if i.intent != "unknown"] + assert len(classified) > 0 + + +class TestGitDynamicMemoryPipelines: + """Tests against ai.platform.pipelines — revert detection.""" + + @pytest.fixture(autouse=True) + def setup(self): + _skip_if_missing(PIPELINES_REPO) + self.config = RecencyConfig(halflife_days=30.0, working_window_days=14) + self.mem = GitDynamicMemory(PIPELINES_REPO, recency_config=self.config) + + def test_revert_detected(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + assert len(hist.reverts) >= 1 + # Known revert: e534b21 reverts 75591dd + revert_hashes = {r.reverting_hash[:7] for r in hist.reverts} + assert "e534b21" in revert_hashes + + def test_effective_history_excludes_reverts(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + effective_hashes = {c.hash[:7] for c in hist.effective_commits} + # Both revert and reverted should be excluded + assert "e534b21" not in effective_hashes + assert "75591dd" not in effective_hashes + + def test_all_commits_contains_reverted(self): + hist = self.mem.file_history(".github/workflows/build-components.yml", limit=25) + all_hashes = {c.hash[:7] for c in hist.all_commits} + # All commits should include both + assert "e534b21" in all_hashes + assert "75591dd" in all_hashes + + def test_branch_resolver_lists_branches(self): + branches = self.mem.branch_resolver.all_branch_names() + assert len(branches) >= 1 + assert "main" in branches + + def test_worktree_list(self): + worktrees = self.mem.branch_resolver.worktree_list() + assert len(worktrees) >= 1 + # Current worktree should be marked + current = [w for w in worktrees if w.is_current] + assert len(current) == 1 + + +class TestRecencyScorer: + """Unit-level tests for recency math.""" + + def test_within_window_full_weight(self): + from datetime import datetime, timedelta, timezone + + scorer = RecencyScorer(RecencyConfig(halflife_days=30, working_window_days=14)) + now = datetime.now(timezone.utc) + assert scorer.score(now - timedelta(days=7), now=now) == 1.0 + + def test_halflife_gives_half_weight(self): + from datetime import datetime, timedelta, timezone + import math + + scorer = RecencyScorer(RecencyConfig(halflife_days=30, working_window_days=14)) + now = datetime.now(timezone.utc) + # 44 days ago = 30 days beyond window = one halflife + score = scorer.score(now - timedelta(days=44), now=now) + assert abs(score - 0.5) < 0.01 diff --git a/tests/unit/test_multi_hop.py b/tests/unit/test_multi_hop.py index f12bccb..f2a953e 100644 --- a/tests/unit/test_multi_hop.py +++ b/tests/unit/test_multi_hop.py @@ -500,6 +500,27 @@ def mock_search_lexical(query, *args, **kwargs): first_query = calls[0] assert "how" not in first_query.lower() or "main" in first_query.lower() + def test_uses_hybrid_variants_when_budgeted_embeddings_enabled(self, backend, sample_chunks): + """Budgeted indexes should seed research via multiple hybrid query variants.""" + backend.store_chunks_batch(sample_chunks) + backend.embedding_enabled = True + backend.embedding_granularity = "budget" + + original_search_hybrid = backend.search_hybrid + calls = [] + + def mock_search_hybrid(query, *args, **kwargs): + calls.append(query) + return original_search_hybrid(query, *args, **kwargs) + + backend.search_hybrid = mock_search_hybrid + + strategy = MultiHopSearchStrategy(backend, max_hops=1) + strategy.research("How does load_config work in main flow?", max_results_per_hop=5) + + assert len(calls) >= 2 + assert any("load_config" in q for q in calls) + class TestNaturalLanguageQueries: """Test that research handles natural language questions.""" diff --git a/tests/unit/test_query_preprocessor.py b/tests/unit/test_query_preprocessor.py index d8aa71e..8725692 100644 --- a/tests/unit/test_query_preprocessor.py +++ b/tests/unit/test_query_preprocessor.py @@ -89,6 +89,29 @@ def test_multiple_code_identifiers(self, preprocessor): assert "load_config" in result assert "use" in result + def test_expand_variants_returns_2_to_3_unique_forms(self, preprocessor): + """Expanded variants should keep raw + focused forms without duplicates.""" + variants = preprocessor.expand_variants( + "How does ChipCountingService use load_config in authentication flow?" + ) + assert 2 <= len(variants) <= 3 + assert variants[0] == "How does ChipCountingService use load_config in authentication flow?" + assert any("ChipCountingService" in v for v in variants) + assert any("load_config" in v for v in variants) + assert len({v.lower() for v in variants}) == len(variants) + + def test_expand_variants_synthesizes_code_forms(self, preprocessor): + variants = preprocessor.expand_variants( + "How does stable diffusion trainer config work?" + ) + combined = " ".join(variants) + assert "StableDiffusionTrainer" in combined or "stable_diffusion_trainer" in combined + + def test_expand_variants_synthesizes_api_forms(self, preprocessor): + variants = preprocessor.expand_variants("How does task REST API work?") + combined = " ".join(variants) + assert any(x in combined for x in ["TaskViewSet", "TaskSerializer", "TaskAPIView"]) + class TestExtractKeywords: """Test keyword extraction specifically.""" diff --git a/tests/unit/test_recency.py b/tests/unit/test_recency.py new file mode 100644 index 0000000..35e4817 --- /dev/null +++ b/tests/unit/test_recency.py @@ -0,0 +1,70 @@ +"""Tests for RecencyScorer.""" + +import math +from datetime import datetime, timedelta, timezone + +from sia_code.memory.recency import RecencyConfig, RecencyScorer + + +def _days_ago(days: float) -> datetime: + """Create a commit date exactly N days ago (truncated to seconds to avoid fp edge).""" + dt = datetime.now(timezone.utc) - timedelta(days=days) + return dt.replace(microsecond=0) + + +class TestRecencyScorer: + def test_within_working_window_full_weight(self): + scorer = RecencyScorer() + assert scorer.score(_days_ago(0)) == 1.0 + assert scorer.score(_days_ago(7)) == 1.0 + assert scorer.score(_days_ago(13.9)) == 1.0 # just inside window + + def test_one_halflife_beyond_window(self): + scorer = RecencyScorer() # halflife=30, window=14 + # 44 days old = 30 days beyond window = one halflife → 0.5 + score = scorer.score(_days_ago(44)) + assert abs(score - 0.5) < 0.01 + + def test_two_halflives_beyond_window(self): + scorer = RecencyScorer() + # 74 days old = 60 days beyond window = two halflives → 0.25 + score = scorer.score(_days_ago(74)) + assert abs(score - 0.25) < 0.01 + + def test_future_commit_full_weight(self): + scorer = RecencyScorer() + future = datetime.now(timezone.utc) + timedelta(hours=1) + assert scorer.score(future) == 1.0 + + def test_custom_config(self): + config = RecencyConfig(halflife_days=7.0, working_window_days=3) + scorer = RecencyScorer(config) + # 2.9 days old → still in window + assert scorer.score(_days_ago(2.9)) == 1.0 + # 10 days old = 7 days beyond window = one halflife → 0.5 + score = scorer.score(_days_ago(10)) + assert abs(score - 0.5) < 0.01 + + def test_weighted_score(self): + scorer = RecencyScorer() + date = _days_ago(44) # one halflife beyond window → recency=0.5 + # base=0.8, recency=0.5, branch=0.8 + result = scorer.weighted_score(0.8, date, branch_relevance=0.8) + expected = 0.8 * 0.5 * 0.8 + assert abs(result - expected) < 0.01 + + def test_deterministic_with_now_param(self): + scorer = RecencyScorer() + now = datetime(2024, 6, 1, tzinfo=timezone.utc) + commit = datetime(2024, 5, 1, tzinfo=timezone.utc) # 31 days ago + # 31 - 14 = 17 days beyond window + expected = math.exp(-math.log(2) / 30.0 * 17) + score = scorer.score(commit, now=now) + assert abs(score - expected) < 0.001 + + def test_naive_datetime_treated_as_utc(self): + scorer = RecencyScorer() + naive_commit = datetime.now() - timedelta(days=7) + # Should not crash, treated as UTC + score = scorer.score(naive_commit) + assert score == 1.0 diff --git a/uv.lock b/uv.lock index 016e90f..802d693 100644 --- a/uv.lock +++ b/uv.lock @@ -2955,7 +2955,7 @@ wheels = [ [[package]] name = "sia-code" -version = "0.7.1" +version = "0.7.2" source = { editable = "." } dependencies = [ { name = "click" },