Add dynamic agent-mode query pipeline

.gitignore

@@ -1,5 +1,9 @@
 # RajShah-1 gitignore
 code-dump.txt
+nohup.out
+out.log
+ideas.md
+index/
 
 # --- Python ---
 __pycache__/

config/config.yaml

@@ -12,4 +12,9 @@ chunk_overlap: 200
 use_hyde: false
 hyde_max_tokens: 300
 use_indexed_chunks: false
-rerank_mode: "cross_encoder"
+rerank_mode: "cross_encoder"
+
+# Agent mode settings
+use_agent: true
+agent_reasoning_limit: 5
+agent_tool_limit: 20

src/agent/__init__.py

@@ -0,0 +1,7 @@
+from src.agent.context import ContextRegistry
+from src.agent.orchestrator import AgentOrchestrator
+from src.agent.logger import AgentLogger
+from src.agent.toolkit import AgentToolkit
+
+__all__ = ["ContextRegistry", "AgentOrchestrator", "AgentLogger", "AgentToolkit"]
+

src/agent/context.py

@@ -0,0 +1,117 @@
+from typing import Dict, List, Optional, Any
+from src.agent.types import ObservationMetadata
+
+class ContextBudgetExceeded(Exception):
+    """Raised when adding an observation would exceed the max context budget."""
+    pass
+
+class ContextRegistry:
+    """
+    Keyed registry for agent observations. 
+    Enforces a rough token budget (approx 3.5 chars per token).
+    """
+
+    def __init__(self, max_tokens: int = 8000):
+        self._observations: Dict[str, str] = {}
+        self._metadata: Dict[str, ObservationMetadata] = {}
+        self._counter: int = 0
+        self._max_tokens = max_tokens
+        self._current_chars = 0
+
+    @property
+    def current_tokens(self) -> int:
+        return int(self._current_chars / 3.5)
+
+    @property
+    def status(self) -> Dict[str, Any]:
+        used = self.current_tokens
+        return {
+            "used": used,
+            "total": self._max_tokens,
+            "usage_percent": (used / self._max_tokens) * 100 if self._max_tokens > 0 else 0.0,
+            "count": len(self._observations)
+        }
+
+    def _check_budget(self, text: str):
+        new_tokens = int(len(text) / 3.5)
+        if (self.current_tokens + new_tokens) > self._max_tokens:
+            raise ContextBudgetExceeded(
+                f"Cannot add observation ({new_tokens} tokens). "
+                f"Registry full: {self.current_tokens}/{self._max_tokens} tokens used."
+            )
+
+    def add(self, text: str, step: Optional[int] = None) -> str:
+        self._check_budget(text)
+        self._counter += 1
+        ref_id = f"obs_{self._counter}"
+        self._observations[ref_id] = text
+        self._metadata[ref_id] = ObservationMetadata(added_in_step=step)
+        self._current_chars += len(text)
+        return ref_id
+
+    def remove(self, ref_id: str, step: Optional[int] = None) -> bool:
+        if ref_id in self._observations:
+            text = self._observations.pop(ref_id)
+            if ref_id in self._metadata:
+                self._metadata[ref_id].removed_in_step = step
+            else:
+                self._metadata[ref_id] = ObservationMetadata(removed_in_step=step)
+            self._current_chars -= len(text)
+            return True
+        return False
+
+    def replace(self, ref_id: str, new_text: str, step: Optional[int] = None) -> None:
+        if ref_id not in self._observations:
+            raise KeyError(f"Observation {ref_id} not found.")
+
+        old_text = self._observations[ref_id]
+        diff_chars = len(new_text) - len(old_text)
+        new_total_tokens = int((self._current_chars + diff_chars) / 3.5)
+
+        if new_total_tokens > self._max_tokens:
+            raise ContextBudgetExceeded(
+                f"Replacement exceeds budget. Total would be {new_total_tokens}/{self._max_tokens}."
+            )
+
+        self._observations[ref_id] = new_text
+        if ref_id in self._metadata:
+            self._metadata[ref_id].replaced_in_step = step
+            self._metadata[ref_id].replaced_with = ref_id
+        self._current_chars += diff_chars
+
+    def get(self, ref_id: str) -> Optional[str]:
+        return self._observations.get(ref_id)
+
+    def list_ids(self) -> List[str]:
+        return list(self._observations.keys())
+
+    def clear(self) -> None:
+        self._observations.clear()
+        self._metadata.clear()
+        self._counter = 0
+        self._current_chars = 0
+
+    def __len__(self) -> int:
+        return len(self._observations)
+
+    def get_all_metadata(self) -> Dict[str, Dict[str, Any]]:
+        result = {}
+        all_ref_ids = set(self._observations.keys()) | set(self._metadata.keys())
+
+        for ref_id in all_ref_ids:
+            meta = self._metadata.get(ref_id, ObservationMetadata())
+            lifecycle = []
+            if meta.added_in_step is not None: lifecycle.append(f"added-in-step-{meta.added_in_step}")
+            if meta.removed_in_step is not None: lifecycle.append(f"removed-in-step-{meta.removed_in_step}")
+            if meta.replaced_in_step is not None: lifecycle.append(f"replaced-in-step-{meta.replaced_in_step}")
+            if meta.kept_in_final: lifecycle.append("kept-in-final-content")
+
+            result[ref_id] = {
+                "content": self._observations.get(ref_id),
+                "lifecycle": "; ".join(lifecycle) if lifecycle else "no-events",
+                "added_in_step": meta.added_in_step,
+                "removed_in_step": meta.removed_in_step,
+                "replaced_in_step": meta.replaced_in_step,
+                "kept_in_final": meta.kept_in_final,
+            }
+        return result

src/agent/generate_summaries.py

@@ -0,0 +1,47 @@
+import json
+from pathlib import Path
+from src.agent.summarizer import ThinkingSummarizer
+from src.agent.summary_db import init_db, save_section_to_db
+
+_PROJECT_ROOT = Path(__file__).parent.parent.parent
+EXTRACTED_PATH = _PROJECT_ROOT / "data" / "extracted_sections.json"
+NAV_DB_PATH = _PROJECT_ROOT / "data" / "nav_index.sqlite3"
+MODEL_PATH = str(_PROJECT_ROOT / "models" / "Qwen3-30B-A3B-Q6_K.gguf")
+
+def process_all():
+    if not EXTRACTED_PATH.exists():
+        print("No extracted sections found.")
+        return
+
+    with open(EXTRACTED_PATH) as f:
+        sections = json.load(f)
+
+    summarizer = ThinkingSummarizer(MODEL_PATH)
+    init_db(NAV_DB_PATH)
+
+    for i, sec in enumerate(sections):
+        print(f"Processing section {i+1}/{len(sections)}...")
+        text = sec.get("content", "")
+        if not text: continue
+
+        # Dense summary
+        dense = summarizer.summarize_recursive(text)
+
+        # One liner
+        one_line = summarizer.one_line(text)
+
+        # Paragraphs
+        paras = []
+        for p_idx, para in enumerate(text.split("\n\n")):
+            if not para.strip(): continue
+            p_summary = summarizer.one_line(para)
+            paras.append({
+                "index": p_idx,
+                "text": para,
+                "summary": p_summary
+            })
+
+        save_section_to_db(NAV_DB_PATH, i+1, sec.get("heading", ""), dense, one_line, paras)
+
+if __name__ == "__main__":
+    process_all()

src/agent/llm.py

@@ -0,0 +1,19 @@
+from typing import List, Optional
+from src.generator import get_llama_model
+
+class AgentLLM:
+    def __init__(self, model_path: str):
+        self.model_path = model_path
+
+    def completion(self, prompt: str, max_tokens: int = 500, temperature: float = 0.1, stop: Optional[List[str]] = None) -> str:
+        model = get_llama_model(self.model_path)
+        if stop is None:
+            stop = ["<|im_end|>"]
+
+        result = model.create_completion(
+            prompt,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            stop=stop,
+        )
+        return result["choices"][0]["text"]

src/agent/logger.py

@@ -0,0 +1,42 @@
+"""Minimal logging for agent pipeline."""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+
+class AgentLogger:
+    """Logs agent interactions to JSONL file."""
+
+    def __init__(self, session_id: Optional[str] = None):
+        self.session_id = session_id or datetime.now().strftime("%Y%m%d_%H%M%S")
+        self.logs_dir = Path("logs") / "agent"
+        self.logs_dir.mkdir(parents=True, exist_ok=True)
+        self.log_file = self.logs_dir / f"agent_{self.session_id}.jsonl"
+
+    def _write(self, data: Dict[str, Any]) -> None:
+        data["ts"] = datetime.now().isoformat()
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(json.dumps(data, ensure_ascii=False) + "\n")
+
+    def log_step(self, step: int, thought: str, tool_name: Optional[str], tool_args: Dict[str, Any], result: Optional[str], success: bool) -> None:
+        """Log a reasoning step with full thought (no truncation)."""
+        self._write({
+            "event": "step",
+            "step": step,
+            "thought": thought,
+            "tool_name": tool_name,
+            "tool_args": tool_args,
+            "result": result,
+            "success": success,
+        })
+
+    def log_query_complete(self, question: str, answer: str, registry_metadata: Dict[str, Dict[str, Any]]) -> None:
+        """Log query completion with full registry lifecycle."""
+        self._write({
+            "event": "query_complete",
+            "question": question,
+            "answer": answer,
+            "registry_entries": registry_metadata,
+        })