From 5174aae50c531de1b6f5f832b8165aa549a43aed Mon Sep 17 00:00:00 2001 From: Tushar Ghosh Date: Sat, 11 Jul 2026 00:34:40 +0530 Subject: [PATCH 1/3] feat: upgrade internal rfp analyst to agentic rag v2 --- .gitignore | 20 +- Makefile | 20 + agent.py | 134 ++----- app.py | 355 ++++++++++-------- config.py | 42 +-- docs/evaluation.md | 40 ++ document_generator.py | 4 +- evals/golden_questions.yaml | 96 +++++ evals/metrics.py | 67 ++++ evals/run_evals.py | 228 +++++++++++ pyproject.toml | 33 ++ rag_engine.py | 187 ++++----- src/internal_rfp_analyst.egg-info/PKG-INFO | 288 ++++++++++++++ src/internal_rfp_analyst.egg-info/SOURCES.txt | 32 ++ .../dependency_links.txt | 1 + .../top_level.txt | 1 + src/rfp_analyst.egg-info/PKG-INFO | 3 + src/rfp_analyst.egg-info/SOURCES.txt | 32 ++ src/rfp_analyst.egg-info/dependency_links.txt | 1 + src/rfp_analyst.egg-info/top_level.txt | 1 + src/rfp_analyst/__init__.py | 1 + src/rfp_analyst/agent/__init__.py | 1 + src/rfp_analyst/agent/graph.py | 86 +++++ src/rfp_analyst/agent/prompts.py | 111 ++++++ src/rfp_analyst/agent/runtime.py | 138 +++++++ src/rfp_analyst/agent/state.py | 20 + src/rfp_analyst/exceptions.py | 31 ++ src/rfp_analyst/health.py | 33 ++ src/rfp_analyst/ingestion/__init__.py | 1 + src/rfp_analyst/ingestion/chunking.py | 57 +++ src/rfp_analyst/ingestion/loaders.py | 123 ++++++ src/rfp_analyst/ingestion/pipeline.py | 84 +++++ src/rfp_analyst/ingestion/registry.py | 42 +++ src/rfp_analyst/retrieval/__init__.py | 1 + src/rfp_analyst/retrieval/vector_store.py | 102 +++++ src/rfp_analyst/schemas.py | 38 ++ src/rfp_analyst/tools/__init__.py | 1 + src/rfp_analyst/tools/compare_projects.py | 65 ++++ src/rfp_analyst/tools/proposal_writer.py | 43 +++ src/rfp_analyst/tools/rfp_gap_analyzer.py | 66 ++++ src/rfp_analyst/tools/search_kb.py | 57 +++ src/rfp_analyst/tools/source_verifier.py | 57 +++ src/rfp_analyst/ui/helpers.py | 22 ++ src/rfp_analyst/uploads.py | 31 ++ tests/conftest.py | 10 + tests/test_agentic_tools.py | 100 +++++ tests/test_config.py | 90 +++++ tests/test_document_generator.py | 46 +++ tests/test_evals.py | 36 ++ tests/test_imports.py | 36 ++ tests/test_ingestion_pipeline.py | 145 +++++++ tests/test_runtime_hardening.py | 81 ++++ 52 files changed, 2925 insertions(+), 415 deletions(-) create mode 100644 Makefile create mode 100644 docs/evaluation.md create mode 100644 evals/golden_questions.yaml create mode 100644 evals/metrics.py create mode 100644 evals/run_evals.py create mode 100644 pyproject.toml create mode 100644 src/internal_rfp_analyst.egg-info/PKG-INFO create mode 100644 src/internal_rfp_analyst.egg-info/SOURCES.txt create mode 100644 src/internal_rfp_analyst.egg-info/dependency_links.txt create mode 100644 src/internal_rfp_analyst.egg-info/top_level.txt create mode 100644 src/rfp_analyst.egg-info/PKG-INFO create mode 100644 src/rfp_analyst.egg-info/SOURCES.txt create mode 100644 src/rfp_analyst.egg-info/dependency_links.txt create mode 100644 src/rfp_analyst.egg-info/top_level.txt create mode 100644 src/rfp_analyst/__init__.py create mode 100644 src/rfp_analyst/agent/__init__.py create mode 100644 src/rfp_analyst/agent/graph.py create mode 100644 src/rfp_analyst/agent/prompts.py create mode 100644 src/rfp_analyst/agent/runtime.py create mode 100644 src/rfp_analyst/agent/state.py create mode 100644 src/rfp_analyst/exceptions.py create mode 100644 src/rfp_analyst/health.py create mode 100644 src/rfp_analyst/ingestion/__init__.py create mode 100644 src/rfp_analyst/ingestion/chunking.py create mode 100644 src/rfp_analyst/ingestion/loaders.py create mode 100644 src/rfp_analyst/ingestion/pipeline.py create mode 100644 src/rfp_analyst/ingestion/registry.py create mode 100644 src/rfp_analyst/retrieval/__init__.py create mode 100644 src/rfp_analyst/retrieval/vector_store.py create mode 100644 src/rfp_analyst/schemas.py create mode 100644 src/rfp_analyst/tools/__init__.py create mode 100644 src/rfp_analyst/tools/compare_projects.py create mode 100644 src/rfp_analyst/tools/proposal_writer.py create mode 100644 src/rfp_analyst/tools/rfp_gap_analyzer.py create mode 100644 src/rfp_analyst/tools/search_kb.py create mode 100644 src/rfp_analyst/tools/source_verifier.py create mode 100644 src/rfp_analyst/ui/helpers.py create mode 100644 src/rfp_analyst/uploads.py create mode 100644 tests/conftest.py create mode 100644 tests/test_agentic_tools.py create mode 100644 tests/test_config.py create mode 100644 tests/test_document_generator.py create mode 100644 tests/test_evals.py create mode 100644 tests/test_imports.py create mode 100644 tests/test_ingestion_pipeline.py create mode 100644 tests/test_runtime_hardening.py diff --git a/.gitignore b/.gitignore index bb5e738..47a896d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,28 @@ -# ─── Environment ─── +# Environment .env +.venv/ venv/ __pycache__/ +.pytest_cache/ +.pytest_tmp/ +.ruff_cache/ *.pyc +.streamlit/secrets.toml -# ─── Vector Store (regenerated on deploy) ─── +# Vector Store (regenerated on deploy) vectorstore/ -# ─── Generated PDFs (regenerated on deploy) ─── +# Generated PDFs (regenerated on deploy) data/documents/*.pdf -# ─── IDE ─── +# Generated evaluation outputs +evals/results.json + +# IDE .vscode/ .idea/ *.swp -# ─── OS ─── +# OS Thumbs.db -.DS_Store +.DS_Store \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1fd3f8b --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +PYTHON ?= python +PIP ?= $(PYTHON) -m pip + +.PHONY: install test lint run generate-docs + +install: + $(PIP) install -r requirements.txt + $(PIP) install pytest ruff + +test: + $(PYTHON) -m pytest -q + +lint: + $(PYTHON) -m ruff check . + +run: + $(PYTHON) -m streamlit run app.py + +generate-docs: + $(PYTHON) document_generator.py diff --git a/agent.py b/agent.py index 6de0d98..ca3f794 100644 --- a/agent.py +++ b/agent.py @@ -1,33 +1,29 @@ -""" -RAG Query Engine — Fast retrieval-augmented generation with streaming. -Uses Groq (primary, fastest) or Gemini (fallback) for LLM generation. -Local embeddings for retrieval — zero API overhead for search. -""" - -from langchain_core.messages import HumanMessage +"""RAG Engine - feature-flagged simple and agentic execution paths.""" from config import ( - GROQ_API_KEY, + GEMINI_MODEL, GOOGLE_API_KEY, + GROQ_API_KEY, GROQ_MODEL, - GEMINI_MODEL, - LLM_TEMPERATURE, LLM_MAX_TOKENS, - AGENT_SYSTEM_PROMPT, - RETRIEVAL_K, + LLM_TEMPERATURE, ) -from rag_engine import similarity_search, get_vectorstore_stats +from rfp_analyst.agent.runtime import prepare_query_payload, run_query, stream_query_response +from rfp_analyst.exceptions import LLMProviderNotConfiguredError -# ─── LLM Initialization ────────────────────────────────────────────────────── - def _get_provider_name(): """Return which LLM provider is active.""" if GROQ_API_KEY: return f"Groq ({GROQ_MODEL})" - elif GOOGLE_API_KEY: + if GOOGLE_API_KEY: return f"Gemini ({GEMINI_MODEL})" - return "None" + return "Not configured" + + +def is_llm_provider_configured() -> bool: + """Return whether any supported LLM provider is configured.""" + return bool(GROQ_API_KEY or GOOGLE_API_KEY) def get_llm(): @@ -40,7 +36,7 @@ def get_llm(): temperature=LLM_TEMPERATURE, max_tokens=LLM_MAX_TOKENS, ) - elif GOOGLE_API_KEY: + if GOOGLE_API_KEY: from langchain_google_genai import ChatGoogleGenerativeAI return ChatGoogleGenerativeAI( model=GEMINI_MODEL, @@ -48,11 +44,10 @@ def get_llm(): temperature=LLM_TEMPERATURE, max_output_tokens=LLM_MAX_TOKENS, ) - else: - raise ValueError( - "No API key found. Set GROQ_API_KEY (recommended, https://console.groq.com) " - "or GOOGLE_API_KEY (https://aistudio.google.com/apikey)" - ) + raise LLMProviderNotConfiguredError( + "No LLM provider is configured. Add GROQ_API_KEY or GOOGLE_API_KEY in your .env file locally, " + "or in Streamlit secrets on deployment." + ) def create_agent(): @@ -60,99 +55,20 @@ def create_agent(): return get_llm() -# ─── Context Retrieval ──────────────────────────────────────────────────────── - -def _retrieve_context(user_query: str): - """Retrieve relevant documents and format context. Local, instant.""" - try: - results = similarity_search(user_query, k=RETRIEVAL_K) - except Exception: - return "", [], [] - - context_parts = [] - sources_used = [] - for doc, score in results: - source = doc.metadata.get("source_file", "Unknown") - page = doc.metadata.get("page", "?") - context_parts.append( - f"[Source: {source}, Page {int(page) + 1}]\n{doc.page_content}" - ) - sources_used.append({"source": source, "page": page, "score": f"{score:.2f}"}) - - context = "\n\n---\n\n".join(context_parts) if context_parts else "No relevant documents found." - return context, context_parts, sources_used - - -def _build_prompt(user_query, context, chat_history=None): - """Build the full prompt with context, history, and question.""" - stats = get_vectorstore_stats() - project_list = "\n".join( - f" - {name}" for name in stats.get("document_names", []) - ) or " No documents ingested yet." - - history_text = "" - if chat_history: - recent = [m for m in chat_history[-6:] if m.get("role") in ("user", "assistant")] - for msg in recent: - role = "User" if msg["role"] == "user" else "Assistant" - history_text += f"{role}: {msg['content'][:300]}\n" - - return f"""{AGENT_SYSTEM_PROMPT} - -── Available Documents ── -{project_list} -Total: {stats.get('total_documents', 0)} documents, {stats.get('total_chunks', 0)} chunks - -── Retrieved Context ── -{context} - -{f"── Recent Conversation ──{chr(10)}{history_text}" if history_text else ""} -── Question ── -{user_query} - -Answer thoroughly with source citations.""" - - -# ─── Query Functions ────────────────────────────────────────────────────────── - def prepare_query(user_query: str, chat_history: list = None): - """Prepare retrieval context and prompt. Returns (sources, prompt).""" - context, context_parts, sources_used = _retrieve_context(user_query) - prompt = _build_prompt(user_query, context, chat_history) - - reasoning_trace = [] - if sources_used: - reasoning_trace.append({ - "tool": "search_knowledge_base", - "input": {"query": user_query}, - }) - for s in sources_used[:3]: - reasoning_trace.append({ - "tool_response": f"{s['source']} (Page {s['page']})", - "snippet": next( - (p[:200] for p in context_parts if s["source"] in p), "..." - ), - }) - - return prompt, reasoning_trace + """Prepare a feature-flagged query payload and visible tool trace.""" + payload = prepare_query_payload(user_query, chat_history) + return payload, payload["reasoning_trace"] def query_agent_stream(llm, prompt): - """Stream LLM response chunks. Yields text as it's generated.""" - for chunk in llm.stream([HumanMessage(content=prompt)]): - if chunk.content: - yield chunk.content + """Stream either the legacy simple path or the new agentic path.""" + yield from stream_query_response(llm, prompt) def query_agent(llm, user_query: str, thread_id: str = "default", chat_history: list = None): - """Non-streaming query (backward compatible). Single LLM call.""" - prompt, reasoning_trace = prepare_query(user_query, chat_history) - response = llm.invoke([HumanMessage(content=prompt)]) - return { - "answer": response.content, - "reasoning_trace": reasoning_trace, - "all_messages": [], - } + """Non-streaming query wrapper.""" + return run_query(llm, user_query, chat_history) if __name__ == "__main__": diff --git a/app.py b/app.py index 34819f2..38fe095 100644 --- a/app.py +++ b/app.py @@ -1,25 +1,46 @@ -""" -Streamlit Dashboard — Internal RFP Analyst Chatbot. -Professional chatbot UI with streaming responses, citations, and source traces. -""" - -import streamlit as st +import json import time from pathlib import Path +import streamlit as st + # Must be first Streamlit command st.set_page_config( page_title="Internal RFP Analyst", - page_icon="🔍", + page_icon="?", layout="wide", initial_sidebar_state="expanded", ) -from config import APP_TITLE, APP_SUBTITLE, SAMPLE_QUESTIONS, DATA_DIR -from rag_engine import ingest_documents, get_vectorstore_stats -from agent import create_agent, prepare_query, query_agent_stream, _get_provider_name - -# ─── Custom CSS ─────────────────────────────────────────────────────────────── +from agent import ( + _get_provider_name, + create_agent, + is_llm_provider_configured, + prepare_query, + query_agent_stream, +) +from config import APP_TITLE, APP_SUBTITLE, DATA_DIR, SAMPLE_QUESTIONS +from rag_engine import get_vectorstore_stats, ingest_documents +from rfp_analyst.exceptions import ( + IngestionError, + KnowledgeBaseNotReadyError, + LLMProviderNotConfiguredError, + RFPAnalystError, + UnsupportedFileError, +) +from rfp_analyst.health import get_app_health +from rfp_analyst.ui.helpers import format_latency_display, get_chat_avatar +from rfp_analyst.uploads import validate_uploaded_pdf + +APP_ROOT = Path(__file__).resolve().parent +EVAL_RESULTS_PATH = APP_ROOT / "evals" / "results.json" +LLM_CONFIGURATION_WARNING = ( + "No LLM provider is configured. Add GROQ_API_KEY or GOOGLE_API_KEY in your .env file locally, " + "or in Streamlit secrets on deployment." +) +KNOWLEDGE_BASE_NOT_READY_MESSAGE = ( + "Knowledge base is not ready. Generate or upload PDFs and click Ingest Documents." +) st.markdown(""" """, unsafe_allow_html=True) - -# ─── Session State Initialization ───────────────────────────────────────────── - if "messages" not in st.session_state: st.session_state.messages = [] if "agent" not in st.session_state: @@ -130,32 +144,66 @@ if "pending_query" not in st.session_state: st.session_state.pending_query = None -# ─── Auto-Setup (for Streamlit Cloud: generate PDFs & ingest if needed) ─────── - -if "auto_setup_done" not in st.session_state: - stats = get_vectorstore_stats() - if stats["status"] == "not_initialized": - with st.spinner("🔧 First-time setup: Generating documents & building knowledge base..."): - from document_generator import generate_all_documents - generate_all_documents() - ingest_documents() - st.session_state.auto_setup_done = True - -# ─── Sidebar ────────────────────────────────────────────────────────────────── +def _load_evaluation_snapshot(): + if not EVAL_RESULTS_PATH.exists(): + return None + try: + return json.loads(EVAL_RESULTS_PATH.read_text(encoding="utf-8")) + except Exception: + return None + + +def _render_reasoning_trace(reasoning_trace: list): + with st.expander("Sources Used", expanded=False): + for step in reasoning_trace: + if "tool" in step: + details = ", ".join( + f"{key}: {value}" for key, value in step.get("input", {}).items() + ) + st.markdown( + f'
Tool: {step["tool"]}
{details}
', + unsafe_allow_html=True, + ) + elif "tool_response" in step: + st.markdown( + f'
{step["tool_response"]}
{step["snippet"][:150]}...
', + unsafe_allow_html=True, + ) + + +def _handle_uploads(uploaded_files): + if not uploaded_files: + return + + DATA_DIR.mkdir(parents=True, exist_ok=True) + uploaded_count = 0 + for uploaded_file in uploaded_files: + try: + safe_name = validate_uploaded_pdf(uploaded_file) + save_path = DATA_DIR / safe_name + with open(save_path, "wb") as handle: + handle.write(uploaded_file.getbuffer()) + uploaded_count += 1 + except UnsupportedFileError as error: + st.warning(str(error)) + if uploaded_count: + st.success(f"Uploaded {uploaded_count} file(s). Click 'Ingest Documents' to index.") + + +provider_name = _get_provider_name() +health = get_app_health(provider_name) +llm_configured = health["llm_provider_configured"] +kb_ready = health["vectorstore_ready"] with st.sidebar: - st.markdown("### ⚙️ Knowledge Base") + st.markdown("### Knowledge Base") st.markdown("---") - # Stats stats = get_vectorstore_stats() if stats["status"] == "ready": - st.markdown( - '● Ready', - unsafe_allow_html=True, - ) + st.markdown('Ready', unsafe_allow_html=True) col1, col2 = st.columns(2) with col1: st.markdown( @@ -170,67 +218,78 @@ unsafe_allow_html=True, ) else: - st.markdown( - '● Not Initialized', - unsafe_allow_html=True, - ) + st.markdown('Not Initialized', unsafe_allow_html=True) + st.info(KNOWLEDGE_BASE_NOT_READY_MESSAGE) st.markdown("---") + st.markdown("### LLM Provider") + st.markdown(f'
{provider_name}
', unsafe_allow_html=True) + if not llm_configured: + st.warning(LLM_CONFIGURATION_WARNING) - # LLM Provider info - st.markdown("### 🤖 LLM Provider") - provider_name = _get_provider_name() - st.markdown(f'
⚡ {provider_name}
', unsafe_allow_html=True) + st.markdown("---") + st.markdown("### App Health") + st.caption(f"Vectorstore ready: {'Yes' if health['vectorstore_ready'] else 'No'}") + st.caption(f"Required directories ready: {'Yes' if all(item['exists'] for item in health['required_directories'].values()) else 'No'}") + evaluation_snapshot = _load_evaluation_snapshot() st.markdown("---") + st.markdown("### Evaluation Snapshot") + if evaluation_snapshot: + metrics = evaluation_snapshot.get("metrics", {}) + st.markdown( + ( + '
' + f"Pass Rate: {evaluation_snapshot.get('passed_questions', 0)}/{evaluation_snapshot.get('total_questions', 0)}
" + f"Retrieval Hit Rate: {metrics.get('retrieval_hit_rate', 0):.2f}
" + f"Citation Coverage: {metrics.get('citation_coverage', 0):.2f}
" + f"Grounded Answer Score: {metrics.get('grounded_answer_score', 0):.2f}
" + f"Average Latency: {format_latency_display(metrics)}
" + f"Tool Call Count: {metrics.get('tool_call_count', 0):.2f}
" + f"Failure Rate: {metrics.get('failure_rate', 0):.2f}" + '
' + ), + unsafe_allow_html=True, + ) + else: + st.info("No evaluation run found") - # Ingestion - st.markdown("### 📥 Document Ingestion") + st.markdown("---") + st.markdown("### Document Ingestion") - if st.button("🚀 Ingest Documents", use_container_width=True, type="primary"): + if st.button("Ingest Documents", use_container_width=True, type="primary"): with st.spinner("Processing documents..."): try: ingest_documents() - st.success("✅ Documents ingested successfully!") + st.success("Documents ingested successfully.") st.session_state.agent = None time.sleep(1) st.rerun() - except Exception as e: - st.error(f"❌ Error: {str(e)}") + except (IngestionError, RFPAnalystError) as error: + st.warning(str(error)) + except Exception as error: + st.warning(f"Ingestion failed: {error}") - # Upload custom PDFs st.markdown("---") - st.markdown("### 📄 Upload Custom PDFs") + st.markdown("### Upload Custom PDFs") uploaded_files = st.file_uploader( "Drop PDFs here", type=["pdf"], accept_multiple_files=True, label_visibility="collapsed", ) - if uploaded_files: - DATA_DIR.mkdir(parents=True, exist_ok=True) - for uf in uploaded_files: - save_path = DATA_DIR / uf.name - with open(save_path, "wb") as f: - f.write(uf.getbuffer()) - st.success(f"Uploaded {len(uploaded_files)} file(s). Click 'Ingest Documents' to index.") - - # Settings + _handle_uploads(uploaded_files) + st.markdown("---") - st.markdown("### 🎛️ Settings") + st.markdown("### Settings") show_reasoning = st.toggle("Show Source Traces", value=True) - # Reset st.markdown("---") - if st.button("🗑️ Clear Chat History", use_container_width=True): + if st.button("Clear Chat History", use_container_width=True): st.session_state.messages = [] st.session_state.agent = None st.rerun() - -# ─── Main Area ──────────────────────────────────────────────────────────────── - -# Header st.markdown( f"""
@@ -241,116 +300,80 @@ unsafe_allow_html=True, ) -# Sample questions (only when no messages) +if not kb_ready: + st.info(KNOWLEDGE_BASE_NOT_READY_MESSAGE) +if not llm_configured: + st.warning(LLM_CONFIGURATION_WARNING) + if not st.session_state.messages: - st.markdown("#### 💡 Try asking:") + st.markdown("#### Try asking:") cols = st.columns(2) - for i, q in enumerate(SAMPLE_QUESTIONS[:6]): - with cols[i % 2]: - if st.button(q, key=f"sample_{i}", use_container_width=True): - st.session_state.pending_query = q - st.session_state.messages.append({"role": "user", "content": q}) + for index, question in enumerate(SAMPLE_QUESTIONS[:6]): + with cols[index % 2]: + if st.button( + question, + key=f"sample_{index}", + use_container_width=True, + disabled=(not llm_configured or not kb_ready), + ): + st.session_state.pending_query = question + st.session_state.messages.append({"role": "user", "content": question}) st.rerun() -# Chat history -for msg in st.session_state.messages: - with st.chat_message(msg["role"], avatar="👤" if msg["role"] == "user" else "🤖"): - st.markdown(msg["content"]) - - # Show reasoning trace if available - if msg["role"] == "assistant" and "reasoning" in msg and msg["reasoning"] and show_reasoning: - with st.expander("📚 Sources Used", expanded=False): - for step in msg["reasoning"]: - if "tool" in step: - st.markdown( - f'
' - f'🔧 Retrieval Query: {step["input"].get("query", "")}' - f'
', - unsafe_allow_html=True, - ) - elif "tool_response" in step: - st.markdown( - f'
' - f'📄 {step["tool_response"]}
' - f'{step["snippet"][:150]}...' - f'
', - unsafe_allow_html=True, - ) - - -# ─── Helper: Process a query with streaming ────────────────────────────────── +for message in st.session_state.messages: + with st.chat_message(message["role"], avatar=get_chat_avatar(message["role"])): + st.markdown(message["content"]) + if message["role"] == "assistant" and message.get("reasoning") and show_reasoning: + _render_reasoning_trace(message["reasoning"]) + def _process_query(user_query: str): - """Process a query: retrieve context, stream LLM response, store result.""" try: - # Initialize agent if needed + if not llm_configured: + raise LLMProviderNotConfiguredError(LLM_CONFIGURATION_WARNING) + if not kb_ready: + raise KnowledgeBaseNotReadyError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) + if st.session_state.agent is None: st.session_state.agent = create_agent() - # Prepare context (local, instant) - prompt, reasoning_trace = prepare_query( - user_query, chat_history=st.session_state.messages - ) - - # Stream the LLM response - with st.chat_message("assistant", avatar="🤖"): - full_response = st.write_stream( - query_agent_stream(st.session_state.agent, prompt) - ) + prompt, reasoning_trace = prepare_query(user_query, chat_history=st.session_state.messages) - # Show sources after streaming completes + with st.chat_message("assistant", avatar=get_chat_avatar("assistant")): + full_response = st.write_stream(query_agent_stream(st.session_state.agent, prompt)) if reasoning_trace and show_reasoning: - with st.expander("📚 Sources Used", expanded=False): - for step in reasoning_trace: - if "tool" in step: - st.markdown( - f'
' - f'🔧 Retrieval Query: {step["input"].get("query", "")}' - f'
', - unsafe_allow_html=True, - ) - elif "tool_response" in step: - st.markdown( - f'
' - f'📄 {step["tool_response"]}
' - f'{step["snippet"][:150]}...' - f'
', - unsafe_allow_html=True, - ) - - # Store message - st.session_state.messages.append({ - "role": "assistant", - "content": full_response, - "reasoning": reasoning_trace, - }) - - except Exception as e: - error_str = str(e) - if "429" in error_str or "RESOURCE_EXHAUSTED" in error_str or "quota" in error_str.lower(): - friendly_msg = ( - "⏳ **Rate limit reached.** Please wait a moment and try again. " + _render_reasoning_trace(reasoning_trace) + + st.session_state.messages.append( + {"role": "assistant", "content": full_response, "reasoning": reasoning_trace} + ) + except (LLMProviderNotConfiguredError, KnowledgeBaseNotReadyError, RFPAnalystError) as error: + friendly_message = str(error) + st.warning(friendly_message) + st.session_state.messages.append({"role": "assistant", "content": friendly_message, "reasoning": []}) + except Exception as error: + error_text = str(error) + if "429" in error_text or "RESOURCE_EXHAUSTED" in error_text or "quota" in error_text.lower(): + friendly_message = ( + "Rate limit reached. Please wait a moment and try again. " "Consider adding a GROQ_API_KEY for faster, more reliable responses." ) - st.warning(friendly_msg) else: - friendly_msg = f"❌ Error: {error_str}" - st.error(friendly_msg) - st.session_state.messages.append( - {"role": "assistant", "content": friendly_msg, "reasoning": []} - ) - + friendly_message = f"Error: {error_text}" + st.warning(friendly_message) + st.session_state.messages.append({"role": "assistant", "content": friendly_message, "reasoning": []}) -# ─── Process pending query from sample buttons ─────────────────────────────── -pending = st.session_state.pending_query -if pending: +pending_query = st.session_state.pending_query +if pending_query: st.session_state.pending_query = None - _process_query(pending) + _process_query(pending_query) -# ─── Chat input ─────────────────────────────────────────────────────────────── -if user_input := st.chat_input("Ask about past projects, tech stacks, proposals..."): +if user_input := st.chat_input( + "Ask about past projects, tech stacks, proposals...", + disabled=(not llm_configured or not kb_ready), +): st.session_state.messages.append({"role": "user", "content": user_input}) - with st.chat_message("user", avatar="👤"): + with st.chat_message("user", avatar=get_chat_avatar("user")): st.markdown(user_input) _process_query(user_input) diff --git a/config.py b/config.py index c423e7e..d9e8d09 100644 --- a/config.py +++ b/config.py @@ -1,22 +1,20 @@ -""" +""" Central Configuration for the Internal RFP Analyst Agent. All paths, model settings, and pipeline parameters are defined here. """ import os from pathlib import Path + from dotenv import load_dotenv load_dotenv() -# ─── Paths ──────────────────────────────────────────────────────────────────── BASE_DIR = Path(__file__).parent DATA_DIR = BASE_DIR / "data" / "documents" VECTORSTORE_DIR = BASE_DIR / "vectorstore" ASSETS_DIR = BASE_DIR / "assets" -# ─── API Keys ───────────────────────────────────────────────────────────────── -# Supports both local .env and Streamlit Cloud secrets try: import streamlit as st GROQ_API_KEY = st.secrets.get("GROQ_API_KEY", os.getenv("GROQ_API_KEY", "")) @@ -25,29 +23,22 @@ GROQ_API_KEY = os.getenv("GROQ_API_KEY", "") GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "") -# ─── LLM Provider Selection ────────────────────────────────────────────────── -# Priority: Groq (fastest free inference) → Gemini (fallback) -# Groq: 30 RPM, 6000 RPD free tier — https://console.groq.com -# Gemini: 15 RPM free tier — https://aistudio.google.com/apikey - -GROQ_MODEL = "llama-3.3-70b-versatile" # Best quality on Groq free tier -GEMINI_MODEL = "gemini-2.0-flash" # Fallback - -LLM_TEMPERATURE = 0.3 # Low temp for factual retrieval +GROQ_MODEL = "llama-3.3-70b-versatile" +GEMINI_MODEL = "gemini-2.0-flash" +LLM_TEMPERATURE = 0.3 LLM_MAX_TOKENS = 2048 - -# ─── Embeddings — Local (no API calls, no rate limits) ──────────────────────── EMBEDDING_MODEL = "BAAI/bge-small-en-v1.5" -# ─── RAG Pipeline ───────────────────────────────────────────────────────────── -CHUNK_SIZE = 512 # tokens per chunk -CHUNK_OVERLAP = 50 # overlap for context continuity +CHUNK_SIZE = 512 +CHUNK_OVERLAP = 50 COLLECTION_NAME = "rfp_kb_v2" -RETRIEVAL_K = 6 # number of chunks to retrieve per query +RETRIEVAL_K = 6 +MAX_UPLOAD_FILE_SIZE_BYTES = 25 * 1024 * 1024 +MAX_UPLOAD_PAGE_COUNT = 250 +AGENT_MODE = os.getenv("AGENT_MODE", "agentic").strip().lower() or "agentic" -# ─── Agent ──────────────────────────────────────────────────────────────────── -AGENT_SYSTEM_PROMPT = """You are the **Internal RFP Analyst**, an AI-powered knowledge agent -for a global fintech consulting firm. Your job is to help internal teams quickly find +AGENT_SYSTEM_PROMPT = """You are the **Internal RFP Analyst**, an AI-powered knowledge agent +for a global fintech consulting firm. Your job is to help internal teams quickly find information from past proposals, project outlines, RFP responses, and case studies. RULES: @@ -55,14 +46,13 @@ 2. Cite your sources clearly using [Source: , Page ] format. 3. If you cannot find relevant information, say so honestly. 4. When comparing projects, present information in a structured table format. -5. Be concise but thorough — consultants are busy people. +5. Be concise but thorough - consultants are busy people. 6. If the user's question is ambiguous, ask a clarifying question before answering. """ -# ─── UI ─────────────────────────────────────────────────────────────────────── -APP_TITLE = "🔍 Internal RFP Analyst" +APP_TITLE = "Internal RFP Analyst" APP_SUBTITLE = "AI-Powered Knowledge Agent for Fintech Consulting" -APP_ICON = "🔍" +APP_ICON = "?" SAMPLE_QUESTIONS = [ "What tech stack did we use for the last banking audit?", "Which projects used Azure services?", diff --git a/docs/evaluation.md b/docs/evaluation.md new file mode 100644 index 0000000..a878add --- /dev/null +++ b/docs/evaluation.md @@ -0,0 +1,40 @@ +# Evaluation Suite + +This project includes a deterministic evaluation suite in `evals/` so we can validate retrieval, tool orchestration, grounding, and fallback behavior without depending on paid model calls. + +## What it covers + +- Direct fact lookup +- Project comparison +- Budget extraction +- Timeline extraction +- Tech stack search +- Compliance framework search +- Ambiguous question handling +- No-answer / insufficient evidence behavior +- Proposal outline generation +- Multi-document synthesis + +## Files + +- `evals/golden_questions.yaml`: the golden question set +- `evals/run_evals.py`: deterministic runner that uses mocked retrieval and non-API answer synthesis +- `evals/metrics.py`: aggregate metric calculations +- `evals/results.json`: generated output after an eval run + +## Metrics + +- `retrieval_hit_rate`: how often expected source documents are retrieved +- `citation_coverage`: how often answers that should cite sources actually include citations +- `grounded_answer_score`: fraction of answers that pass grounding verification +- `average_latency`: average per-question runtime in milliseconds +- `tool_call_count`: average number of tool steps used per question +- `failure_rate`: fraction of eval cases that fail expectations + +## UI snapshot + +If `evals/results.json` exists, the Streamlit sidebar shows an `Evaluation Snapshot` section with the latest metric summary and pass count. + +## Notes + +The eval runner is intentionally deterministic. It uses a small in-memory corpus and mocked retrieval behavior so the suite can run in CI or on local machines without external API dependencies. diff --git a/document_generator.py b/document_generator.py index 5e122b2..149112f 100644 --- a/document_generator.py +++ b/document_generator.py @@ -3,10 +3,10 @@ Generates realistic fintech consulting documents as PDFs. """ -import os -from fpdf import FPDF from pathlib import Path +from fpdf import FPDF + DATA_DIR = Path(__file__).parent / "data" / "documents" DOCUMENTS = [ diff --git a/evals/golden_questions.yaml b/evals/golden_questions.yaml new file mode 100644 index 0000000..830b791 --- /dev/null +++ b/evals/golden_questions.yaml @@ -0,0 +1,96 @@ +id: q1 +category: direct_fact_lookup +question: What tech stack did we use for the last banking audit? +expected_sources: + - Banking_Audit.pdf +expected_keywords: + - Azure SQL + - Power BI +expect_citations: true +--- +id: q2 +category: project_comparison +question: Compare the banking and insurance projects. +expected_sources: + - Banking_Audit.pdf + - Insurance_Automation.pdf +expected_keywords: + - budget + - timeline +expect_citations: true +--- +id: q3 +category: budget_extraction +question: What was the budget for the healthcare migration project? +expected_sources: + - Healthcare_Migration.pdf +expected_keywords: + - $1,200,000 +expect_citations: true +--- +id: q4 +category: timeline_extraction +question: What was the timeline for the retail supply chain analytics platform? +expected_sources: + - Retail_Supply_Chain.pdf +expected_keywords: + - 24 weeks +expect_citations: true +--- +id: q5 +category: tech_stack_search +question: Which projects used Azure services? +expected_sources: + - Banking_Audit.pdf + - Healthcare_Migration.pdf + - Insurance_Automation.pdf +expected_keywords: + - Azure +expect_citations: true +--- +id: q6 +category: compliance_framework_search +question: What compliance frameworks did we follow in pharma projects? +expected_sources: + - Pharma_Clinical_Trials.pdf +expected_keywords: + - FDA 21 CFR Part 11 + - GDPR +expect_citations: true +--- +id: q7 +category: ambiguous_question_handling +question: Can you compare it with the other one? +expected_keywords: + - ambiguous + - clarify +expect_citations: false +--- +id: q8 +category: no_answer_behavior +question: What was the blockchain architecture used in the aerospace project? +expected_keywords: + - couldn't find + - refine the question +expect_citations: false +--- +id: q9 +category: proposal_outline_generation +question: Write a proposal outline for an Azure migration RFP that requires dashboards and HIPAA compliance. +expected_sources: + - Healthcare_Migration.pdf +expected_keywords: + - Executive Summary + - Client Requirements +expect_citations: true +--- +id: q10 +category: multi_document_synthesis +question: Which projects combined Azure with compliance-heavy delivery, and what outcomes did they achieve? +expected_sources: + - Healthcare_Migration.pdf + - Pharma_Clinical_Trials.pdf +expected_keywords: + - HIPAA + - FDA 21 CFR Part 11 +expect_citations: true diff --git a/evals/metrics.py b/evals/metrics.py new file mode 100644 index 0000000..cc28be7 --- /dev/null +++ b/evals/metrics.py @@ -0,0 +1,67 @@ +"""Evaluation metrics for deterministic RAG checks.""" + +from __future__ import annotations + +import re +from statistics import mean + +_CITATION_PATTERN = re.compile(r"\[Source:\s*[^,\]]+\s*,\s*Page\s*\d+\]") + + +def _safe_mean(values: list[float]) -> float: + return round(mean(values), 4) if values else 0.0 + + +def compute_retrieval_hit_rate(results: list[dict]) -> float: + scores = [] + for result in results: + expected_sources = result.get("expected_sources", []) + if not expected_sources: + continue + retrieved_sources = set(result.get("retrieved_sources", [])) + hit_count = sum(1 for source in expected_sources if source in retrieved_sources) + scores.append(hit_count / len(expected_sources)) + return _safe_mean(scores) + + +def compute_citation_coverage(results: list[dict]) -> float: + scores = [] + for result in results: + if not result.get("expect_citations", False): + continue + answer = result.get("answer", "") + scores.append(1.0 if _CITATION_PATTERN.search(answer) else 0.0) + return _safe_mean(scores) + + +def compute_grounded_answer_score(results: list[dict]) -> float: + scores = [1.0 if result.get("grounded", False) else 0.0 for result in results] + return _safe_mean(scores) + + +def compute_average_latency(results: list[dict]) -> float: + latencies = [float(result.get("latency_ms", 0.0)) for result in results] + return _safe_mean(latencies) + + +def compute_tool_call_count(results: list[dict]) -> float: + counts = [float(result.get("tool_call_count", 0)) for result in results] + return _safe_mean(counts) + + +def compute_failure_rate(results: list[dict]) -> float: + failures = [1.0 if not result.get("passed", False) else 0.0 for result in results] + return _safe_mean(failures) + + +def build_metrics_summary(results: list[dict]) -> dict: + average_latency_ms = compute_average_latency(results) + return { + "retrieval_hit_rate": compute_retrieval_hit_rate(results), + "citation_coverage": compute_citation_coverage(results), + "grounded_answer_score": compute_grounded_answer_score(results), + "average_latency": average_latency_ms, + "average_latency_ms": average_latency_ms, + "tool_call_count": compute_tool_call_count(results), + "failure_rate": compute_failure_rate(results), + } diff --git a/evals/run_evals.py b/evals/run_evals.py new file mode 100644 index 0000000..45d7765 --- /dev/null +++ b/evals/run_evals.py @@ -0,0 +1,228 @@ +"""Deterministic evaluation runner for the RAG system.""" + +from __future__ import annotations + +import json +import re +import sys +import time +from pathlib import Path + +import yaml +from langchain_core.documents import Document + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = PROJECT_ROOT / "src" +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) +if str(SRC_ROOT) not in sys.path: + sys.path.insert(0, str(SRC_ROOT)) + +from evals.metrics import build_metrics_summary +from rfp_analyst.agent.graph import run_agent_graph +from rfp_analyst.agent.state import AgentState +from rfp_analyst.tools.source_verifier import verify_answer_grounding + +GOLDEN_PATH = PROJECT_ROOT / "evals" / "golden_questions.yaml" +RESULTS_PATH = PROJECT_ROOT / "evals" / "results.json" + + +def build_mock_corpus() -> list[Document]: + return [ + Document( + page_content=( + "Banking sector digital audit. Technology Stack: Azure SQL, Azure Data Factory, Power BI. " + "Timeline & Milestones: 16 weeks. Budget Range: $850,000. " + "Key Outcomes: improved data quality and faster audit cycles." + ), + metadata={"source_file": "Banking_Audit.pdf", "page": 0}, + ), + Document( + page_content=( + "Healthcare migration proposal. Technology Stack: Azure SQL, Azure Databricks, Power BI Premium. " + "Timeline & Milestones: 18 weeks. Budget Range: $1,200,000. " + "Compliance: HIPAA and SOC 2 Type II. Key Outcomes: lower infrastructure costs and real-time dashboards." + ), + metadata={"source_file": "Healthcare_Migration.pdf", "page": 0}, + ), + Document( + page_content=( + "Retail supply chain analytics platform. Technology Stack: Snowflake, Tableau, Databricks. " + "Timeline & Milestones: 24 weeks. Budget Range: $1,800,000. Key Outcomes: 34% reduction in stockouts." + ), + metadata={"source_file": "Retail_Supply_Chain.pdf", "page": 0}, + ), + Document( + page_content=( + "Insurance claims processing automation. Technology Stack: Azure Functions, UiPath, Azure Cognitive Services. " + "Timeline & Milestones: 12 weeks. Budget Range: $650,000. Key Outcomes: claims automation and lower operating costs." + ), + metadata={"source_file": "Insurance_Automation.pdf", "page": 1}, + ), + Document( + page_content=( + "Pharma clinical trial data platform. Compliance: FDA 21 CFR Part 11, GDPR, ICH E6(R2) GCP. " + "Technology Stack: Amazon Redshift, Apache Airflow, Python dashboards. " + "Key Outcomes: faster submissions and time-to-insight reduced to hours." + ), + metadata={"source_file": "Pharma_Clinical_Trials.pdf", "page": 2}, + ), + ] + + +def tokenize(text: str) -> set[str]: + return {token for token in re.findall(r"[a-z0-9]{3,}", text.lower())} + + +def build_search_fn(corpus: list[Document]): + def search_fn(query: str, k: int = 6): + query_tokens = tokenize(query) + scored = [] + for document in corpus: + haystack = f"{document.metadata.get('source_file', '')} {document.page_content}" + score = len(query_tokens.intersection(tokenize(haystack))) + if score > 0: + scored.append((document, min(0.99, score / max(len(query_tokens), 1)))) + scored.sort(key=lambda item: item[1], reverse=True) + return scored[:k] + + return search_fn + + +def load_golden_questions() -> list[dict]: + with GOLDEN_PATH.open("r", encoding="utf-8") as handle: + return list(yaml.safe_load_all(handle)) + + +def stats_fn() -> dict: + return { + "status": "ready", + "total_documents": 5, + "total_chunks": 5, + "document_names": [ + "Banking_Audit.pdf", + "Healthcare_Migration.pdf", + "Insurance_Automation.pdf", + "Pharma_Clinical_Trials.pdf", + "Retail_Supply_Chain.pdf", + ], + } + + +def _cited_snippet(source: dict) -> str: + return f"{source['snippet']} [Source: {source['source']}, Page {source['page'] + 1}]" + + +def synthesize_answer(state) -> str: + if state.final_answer: + return state.final_answer + + if state.intent == "compare_projects": + return "\n".join(_cited_snippet(source) for source in state.sources[:2]) + + if state.intent == "proposal_writer": + healthcare = next((source for source in state.sources if source['source'] == 'Healthcare_Migration.pdf'), None) + fallback = state.sources[0] if state.sources else None + evidence = healthcare or fallback + evidence_line = _cited_snippet(evidence) if evidence else "No grounded case study evidence found." + return "\n".join([ + "## Executive Summary", + evidence_line, + "## Client Requirements", + evidence_line, + "## Relevant Case Studies", + evidence_line, + ]) + + if state.intent == "rfp_gap_analysis": + matches = state.tool_outputs.get("find_relevant_case_studies", {}).get("matches", []) + if not matches: + return "I couldn't find enough evidence to map these requirements to prior work." + return "\n".join( + f"{match['source']} supports requirements {', '.join(match['matched_requirements'])} [Source: {match['source']}, Page {match['pages'][0] + 1}]" + for match in matches + ) + + if not state.sources: + return "I couldn't find relevant documents for this request. Please ingest more documents or refine the question." + + lower_query = state.query.lower() + if "which projects" in lower_query and "compliance" in lower_query: + prioritized = [ + source for source in state.sources + if source['source'] in {"Healthcare_Migration.pdf", "Pharma_Clinical_Trials.pdf"} + ] + return "\n".join(_cited_snippet(source) for source in prioritized[:2]) + + if "which projects" in lower_query: + unique_sources = [] + seen = set() + for source in state.sources: + if source['source'] in seen: + continue + seen.add(source['source']) + unique_sources.append(source) + return "\n".join(_cited_snippet(source) for source in unique_sources[:3]) + + return _cited_snippet(state.sources[0]) + + +def evaluate_question(question: dict, search_fn) -> dict: + started = time.perf_counter() + state = run_agent_graph(AgentState(query=question['question']), search_fn=search_fn, stats_fn=stats_fn) + answer = synthesize_answer(state) + verification = verify_answer_grounding(answer, state.retrieved_documents) if state.retrieved_documents else {"is_grounded": not question.get('expect_citations', False), "unsupported_claims": []} + elapsed_ms = (time.perf_counter() - started) * 1000 + + retrieved_sources = sorted({source['source'] for source in state.sources}) + keyword_hits = [keyword for keyword in question.get('expected_keywords', []) if keyword.lower() in answer.lower()] + citations_present = "[Source:" in answer + + passed = True + if question.get('expected_sources'): + passed = passed and all(source in retrieved_sources for source in question['expected_sources']) + if question.get('expected_keywords'): + passed = passed and len(keyword_hits) >= max(1, len(question['expected_keywords']) // 2) + if question.get('expect_citations', False): + passed = passed and citations_present + passed = passed and verification.get('is_grounded', True) + + return { + "id": question['id'], + "category": question['category'], + "question": question['question'], + "answer": answer, + "expected_sources": question.get('expected_sources', []), + "retrieved_sources": retrieved_sources, + "keyword_hits": keyword_hits, + "expect_citations": question.get('expect_citations', False), + "citations_present": citations_present, + "grounded": verification.get('is_grounded', True), + "unsupported_claims": verification.get('unsupported_claims', []), + "tool_call_count": len([step for step in state.tool_trace if step.get('tool')]), + "tool_trace": state.tool_trace, + "latency_ms": round(elapsed_ms, 2), + "passed": passed, + } + + +def run_all_evals() -> dict: + corpus = build_mock_corpus() + search_fn = build_search_fn(corpus) + questions = load_golden_questions() + results = [evaluate_question(question, search_fn) for question in questions] + metrics = build_metrics_summary(results) + payload = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "total_questions": len(results), + "passed_questions": sum(1 for result in results if result['passed']), + "metrics": metrics, + "results": results, + } + RESULTS_PATH.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return payload + + +if __name__ == "__main__": + summary = run_all_evals() + print(json.dumps(summary["metrics"], indent=2)) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..852739e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "internal-rfp-analyst" +version = "0.1.0" +description = "Internal RFP Analyst agentic RAG application" +readme = "README.md" +requires-python = ">=3.11" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] +include = ["rfp_analyst*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--basetemp=.pytest_tmp" + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F"] +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +"app.py" = ["E402"] +"evals/run_evals.py" = ["E402"] \ No newline at end of file diff --git a/rag_engine.py b/rag_engine.py index 2941b83..e24da67 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -1,93 +1,87 @@ -""" -RAG Engine — Document Ingestion, Embedding & Retrieval Pipeline. -Uses PyMuPDF for PDF loading, ChromaDB for vector storage. -Embeddings run locally via FastEmbed (ONNX) — no API calls, no rate limits. +"""RAG Engine - Document Ingestion, Embedding & Retrieval Pipeline. +Backward-compatible wrappers for the production ingestion package. """ -import os from pathlib import Path -from langchain_community.document_loaders import PyMuPDFLoader -from langchain_text_splitters import RecursiveCharacterTextSplitter -from langchain_community.embeddings.fastembed import FastEmbedEmbeddings -from langchain_chroma import Chroma + from config import ( - EMBEDDING_MODEL, - CHUNK_SIZE, CHUNK_OVERLAP, + CHUNK_SIZE, COLLECTION_NAME, DATA_DIR, - VECTORSTORE_DIR, RETRIEVAL_K, + VECTORSTORE_DIR, ) +from rfp_analyst.exceptions import IngestionError, KnowledgeBaseNotReadyError, RetrievalError +from rfp_analyst.ingestion.chunking import chunk_loaded_sources +from rfp_analyst.ingestion.loaders import load_pdf_sources +from rfp_analyst.ingestion.pipeline import IngestionPipeline +from rfp_analyst.retrieval.vector_store import VectorStoreManager +from rfp_analyst.retrieval.vector_store import get_embeddings as _get_embeddings +from rfp_analyst.schemas import LoadedSource def get_embeddings(): - """Initialize local embedding model. No API key needed, no rate limits.""" - return FastEmbedEmbeddings(model_name=EMBEDDING_MODEL) + """Backward-compatible embeddings wrapper.""" + return _get_embeddings() def load_pdfs(doc_dir: Path = DATA_DIR): - """Load all PDFs from the given directory using PyMuPDF.""" - pdf_files = sorted(doc_dir.glob("*.pdf")) - if not pdf_files: - raise FileNotFoundError(f"No PDF files found in {doc_dir}") - + """Backward-compatible PDF loading wrapper.""" + loaded_sources = load_pdf_sources(doc_dir) all_docs = [] - for pdf_path in pdf_files: - loader = PyMuPDFLoader(str(pdf_path)) - docs = loader.load() - # Enrich metadata - for doc in docs: - doc.metadata["source_file"] = pdf_path.name - doc.metadata["source_path"] = str(pdf_path) - all_docs.extend(docs) - print(f" Loaded: {pdf_path.name} ({len(docs)} pages)") - - print(f"Total pages loaded: {len(all_docs)}") + for source in loaded_sources: + all_docs.extend(source.documents) return all_docs def chunk_documents(documents): - """Split documents into chunks with overlap for context continuity.""" - splitter = RecursiveCharacterTextSplitter( + """Backward-compatible chunking wrapper.""" + if not documents: + return [] + + grouped_sources = {} + for document in documents: + file_hash = document.metadata.get("file_hash", "legacy") + grouped_sources.setdefault(file_hash, []).append(document) + + loaded_sources = [] + for file_hash, source_documents in grouped_sources.items(): + first = source_documents[0] + loaded_sources.append( + LoadedSource( + source_file=first.metadata.get("source_file", "unknown.pdf"), + source_path=first.metadata.get("source_path", ""), + file_hash=file_hash, + page_count=len(source_documents), + document_type=first.metadata.get("document_type"), + documents=source_documents, + ) + ) + + return chunk_loaded_sources( + loaded_sources, chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP, - length_function=len, - separators=["\n\n", "\n", ". ", " ", ""], ) - chunks = splitter.split_documents(documents) - print(f"Created {len(chunks)} chunks (size={CHUNK_SIZE}, overlap={CHUNK_OVERLAP})") - return chunks def create_vectorstore(chunks, persist_dir: Path = VECTORSTORE_DIR): - """Embed chunks and store in ChromaDB. Uses local embeddings — fast, no API calls.""" - persist_dir.mkdir(parents=True, exist_ok=True) - embeddings = get_embeddings() - - vectorstore = Chroma.from_documents( - documents=chunks, - embedding=embeddings, + """Backward-compatible vector store wrapper.""" + manager = VectorStoreManager( + persist_dir=persist_dir, collection_name=COLLECTION_NAME, - persist_directory=str(persist_dir), ) - print(f"Vector store created with {vectorstore._collection.count()} vectors") - print(f"Persisted to: {persist_dir}") - return vectorstore + return manager.upsert_documents(chunks) def load_vectorstore(persist_dir: Path = VECTORSTORE_DIR): """Load an existing ChromaDB vector store from disk.""" - if not persist_dir.exists(): - raise FileNotFoundError( - f"Vector store not found at {persist_dir}. Run ingestion first." - ) - embeddings = get_embeddings() - vectorstore = Chroma( + manager = VectorStoreManager( + persist_dir=persist_dir, collection_name=COLLECTION_NAME, - embedding_function=embeddings, - persist_directory=str(persist_dir), ) + vectorstore = manager.load(create_if_missing=False) count = vectorstore._collection.count() print(f"Loaded vector store with {count} vectors") return vectorstore @@ -95,68 +89,49 @@ def load_vectorstore(persist_dir: Path = VECTORSTORE_DIR): def get_retriever(k: int = RETRIEVAL_K): """Get a LangChain retriever from the persisted vector store.""" - vectorstore = load_vectorstore() - return vectorstore.as_retriever( - search_type="similarity", - search_kwargs={"k": k}, + manager = VectorStoreManager( + persist_dir=VECTORSTORE_DIR, + collection_name=COLLECTION_NAME, ) + return manager.get_retriever(k=k) def similarity_search(query: str, k: int = RETRIEVAL_K): """Direct similarity search returning documents with scores.""" - vectorstore = load_vectorstore() - results = vectorstore.similarity_search_with_relevance_scores(query, k=k) - return results + manager = VectorStoreManager( + persist_dir=VECTORSTORE_DIR, + collection_name=COLLECTION_NAME, + ) + try: + return manager.similarity_search(query, k=k) + except KnowledgeBaseNotReadyError: + raise + except Exception as error: + raise RetrievalError(str(error)) from error def get_vectorstore_stats(): """Get statistics about the current vector store.""" - try: - vectorstore = load_vectorstore() - count = vectorstore._collection.count() - # Get unique sources - all_metadata = vectorstore._collection.get()["metadatas"] - sources = set() - for m in all_metadata: - if "source_file" in m: - sources.add(m["source_file"]) - return { - "total_chunks": count, - "total_documents": len(sources), - "document_names": sorted(sources), - "status": "ready", - } - except Exception: - return { - "total_chunks": 0, - "total_documents": 0, - "document_names": [], - "status": "not_initialized", - } + manager = VectorStoreManager( + persist_dir=VECTORSTORE_DIR, + collection_name=COLLECTION_NAME, + ) + return manager.get_stats() def ingest_documents(doc_dir: Path = DATA_DIR): - """Full ingestion pipeline: load → chunk → embed → store.""" - print("=" * 60) - print("DOCUMENT INGESTION PIPELINE") - print("=" * 60) - - print("\n[1/3] Loading PDFs...") - documents = load_pdfs(doc_dir) - - print("\n[2/3] Chunking documents...") - chunks = chunk_documents(documents) - - print("\n[3/3] Embedding & storing in ChromaDB (local embeddings)...") - vectorstore = create_vectorstore(chunks) - - stats = get_vectorstore_stats() - print("\n" + "=" * 60) - print("INGESTION COMPLETE") - print(f" Documents: {stats['total_documents']}") - print(f" Chunks: {stats['total_chunks']}") - print("=" * 60) - return vectorstore + """Full ingestion pipeline wrapper.""" + pipeline = IngestionPipeline( + doc_dir=doc_dir, + persist_dir=VECTORSTORE_DIR, + collection_name=COLLECTION_NAME, + ) + try: + return pipeline.run() + except Exception as error: + if isinstance(error, IngestionError): + raise + raise IngestionError(str(error)) from error if __name__ == "__main__": diff --git a/src/internal_rfp_analyst.egg-info/PKG-INFO b/src/internal_rfp_analyst.egg-info/PKG-INFO new file mode 100644 index 0000000..d082467 --- /dev/null +++ b/src/internal_rfp_analyst.egg-info/PKG-INFO @@ -0,0 +1,288 @@ +Metadata-Version: 2.4 +Name: internal-rfp-analyst +Version: 0.1.0 +Summary: Internal RFP Analyst agentic RAG application +Requires-Python: >=3.11 +Description-Content-Type: text/markdown + +
+ +# 🔍 Internal RFP Analyst + +### AI-Powered RAG Knowledge Agent for Enterprise Consulting + +[![Live Demo](https://img.shields.io/badge/🚀_Live_Demo-Streamlit_Cloud-FF4B4B?style=for-the-badge&logo=streamlit&logoColor=white)](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) +[![Python](https://img.shields.io/badge/Python-3.10+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://python.org) +[![LangChain](https://img.shields.io/badge/LangChain-0.3+-1C3C3C?style=for-the-badge&logo=langchain&logoColor=white)](https://langchain.com) +[![Groq](https://img.shields.io/badge/Groq-LPU_Inference-F55036?style=for-the-badge)](https://groq.com) +[![License](https://img.shields.io/badge/License-MIT-green?style=for-the-badge)](LICENSE) + +**Instantly search past proposals, RFP responses, project outlines, and case studies using natural language.** Built with a production-grade RAG pipeline featuring local ONNX embeddings, streaming LLM responses, and multi-provider failover. + +[Live Demo](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) · [Architecture](#-architecture) · [Quick Start](#-quick-start) · [Challenges & Solutions](#-engineering-challenges--solutions) + +
+ +--- + +## ✨ Key Features + +| Feature | Description | +|---|---| +| 🔍 **Semantic Search** | Natural language queries over a ChromaDB vector store with relevance-scored retrieval | +| ⚡ **Streaming Responses** | Word-by-word response streaming via `st.write_stream` for instant perceived performance | +| 🧠 **Local Embeddings** | ONNX-based FastEmbed (`bge-small-en-v1.5`) — zero API calls, zero rate limits for retrieval | +| 🔄 **Multi-Provider LLM** | Groq (Llama 3.3 70B) primary + Gemini fallback — automatic provider selection | +| 📄 **PDF Ingestion** | Upload custom PDFs or use the built-in 10-document consulting knowledge base | +| 📚 **Source Citations** | Every answer cites exact document name and page number | +| 💬 **Conversation Memory** | Chat history maintained in-session for contextual follow-ups | +| ☁️ **Zero-Config Deploy** | Auto-generates sample documents and ingests on first Streamlit Cloud boot | + +--- + +## 🏗️ Architecture + +```mermaid +flowchart TB + subgraph UI["🖥️ Streamlit UI"] + A["User Query"] --> B["Chat Interface"] + B --> C["st.write_stream"] + end + + subgraph RAG["⚡ RAG Pipeline"] + D["FastEmbed ONNX
bge-small-en-v1.5"] --> E["ChromaDB
Vector Store"] + E --> F["Top-K Retrieval
k=6 chunks"] + end + + subgraph LLM["🤖 LLM Layer"] + G{"Provider
Selection"} + G -->|"Primary"| H["Groq LPU
Llama 3.3 70B"] + G -->|"Fallback"| I["Google Gemini
2.0 Flash"] + end + + subgraph INGEST["📥 Ingestion Pipeline"] + J["PDF Documents"] --> K["PyMuPDF Loader"] + K --> L["Recursive Chunking
512 tokens, 50 overlap"] + L --> D + end + + A --> F + F --> |"Context + Prompt"| G + H --> C + I --> C + + style UI fill:#1a1a2e,stroke:#667eea,color:#fff + style RAG fill:#16213e,stroke:#0f3460,color:#fff + style LLM fill:#1a1a2e,stroke:#e94560,color:#fff + style INGEST fill:#16213e,stroke:#533483,color:#fff +``` + +### Request Flow (Single Query) + +```mermaid +sequenceDiagram + participant U as 👤 User + participant S as 🖥️ Streamlit + participant E as ⚡ FastEmbed (Local) + participant C as 🗄️ ChromaDB + participant L as 🤖 Groq/Gemini + + U->>S: "What tech stack did we use for banking?" + S->>E: Embed query (local, ~5ms) + E->>C: Similarity search (k=6) + C-->>S: Top 6 relevant chunks + metadata + S->>L: Single prompt with context + L-->>S: Streaming response tokens + S-->>U: Word-by-word answer with citations + + Note over E,C: Zero API calls for retrieval + Note over L: Single LLM call per query +``` + +--- + +## 🛠️ Technology Stack + +| Layer | Technology | Why This Choice | +|---|---|---| +| **LLM (Primary)** | Groq — Llama 3.3 70B | Fastest free inference (LPU), 30 RPM, sub-second latency | +| **LLM (Fallback)** | Google Gemini 2.0 Flash | Free tier backup, 15 RPM | +| **Embeddings** | FastEmbed (ONNX) — `bge-small-en-v1.5` | Local execution, no API calls, no rate limits | +| **Vector Store** | ChromaDB (persistent) | Lightweight, embedded, perfect for document-scale RAG | +| **RAG Framework** | LangChain 0.3+ | Industry-standard abstractions for retrieval chains | +| **PDF Processing** | PyMuPDF | Fastest Python PDF parser, preserves layout metadata | +| **UI** | Streamlit | Rapid prototyping with built-in streaming support | +| **Deployment** | Streamlit Community Cloud | Free hosting with GitHub auto-deploy | + +--- + +## 🚀 Quick Start + +### Option 1: Use the Live Demo +👉 **[app-rfp-analyst.streamlit.app](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/)** — No setup required. The app auto-generates sample documents on first load. + +### Option 2: Run Locally + +#### 1. Get a Free API Key (Choose One) + +| Provider | Speed | Free Limit | Get Key | +|---|---|---|---| +| **Groq** ⭐ Recommended | ~100 tok/s | 30 RPM, 6000 RPD | [console.groq.com/keys](https://console.groq.com/keys) | +| Google Gemini | ~30 tok/s | 15 RPM | [aistudio.google.com/apikey](https://aistudio.google.com/apikey) | + +#### 2. Setup + +```bash +# Clone the repository +git clone https://github.com/tusharg007/Internal-RFP-Analyst.git +cd Internal-RFP-Analyst + +# Create virtual environment +python -m venv venv +venv\Scripts\activate # Windows +# source venv/bin/activate # Mac/Linux + +# Install dependencies +pip install -r requirements.txt + +# Configure API key +copy .env.example .env +# Edit .env → add your GROQ_API_KEY (or GOOGLE_API_KEY) +``` + +#### 3. Launch + +```bash +streamlit run app.py +``` + +The app will auto-generate 10 sample consulting documents and build the vector store on first launch. + +--- + +## 💬 Example Queries + +| Query | What It Tests | +|---|---| +| *"List all projects with their timelines"* | Full knowledge base traversal | +| *"What tech stack did we use for the banking audit?"* | Precise document retrieval | +| *"Compare the healthcare and insurance projects"* | Cross-document synthesis | +| *"Which projects used Azure services?"* | Multi-document filtering | +| *"What was the budget for the supply chain platform?"* | Specific fact extraction | +| *"What compliance frameworks did we follow in pharma?"* | Domain-specific retrieval | + +--- + +## 🧪 Engineering Challenges & Solutions + +### Challenge 1: Gemini API Rate Limits Killed the App + +**Problem:** The original architecture used Google Gemini for *both* embeddings and LLM generation. The free tier (100 embedding req/min, 15 LLM req/min) was exhausted within minutes, returning `429 RESOURCE_EXHAUSTED` errors. The multi-step ReAct agent made 3-5 LLM calls per query, compounding the problem. + +**Solution: Hybrid local + cloud architecture** + +```mermaid +flowchart LR + subgraph BEFORE["❌ Before — All API Calls"] + A1["Gemini Embeddings API"] -->|"Rate Limited"| B1["429 Error"] + A2["Gemini LLM x 3-5 calls"] -->|"Rate Limited"| B1 + end + + subgraph AFTER["✅ After — Minimal API Calls"] + C1["FastEmbed Local
Zero API calls"] -->|"Instant"| D1["Success"] + C2["Groq LLM x 1 call
30 RPM limit"] -->|"Sub-second"| D1 + end + + style BEFORE fill:#2d1117,stroke:#f85149,color:#fff + style AFTER fill:#0d1117,stroke:#3fb950,color:#fff +``` + +| Metric | Before | After | Improvement | +|---|---|---|---| +| API calls per query | 4-6 (embed + 3-5 LLM) | **1** (LLM only) | **83% reduction** | +| Embedding rate limits | 100/min (API) | **∞** (local) | **Eliminated** | +| LLM rate limits | 15 RPM (Gemini) | **30 RPM** (Groq) | **2x headroom** | + +### Challenge 2: 10+ Minute Response Times + +**Problem:** The ReAct agent architecture (LangGraph) made multiple sequential LLM calls — tool selection → execution → result processing → possibly more tools → final answer. Each call could trigger a rate-limit retry with exponential backoff (10s → 20s → 40s), compounding to 10+ minute waits. + +**Solution: Single-call RAG with streaming** + +- Replaced multi-step ReAct agent with a **single LLM call** architecture +- All context (retrieved chunks + project list + chat history) is assembled locally and sent in one prompt +- **Streaming responses** via `st.write_stream()` — text appears word-by-word, so the user sees output within 500ms even if full generation takes 3-5s + +| Metric | Before (ReAct) | After (Single-Call RAG) | +|---|---|---| +| LLM calls per query | 3-5 | **1** | +| Worst-case response time | 10+ minutes | **3-8 seconds** | +| Perceived latency | Full wait → wall of text | **~500ms** (streaming) | + +### Challenge 3: Sample Question Buttons Did Nothing + +**Problem:** Clicking a sample question button added the message to chat history and triggered `st.rerun()`, but after the rerun, only the `st.chat_input()` code path processed queries — sample button clicks were silently ignored. + +**Solution:** Introduced a `pending_query` session state flag. Button clicks set this flag before rerun. After rerun, a dedicated handler detects the pending query and routes it through the same processing pipeline as typed messages. + +### Challenge 4: Ephemeral Filesystem on Streamlit Cloud + +**Problem:** Streamlit Cloud's filesystem resets on every cold start, losing the vector store and requiring re-ingestion. + +**Solution:** Auto-setup pipeline — on first load, the app detects an empty vector store, generates 10 sample PDFs via `document_generator.py`, and ingests them automatically. With local embeddings, this entire process completes in **under 15 seconds** (vs. minutes with API-based embeddings). + +--- + +## 📁 Project Structure + +``` +Internal-RFP-Analyst/ +├── app.py # Streamlit UI with streaming chat +├── agent.py # RAG query engine (Groq/Gemini + retrieval) +├── rag_engine.py # Ingestion pipeline (FastEmbed + ChromaDB) +├── config.py # Central configuration & provider selection +├── document_generator.py # Generates 10 realistic consulting PDFs +├── requirements.txt # Python dependencies +├── .env.example # API key template +├── .streamlit/ +│ └── config.toml # Streamlit theme configuration +├── data/documents/ # PDF documents (auto-generated) +└── vectorstore/ # ChromaDB persistent storage +``` + +### Module Responsibilities + +| Module | Lines | Responsibility | +|---|---|---| +| `config.py` | ~75 | API keys, model selection, RAG parameters, system prompt | +| `rag_engine.py` | ~165 | PDF loading → chunking → local embedding → ChromaDB storage/retrieval | +| `agent.py` | ~155 | LLM provider selection, prompt assembly, streaming query execution | +| `app.py` | ~280 | Streamlit UI, session management, chat rendering, error handling | +| `document_generator.py` | ~550 | Generates 10 industry-specific consulting PDFs with realistic content | + +--- + +## 🔧 Configuration + +### Environment Variables + +| Variable | Required | Description | +|---|---|---| +| `GROQ_API_KEY` | ⭐ Recommended | Groq API key for fastest inference ([get free key](https://console.groq.com/keys)) | +| `GOOGLE_API_KEY` | Optional | Google Gemini key as fallback ([get free key](https://aistudio.google.com/apikey)) | + +### For Streamlit Cloud Deployment + +Add secrets in **Settings → Secrets**: + +```toml +GROQ_API_KEY = "gsk_your_key_here" +# Optional fallback: +# GOOGLE_API_KEY = "your_google_key_here" +``` + +--- + +## 📜 License + +This project is for educational and portfolio demonstration purposes. Built by [Tushar Ghosh](https://github.com/tusharg007). diff --git a/src/internal_rfp_analyst.egg-info/SOURCES.txt b/src/internal_rfp_analyst.egg-info/SOURCES.txt new file mode 100644 index 0000000..7adaf40 --- /dev/null +++ b/src/internal_rfp_analyst.egg-info/SOURCES.txt @@ -0,0 +1,32 @@ +README.md +pyproject.toml +src/internal_rfp_analyst.egg-info/PKG-INFO +src/internal_rfp_analyst.egg-info/SOURCES.txt +src/internal_rfp_analyst.egg-info/dependency_links.txt +src/internal_rfp_analyst.egg-info/top_level.txt +src/rfp_analyst/__init__.py +src/rfp_analyst/schemas.py +src/rfp_analyst/agent/__init__.py +src/rfp_analyst/agent/graph.py +src/rfp_analyst/agent/prompts.py +src/rfp_analyst/agent/runtime.py +src/rfp_analyst/agent/state.py +src/rfp_analyst/ingestion/__init__.py +src/rfp_analyst/ingestion/chunking.py +src/rfp_analyst/ingestion/loaders.py +src/rfp_analyst/ingestion/pipeline.py +src/rfp_analyst/ingestion/registry.py +src/rfp_analyst/retrieval/__init__.py +src/rfp_analyst/retrieval/vector_store.py +src/rfp_analyst/tools/__init__.py +src/rfp_analyst/tools/compare_projects.py +src/rfp_analyst/tools/proposal_writer.py +src/rfp_analyst/tools/rfp_gap_analyzer.py +src/rfp_analyst/tools/search_kb.py +src/rfp_analyst/tools/source_verifier.py +tests/test_agentic_tools.py +tests/test_config.py +tests/test_document_generator.py +tests/test_evals.py +tests/test_imports.py +tests/test_ingestion_pipeline.py \ No newline at end of file diff --git a/src/internal_rfp_analyst.egg-info/dependency_links.txt b/src/internal_rfp_analyst.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/internal_rfp_analyst.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/internal_rfp_analyst.egg-info/top_level.txt b/src/internal_rfp_analyst.egg-info/top_level.txt new file mode 100644 index 0000000..9bd1376 --- /dev/null +++ b/src/internal_rfp_analyst.egg-info/top_level.txt @@ -0,0 +1 @@ +rfp_analyst diff --git a/src/rfp_analyst.egg-info/PKG-INFO b/src/rfp_analyst.egg-info/PKG-INFO new file mode 100644 index 0000000..e6bf909 --- /dev/null +++ b/src/rfp_analyst.egg-info/PKG-INFO @@ -0,0 +1,3 @@ +Metadata-Version: 2.4 +Name: rfp_analyst +Version: 0.0.0 diff --git a/src/rfp_analyst.egg-info/SOURCES.txt b/src/rfp_analyst.egg-info/SOURCES.txt new file mode 100644 index 0000000..d7fc1d6 --- /dev/null +++ b/src/rfp_analyst.egg-info/SOURCES.txt @@ -0,0 +1,32 @@ +README.md +pyproject.toml +src/rfp_analyst/__init__.py +src/rfp_analyst/schemas.py +src/rfp_analyst.egg-info/PKG-INFO +src/rfp_analyst.egg-info/SOURCES.txt +src/rfp_analyst.egg-info/dependency_links.txt +src/rfp_analyst.egg-info/top_level.txt +src/rfp_analyst/agent/__init__.py +src/rfp_analyst/agent/graph.py +src/rfp_analyst/agent/prompts.py +src/rfp_analyst/agent/runtime.py +src/rfp_analyst/agent/state.py +src/rfp_analyst/ingestion/__init__.py +src/rfp_analyst/ingestion/chunking.py +src/rfp_analyst/ingestion/loaders.py +src/rfp_analyst/ingestion/pipeline.py +src/rfp_analyst/ingestion/registry.py +src/rfp_analyst/retrieval/__init__.py +src/rfp_analyst/retrieval/vector_store.py +src/rfp_analyst/tools/__init__.py +src/rfp_analyst/tools/compare_projects.py +src/rfp_analyst/tools/proposal_writer.py +src/rfp_analyst/tools/rfp_gap_analyzer.py +src/rfp_analyst/tools/search_kb.py +src/rfp_analyst/tools/source_verifier.py +tests/test_agentic_tools.py +tests/test_config.py +tests/test_document_generator.py +tests/test_evals.py +tests/test_imports.py +tests/test_ingestion_pipeline.py \ No newline at end of file diff --git a/src/rfp_analyst.egg-info/dependency_links.txt b/src/rfp_analyst.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/rfp_analyst.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/rfp_analyst.egg-info/top_level.txt b/src/rfp_analyst.egg-info/top_level.txt new file mode 100644 index 0000000..9bd1376 --- /dev/null +++ b/src/rfp_analyst.egg-info/top_level.txt @@ -0,0 +1 @@ +rfp_analyst diff --git a/src/rfp_analyst/__init__.py b/src/rfp_analyst/__init__.py new file mode 100644 index 0000000..ce8278c --- /dev/null +++ b/src/rfp_analyst/__init__.py @@ -0,0 +1 @@ +"""Internal RFP Analyst package.""" diff --git a/src/rfp_analyst/agent/__init__.py b/src/rfp_analyst/agent/__init__.py new file mode 100644 index 0000000..c84763c --- /dev/null +++ b/src/rfp_analyst/agent/__init__.py @@ -0,0 +1 @@ +"""Agent orchestration package.""" diff --git a/src/rfp_analyst/agent/graph.py b/src/rfp_analyst/agent/graph.py new file mode 100644 index 0000000..6d16af9 --- /dev/null +++ b/src/rfp_analyst/agent/graph.py @@ -0,0 +1,86 @@ +"""Deterministic agent graph for orchestration.""" + +from __future__ import annotations + +from rag_engine import get_vectorstore_stats +from rfp_analyst.agent.prompts import ( + build_agentic_prompt, + build_ambiguous_question_message, + build_no_documents_message, + classify_query_intent, + is_ambiguous_query, +) +from rfp_analyst.agent.state import AgentState +from rfp_analyst.tools.compare_projects import compare_projects +from rfp_analyst.tools.proposal_writer import generate_proposal_outline +from rfp_analyst.tools.rfp_gap_analyzer import extract_rfp_requirements, find_relevant_case_studies +from rfp_analyst.tools.search_kb import search_knowledge_base + + +def _add_source_trace(state: AgentState, sources: list[dict]) -> None: + for source in sources[:5]: + state.tool_trace.append( + { + "tool_response": f"{source['source']} (Page {source['page'] + 1})", + "snippet": source["snippet"], + } + ) + + +def run_agent_graph(state: AgentState, search_fn=None, stats_fn=None) -> AgentState: + """Run the agent workflow from intent classification to synthesis prompt creation.""" + state.stats = stats_fn() if stats_fn else get_vectorstore_stats() + if state.stats.get("status") != "ready" or state.stats.get("total_documents", 0) == 0: + state.tool_trace.append({"tool": "knowledge_base_status", "input": {"status": "not_initialized"}}) + state.final_answer = build_no_documents_message(state.query) + return state + + if is_ambiguous_query(state.query): + state.tool_trace.append({"tool": "ambiguity_check", "input": {"status": "ambiguous"}}) + state.final_answer = build_ambiguous_question_message(state.query) + return state + + state.intent = classify_query_intent(state.query) + state.tool_trace.append({"tool": "classify_intent", "input": {"query": state.query, "intent": state.intent}}) + + if state.intent == "compare_projects": + comparison = compare_projects(state.query, search_fn=search_fn) + state.tool_outputs["compare_projects"] = comparison + state.retrieved_documents = comparison["documents"] + state.sources = comparison["sources"] + state.tool_trace.append({"tool": "compare_projects", "input": {"query": state.query}}) + _add_source_trace(state, state.sources) + elif state.intent in {"rfp_gap_analysis", "proposal_writer"}: + requirements = extract_rfp_requirements(state.query) + state.tool_outputs["extract_rfp_requirements"] = requirements + state.tool_trace.append({"tool": "extract_rfp_requirements", "input": {"count": len(requirements['requirements'])}}) + + case_studies = find_relevant_case_studies(requirements["requirements"], search_fn=search_fn) + state.tool_outputs["find_relevant_case_studies"] = case_studies + state.retrieved_documents = case_studies["documents"] + state.sources = [ + {"source": match["source"], "page": match["pages"][0] if match["pages"] else 0, "snippet": match["snippets"][0] if match["snippets"] else ""} + for match in case_studies["matches"] + ] + state.tool_trace.append({"tool": "find_relevant_case_studies", "input": {"matches": len(case_studies['matches'])}}) + _add_source_trace(state, state.sources) + + if state.intent == "proposal_writer": + outline = generate_proposal_outline(state.query, case_studies, requirements["requirements"]) + state.tool_outputs["generate_proposal_outline"] = outline + state.tool_trace.append({"tool": "generate_proposal_outline", "input": {"sections": 5}}) + else: + search_result = search_knowledge_base(state.query, search_fn=search_fn) + state.tool_outputs["search_knowledge_base"] = search_result + state.retrieved_documents = search_result["documents"] + state.sources = search_result["sources"] + state.tool_trace.append({"tool": "search_knowledge_base", "input": {"query": state.query}}) + _add_source_trace(state, state.sources) + + if not state.retrieved_documents and not state.sources: + state.final_answer = "I couldn't find relevant documents for this request. Please ingest more documents or refine the question." + return state + + state.prompt = build_agentic_prompt(state) + state.tool_trace.append({"tool": "verify_answer_grounding", "input": {"status": "planned"}}) + return state diff --git a/src/rfp_analyst/agent/prompts.py b/src/rfp_analyst/agent/prompts.py new file mode 100644 index 0000000..5250108 --- /dev/null +++ b/src/rfp_analyst/agent/prompts.py @@ -0,0 +1,111 @@ +"""Prompts and deterministic intent classification.""" + +from __future__ import annotations + +import re + +from config import AGENT_SYSTEM_PROMPT + + +def classify_query_intent(query: str) -> str: + lower = query.lower() + if any(keyword in lower for keyword in ("compare", "versus", "vs", "difference")): + return "compare_projects" + if any(keyword in lower for keyword in ("proposal", "outline", "respond to rfp", "write proposal")): + return "proposal_writer" + if any(keyword in lower for keyword in ("rfp", "requirement", "gap", "case study", "fit")): + return "rfp_gap_analysis" + return "search" + + +def is_ambiguous_query(query: str) -> bool: + lower = query.lower().strip() + pronouns = {"it", "that", "this", "they", "them", "those", "one", "ones", "other"} + tokens = re.findall(r"[a-zA-Z]+", lower) + if len(tokens) <= 3: + return True + if any(token in pronouns for token in tokens) and not any( + keyword in lower for keyword in ("banking", "healthcare", "insurance", "project", "proposal", "rfp") + ): + return True + return False + + +def _render_history(chat_history: list | None) -> str: + if not chat_history: + return "" + rendered = [] + for item in chat_history[-6:]: + role = item.get("role", "user").title() + rendered.append(f"{role}: {item.get('content', '')[:300]}") + return "\n".join(rendered) + + +def build_no_documents_message(query: str) -> str: + return ( + "I couldn't find any indexed project documents to answer this yet. " + "Please ingest documents first, then try your question again. " + f"Your question was: {query}" + ) + + +def build_ambiguous_question_message(query: str) -> str: + return ( + "Your question is a bit ambiguous. Please clarify which project, proposal, or document set you mean " + f"before I answer: {query}" + ) + + +def build_simple_prompt(user_query: str, context: str, stats: dict, chat_history: list | None = None) -> str: + history = _render_history(chat_history) + project_list = "\n".join(f" - {name}" for name in stats.get("document_names", [])) or " No documents ingested yet." + history_block = f"\nRecent Conversation:\n{history}\n" if history else "" + return f"""{AGENT_SYSTEM_PROMPT} + +Available Documents: +{project_list} +Total: {stats.get('total_documents', 0)} documents, {stats.get('total_chunks', 0)} chunks + +Retrieved Context: +{context} +{history_block} +Question: +{user_query} + +Answer thoroughly with source citations.""" + + +def build_agentic_prompt(state) -> str: + history = _render_history(state.chat_history) + tool_summaries = [] + for name, payload in state.tool_outputs.items(): + if isinstance(payload, dict): + if "comparison_markdown" in payload: + tool_summaries.append(f"Tool {name}:\n{payload['comparison_markdown']}") + elif "outline" in payload: + tool_summaries.append(f"Tool {name}:\n{payload['outline']}") + elif "summary" in payload: + tool_summaries.append(f"Tool {name}: {payload['summary']}") + + evidence = "\n\n".join( + f"[Source: {source['source']}, Page {source['page'] + 1}]\n{source['snippet']}" + for source in state.sources[:8] + ) or "No evidence retrieved." + + history_block = f"\nRecent Conversation:\n{history}\n" if history else "" + tool_block = "\n\n".join(tool_summaries) if tool_summaries else "No specialized tool outputs." + return f"""{AGENT_SYSTEM_PROMPT} + +Intent: {state.intent} +Available Documents: {state.stats.get('total_documents', 0)} + +Tool Outputs: +{tool_block} + +Evidence: +{evidence} +{history_block} +Question: +{state.query} + +Write a grounded final answer with explicit [Source: , Page ] citations for every major claim.""" diff --git a/src/rfp_analyst/agent/runtime.py b/src/rfp_analyst/agent/runtime.py new file mode 100644 index 0000000..2a7dd48 --- /dev/null +++ b/src/rfp_analyst/agent/runtime.py @@ -0,0 +1,138 @@ +"""Runtime helpers for simple and agentic RAG modes.""" + +from __future__ import annotations + +from langchain_core.messages import HumanMessage + +from config import AGENT_MODE +from rag_engine import get_vectorstore_stats +from rfp_analyst.agent.graph import run_agent_graph +from rfp_analyst.agent.prompts import build_simple_prompt +from rfp_analyst.agent.state import AgentState +from rfp_analyst.exceptions import KnowledgeBaseNotReadyError, RetrievalError +from rfp_analyst.tools.search_kb import search_knowledge_base +from rfp_analyst.tools.source_verifier import verify_answer_grounding + + +KNOWLEDGE_BASE_NOT_READY_MESSAGE = ( + "Knowledge base is not ready. Generate or upload PDFs and click Ingest Documents." +) + + +def get_agent_mode() -> str: + return AGENT_MODE if AGENT_MODE in {"simple", "agentic"} else "agentic" + + +def _stream_text(text: str, chunk_size: int = 120): + for index in range(0, len(text), chunk_size): + yield text[index:index + chunk_size] + + +def prepare_simple_query(user_query: str, chat_history: list | None = None) -> dict: + stats = get_vectorstore_stats() + if stats.get("status") != "ready": + return { + "mode": "simple", + "prompt": "", + "documents": [], + "reasoning_trace": [{"tool": "knowledge_base_status", "input": {"status": "not_initialized"}}], + "prebuilt_answer": KNOWLEDGE_BASE_NOT_READY_MESSAGE, + } + + result = search_knowledge_base(user_query) + if not result["documents"]: + return { + "mode": "simple", + "prompt": "", + "documents": [], + "reasoning_trace": [{"tool": "search_knowledge_base", "input": {"query": user_query, "mode": "simple"}}], + "prebuilt_answer": "I couldn't find relevant documents for this request. Please ingest more documents or refine the question.", + } + + prompt = build_simple_prompt(user_query, result["context"], stats, chat_history) + reasoning_trace = [{"tool": "search_knowledge_base", "input": {"query": user_query, "mode": "simple"}}] + for source in result["sources"][:3]: + reasoning_trace.append( + { + "tool_response": f"{source['source']} (Page {source['page'] + 1})", + "snippet": source["snippet"], + } + ) + return { + "mode": "simple", + "prompt": prompt, + "documents": result["documents"], + "reasoning_trace": reasoning_trace, + "prebuilt_answer": None, + } + + +def prepare_agentic_query(user_query: str, chat_history: list | None = None) -> dict: + state = run_agent_graph(AgentState(query=user_query, chat_history=chat_history or [])) + return { + "mode": "agentic", + "prompt": state.prompt, + "documents": state.retrieved_documents, + "reasoning_trace": state.tool_trace, + "prebuilt_answer": state.final_answer or None, + } + + +def prepare_query_payload(user_query: str, chat_history: list | None = None) -> dict: + if get_agent_mode() == "simple": + return prepare_simple_query(user_query, chat_history) + return prepare_agentic_query(user_query, chat_history) + + +def stream_query_response(llm, payload): + if isinstance(payload, str): + for chunk in llm.stream([HumanMessage(content=payload)]): + if chunk.content: + yield chunk.content + return + + prebuilt_answer = payload.get("prebuilt_answer") + if prebuilt_answer: + yield from _stream_text(prebuilt_answer) + return + + prompt = payload.get("prompt", "") + if not prompt: + raise KnowledgeBaseNotReadyError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) + + full_response = "" + for chunk in llm.stream([HumanMessage(content=prompt)]): + if chunk.content: + full_response += chunk.content + yield chunk.content + + verification = verify_answer_grounding(full_response, payload.get("documents", [])) + if not verification["is_grounded"]: + warning_lines = ["\n\nGrounding check: some claims may not be fully supported:"] + warning_lines.extend(f"- {claim}" for claim in verification["unsupported_claims"][:3]) + yield "\n".join(warning_lines) + + +def run_query(llm, user_query: str, chat_history: list | None = None) -> dict: + payload = prepare_query_payload(user_query, chat_history) + if payload.get("prebuilt_answer"): + return { + "answer": payload["prebuilt_answer"], + "reasoning_trace": payload["reasoning_trace"], + "all_messages": [], + } + + prompt = payload.get("prompt", "") + if not prompt: + raise RetrievalError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) + + response = llm.invoke([HumanMessage(content=prompt)]) + answer = response.content + verification = verify_answer_grounding(answer, payload.get("documents", [])) + if not verification["is_grounded"]: + answer += "\n\nGrounding check: some claims may not be fully supported." + return { + "answer": answer, + "reasoning_trace": payload["reasoning_trace"], + "all_messages": [], + } diff --git a/src/rfp_analyst/agent/state.py b/src/rfp_analyst/agent/state.py new file mode 100644 index 0000000..3390ca2 --- /dev/null +++ b/src/rfp_analyst/agent/state.py @@ -0,0 +1,20 @@ +"""Agent state container.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class AgentState: + query: str + chat_history: list | None = None + intent: str = "search" + tool_trace: list[dict] = field(default_factory=list) + retrieved_documents: list = field(default_factory=list) + sources: list[dict] = field(default_factory=list) + tool_outputs: dict = field(default_factory=dict) + stats: dict = field(default_factory=dict) + prompt: str = "" + final_answer: str = "" + verification: dict = field(default_factory=dict) diff --git a/src/rfp_analyst/exceptions.py b/src/rfp_analyst/exceptions.py new file mode 100644 index 0000000..8706084 --- /dev/null +++ b/src/rfp_analyst/exceptions.py @@ -0,0 +1,31 @@ +"""Project-specific exceptions.""" + +from __future__ import annotations + + +class RFPAnalystError(Exception): + """Base application exception.""" + + +class LLMProviderNotConfiguredError(RFPAnalystError): + """Raised when no LLM provider credentials are configured.""" + + +class KnowledgeBaseNotReadyError(RFPAnalystError): + """Raised when retrieval is attempted before the knowledge base is ready.""" + + +class NoDocumentsFoundError(RFPAnalystError): + """Raised when no source documents are available for ingestion.""" + + +class IngestionError(RFPAnalystError): + """Raised when document ingestion fails.""" + + +class RetrievalError(RFPAnalystError): + """Raised when retrieval fails.""" + + +class UnsupportedFileError(RFPAnalystError): + """Raised when an uploaded file is invalid or unsupported.""" diff --git a/src/rfp_analyst/health.py b/src/rfp_analyst/health.py new file mode 100644 index 0000000..4733bfa --- /dev/null +++ b/src/rfp_analyst/health.py @@ -0,0 +1,33 @@ +"""App health checks.""" + +from __future__ import annotations + +from pathlib import Path + +from config import ASSETS_DIR, DATA_DIR, VECTORSTORE_DIR +from rfp_analyst.retrieval.vector_store import VectorStoreManager + + +REQUIRED_DIRECTORIES = { + "data_dir": DATA_DIR, + "vectorstore_dir": VECTORSTORE_DIR, + "assets_dir": ASSETS_DIR, +} + + +def get_app_health(llm_provider_name: str = "Not configured") -> dict: + """Return high-level application health without raising UI-breaking errors.""" + directories = { + name: {"path": str(path), "exists": Path(path).exists()} + for name, path in REQUIRED_DIRECTORIES.items() + } + + stats = VectorStoreManager(persist_dir=VECTORSTORE_DIR).get_stats() + return { + "vectorstore_ready": stats.get("status") == "ready", + "document_count": stats.get("total_documents", 0), + "chunk_count": stats.get("total_chunks", 0), + "llm_provider_configured": llm_provider_name != "Not configured", + "llm_provider_name": llm_provider_name, + "required_directories": directories, + } diff --git a/src/rfp_analyst/ingestion/__init__.py b/src/rfp_analyst/ingestion/__init__.py new file mode 100644 index 0000000..be89c85 --- /dev/null +++ b/src/rfp_analyst/ingestion/__init__.py @@ -0,0 +1 @@ +"""Ingestion pipeline components.""" diff --git a/src/rfp_analyst/ingestion/chunking.py b/src/rfp_analyst/ingestion/chunking.py new file mode 100644 index 0000000..c4850a9 --- /dev/null +++ b/src/rfp_analyst/ingestion/chunking.py @@ -0,0 +1,57 @@ +"""Deterministic document chunking.""" + +from __future__ import annotations + +import hashlib + +from langchain_core.documents import Document +from langchain_text_splitters import RecursiveCharacterTextSplitter + +from config import CHUNK_OVERLAP, CHUNK_SIZE +from rfp_analyst.schemas import LoadedSource + + +def build_chunk_id(file_hash: str, page: int, chunk_index: int, content: str) -> str: + """Build a deterministic chunk ID from stable source data.""" + fingerprint = f"{file_hash}:{page}:{chunk_index}:{content}".encode("utf-8") + return hashlib.sha256(fingerprint).hexdigest() + + +def chunk_loaded_sources( + loaded_sources: list[LoadedSource], + chunk_size: int = CHUNK_SIZE, + chunk_overlap: int = CHUNK_OVERLAP, +) -> list[Document]: + """Split source documents into chunks with deterministic metadata.""" + splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + length_function=len, + separators=["\n\n", "\n", ". ", " ", ""], + ) + + chunks: list[Document] = [] + for source in loaded_sources: + source_chunks = splitter.split_documents(source.documents) + for chunk_index, chunk in enumerate(source_chunks): + page = int(chunk.metadata.get("page", 0)) + chunk_id = build_chunk_id( + file_hash=source.file_hash, + page=page, + chunk_index=chunk_index, + content=chunk.page_content, + ) + chunk.metadata.update( + { + "source_file": source.source_file, + "source_path": source.source_path, + "file_hash": source.file_hash, + "page": page, + "chunk_id": chunk_id, + "document_type": source.document_type, + } + ) + chunks.append(chunk) + + print(f"Created {len(chunks)} chunks (size={chunk_size}, overlap={chunk_overlap})") + return chunks diff --git a/src/rfp_analyst/ingestion/loaders.py b/src/rfp_analyst/ingestion/loaders.py new file mode 100644 index 0000000..33b22eb --- /dev/null +++ b/src/rfp_analyst/ingestion/loaders.py @@ -0,0 +1,123 @@ +"""Validated document loading utilities.""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +import fitz +from langchain_community.document_loaders import PyMuPDFLoader + +from config import DATA_DIR, MAX_UPLOAD_FILE_SIZE_BYTES, MAX_UPLOAD_PAGE_COUNT +from rfp_analyst.exceptions import IngestionError, NoDocumentsFoundError +from rfp_analyst.schemas import LoadedSource + +SAFE_FILENAME_PATTERN = re.compile(r"[^A-Za-z0-9._-]+") + + +def sanitize_filename(filename: str) -> str: + """Normalize uploaded filenames to a safe PDF filename.""" + original = Path(filename) + stem = SAFE_FILENAME_PATTERN.sub("_", original.stem).strip("._") or "document" + suffix = original.suffix.lower() if original.suffix else ".pdf" + if suffix != ".pdf": + suffix = ".pdf" + return f"{stem}{suffix}" + + +def ensure_safe_pdf_path(pdf_path: Path) -> Path: + """Rename files with unsafe names before ingestion.""" + safe_name = sanitize_filename(pdf_path.name) + target_path = pdf_path.with_name(safe_name) + + if target_path == pdf_path: + return pdf_path + + if target_path.exists(): + short_hash = hashlib.sha256(pdf_path.name.encode("utf-8")).hexdigest()[:8] + target_path = pdf_path.with_name(f"{Path(safe_name).stem}_{short_hash}.pdf") + + pdf_path.rename(target_path) + return target_path + + +def sha256_file(file_path: Path) -> str: + """Return the SHA256 hash for the file contents.""" + digest = hashlib.sha256() + with file_path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def infer_document_type(pdf_path: Path) -> str | None: + """Infer document type from the filename when possible.""" + normalized = pdf_path.stem.replace("_", " ").lower() + candidates = ("proposal", "project outline", "case study", "rfp response") + for candidate in candidates: + if candidate in normalized: + return candidate.title() + return None + + +def validate_pdf(pdf_path: Path) -> int: + """Validate file size and page count before loading.""" + file_size = pdf_path.stat().st_size + if file_size > MAX_UPLOAD_FILE_SIZE_BYTES: + raise IngestionError( + f"{pdf_path.name} exceeds max file size of {MAX_UPLOAD_FILE_SIZE_BYTES} bytes" + ) + + with fitz.open(pdf_path) as pdf_document: + page_count = pdf_document.page_count + + if page_count > MAX_UPLOAD_PAGE_COUNT: + raise IngestionError( + f"{pdf_path.name} exceeds max page count of {MAX_UPLOAD_PAGE_COUNT}" + ) + return page_count + + +def load_pdf_sources(doc_dir: Path = DATA_DIR) -> list[LoadedSource]: + """Load, validate, and enrich all PDFs in a directory.""" + pdf_files = sorted(doc_dir.glob("*.pdf")) + if not pdf_files: + raise NoDocumentsFoundError(f"No PDF files found in {doc_dir}") + + loaded_sources: list[LoadedSource] = [] + try: + for raw_pdf_path in pdf_files: + pdf_path = ensure_safe_pdf_path(raw_pdf_path) + page_count = validate_pdf(pdf_path) + file_hash = sha256_file(pdf_path) + document_type = infer_document_type(pdf_path) + + loader = PyMuPDFLoader(str(pdf_path)) + documents = loader.load() + + for page_index, document in enumerate(documents): + document.metadata["source_file"] = pdf_path.name + document.metadata["source_path"] = str(pdf_path.resolve()) + document.metadata["file_hash"] = file_hash + document.metadata["page"] = int(document.metadata.get("page", page_index)) + document.metadata["document_type"] = document_type + + loaded_sources.append( + LoadedSource( + source_file=pdf_path.name, + source_path=str(pdf_path.resolve()), + file_hash=file_hash, + page_count=page_count, + document_type=document_type, + documents=documents, + ) + ) + print(f" Loaded: {pdf_path.name} ({page_count} pages)") + except NoDocumentsFoundError: + raise + except Exception as error: + raise IngestionError(str(error)) from error + + print(f"Total pages loaded: {sum(source.page_count for source in loaded_sources)}") + return loaded_sources diff --git a/src/rfp_analyst/ingestion/pipeline.py b/src/rfp_analyst/ingestion/pipeline.py new file mode 100644 index 0000000..f6334ad --- /dev/null +++ b/src/rfp_analyst/ingestion/pipeline.py @@ -0,0 +1,84 @@ +"""High-level ingestion pipeline orchestration.""" + +from __future__ import annotations + +from pathlib import Path + +from config import COLLECTION_NAME, DATA_DIR, VECTORSTORE_DIR +from rfp_analyst.ingestion.chunking import chunk_loaded_sources +from rfp_analyst.ingestion.loaders import load_pdf_sources +from rfp_analyst.ingestion.registry import IngestionRegistry +from rfp_analyst.retrieval.vector_store import VectorStoreManager +from rfp_analyst.schemas import IngestionRecord + + +class IngestionPipeline: + """Production-grade ingestion pipeline with duplicate prevention.""" + + def __init__( + self, + doc_dir: Path = DATA_DIR, + persist_dir: Path = VECTORSTORE_DIR, + collection_name: str = COLLECTION_NAME, + registry: IngestionRegistry | None = None, + vector_store_manager: VectorStoreManager | None = None, + ): + self.doc_dir = Path(doc_dir) + self.persist_dir = Path(persist_dir) + self.registry = registry or IngestionRegistry(self.persist_dir / "ingestion_registry.json") + self.vector_store_manager = vector_store_manager or VectorStoreManager( + persist_dir=self.persist_dir, + collection_name=collection_name, + ) + + def run(self): + """Load, dedupe, chunk, and persist documents.""" + print("=" * 60) + print("DOCUMENT INGESTION PIPELINE") + print("=" * 60) + + print("\n[1/3] Loading PDFs...") + loaded_sources = load_pdf_sources(self.doc_dir) + + new_sources = [ + source + for source in loaded_sources + if not self.registry.contains_hash(source.file_hash) + ] + + skipped_count = len(loaded_sources) - len(new_sources) + if skipped_count: + print(f"Skipped {skipped_count} previously ingested file(s)") + + print("\n[2/3] Chunking documents...") + chunks = chunk_loaded_sources(new_sources) if new_sources else [] + + print("\n[3/3] Embedding & storing in ChromaDB (local embeddings)...") + vectorstore = self.vector_store_manager.upsert_documents(chunks) + + if new_sources: + chunk_ids_by_hash: dict[str, list[str]] = {} + for chunk in chunks: + chunk_ids_by_hash.setdefault(chunk.metadata["file_hash"], []).append( + chunk.metadata["chunk_id"] + ) + + for source in new_sources: + self.registry.register( + IngestionRecord( + source_file=source.source_file, + source_path=source.source_path, + file_hash=source.file_hash, + page_count=source.page_count, + document_type=source.document_type, + chunk_ids=chunk_ids_by_hash.get(source.file_hash, []), + ) + ) + + stats = self.vector_store_manager.get_stats() + print("\n" + "=" * 60) + print("INGESTION COMPLETE") + print(f" Documents: {stats['total_documents']}") + print(f" Chunks: {stats['total_chunks']}") + print("=" * 60) + return vectorstore diff --git a/src/rfp_analyst/ingestion/registry.py b/src/rfp_analyst/ingestion/registry.py new file mode 100644 index 0000000..70e72c6 --- /dev/null +++ b/src/rfp_analyst/ingestion/registry.py @@ -0,0 +1,42 @@ +"""Simple JSON-backed ingestion registry.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from rfp_analyst.schemas import IngestionRecord + + +class IngestionRegistry: + """Track ingested files so repeat runs can skip duplicates safely.""" + + def __init__(self, registry_path: Path): + self.registry_path = registry_path + + def _read(self) -> dict[str, dict]: + if not self.registry_path.exists(): + return {} + return json.loads(self.registry_path.read_text(encoding="utf-8")) + + def _write(self, payload: dict[str, dict]) -> None: + self.registry_path.parent.mkdir(parents=True, exist_ok=True) + self.registry_path.write_text( + json.dumps(payload, indent=2, sort_keys=True), + encoding="utf-8", + ) + + def contains_hash(self, file_hash: str) -> bool: + return file_hash in self._read() + + def get_record(self, file_hash: str) -> IngestionRecord | None: + payload = self._read().get(file_hash) + return IngestionRecord.from_dict(payload) if payload else None + + def register(self, record: IngestionRecord) -> None: + payload = self._read() + payload[record.file_hash] = record.to_dict() + self._write(payload) + + def records(self) -> list[IngestionRecord]: + return [IngestionRecord.from_dict(item) for item in self._read().values()] diff --git a/src/rfp_analyst/retrieval/__init__.py b/src/rfp_analyst/retrieval/__init__.py new file mode 100644 index 0000000..53f9796 --- /dev/null +++ b/src/rfp_analyst/retrieval/__init__.py @@ -0,0 +1 @@ +"""Retrieval and vector store integrations.""" diff --git a/src/rfp_analyst/retrieval/vector_store.py b/src/rfp_analyst/retrieval/vector_store.py new file mode 100644 index 0000000..40da3ed --- /dev/null +++ b/src/rfp_analyst/retrieval/vector_store.py @@ -0,0 +1,102 @@ +"""Vector store management for ingestion and retrieval.""" + +from __future__ import annotations + +from pathlib import Path + +from langchain_chroma import Chroma +from langchain_community.embeddings.fastembed import FastEmbedEmbeddings +from langchain_core.documents import Document + +from config import COLLECTION_NAME, EMBEDDING_MODEL, RETRIEVAL_K, VECTORSTORE_DIR +from rfp_analyst.exceptions import KnowledgeBaseNotReadyError + + +def get_embeddings(): + """Initialize local embeddings for ingestion and search.""" + return FastEmbedEmbeddings(model_name=EMBEDDING_MODEL) + + +class VectorStoreManager: + """Encapsulate Chroma persistence and deduplicated upserts.""" + + def __init__( + self, + persist_dir: Path = VECTORSTORE_DIR, + collection_name: str = COLLECTION_NAME, + embedding_function=None, + ): + self.persist_dir = Path(persist_dir) + self.collection_name = collection_name + self.embedding_function = embedding_function or get_embeddings() + + def load(self, create_if_missing: bool = True) -> Chroma: + """Load or initialize the Chroma collection.""" + if not self.persist_dir.exists(): + if not create_if_missing: + raise KnowledgeBaseNotReadyError( + "Knowledge base is not ready. Generate or upload PDFs and click Ingest Documents." + ) + self.persist_dir.mkdir(parents=True, exist_ok=True) + return Chroma( + collection_name=self.collection_name, + embedding_function=self.embedding_function, + persist_directory=str(self.persist_dir), + ) + + def upsert_documents(self, documents: list[Document]) -> Chroma: + """Add only new chunk IDs into the collection.""" + vectorstore = self.load(create_if_missing=True) + existing_ids = set(vectorstore._collection.get().get("ids", [])) + + new_documents = [] + new_ids = [] + for document in documents: + chunk_id = document.metadata["chunk_id"] + if chunk_id in existing_ids: + continue + new_documents.append(document) + new_ids.append(chunk_id) + + if new_documents: + vectorstore.add_documents(new_documents, ids=new_ids) + + print(f"Vector store contains {vectorstore._collection.count()} vectors") + print(f"Persisted to: {self.persist_dir}") + return vectorstore + + def get_retriever(self, k: int = RETRIEVAL_K): + """Get a retriever for similarity search.""" + return self.load(create_if_missing=False).as_retriever( + search_type="similarity", + search_kwargs={"k": k}, + ) + + def similarity_search(self, query: str, k: int = RETRIEVAL_K): + """Run similarity search with relevance scores.""" + return self.load(create_if_missing=False).similarity_search_with_relevance_scores(query, k=k) + + def get_stats(self) -> dict: + """Return collection stats for the UI.""" + try: + vectorstore = self.load(create_if_missing=False) + count = vectorstore._collection.count() + all_metadata = vectorstore._collection.get().get("metadatas", []) + sources = { + metadata["source_file"] + for metadata in all_metadata + if metadata and metadata.get("source_file") + } + return { + "total_chunks": count, + "total_documents": len(sources), + "document_names": sorted(sources), + "status": "ready" if count else "not_initialized", + } + except Exception: + return { + "total_chunks": 0, + "total_documents": 0, + "document_names": [], + "status": "not_initialized", + } diff --git a/src/rfp_analyst/schemas.py b/src/rfp_analyst/schemas.py new file mode 100644 index 0000000..21b774a --- /dev/null +++ b/src/rfp_analyst/schemas.py @@ -0,0 +1,38 @@ +"""Shared schemas for ingestion and retrieval.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field + +from langchain_core.documents import Document + + +@dataclass(frozen=True) +class IngestionRecord: + """Persistent ingestion metadata used for duplicate prevention.""" + + source_file: str + source_path: str + file_hash: str + page_count: int + document_type: str | None = None + chunk_ids: list[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return asdict(self) + + @classmethod + def from_dict(cls, payload: dict) -> "IngestionRecord": + return cls(**payload) + + +@dataclass(frozen=True) +class LoadedSource: + """Represents a validated source PDF and its extracted page documents.""" + + source_file: str + source_path: str + file_hash: str + page_count: int + document_type: str | None + documents: list[Document] diff --git a/src/rfp_analyst/tools/__init__.py b/src/rfp_analyst/tools/__init__.py new file mode 100644 index 0000000..bd7ede6 --- /dev/null +++ b/src/rfp_analyst/tools/__init__.py @@ -0,0 +1 @@ +"""Tooling package for agentic workflows.""" diff --git a/src/rfp_analyst/tools/compare_projects.py b/src/rfp_analyst/tools/compare_projects.py new file mode 100644 index 0000000..b04cb0e --- /dev/null +++ b/src/rfp_analyst/tools/compare_projects.py @@ -0,0 +1,65 @@ +"""Project-comparison tool.""" + +from __future__ import annotations + +import re + +from config import RETRIEVAL_K +from rfp_analyst.tools.search_kb import search_knowledge_base + + +_FIELDS = { + "timeline": "Timeline & Milestones", + "budget": "Budget Range", + "tech_stack": "Technology Stack", + "outcomes": "Key Outcomes", +} + + +def _extract_field(text: str, label: str) -> str: + pattern = re.compile(rf"{re.escape(label)}[:\s-]*(.+?)(?:\n[A-Z][A-Za-z &]+[:\s]|$)", re.IGNORECASE | re.DOTALL) + match = pattern.search(text) + if match: + return " ".join(match.group(1).split()) + return "Not found in retrieved evidence" + + +def compare_projects(query: str, search_fn=None, k: int = RETRIEVAL_K) -> dict: + """Compare projects across key delivery dimensions.""" + search_result = search_knowledge_base(query, k=k, search_fn=search_fn) + grouped = {} + for document in search_result["documents"]: + source = document.metadata.get("source_file", "Unknown") + grouped.setdefault(source, []).append(document.page_content) + + rows = [] + for source, chunks in grouped.items(): + combined = "\n".join(chunks) + rows.append( + { + "source": source, + "timeline": _extract_field(combined, _FIELDS["timeline"]), + "budget": _extract_field(combined, _FIELDS["budget"]), + "tech_stack": _extract_field(combined, _FIELDS["tech_stack"]), + "outcomes": _extract_field(combined, _FIELDS["outcomes"]), + } + ) + + if not rows: + comparison_markdown = "No comparable project evidence found." + else: + header = "| Project | Timeline | Budget | Tech Stack | Outcomes |" + separator = "| --- | --- | --- | --- | --- |" + body = [ + f"| {row['source']} | {row['timeline']} | {row['budget']} | {row['tech_stack']} | {row['outcomes']} |" + for row in rows + ] + comparison_markdown = "\n".join([header, separator, *body]) + + return { + "query": query, + "comparison_markdown": comparison_markdown, + "rows": rows, + "documents": search_result["documents"], + "sources": search_result["sources"], + } diff --git a/src/rfp_analyst/tools/proposal_writer.py b/src/rfp_analyst/tools/proposal_writer.py new file mode 100644 index 0000000..208c7e2 --- /dev/null +++ b/src/rfp_analyst/tools/proposal_writer.py @@ -0,0 +1,43 @@ +"""Proposal drafting helpers.""" + +from __future__ import annotations + + +def generate_proposal_outline(user_query: str, case_studies: dict, requirements: list[dict] | None = None) -> dict: + """Create a grounded proposal outline from retrieved evidence.""" + requirements = requirements or [] + matches = case_studies.get("matches", []) + + requirement_lines = "\n".join( + f"- {requirement['id']}: {requirement['text']}" for requirement in requirements + ) or "- No structured requirements were extracted." + + evidence_lines = "\n".join( + f"- {match['source']} [Source: {match['source']}, Page {match['pages'][0] + 1}]" + for match in matches[:5] + ) or "- No matching case studies found." + + outline = f"""## Executive Summary +Address the request: {user_query} + +## Client Requirements +{requirement_lines} + +## Relevant Case Studies +{evidence_lines} + +## Proposed Approach +- Reuse proven delivery patterns from the cited projects. +- Map each workstream to the structured client requirements. +- Highlight measurable outcomes supported by prior engagements. + +## Delivery Plan +- Discovery and requirements confirmation +- Solution design and implementation +- Validation, rollout, and stakeholder enablement + +## Risks and Mitigations +- Call out delivery risks only when supported by cited prior work. +- Add compliance and implementation assumptions explicitly. +""" + return {"outline": outline, "requirements": requirements, "matches": matches} diff --git a/src/rfp_analyst/tools/rfp_gap_analyzer.py b/src/rfp_analyst/tools/rfp_gap_analyzer.py new file mode 100644 index 0000000..d06683e --- /dev/null +++ b/src/rfp_analyst/tools/rfp_gap_analyzer.py @@ -0,0 +1,66 @@ +"""RFP requirement extraction and matching tools.""" + +from __future__ import annotations + +import re + +from config import RETRIEVAL_K +from rfp_analyst.tools.search_kb import search_knowledge_base + + +def extract_rfp_requirements(rfp_text: str) -> dict: + """Turn raw RFP text into a simple structured requirements list.""" + lines = [line.strip(" -\t") for line in rfp_text.splitlines() if line.strip()] + candidates = [] + for line in lines: + segments = [segment.strip() for segment in re.split(r"(?<=[.!?])\s+", line) if segment.strip()] + for segment in segments: + lower = segment.lower() + if any(keyword in lower for keyword in ("must", "should", "require", "need", "include", "support")): + candidates.append(segment) + + if not candidates: + candidates = [segment.strip() for segment in re.split(r"(?<=[.!?])\s+", rfp_text) if segment.strip()] + + requirements = [] + for index, text in enumerate(candidates, start=1): + requirements.append({"id": f"REQ-{index:02d}", "text": text}) + + return { + "requirements": requirements, + "summary": f"Extracted {len(requirements)} requirement(s).", + } + + +def find_relevant_case_studies(requirements: list[dict], search_fn=None, k: int = RETRIEVAL_K) -> dict: + """Find matching prior work for a set of requirements.""" + matches = {} + supporting_documents = [] + for requirement in requirements: + result = search_knowledge_base(requirement["text"], k=k, search_fn=search_fn) + for source in result["sources"]: + entry = matches.setdefault( + source["source"], + {"source": source["source"], "pages": set(), "matched_requirements": [], "snippets": []}, + ) + entry["pages"].add(source["page"]) + entry["matched_requirements"].append(requirement["id"]) + entry["snippets"].append(source["snippet"]) + supporting_documents.extend(result["documents"]) + + normalized_matches = [] + for item in matches.values(): + normalized_matches.append( + { + "source": item["source"], + "pages": sorted(item["pages"]), + "matched_requirements": sorted(set(item["matched_requirements"])), + "snippets": item["snippets"][:3], + } + ) + + normalized_matches.sort(key=lambda item: (-len(item["matched_requirements"]), item["source"])) + return { + "matches": normalized_matches, + "documents": supporting_documents, + } diff --git a/src/rfp_analyst/tools/search_kb.py b/src/rfp_analyst/tools/search_kb.py new file mode 100644 index 0000000..d866291 --- /dev/null +++ b/src/rfp_analyst/tools/search_kb.py @@ -0,0 +1,57 @@ +"""Knowledge-base search tool.""" + +from __future__ import annotations + +from typing import Callable + +from config import RETRIEVAL_K +from rfp_analyst.retrieval.vector_store import VectorStoreManager + + +SearchFn = Callable[[str, int], list[tuple[object, float]]] + + +def _default_search(query: str, k: int = RETRIEVAL_K): + return VectorStoreManager().similarity_search(query, k=k) + + +def search_knowledge_base(query: str, k: int = RETRIEVAL_K, search_fn: SearchFn | None = None) -> dict: + """Retrieve relevant chunks and normalize them for downstream tools.""" + search = search_fn or _default_search + try: + results = search(query, k=k) + except Exception: + return { + "query": query, + "documents": [], + "sources": [], + "context_parts": [], + "context": "No relevant documents found.", + } + + context_parts = [] + sources = [] + documents = [] + for doc, score in results: + source = doc.metadata.get("source_file", "Unknown") + page = int(doc.metadata.get("page", 0)) + snippet = doc.page_content.strip() + context_parts.append(f"[Source: {source}, Page {page + 1}]\n{snippet}") + sources.append( + { + "source": source, + "page": page, + "score": f"{score:.2f}", + "snippet": snippet[:220], + } + ) + documents.append(doc) + + context = "\n\n---\n\n".join(context_parts) if context_parts else "No relevant documents found." + return { + "query": query, + "documents": documents, + "sources": sources, + "context_parts": context_parts, + "context": context, + } diff --git a/src/rfp_analyst/tools/source_verifier.py b/src/rfp_analyst/tools/source_verifier.py new file mode 100644 index 0000000..a42c14d --- /dev/null +++ b/src/rfp_analyst/tools/source_verifier.py @@ -0,0 +1,57 @@ +"""Grounding verification helpers.""" + +from __future__ import annotations + +import re + + +_CITATION_PATTERN = re.compile(r"\[Source:\s*(?P[^,\]]+)\s*,\s*Page\s*(?P\d+)\]") + + +def _tokenize(text: str) -> set[str]: + return {token for token in re.findall(r"[A-Za-z0-9]{4,}", text.lower())} + + +def verify_answer_grounding(answer: str, supporting_documents: list[object]) -> dict: + """Check whether the answer's major claims are backed by cited evidence.""" + source_index = {} + for document in supporting_documents: + source_name = document.metadata.get("source_file", "Unknown") + source_index.setdefault(source_name, []).append(document.page_content) + + unsupported_claims = [] + checked_claims = [] + for raw_line in answer.splitlines(): + line = raw_line.strip() + if len(line) <= 20 or line.startswith("##"): + continue + + line_citations = [match.group("source").strip() for match in _CITATION_PATTERN.finditer(line)] + line_without_citations = _CITATION_PATTERN.sub("", line) + sentence_candidates = [segment.strip() for segment in re.split(r"(?<=[.!?])\s+", line_without_citations) if segment.strip()] + + for claim in sentence_candidates: + if len(claim) <= 20: + continue + checked_claims.append(claim) + claim_tokens = _tokenize(claim) + if not claim_tokens: + continue + + candidate_sources = line_citations or list(source_index) + supported = False + for source_name in candidate_sources: + combined_source_text = " ".join(source_index.get(source_name, [])) + source_tokens = _tokenize(combined_source_text) + if len(claim_tokens.intersection(source_tokens)) >= 2: + supported = True + break + + if not supported: + unsupported_claims.append(claim) + + return { + "is_grounded": not unsupported_claims, + "checked_claims": checked_claims, + "unsupported_claims": unsupported_claims, + } diff --git a/src/rfp_analyst/ui/helpers.py b/src/rfp_analyst/ui/helpers.py new file mode 100644 index 0000000..703f989 --- /dev/null +++ b/src/rfp_analyst/ui/helpers.py @@ -0,0 +1,22 @@ +"""UI helpers for Streamlit rendering.""" + +from __future__ import annotations + + +def get_chat_avatar(role: str) -> str: + """Return a Streamlit-safe avatar for a given chat role.""" + return "👤" if role == "user" else "🤖" + + +def format_latency_display(metrics: dict) -> str: + """Render evaluation latency safely with correct units.""" + if not metrics: + return "N/A" + + if "average_latency_ms" in metrics: + return f"{float(metrics['average_latency_ms']):.2f} ms" + + latency_value = float(metrics.get("average_latency", 0.0)) + if latency_value >= 1: + return f"{latency_value:.2f} ms" + return f"{latency_value:.3f} s" diff --git a/src/rfp_analyst/uploads.py b/src/rfp_analyst/uploads.py new file mode 100644 index 0000000..df11018 --- /dev/null +++ b/src/rfp_analyst/uploads.py @@ -0,0 +1,31 @@ +"""Upload validation helpers.""" + +from __future__ import annotations + +from pathlib import Path + +from config import MAX_UPLOAD_FILE_SIZE_BYTES +from rfp_analyst.exceptions import UnsupportedFileError +from rfp_analyst.ingestion.loaders import sanitize_filename + +VALID_PDF_MIME_TYPES = {"application/pdf", "application/x-pdf"} + + +def validate_uploaded_pdf(uploaded_file) -> str: + """Validate upload metadata and return a sanitized filename.""" + original_suffix = Path(uploaded_file.name).suffix.lower() + if original_suffix != ".pdf": + raise UnsupportedFileError("Only PDF files are supported.") + + sanitized_name = sanitize_filename(uploaded_file.name) + file_type = getattr(uploaded_file, "type", "") or "" + if file_type and file_type not in VALID_PDF_MIME_TYPES: + raise UnsupportedFileError("The uploaded file does not appear to be a valid PDF.") + + file_size = getattr(uploaded_file, "size", None) + if file_size is not None and int(file_size) > MAX_UPLOAD_FILE_SIZE_BYTES: + raise UnsupportedFileError( + f"Uploaded PDF exceeds the max size of {MAX_UPLOAD_FILE_SIZE_BYTES} bytes." + ) + + return sanitized_name diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0908ab2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +from pathlib import Path +import sys + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = PROJECT_ROOT / "src" + +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) +if str(SRC_ROOT) not in sys.path: + sys.path.insert(0, str(SRC_ROOT)) diff --git a/tests/test_agentic_tools.py b/tests/test_agentic_tools.py new file mode 100644 index 0000000..9a74613 --- /dev/null +++ b/tests/test_agentic_tools.py @@ -0,0 +1,100 @@ +from langchain_core.documents import Document + +from rfp_analyst.agent.graph import run_agent_graph +from rfp_analyst.agent.state import AgentState +from rfp_analyst.tools.compare_projects import compare_projects +from rfp_analyst.tools.proposal_writer import generate_proposal_outline +from rfp_analyst.tools.rfp_gap_analyzer import extract_rfp_requirements, find_relevant_case_studies +from rfp_analyst.tools.search_kb import search_knowledge_base +from rfp_analyst.tools.source_verifier import verify_answer_grounding + + +def make_doc(source: str, page: int, content: str): + return Document(page_content=content, metadata={"source_file": source, "page": page}) + + +def fake_search_fn(query: str, k: int = 6): + docs = [ + ( + make_doc( + "Banking_Audit.pdf", + 0, + "Timeline & Milestones: 16 weeks. Budget Range: $850,000. Technology Stack: Azure SQL, Power BI. Key Outcomes: Improved data quality.", + ), + 0.93, + ), + ( + make_doc( + "Insurance_Automation.pdf", + 1, + "Timeline & Milestones: 12 weeks. Budget Range: $650,000. Technology Stack: Azure Functions, UiPath. Key Outcomes: Claims automation.", + ), + 0.88, + ), + ] + return docs[:k] + + +def test_search_knowledge_base_with_mocked_retriever(): + result = search_knowledge_base("azure projects", search_fn=fake_search_fn) + assert len(result["documents"]) == 2 + assert result["sources"][0]["source"] == "Banking_Audit.pdf" + assert "[Source: Banking_Audit.pdf, Page 1]" in result["context"] + + +def test_compare_projects_with_mocked_retriever(): + result = compare_projects("compare banking and insurance", search_fn=fake_search_fn) + assert "| Project | Timeline | Budget | Tech Stack | Outcomes |" in result["comparison_markdown"] + assert "Banking_Audit.pdf" in result["comparison_markdown"] + + +def test_extract_rfp_requirements(): + result = extract_rfp_requirements("The solution must support Azure. The vendor should include dashboards.") + assert len(result["requirements"]) >= 2 + assert result["requirements"][0]["id"].startswith("REQ-") + + +def test_find_relevant_case_studies_with_mocked_retriever(): + requirements = [{"id": "REQ-01", "text": "Azure migration"}] + result = find_relevant_case_studies(requirements, search_fn=fake_search_fn) + assert result["matches"] + assert result["matches"][0]["source"] == "Banking_Audit.pdf" + + +def test_generate_proposal_outline(): + case_studies = { + "matches": [{"source": "Banking_Audit.pdf", "pages": [0], "snippets": ["Azure SQL and Power BI"]}] + } + outline = generate_proposal_outline("Write a proposal", case_studies, [{"id": "REQ-01", "text": "Azure"}]) + assert "## Executive Summary" in outline["outline"] + assert "[Source: Banking_Audit.pdf, Page 1]" in outline["outline"] + + +def test_verify_answer_grounding_catches_unsupported_claims(): + answer = ( + "The banking project used Azure SQL and Power BI [Source: Banking_Audit.pdf, Page 1]. " + "It also deployed Kubernetes to 30 regions." + ) + verification = verify_answer_grounding(answer, [fake_search_fn("x")[0][0]]) + assert not verification["is_grounded"] + assert any("Kubernetes" in claim for claim in verification["unsupported_claims"]) + + +def test_graph_happy_path(): + state = run_agent_graph( + AgentState(query="Compare the banking and insurance projects"), + search_fn=fake_search_fn, + stats_fn=lambda: {"status": "ready", "total_documents": 2, "total_chunks": 4, "document_names": ["Banking_Audit.pdf", "Insurance_Automation.pdf"]}, + ) + assert state.intent == "compare_projects" + assert state.prompt + assert any(step.get("tool") == "compare_projects" for step in state.tool_trace) + + +def test_no_documents_fallback(): + state = run_agent_graph( + AgentState(query="What projects used Azure?"), + search_fn=fake_search_fn, + stats_fn=lambda: {"status": "not_initialized", "total_documents": 0, "total_chunks": 0, "document_names": []}, + ) + assert "Please ingest documents first" in state.final_answer diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..3f42765 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,90 @@ +import importlib +import sys +from pathlib import Path +from types import SimpleNamespace + + +def load_fresh_config(): + sys.modules.pop("config", None) + return importlib.import_module("config") + + +def load_fresh_agent(): + sys.modules.pop("agent", None) + return importlib.import_module("agent") + + +def test_config_default_paths_and_values(monkeypatch): + monkeypatch.delenv("GROQ_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + monkeypatch.delenv("AGENT_MODE", raising=False) + monkeypatch.setitem(sys.modules, "streamlit", SimpleNamespace(secrets={})) + + config = load_fresh_config() + + assert config.BASE_DIR == Path(config.__file__).resolve().parent + assert config.DATA_DIR == config.BASE_DIR / "data" / "documents" + assert config.VECTORSTORE_DIR == config.BASE_DIR / "vectorstore" + assert config.ASSETS_DIR == config.BASE_DIR / "assets" + assert config.GROQ_MODEL == "llama-3.3-70b-versatile" + assert config.GEMINI_MODEL == "gemini-2.0-flash" + assert config.LLM_TEMPERATURE == 0.3 + assert config.LLM_MAX_TOKENS == 2048 + assert config.CHUNK_SIZE == 512 + assert config.CHUNK_OVERLAP == 50 + assert config.COLLECTION_NAME == "rfp_kb_v2" + assert config.RETRIEVAL_K == 6 + assert config.MAX_UPLOAD_FILE_SIZE_BYTES == 25 * 1024 * 1024 + assert config.MAX_UPLOAD_PAGE_COUNT == 250 + assert config.AGENT_MODE == "agentic" + assert config.SAMPLE_QUESTIONS + + +def test_config_prefers_streamlit_secrets(monkeypatch): + monkeypatch.setenv("GROQ_API_KEY", "env-groq") + monkeypatch.setenv("GOOGLE_API_KEY", "env-google") + monkeypatch.setitem( + sys.modules, + "streamlit", + SimpleNamespace( + secrets={ + "GROQ_API_KEY": "secret-groq", + "GOOGLE_API_KEY": "secret-google", + } + ), + ) + + config = load_fresh_config() + + assert config.GROQ_API_KEY == "secret-groq" + assert config.GOOGLE_API_KEY == "secret-google" + + +def test_config_falls_back_to_env_when_streamlit_secrets_fail(monkeypatch): + class BrokenSecrets: + def get(self, *_args, **_kwargs): + raise RuntimeError("secrets unavailable") + + monkeypatch.setenv("GROQ_API_KEY", "env-groq") + monkeypatch.setenv("GOOGLE_API_KEY", "env-google") + monkeypatch.setitem( + sys.modules, + "streamlit", + SimpleNamespace(secrets=BrokenSecrets()), + ) + + config = load_fresh_config() + + assert config.GROQ_API_KEY == "env-groq" + assert config.GOOGLE_API_KEY == "env-google" + + +def test_no_llm_provider_state(monkeypatch): + monkeypatch.delenv("GROQ_API_KEY", raising=False) + monkeypatch.delenv("GOOGLE_API_KEY", raising=False) + monkeypatch.setitem(sys.modules, "streamlit", SimpleNamespace(secrets={})) + load_fresh_config() + agent = load_fresh_agent() + + assert agent._get_provider_name() == "Not configured" + assert agent.is_llm_provider_configured() is False diff --git a/tests/test_document_generator.py b/tests/test_document_generator.py new file mode 100644 index 0000000..f517b42 --- /dev/null +++ b/tests/test_document_generator.py @@ -0,0 +1,46 @@ +import document_generator + + +def test_document_definitions_have_required_fields(): + assert document_generator.DOCUMENTS + + for document in document_generator.DOCUMENTS: + assert {"title", "type", "client_industry", "sections"} <= document.keys() + assert document["sections"] + + +def test_generate_all_documents_creates_expected_pdfs(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(document_generator, "DATA_DIR", tmp_path) + + document_generator.generate_all_documents() + + generated_files = sorted(tmp_path.glob("*.pdf")) + + assert len(generated_files) == len(document_generator.DOCUMENTS) + assert all(file.stat().st_size > 0 for file in generated_files) + assert all("/" not in file.name for file in generated_files) + + output = capsys.readouterr().out + assert f"All {len(document_generator.DOCUMENTS)} documents saved to:" in output + + +def test_pdf_document_can_render_single_document(tmp_path): + sample = document_generator.DOCUMENTS[0] + pdf = document_generator.PDFDocument( + sample["title"], + sample["type"], + sample["client_industry"], + ) + pdf.alias_nb_pages() + pdf.add_title_page() + pdf.add_page() + + for section_title, section_content in sample["sections"].items(): + pdf.add_section(section_title, section_content) + + output_path = tmp_path / "single_document.pdf" + pdf.output(str(output_path)) + + assert output_path.exists() + assert output_path.stat().st_size > 0 + assert pdf.page_no() >= 2 diff --git a/tests/test_evals.py b/tests/test_evals.py new file mode 100644 index 0000000..6c5ff0d --- /dev/null +++ b/tests/test_evals.py @@ -0,0 +1,36 @@ +from evals.metrics import build_metrics_summary + + +def test_build_metrics_summary(): + results = [ + { + "expected_sources": ["A.pdf"], + "retrieved_sources": ["A.pdf"], + "expect_citations": True, + "answer": "Fact [Source: A.pdf, Page 1]", + "grounded": True, + "latency_ms": 10, + "tool_call_count": 2, + "passed": True, + }, + { + "expected_sources": ["B.pdf"], + "retrieved_sources": [], + "expect_citations": False, + "answer": "No answer", + "grounded": False, + "latency_ms": 30, + "tool_call_count": 1, + "passed": False, + }, + ] + + metrics = build_metrics_summary(results) + + assert metrics["retrieval_hit_rate"] == 0.5 + assert metrics["citation_coverage"] == 1.0 + assert metrics["grounded_answer_score"] == 0.5 + assert metrics["average_latency"] == 20.0 + assert metrics["average_latency_ms"] == 20.0 + assert metrics["tool_call_count"] == 1.5 + assert metrics["failure_rate"] == 0.5 diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..c9def2c --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,36 @@ +import importlib + +import pytest + + +@pytest.mark.parametrize( + "module_name", + [ + "config", + "document_generator", + "agent", + "rag_engine", + "rfp_analyst.exceptions", + "rfp_analyst.health", + "rfp_analyst.uploads", + "rfp_analyst.ui.helpers", + "rfp_analyst.ingestion.loaders", + "rfp_analyst.ingestion.chunking", + "rfp_analyst.ingestion.registry", + "rfp_analyst.ingestion.pipeline", + "rfp_analyst.retrieval.vector_store", + "rfp_analyst.schemas", + "rfp_analyst.tools.search_kb", + "rfp_analyst.tools.compare_projects", + "rfp_analyst.tools.rfp_gap_analyzer", + "rfp_analyst.tools.proposal_writer", + "rfp_analyst.tools.source_verifier", + "rfp_analyst.agent.state", + "rfp_analyst.agent.prompts", + "rfp_analyst.agent.graph", + "rfp_analyst.agent.runtime", + ], +) +def test_module_imports(module_name): + module = importlib.import_module(module_name) + assert module is not None diff --git a/tests/test_ingestion_pipeline.py b/tests/test_ingestion_pipeline.py new file mode 100644 index 0000000..ebe68ea --- /dev/null +++ b/tests/test_ingestion_pipeline.py @@ -0,0 +1,145 @@ +from pathlib import Path + +from langchain_core.documents import Document + +from rfp_analyst.ingestion.chunking import chunk_loaded_sources +from rfp_analyst.ingestion.pipeline import IngestionPipeline +from rfp_analyst.ingestion.registry import IngestionRegistry +from rfp_analyst.schemas import LoadedSource + + +class FakeCollection: + def __init__(self): + self.documents = {} + + def get(self): + return { + "ids": list(self.documents), + "metadatas": [doc.metadata for doc in self.documents.values()], + } + + def count(self): + return len(self.documents) + + +class FakeVectorStore: + def __init__(self): + self._collection = FakeCollection() + + def add_documents(self, documents, ids): + for doc, chunk_id in zip(documents, ids): + self._collection.documents[chunk_id] = doc + + +class FakeVectorStoreManager: + def __init__(self): + self.vectorstore = FakeVectorStore() + + def upsert_documents(self, documents): + existing_ids = set(self.vectorstore._collection.get()["ids"]) + new_docs = [] + new_ids = [] + for document in documents: + chunk_id = document.metadata["chunk_id"] + if chunk_id in existing_ids: + continue + new_docs.append(document) + new_ids.append(chunk_id) + if new_docs: + self.vectorstore.add_documents(new_docs, new_ids) + return self.vectorstore + + def get_stats(self): + metadatas = self.vectorstore._collection.get()["metadatas"] + source_files = {metadata["source_file"] for metadata in metadatas} + return { + "total_chunks": self.vectorstore._collection.count(), + "total_documents": len(source_files), + "document_names": sorted(source_files), + "status": "ready" if metadatas else "not_initialized", + } + + +def make_loaded_source(name="Client Proposal.pdf", file_hash="abc123"): + return LoadedSource( + source_file=name, + source_path=str(Path("C:/tmp") / name), + file_hash=file_hash, + page_count=1, + document_type="Proposal", + documents=[ + Document( + page_content="Alpha beta gamma delta epsilon zeta eta theta.", + metadata={"page": 0}, + ) + ], + ) + + +def test_chunking_adds_required_metadata(): + chunks = chunk_loaded_sources( + [make_loaded_source()], + chunk_size=20, + chunk_overlap=0, + ) + + assert chunks + assert all(chunk.metadata["source_file"] == "Client Proposal.pdf" for chunk in chunks) + assert all(chunk.metadata["file_hash"] == "abc123" for chunk in chunks) + assert all(chunk.metadata["document_type"] == "Proposal" for chunk in chunks) + assert all(chunk.metadata["chunk_id"] for chunk in chunks) + assert len({chunk.metadata["chunk_id"] for chunk in chunks}) == len(chunks) + + +def test_registry_prevents_duplicate_ingestion(monkeypatch, tmp_path): + registry = IngestionRegistry(tmp_path / "ingestion_registry.json") + vector_store_manager = FakeVectorStoreManager() + source = make_loaded_source() + + monkeypatch.setattr( + "rfp_analyst.ingestion.pipeline.load_pdf_sources", + lambda _doc_dir: [source], + ) + + pipeline = IngestionPipeline( + doc_dir=tmp_path, + persist_dir=tmp_path / "vectorstore", + registry=registry, + vector_store_manager=vector_store_manager, + ) + + pipeline.run() + first_count = vector_store_manager.get_stats()["total_chunks"] + + pipeline.run() + second_count = vector_store_manager.get_stats()["total_chunks"] + + assert first_count > 0 + assert second_count == first_count + assert registry.contains_hash(source.file_hash) + + +def test_repeated_ingestion_does_not_double_chunk_count(monkeypatch, tmp_path): + registry = IngestionRegistry(tmp_path / "ingestion_registry.json") + vector_store_manager = FakeVectorStoreManager() + source = make_loaded_source(file_hash="hash-repeat") + + monkeypatch.setattr( + "rfp_analyst.ingestion.pipeline.load_pdf_sources", + lambda _doc_dir: [source], + ) + + pipeline = IngestionPipeline( + doc_dir=tmp_path, + persist_dir=tmp_path / "vectorstore", + registry=registry, + vector_store_manager=vector_store_manager, + ) + + pipeline.run() + stats_after_first_run = vector_store_manager.get_stats() + pipeline.run() + stats_after_second_run = vector_store_manager.get_stats() + + assert stats_after_first_run["total_chunks"] == stats_after_second_run["total_chunks"] + assert stats_after_first_run["total_documents"] == stats_after_second_run["total_documents"] diff --git a/tests/test_runtime_hardening.py b/tests/test_runtime_hardening.py new file mode 100644 index 0000000..34b57e7 --- /dev/null +++ b/tests/test_runtime_hardening.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import pytest + +from rfp_analyst.exceptions import KnowledgeBaseNotReadyError, UnsupportedFileError +from rfp_analyst.health import get_app_health +from rfp_analyst.retrieval.vector_store import VectorStoreManager +from rfp_analyst.ui.helpers import format_latency_display, get_chat_avatar +from rfp_analyst.uploads import validate_uploaded_pdf + + +class FakeUpload: + def __init__(self, name: str, file_type: str = "application/pdf", size: int = 10): + self.name = name + self.type = file_type + self.size = size + + +class FakeVectorStoreManager: + def __init__(self, *_args, **_kwargs): + pass + + def get_stats(self): + return { + "status": "ready", + "total_documents": 3, + "total_chunks": 9, + "document_names": ["a.pdf"], + } + + +def test_invalid_avatar_helper_values(): + assert get_chat_avatar("user") == "👤" + assert get_chat_avatar("assistant") == "🤖" + assert get_chat_avatar("user") not in {"User", "AI"} + assert get_chat_avatar("assistant") not in {"User", "AI"} + + +def test_missing_vectorstore_raises_graceful_error(tmp_path): + manager = VectorStoreManager(persist_dir=tmp_path / "missing_vectorstore") + with pytest.raises(KnowledgeBaseNotReadyError): + manager.load(create_if_missing=False) + + +def test_invalid_upload_filename_sanitization(): + safe_name = validate_uploaded_pdf(FakeUpload("../My bad file!!.pdf")) + assert safe_name == "My_bad_file.pdf" + + +def test_invalid_upload_extension_raises(): + with pytest.raises(UnsupportedFileError): + validate_uploaded_pdf(FakeUpload("report.txt")) + + +def test_invalid_upload_mime_type_raises(): + with pytest.raises(UnsupportedFileError): + validate_uploaded_pdf(FakeUpload("report.pdf", file_type="text/plain")) + + +def test_eval_latency_formatting(): + assert format_latency_display({"average_latency": 0.23}) == "0.230 s" + assert format_latency_display({"average_latency_ms": 12.5}) == "12.50 ms" + + +def test_app_health_check(monkeypatch, tmp_path): + monkeypatch.setattr("rfp_analyst.health.REQUIRED_DIRECTORIES", { + "data_dir": tmp_path / "data", + "vectorstore_dir": tmp_path / "vectorstore", + "assets_dir": tmp_path / "assets", + }) + for path in (tmp_path / "data", tmp_path / "vectorstore", tmp_path / "assets"): + path.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr("rfp_analyst.health.VectorStoreManager", FakeVectorStoreManager) + + health = get_app_health("Not configured") + + assert health["vectorstore_ready"] is True + assert health["document_count"] == 3 + assert health["chunk_count"] == 9 + assert health["llm_provider_configured"] is False + assert all(entry["exists"] for entry in health["required_directories"].values()) From d0cea369806728f54c13dbdf76e9d3ecb74e0405 Mon Sep 17 00:00:00 2001 From: Tushar Ghosh Date: Sun, 12 Jul 2026 05:42:46 +0530 Subject: [PATCH 2/3] feat: complete production-oriented agentic RAG platform --- .env.example | 33 +- .github/workflows/ci.yml | 36 + .gitignore | 28 +- .streamlit/config.toml | 1 + README.md | 576 +++--- agent.py | 248 ++- app.py | 627 +++++-- config.py | 109 +- docs/AGENTIC_RAG.md | 134 ++ docs/ARCHITECTURE.md | 249 +++ docs/FILE_MAP.md | 133 ++ docs/REPRODUCIBILITY.md | 126 ++ docs/TESTING_AND_EVALUATION.md | 135 ++ docs/TROUBLESHOOTING.md | 205 +++ docs/evaluation.md | 88 +- document_generator.py | 5 +- evals/__init__.py | 1 + evals/golden_questions.yaml | 151 +- evals/run_evals.py | 279 +-- evals/run_kb_evals.py | 232 +++ rag_engine.py | 506 +++++- requirements.txt | 12 +- src/internal_rfp_analyst.egg-info/PKG-INFO | 288 --- src/internal_rfp_analyst.egg-info/SOURCES.txt | 32 - .../dependency_links.txt | 1 - .../top_level.txt | 1 - src/rfp_analyst.egg-info/PKG-INFO | 3 - src/rfp_analyst.egg-info/SOURCES.txt | 32 - src/rfp_analyst.egg-info/dependency_links.txt | 1 - src/rfp_analyst.egg-info/top_level.txt | 1 - src/rfp_analyst/__init__.py | 22 +- src/rfp_analyst/agent/__init__.py | 18 +- src/rfp_analyst/agent/graph.py | 1601 ++++++++++++++++- src/rfp_analyst/agent/runtime.py | 64 +- src/rfp_analyst/evals.py | 40 + src/rfp_analyst/exceptions.py | 14 +- src/rfp_analyst/health.py | 93 +- src/rfp_analyst/ingestion/chunking.py | 39 +- src/rfp_analyst/ingestion/loaders.py | 11 +- src/rfp_analyst/retrieval/vector_store.py | 42 +- src/rfp_analyst/schemas.py | 2 + src/rfp_analyst/tools/proposal_writer.py | 55 +- src/rfp_analyst/tools/rfp_gap_analyzer.py | 99 +- src/rfp_analyst/tools/source_verifier.py | 28 +- src/rfp_analyst/ui/__init__.py | 5 + src/rfp_analyst/uploads.py | 54 +- tests/__init__.py | 1 + tests/test_agent_and_ui.py | 130 ++ tests/test_agentic_tools.py | 69 +- tests/test_document_scope.py | 482 +++++ tests/test_evals.py | 70 +- tests/test_health_and_uploads.py | 85 + tests/test_kb_evals.py | 74 + tests/test_langgraph_agent.py | 769 ++++++++ tests/test_prompt_builder_py311.py | 25 + tests/test_streamlit_app_smoke.py | 84 + 56 files changed, 6783 insertions(+), 1466 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 docs/AGENTIC_RAG.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/FILE_MAP.md create mode 100644 docs/REPRODUCIBILITY.md create mode 100644 docs/TESTING_AND_EVALUATION.md create mode 100644 docs/TROUBLESHOOTING.md create mode 100644 evals/__init__.py create mode 100644 evals/run_kb_evals.py delete mode 100644 src/internal_rfp_analyst.egg-info/PKG-INFO delete mode 100644 src/internal_rfp_analyst.egg-info/SOURCES.txt delete mode 100644 src/internal_rfp_analyst.egg-info/dependency_links.txt delete mode 100644 src/internal_rfp_analyst.egg-info/top_level.txt delete mode 100644 src/rfp_analyst.egg-info/PKG-INFO delete mode 100644 src/rfp_analyst.egg-info/SOURCES.txt delete mode 100644 src/rfp_analyst.egg-info/dependency_links.txt delete mode 100644 src/rfp_analyst.egg-info/top_level.txt create mode 100644 src/rfp_analyst/evals.py create mode 100644 src/rfp_analyst/ui/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/test_agent_and_ui.py create mode 100644 tests/test_document_scope.py create mode 100644 tests/test_health_and_uploads.py create mode 100644 tests/test_kb_evals.py create mode 100644 tests/test_langgraph_agent.py create mode 100644 tests/test_prompt_builder_py311.py create mode 100644 tests/test_streamlit_app_smoke.py diff --git a/.env.example b/.env.example index 0ca8415..4750536 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,27 @@ -# ─── Groq API Key (Primary — Fastest, Recommended) ─── -# Get yours FREE at: https://console.groq.com/keys -# 1. Sign up at https://console.groq.com -# 2. Go to API Keys → Create API Key -# 3. Copy the key and paste below +# Internal RFP Analyst environment settings +# +# Configure one provider key for chat. Document generation, ingestion, +# health checks, and retrieval-only evaluations can run without an LLM key. -GROQ_API_KEY=your_groq_api_key_here +GROQ_API_KEY= +GOOGLE_API_KEY= -# ─── Google Gemini API Key (Fallback) ─── -# Get yours at: https://aistudio.google.com/apikey +# Optional model/runtime settings +AGENT_MODE=agentic +MIN_RELEVANCE_SCORE=0.50 -GOOGLE_API_KEY=your_google_api_key_here +# Prompt-budget settings +MAX_PROMPT_TOKENS=6500 +RFP_ANALYSIS_MAX_OUTPUT_TOKENS=1200 +MAX_CONTEXT_CHARS_PER_CHUNK=1000 +MAX_HISTORY_MESSAGES=3 +MAX_TARGET_CHUNKS=4 +MAX_CASE_STUDIES=3 +MAX_CHUNKS_PER_CASE_STUDY=2 + +# Upload limits +MAX_UPLOAD_SIZE_MB=25 +MAX_UPLOAD_PAGE_COUNT=250 + +# Set to 1 only when debugging provider errors locally. +RFP_ANALYST_DEBUG=0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..30d6ea2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install -e . + + - name: Compile critical entry points + run: python -m py_compile app.py agent.py rag_engine.py config.py document_generator.py + + - name: Compile source tree + run: python -m compileall -f src tests evals + + - name: Run pytest + run: python -m pytest -q + + - name: Run offline smoke evaluation + run: python -m evals.run_evals diff --git a/.gitignore b/.gitignore index 47a896d..2800826 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,30 @@ -# Environment +# Environment and secrets .env +.streamlit/secrets.toml .venv/ venv/ +.python_packages/ + +# Python cache and build output __pycache__/ +*.pyc +*.egg-info/ +.ruff_cache/ .pytest_cache/ .pytest_tmp/ -.ruff_cache/ -*.pyc -.streamlit/secrets.toml -# Vector Store (regenerated on deploy) -vectorstore/ +# Logs +*.log -# Generated PDFs (regenerated on deploy) +# Generated runtime artifacts +vectorstore/ +vectorstore_build_*/ data/documents/*.pdf - -# Generated evaluation outputs +data/uploads/ evals/results.json +evals/offline_smoke_results.json +evals/kb_eval_results.json +evals/real_kb_results.json # IDE .vscode/ @@ -25,4 +33,4 @@ evals/results.json # OS Thumbs.db -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.streamlit/config.toml b/.streamlit/config.toml index bfa7bb5..bbd5d8a 100644 --- a/.streamlit/config.toml +++ b/.streamlit/config.toml @@ -7,3 +7,4 @@ font = "sans serif" [server] headless = true +maxUploadSize = 25 diff --git a/README.md b/README.md index b8d1b6e..51ad933 100644 --- a/README.md +++ b/README.md @@ -1,281 +1,441 @@ -
+[![CI](https://github.com/tusharg007/Internal-RFP-Analyst/actions/workflows/ci.yml/badge.svg)](https://github.com/tusharg007/Internal-RFP-Analyst/actions/workflows/ci.yml) -# 🔍 Internal RFP Analyst +# Internal RFP Analyst -### AI-Powered RAG Knowledge Agent for Enterprise Consulting +Tool-orchestrated Agentic RAG for requirement extraction, internal case-study matching, evidence-backed proposal generation, and post-generation grounding. -[![Live Demo](https://img.shields.io/badge/🚀_Live_Demo-Streamlit_Cloud-FF4B4B?style=for-the-badge&logo=streamlit&logoColor=white)](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) -[![Python](https://img.shields.io/badge/Python-3.10+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://python.org) -[![LangChain](https://img.shields.io/badge/LangChain-0.3+-1C3C3C?style=for-the-badge&logo=langchain&logoColor=white)](https://langchain.com) -[![Groq](https://img.shields.io/badge/Groq-LPU_Inference-F55036?style=for-the-badge)](https://groq.com) -[![License](https://img.shields.io/badge/License-MIT-green?style=for-the-badge)](LICENSE) +Internal RFP Analyst is a Python 3.11+ Streamlit application for analyzing uploaded target documents against an internal case-study corpus. It combines PDF ingestion, ChromaDB retrieval, a LangGraph-based orchestration runtime, deterministic specialized tools, prompt compaction, LLM synthesis, and post-generation verification with optional bounded repair. -**Instantly search past proposals, RFP responses, project outlines, and case studies using natural language.** Built with a production-grade RAG pipeline featuring local ONNX embeddings, streaming LLM responses, and multi-provider failover. +The repository is structured as a reproducible single-user demonstration. It includes tests, evaluation runners, and deployment guidance, but it is not presented as a production-scale multi-tenant service. -[Live Demo](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) · [Architecture](#-architecture) · [Quick Start](#-quick-start) · [Challenges & Solutions](#-engineering-challenges--solutions) +## Project Overview -
+Internal RFP analysis needs more than document search. A user may need to treat one uploaded document as the target requirements, search internal case studies separately, identify missing details needed for a proposal, compare prior work, and generate a cited response. A basic retrieve-and-generate workflow tends to blend those roles into one context window and makes it easy to confuse target requirements with internal examples. ---- +This project keeps those responsibilities separate: -## ✨ Key Features +- Uploaded PDFs are the target corpus. +- Generated sample PDFs are the internal case-study corpus. +- Retrieval can be scoped to uploaded documents, sample documents, or all indexed documents. +- Cross-corpus RFP analysis retrieves target evidence from uploads and case-study evidence from samples. +- The LangGraph runtime coordinates health checks, intent routing, tool planning, retrieval, tool execution, prompt budgeting, LLM generation, and grounding verification. -| Feature | Description | -|---|---| -| 🔍 **Semantic Search** | Natural language queries over a ChromaDB vector store with relevance-scored retrieval | -| ⚡ **Streaming Responses** | Word-by-word response streaming via `st.write_stream` for instant perceived performance | -| 🧠 **Local Embeddings** | ONNX-based FastEmbed (`bge-small-en-v1.5`) — zero API calls, zero rate limits for retrieval | -| 🔄 **Multi-Provider LLM** | Groq (Llama 3.3 70B) primary + Gemini fallback — automatic provider selection | -| 📄 **PDF Ingestion** | Upload custom PDFs or use the built-in 10-document consulting knowledge base | -| 📚 **Source Citations** | Every answer cites exact document name and page number | -| 💬 **Conversation Memory** | Chat history maintained in-session for contextual follow-ups | -| ☁️ **Zero-Config Deploy** | Auto-generates sample documents and ingests on first Streamlit Cloud boot | +The result is not just search. The user receives requirement summaries, inferred gaps, ranked case studies, comparison outputs, proposal outlines, page-level citations, grouped source traces, and graceful fallbacks when evidence or configuration is missing. ---- +## Key Capabilities -## 🏗️ Architecture +- PDF upload validation with filename sanitization, MIME checks, size limits, and page-count limits. +- Generated synthetic sample case-study corpus for repeatable local setup. +- Deterministic and idempotent ingestion with duplicate-file and duplicate-chunk handling. +- Unique chunk identifiers based on origin, file identity, page, chunk position, and content hash. +- ChromaDB persistence with local FastEmbed embeddings. +- Retrieval scopes for uploaded documents, sample documents, and all documents. +- Relevance filtering with explicit insufficient-evidence fallbacks. +- Conversational follow-up resolution using recent cited entities. +- Previous-answer source recall without another retrieval call. +- Requirement extraction with inferred absent/ambiguous gap identification. +- Evidence-based case-study scoring across Azure migration, regulatory/HIPAA fit, dashboards and analytics, phased delivery, and measurable outcomes. +- Project comparison across timeline, budget, technology stack, and outcomes. +- Six-section proposal-outline generation with cited evidence. +- Prompt compaction and prompt-budget tracing before LLM calls. +- Post-generation grounding verification with one bounded repair pass when needed. +- Visible execution traces that expose tool names and summaries without private chain-of-thought. +- Offline smoke evaluation and real knowledge-base evaluation. +- Friendly handling for missing providers, invalid API keys, oversized model requests, empty scopes, low relevance, and Windows vectorstore locks. + +## Architecture Diagrams + +### System Architecture ```mermaid -flowchart TB - subgraph UI["🖥️ Streamlit UI"] - A["User Query"] --> B["Chat Interface"] - B --> C["st.write_stream"] - end - - subgraph RAG["⚡ RAG Pipeline"] - D["FastEmbed ONNX
bge-small-en-v1.5"] --> E["ChromaDB
Vector Store"] - E --> F["Top-K Retrieval
k=6 chunks"] - end - - subgraph LLM["🤖 LLM Layer"] - G{"Provider
Selection"} - G -->|"Primary"| H["Groq LPU
Llama 3.3 70B"] - G -->|"Fallback"| I["Google Gemini
2.0 Flash"] - end - - subgraph INGEST["📥 Ingestion Pipeline"] - J["PDF Documents"] --> K["PyMuPDF Loader"] - K --> L["Recursive Chunking
512 tokens, 50 overlap"] - L --> D - end - - A --> F - F --> |"Context + Prompt"| G - H --> C - I --> C - - style UI fill:#1a1a2e,stroke:#667eea,color:#fff - style RAG fill:#16213e,stroke:#0f3460,color:#fff - style LLM fill:#1a1a2e,stroke:#e94560,color:#fff - style INGEST fill:#16213e,stroke:#533483,color:#fff +flowchart TD + UI["Streamlit UI
app.py"] --> Adapter["Application adapter
agent.py"] + Adapter --> Graph["LangGraph orchestration
src/rfp_analyst/agent/graph.py"] + Graph --> Routing["Intent routing and tool planning"] + Routing --> Tools["Specialized tools"] + Tools --> Retrieval["Scoped retrieval"] + Retrieval --> Store["ChromaDB / indexed PDFs"] + Graph --> Budget["Prompt compaction and budgeting"] + Budget --> LLM["Groq or Gemini generation"] + LLM --> Verify["Grounding verification and optional repair"] + Verify --> Answer["Final answer with citations and traces"] ``` -### Request Flow (Single Query) +### Document-Ingestion Pipeline ```mermaid -sequenceDiagram - participant U as 👤 User - participant S as 🖥️ Streamlit - participant E as ⚡ FastEmbed (Local) - participant C as 🗄️ ChromaDB - participant L as 🤖 Groq/Gemini - - U->>S: "What tech stack did we use for banking?" - S->>E: Embed query (local, ~5ms) - E->>C: Similarity search (k=6) - C-->>S: Top 6 relevant chunks + metadata - S->>L: Single prompt with context - L-->>S: Streaming response tokens - S-->>U: Word-by-word answer with citations - - Note over E,C: Zero API calls for retrieval - Note over L: Single LLM call per query +flowchart TD + Sample["Generated sample PDFs"] --> Validate["Validation and origin assignment"] + Upload["Uploaded PDFs"] --> Validate + Validate --> Load["PDF loading"] + Load --> Chunk["Chunking"] + Chunk --> Metadata["Metadata and deterministic IDs"] + Metadata --> Dedup["Duplicate file and chunk handling"] + Dedup --> Embed["Embedding"] + Embed --> Chroma["ChromaDB"] + Chroma --> Stats["Knowledge-base statistics"] ``` ---- +### Agentic RAG Workflow -## 🛠️ Technology Stack +```mermaid +flowchart TD + Start["START"] --> Health["health_check"] + Health --> Classify["classify_intent"] + Classify --> Plan["plan_tools"] + Plan --> Retrieve["execute_retrieval"] + Retrieve --> Tools["execute_specialized_tool"] + Tools --> Prompt["synthesize_prompt"] + Prompt --> Evidence["evidence_availability_check"] + Evidence --> Final["final_response"] + Final --> Generate["LLM generation"] + Generate --> Verify["grounding_verifier"] + Verify --> Repair{"repair needed?"} + Repair -->|yes| AnswerRepair["answer_repair"] + AnswerRepair --> FinalVerify["final_grounding_verifier"] + Repair -->|no| FinalVerify +``` -| Layer | Technology | Why This Choice | -|---|---|---| -| **LLM (Primary)** | Groq — Llama 3.3 70B | Fastest free inference (LPU), 30 RPM, sub-second latency | -| **LLM (Fallback)** | Google Gemini 2.0 Flash | Free tier backup, 15 RPM | -| **Embeddings** | FastEmbed (ONNX) — `bge-small-en-v1.5` | Local execution, no API calls, no rate limits | -| **Vector Store** | ChromaDB (persistent) | Lightweight, embedded, perfect for document-scale RAG | -| **RAG Framework** | LangChain 0.3+ | Industry-standard abstractions for retrieval chains | -| **PDF Processing** | PyMuPDF | Fastest Python PDF parser, preserves layout metadata | -| **UI** | Streamlit | Rapid prototyping with built-in streaming support | -| **Deployment** | Streamlit Community Cloud | Free hosting with GitHub auto-deploy | +### Cross-Corpus RFP Analysis ---- +```mermaid +flowchart LR + Uploads["Uploaded target documents"] --> Target["Target-context retrieval"] + Target --> Requirements["Requirement extraction"] + Requirements --> Gaps["Inferred gap analysis"] + + Samples["Sample case studies"] --> Cases["Case-study retrieval and scoring"] + Cases --> Compare["Fit comparison"] + + Gaps --> Proposal["Proposal generation"] + Compare --> Proposal + Proposal --> Budget["Prompt budgeting"] + Budget --> LLM["LLM synthesis"] + LLM --> Verify["Verification and optional repair"] + Verify --> Response["Cited response"] +``` -## 🚀 Quick Start +## How the Agentic RAG Workflow Works -### Option 1: Use the Live Demo -👉 **[app-rfp-analyst.streamlit.app](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/)** — No setup required. The app auto-generates sample documents on first load. +The canonical runtime lives in [src/rfp_analyst/agent/graph.py](src/rfp_analyst/agent/graph.py). It uses a real LangGraph `StateGraph` when `langgraph` is available and falls back to the same node functions in deterministic order otherwise. -### Option 2: Run Locally +### State management -#### 1. Get a Free API Key (Choose One) +The graph state carries the user query, chat history, retrieval scope, vectorstore statistics, retrieval function, planned tools, retrieved documents, compact retrieval context, tool outputs, traces, prompt text, answer state, resolved conversational entities, and prompt-budget metadata. -| Provider | Speed | Free Limit | Get Key | -|---|---|---|---| -| **Groq** ⭐ Recommended | ~100 tok/s | 30 RPM, 6000 RPD | [console.groq.com/keys](https://console.groq.com/keys) | -| Google Gemini | ~30 tok/s | 15 RPM | [aistudio.google.com/apikey](https://aistudio.google.com/apikey) | +### Intent routing -#### 2. Setup +Implemented intents are: -```bash -# Clone the repository -git clone https://github.com/tusharg007/Internal-RFP-Analyst.git -cd Internal-RFP-Analyst +- `search` +- `compare` +- `proposal` +- `rfp_analysis` +- `previous_sources` +- `ambiguous` -# Create virtual environment -python -m venv venv -venv\Scripts\activate # Windows -# source venv/bin/activate # Mac/Linux +Requests are routed through explicit node execution rather than a single free-form chain. Follow-up resolution happens before retrieval, and ambiguous references produce a clarification message instead of broad retrieval. -# Install dependencies -pip install -r requirements.txt +### Tool orchestration -# Configure API key -copy .env.example .env -# Edit .env → add your GROQ_API_KEY (or GOOGLE_API_KEY) -``` +The runtime executes real deterministic helpers for: -#### 3. Launch +- scoped knowledge-base search +- requirement extraction +- inferred gap identification +- case-study ranking +- project comparison +- proposal-outline generation +- source verification -```bash -streamlit run app.py -``` +Tool outputs feed later graph nodes and also appear in compact form in the final prompt. -The app will auto-generate 10 sample consulting documents and build the vector store on first launch. +### Scoped retrieval ---- +Retrieval can run against: -## 💬 Example Queries +- `upload` +- `sample` +- `all` -| Query | What It Tests | -|---|---| -| *"List all projects with their timelines"* | Full knowledge base traversal | -| *"What tech stack did we use for the banking audit?"* | Precise document retrieval | -| *"Compare the healthcare and insurance projects"* | Cross-document synthesis | -| *"Which projects used Azure services?"* | Multi-document filtering | -| *"What was the budget for the supply chain platform?"* | Specific fact extraction | -| *"What compliance frameworks did we follow in pharma?"* | Domain-specific retrieval | +The vector store filters by `document_origin` for `upload` and `sample`. In `rfp_analysis`, uploads are always the target context and sample documents are always internal case studies. The workflow never concludes that uploads are missing just because retrieved scores fell below the threshold. ---- +### Cross-corpus reasoning -## 🧪 Engineering Challenges & Solutions +`rfp_analysis` keeps uploaded target evidence and sample case-study evidence separate all the way through: -### Challenge 1: Gemini API Rate Limits Killed the App +1. retrieve uploaded target evidence +2. extract requirements and inferred gaps from uploads +3. search sample documents for relevant case studies +4. compare fit using deterministic scoring +5. generate a proposal outline from both branches -**Problem:** The original architecture used Google Gemini for *both* embeddings and LLM generation. The free tier (100 embedding req/min, 15 LLM req/min) was exhausted within minutes, returning `429 RESOURCE_EXHAUSTED` errors. The multi-step ReAct agent made 3-5 LLM calls per query, compounding the problem. +This is what makes the workflow materially different from a basic "chat with documents" setup. -**Solution: Hybrid local + cloud architecture** +### Prompt budgeting -```mermaid -flowchart LR - subgraph BEFORE["❌ Before — All API Calls"] - A1["Gemini Embeddings API"] -->|"Rate Limited"| B1["429 Error"] - A2["Gemini LLM x 3-5 calls"] -->|"Rate Limited"| B1 - end - - subgraph AFTER["✅ After — Minimal API Calls"] - C1["FastEmbed Local
Zero API calls"] -->|"Instant"| D1["Success"] - C2["Groq LLM x 1 call
30 RPM limit"] -->|"Sub-second"| D1 - end - - style BEFORE fill:#2d1117,stroke:#f85149,color:#fff - style AFTER fill:#0d1117,stroke:#3fb950,color:#fff -``` +The prompt builder uses compact sections rather than raw Python object serialization. It keeps: -| Metric | Before | After | Improvement | -|---|---|---|---| -| API calls per query | 4-6 (embed + 3-5 LLM) | **1** (LLM only) | **83% reduction** | -| Embedding rate limits | 100/min (API) | **∞** (local) | **Eliminated** | -| LLM rate limits | 15 RPM (Gemini) | **30 RPM** (Groq) | **2x headroom** | +- current user request +- compact uploaded target evidence +- compact inferred gaps and requirement summaries +- compact sample case-study evidence +- compact comparison/proposal outputs +- short conversation history -### Challenge 2: 10+ Minute Response Times +It drops lower-value context when needed and records a visible `prompt_budget` trace with estimated input tokens, reserved output tokens, projected totals, and included evidence counts. -**Problem:** The ReAct agent architecture (LangGraph) made multiple sequential LLM calls — tool selection → execution → result processing → possibly more tools → final answer. Each call could trigger a rate-limit retry with exponential backoff (10s → 20s → 40s), compounding to 10+ minute waits. +### Grounding -**Solution: Single-call RAG with streaming** +The answer is generated first. Afterwards, `verify_answer_grounding` validates claims against retrieved document text, exact filenames, page numbers, numeric evidence, and named technologies. If unsupported or vague-cited claims remain, the runtime performs one bounded repair pass and verifies again. -- Replaced multi-step ReAct agent with a **single LLM call** architecture -- All context (retrieved chunks + project list + chat history) is assembled locally and sent in one prompt -- **Streaming responses** via `st.write_stream()` — text appears word-by-word, so the user sees output within 500ms even if full generation takes 3-5s +### Observability -| Metric | Before (ReAct) | After (Single-Call RAG) | -|---|---|---| -| LLM calls per query | 3-5 | **1** | -| Worst-case response time | 10+ minutes | **3-8 seconds** | -| Perceived latency | Full wait → wall of text | **~500ms** (streaming) | +The UI exposes safe traces only: -### Challenge 3: Sample Question Buttons Did Nothing +- tool names +- input summaries +- output summaries +- retrieval scope and source selections +- prompt-budget status +- verification status -**Problem:** Clicking a sample question button added the message to chat history and triggered `st.rerun()`, but after the rerun, only the `st.chat_input()` code path processed queries — sample button clicks were silently ignored. +Private chain-of-thought is not shown. -**Solution:** Introduced a `pending_query` session state flag. Button clicks set this flag before rerun. After rerun, a dedicated handler detects the pending query and routes it through the same processing pipeline as typed messages. +## Feature Walkthrough -### Challenge 4: Ephemeral Filesystem on Streamlit Cloud +1. Generate sample PDFs from the sidebar. +2. Upload custom PDFs into the dedicated uploads directory. +3. Click `Ingest Documents` to build or rebuild the knowledge base. +4. Select a document scope. +5. Ask an evidence-backed question. +6. Compare internal case studies. +7. Ask a conversational follow-up. +8. Run cross-corpus RFP analysis. +9. Inspect grouped sources and tool traces. +10. Run offline and real KB evaluations. -**Problem:** Streamlit Cloud's filesystem resets on every cold start, losing the vector store and requiring re-ingestion. +Example prompts: -**Solution:** Auto-setup pipeline — on first load, the app detects an empty vector store, generates 10 sample PDFs via `document_generator.py`, and ingests them automatically. With local embeddings, this entire process completes in **under 15 seconds** (vs. minutes with API-based embeddings). +```text +Compare the healthcare cloud migration and insurance automation projects. +Which documents were used for the previous answer? +Treat uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline. +What is the CEO's private phone number? +``` ---- +Unsupported questions are expected to return an insufficient-evidence fallback rather than a fabricated answer. -## 📁 Project Structure +## Repository Structure -``` +```text Internal-RFP-Analyst/ -├── app.py # Streamlit UI with streaming chat -├── agent.py # RAG query engine (Groq/Gemini + retrieval) -├── rag_engine.py # Ingestion pipeline (FastEmbed + ChromaDB) -├── config.py # Central configuration & provider selection -├── document_generator.py # Generates 10 realistic consulting PDFs -├── requirements.txt # Python dependencies -├── .env.example # API key template -├── .streamlit/ -│ └── config.toml # Streamlit theme configuration -├── data/documents/ # PDF documents (auto-generated) -└── vectorstore/ # ChromaDB persistent storage +|-- app.py +|-- agent.py +|-- rag_engine.py +|-- config.py +|-- document_generator.py +|-- src/rfp_analyst/ +| |-- agent/ +| |-- ingestion/ +| |-- retrieval/ +| |-- tools/ +| `-- ui/ +|-- evals/ +|-- tests/ +|-- docs/ +|-- pyproject.toml +`-- requirements.txt ``` -### Module Responsibilities +See [docs/FILE_MAP.md](docs/FILE_MAP.md) for the complete file-by-file explanation. -| Module | Lines | Responsibility | -|---|---|---| -| `config.py` | ~75 | API keys, model selection, RAG parameters, system prompt | -| `rag_engine.py` | ~165 | PDF loading → chunking → local embedding → ChromaDB storage/retrieval | -| `agent.py` | ~155 | LLM provider selection, prompt assembly, streaming query execution | -| `app.py` | ~280 | Streamlit UI, session management, chat rendering, error handling | -| `document_generator.py` | ~550 | Generates 10 industry-specific consulting PDFs with realistic content | +## Quick Start ---- +### Windows PowerShell -## 🔧 Configuration +```powershell +git clone https://github.com/tusharg007/Internal-RFP-Analyst.git +cd Internal-RFP-Analyst +git checkout agentic-rag-v2 +py -3.11 -m venv .venv +Set-ExecutionPolicy -Scope Process -ExecutionPolicy RemoteSigned +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +Copy-Item .env.example .env +python -m streamlit run app.py +``` -### Environment Variables +### POSIX / macOS / Linux -| Variable | Required | Description | -|---|---|---| -| `GROQ_API_KEY` | ⭐ Recommended | Groq API key for fastest inference ([get free key](https://console.groq.com/keys)) | -| `GOOGLE_API_KEY` | Optional | Google Gemini key as fallback ([get free key](https://aistudio.google.com/apikey)) | +```bash +git clone https://github.com/tusharg007/Internal-RFP-Analyst.git +cd Internal-RFP-Analyst +git checkout agentic-rag-v2 +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +cp .env.example .env +python -m streamlit run app.py +``` -### For Streamlit Cloud Deployment +## Configuration Reference + +All meaningful environment variables come from [config.py](config.py) and [.env.example](.env.example). + +| Variable | Purpose | Default | Required | Accepted values | Security notes | +| --- | --- | --- | --- | --- | --- | +| `GROQ_API_KEY` | Enables Groq chat generation | empty | No | Groq API key string | Secret, never commit | +| `GOOGLE_API_KEY` | Enables Gemini chat generation | empty | No | Google API key string | Secret, never commit | +| `AGENT_MODE` | Compatibility switch for simple vs agentic helpers | `agentic` | No | `simple`, `agentic` | Not sensitive | +| `MIN_RELEVANCE_SCORE` | Retrieval relevance threshold | `0.50` | No | float string | Not sensitive | +| `MAX_PROMPT_TOKENS` | Maximum estimated prompt tokens | `6500` | No | integer | Not sensitive | +| `RFP_ANALYSIS_MAX_OUTPUT_TOKENS` | Reserved output token budget for RFP analysis | `1200` | No | integer | Not sensitive | +| `MAX_CONTEXT_CHARS_PER_CHUNK` | Max characters kept per chunk in prompt compaction | `1000` | No | integer | Not sensitive | +| `MAX_HISTORY_MESSAGES` | Max recent messages included in prompt compaction | `3` | No | integer | Not sensitive | +| `MAX_TARGET_CHUNKS` | Max uploaded target chunks in compact prompt | `4` | No | integer | Not sensitive | +| `MAX_CASE_STUDIES` | Max case studies retained for compact prompt | `3` | No | integer | Not sensitive | +| `MAX_CHUNKS_PER_CASE_STUDY` | Max chunks retained per case study in compact prompt | `2` | No | integer | Not sensitive | +| `MAX_UPLOAD_SIZE_MB` | Upload size limit | `25` | No | integer | Not sensitive | +| `MAX_UPLOAD_PAGE_COUNT` | Upload page-count limit | `250` | No | integer | Not sensitive | +| `RFP_ANALYST_DEBUG` | Allows raw provider details in known error messages | off | No | `0/1`, `false/true`, `no/yes`, `off/on` | Enable only locally | + +Other meaningful constants in `config.py` are code-level configuration rather than environment variables: + +- `GROQ_MODEL` +- `GEMINI_MODEL` +- `LLM_TEMPERATURE` +- `LLM_MAX_TOKENS` +- `EMBEDDING_MODEL` +- `CHUNK_SIZE` +- `CHUNK_OVERLAP` +- `COLLECTION_NAME` +- directory paths and evaluation output paths + +## Running the Application + +On first startup: + +1. launch Streamlit +2. optionally generate sample PDFs +3. optionally upload PDFs +4. click `Ingest Documents` +5. wait for the health panel to show the knowledge base as ready + +Important runtime behaviors: + +- Uploaded files are stored but not searchable until ingestion succeeds. +- If the vectorstore is missing, the app shows a clean readiness message instead of crashing. +- Document scope controls which corpus is searched. +- Source traces can be toggled from the sidebar. +- Clearing chat history removes session messages and cached runtime objects. +- If no provider key is configured, chat stays disabled but upload, generation, and ingestion still work. + +## Testing and Evaluation + +Validation commands: + +```powershell +python -m py_compile app.py agent.py rag_engine.py config.py document_generator.py +python -m compileall -f src tests evals +python -m pytest -q +python -m evals.run_evals +python -m evals.run_kb_evals +``` -Add secrets in **Settings → Secrets**: +Latest verified local test count in this repository pass: `119 passed`. -```toml -GROQ_API_KEY = "gsk_your_key_here" -# Optional fallback: -# GOOGLE_API_KEY = "your_google_key_here" -``` +Evaluation modes: + +- Unit and integration tests exercise runtime behavior, graph routing, ingestion, UI safety, and regression cases. +- Offline deterministic smoke evaluation checks evaluation plumbing with a mock corpus and deterministic answers. +- Real knowledge-base evaluation exercises the actual ingestion, retrieval, and graph path against the generated corpus and a temporary evaluation upload fixture. +- Manual LLM answer-quality validation is still useful for judging writing quality and provider-specific behavior after the deterministic checks pass. + +The smoke evaluation is not evidence of real RAG quality. + +## Deployment + +### Streamlit Community Cloud + +Use: + +- repository: `tusharg007/Internal-RFP-Analyst` +- branch: `agentic-rag-v2` +- entry point: `app.py` + +Deployment notes: + +- install dependencies from `requirements.txt` +- configure `GROQ_API_KEY` or `GOOGLE_API_KEY` in Streamlit Secrets if chat is needed +- the filesystem is ephemeral, so uploaded files and vectorstores do not persist like a managed storage layer +- sample documents or uploads may need to be regenerated and reingested after redeployments +- the architecture is intended as a single-user demonstration and does not provide multi-tenant data isolation + +## Reliability and Safety + +- API keys are resolved from Streamlit secrets, then environment variables, then `.env` during normal local runtime. +- Uploaded documents are ignored by chat until ingestion succeeds. +- Upload validation enforces PDF type, safe filenames, file-size limits, and page-count limits. +- Ingestion builds into a temporary directory before replacing the active vectorstore. +- Duplicate files and duplicate chunks are filtered before Chroma upsert. +- Retrieval uses scope filters and a configurable relevance threshold. +- Low-evidence or unsupported requests return a fallback instead of fabricated facts. +- Prompt-size protection compacts evidence and formats token-limit provider errors into user-facing guidance. +- Grounding verification checks filename/page citations, numeric claims, and named technologies. +- Automated grounding helps, but it does not guarantee that every unsupported claim is caught. + +## Known Limitations + +- Single-user Streamlit storage model +- Local persistent vectorstore design +- Synthetic sample PDF corpus +- Small evaluation corpus +- Embedding and retrieval quality depend on the local model and indexed content +- LLM answer quality still depends on the configured provider +- No authentication or authorization +- No distributed task queue or background ingestion workers +- No managed observability backend +- Grounding verification is bounded and heuristic, not formal proof + +## Future Extensions + +Future work could include: + +- hybrid BM25 plus dense retrieval +- reranking +- managed vector databases +- authentication and tenant isolation +- background ingestion workers +- LangSmith or OpenTelemetry tracing +- larger evaluation datasets +- human approval checkpoints for proposal generation +- structured export formats for proposal outputs + +These are roadmap ideas, not implemented features. + +## Additional Documentation + +- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) +- [docs/AGENTIC_RAG.md](docs/AGENTIC_RAG.md) +- [docs/REPRODUCIBILITY.md](docs/REPRODUCIBILITY.md) +- [docs/TESTING_AND_EVALUATION.md](docs/TESTING_AND_EVALUATION.md) +- [docs/FILE_MAP.md](docs/FILE_MAP.md) +- [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) + +## Screenshots + +No safe repository screenshots are currently committed. ---- +Useful future screenshots would be: -## 📜 License +- the Streamlit home screen with a ready knowledge base +- the upload and ingestion sidebar flow +- a cross-corpus RFP analysis answer with grouped traces +- the evaluation snapshot panel -This project is for educational and portfolio demonstration purposes. Built by [Tushar Ghosh](https://github.com/tusharg007). +Only commit screenshots that exclude API keys, private uploads, and machine-specific sensitive information. diff --git a/agent.py b/agent.py index ca3f794..9e41445 100644 --- a/agent.py +++ b/agent.py @@ -1,53 +1,125 @@ -"""RAG Engine - feature-flagged simple and agentic execution paths.""" - -from config import ( - GEMINI_MODEL, - GOOGLE_API_KEY, - GROQ_API_KEY, - GROQ_MODEL, - LLM_MAX_TOKENS, - LLM_TEMPERATURE, +"""RAG query engine with provider selection and streaming generation.""" + +import os + +from config import GEMINI_MODEL, GROQ_MODEL, LLM_MAX_TOKENS, LLM_TEMPERATURE, get_api_keys +from rfp_analyst.agent.graph import ( + KB_NOT_READY_MESSAGE, + NO_SCOPE_DOCUMENTS_MESSAGE, + prepare_query_payload, + run_query, + stream_query_response, ) -from rfp_analyst.agent.runtime import prepare_query_payload, run_query, stream_query_response from rfp_analyst.exceptions import LLMProviderNotConfiguredError +LLM_CONFIG_WARNING = ( + "No LLM provider is configured. Add GROQ_API_KEY or GOOGLE_API_KEY in your .env " + "file locally, or in Streamlit secrets when deployed." +) +LLM_AUTH_ERROR_MESSAGE = ( + "LLM authentication failed. Your API key is invalid or expired. Update " + "GROQ_API_KEY or GOOGLE_API_KEY in .env locally or Streamlit Secrets on deployment." +) +LLM_TOKEN_BUDGET_ERROR_MESSAGE = ( + "The analysis exceeded the current model's token budget. The request was " + "compacted automatically, but the remaining evidence is still too large. " + "Reduce the document scope or retry with a higher-capacity provider." +) +LLM_AUTH_ERROR_MARKERS = ( + "401", + "invalid_api_key", + "invalid api key", + "authentication", + "unauthorized", +) +LLM_TOKEN_BUDGET_ERROR_MARKERS = ( + "413", + "request too large", + "tokens per minute", + "rate_limit_exceeded", + "context length", + "maximum context", +) + + +def is_debug_mode_enabled() -> bool: + """Return whether raw provider error details may be shown.""" + return os.getenv("RFP_ANALYST_DEBUG", "").lower() in {"1", "true", "yes", "on"} + + +def is_llm_auth_error(error: Exception) -> bool: + """Detect provider authentication failures without exposing secret-bearing details.""" + error_text = str(error).lower() + return any(marker in error_text for marker in LLM_AUTH_ERROR_MARKERS) + + +def is_llm_token_budget_error(error: Exception) -> bool: + """Detect token-limit and request-size failures without exposing raw provider payloads.""" + error_text = str(error).lower() + return any(marker in error_text for marker in LLM_TOKEN_BUDGET_ERROR_MARKERS) + + +def format_llm_error(error: Exception, debug: bool = False) -> str | None: + """Return a safe user-facing LLM error message when the error is recognized.""" + if not is_llm_auth_error(error): + if not is_llm_token_budget_error(error): + return None + if debug: + return f"{LLM_TOKEN_BUDGET_ERROR_MESSAGE}\n\nDebug details: {error}" + return LLM_TOKEN_BUDGET_ERROR_MESSAGE + if debug: + return f"{LLM_AUTH_ERROR_MESSAGE}\n\nDebug details: {error}" + + return LLM_AUTH_ERROR_MESSAGE + + +def sanitize_answer_text(answer: str) -> str: + """Remove unsupported placeholder citations before showing an answer.""" + return str(answer or "").replace( + "[Source: None]", + "The retrieved evidence does not support this claim.", + ) + + def _get_provider_name(): """Return which LLM provider is active.""" - if GROQ_API_KEY: + groq_api_key, google_api_key = get_api_keys() + if groq_api_key: return f"Groq ({GROQ_MODEL})" - if GOOGLE_API_KEY: + if google_api_key: return f"Gemini ({GEMINI_MODEL})" return "Not configured" def is_llm_provider_configured() -> bool: """Return whether any supported LLM provider is configured.""" - return bool(GROQ_API_KEY or GOOGLE_API_KEY) + groq_api_key, google_api_key = get_api_keys() + return bool(groq_api_key or google_api_key) def get_llm(): - """Get LLM with automatic provider selection. Groq preferred (faster).""" - if GROQ_API_KEY: + """Get the active LLM instance.""" + groq_api_key, google_api_key = get_api_keys() + if groq_api_key: from langchain_groq import ChatGroq + return ChatGroq( model=GROQ_MODEL, - api_key=GROQ_API_KEY, + api_key=groq_api_key, temperature=LLM_TEMPERATURE, max_tokens=LLM_MAX_TOKENS, ) - if GOOGLE_API_KEY: + if google_api_key: from langchain_google_genai import ChatGoogleGenerativeAI + return ChatGoogleGenerativeAI( model=GEMINI_MODEL, - google_api_key=GOOGLE_API_KEY, + google_api_key=google_api_key, temperature=LLM_TEMPERATURE, max_output_tokens=LLM_MAX_TOKENS, ) - raise LLMProviderNotConfiguredError( - "No LLM provider is configured. Add GROQ_API_KEY or GOOGLE_API_KEY in your .env file locally, " - "or in Streamlit secrets on deployment." - ) + raise LLMProviderNotConfiguredError(LLM_CONFIG_WARNING) def create_agent(): @@ -55,20 +127,132 @@ def create_agent(): return get_llm() -def prepare_query(user_query: str, chat_history: list = None): - """Prepare a feature-flagged query payload and visible tool trace.""" - payload = prepare_query_payload(user_query, chat_history) - return payload, payload["reasoning_trace"] +def _payload_to_reasoning_trace(payload: dict) -> list[dict]: + traces = [] + for step in payload.get("traces", []): + tool_name = step.get("tool") + if tool_name == "search_knowledge_base": + traces.append( + { + "tool": tool_name, + "input": step.get("input", {"query": payload.get("user_query", "")}), + "input_summary": step.get("input_summary", ""), + "output_summary": step.get("output_summary", ""), + } + ) + grouped_sources = {} + for document in step.get("documents", []): + dedupe_key = ( + document.get("source"), + document.get("page"), + ) + if dedupe_key in grouped_sources: + grouped_sources[dedupe_key]["match_count"] += 1 + grouped_sources[dedupe_key]["chunk_ids"].append(document.get("chunk_id", "")) + if float(document.get("score", 0) or 0) > float(grouped_sources[dedupe_key]["score"] or 0): + grouped_sources[dedupe_key]["score"] = document.get("score", "0.00") + continue + grouped_sources[dedupe_key] = { + "source": document.get("source", "Unknown"), + "page": int(document.get("page", 1) or 1), + "score": document.get("score", "0.00"), + "chunk_id": document.get("chunk_id", ""), + "chunk_ids": [document.get("chunk_id", "")], + "document_origin": document.get("document_origin", "sample"), + "match_count": 1, + } + for source_card in grouped_sources.values(): + match_suffix = ( + f" · {source_card['match_count']} matching chunks" + if source_card["match_count"] > 1 + else "" + ) + traces.append( + { + "tool_response": f"{source_card['source']} (Page {source_card['page']})", + "snippet": f"retrieval score {source_card['score']}{match_suffix}", + "source": source_card["source"], + "page": source_card["page"], + "score": source_card["score"], + "chunk_id": source_card["chunk_id"], + "chunk_ids": source_card["chunk_ids"], + "document_origin": source_card["document_origin"], + "match_count": source_card["match_count"], + } + ) + elif tool_name and tool_name not in {"health_check"}: + traces.append( + { + "tool": tool_name, + "input": {"query": payload.get("user_query", "")}, + "input_summary": step.get("input_summary", ""), + "output_summary": step.get("output_summary", step.get("verification_status", "")), + } + ) + return traces or payload.get("traces", []) -def query_agent_stream(llm, prompt): - """Stream either the legacy simple path or the new agentic path.""" - yield from stream_query_response(llm, prompt) +def prepare_query( + user_query: str, + chat_history: list | None = None, + retrieval_scope: str = "all", + vectorstore_stats: dict | None = None, +): + """Prepare the graph payload and convert traces for the existing UI.""" + payload = prepare_query_payload( + user_query=user_query, + chat_history=chat_history, + retrieval_scope=retrieval_scope, + vectorstore_stats=vectorstore_stats, + ) + reasoning_trace = _payload_to_reasoning_trace(payload) + payload["_ui_reasoning_trace"] = reasoning_trace + return payload, reasoning_trace + +def query_agent_stream(llm, prompt_or_payload): + """Stream query output using the graph payload or a legacy prompt string.""" + try: + if isinstance(prompt_or_payload, dict): + yield from stream_query_response(llm, prompt_or_payload) + return -def query_agent(llm, user_query: str, thread_id: str = "default", chat_history: list = None): - """Non-streaming query wrapper.""" - return run_query(llm, user_query, chat_history) + payload = { + "prompt": str(prompt_or_payload), + "response_mode": "llm", + } + yield from stream_query_response(llm, payload) + except Exception as exc: + safe_message = format_llm_error(exc, debug=is_debug_mode_enabled()) + if safe_message: + yield safe_message + return + raise + + +def query_agent(llm, user_query: str, thread_id: str = "default", chat_history: list | None = None, retrieval_scope: str = "all"): + """Non-streaming query retained for backward compatibility.""" + try: + result = run_query( + llm, + user_query, + thread_id=thread_id, + chat_history=chat_history, + retrieval_scope=retrieval_scope, + ) + except Exception as exc: + safe_message = format_llm_error(exc, debug=is_debug_mode_enabled()) + if safe_message: + return { + "answer": safe_message, + "reasoning_trace": [], + "all_messages": [], + "payload": {}, + } + raise + result["answer"] = sanitize_answer_text(result.get("answer", "")) + result["reasoning_trace"] = _payload_to_reasoning_trace(result.get("payload", {})) + return result if __name__ == "__main__": diff --git a/app.py b/app.py index 38fe095..09aff37 100644 --- a/app.py +++ b/app.py @@ -1,48 +1,71 @@ -import json +"""Streamlit dashboard for the Internal RFP Analyst app.""" + +import gc import time -from pathlib import Path import streamlit as st -# Must be first Streamlit command -st.set_page_config( - page_title="Internal RFP Analyst", - page_icon="?", - layout="wide", - initial_sidebar_state="expanded", -) - from agent import ( + KB_NOT_READY_MESSAGE, + LLM_CONFIG_WARNING, + NO_SCOPE_DOCUMENTS_MESSAGE, _get_provider_name, create_agent, - is_llm_provider_configured, + format_llm_error, + is_debug_mode_enabled, prepare_query, query_agent_stream, ) -from config import APP_TITLE, APP_SUBTITLE, DATA_DIR, SAMPLE_QUESTIONS -from rag_engine import get_vectorstore_stats, ingest_documents +from config import ( + APP_SUBTITLE, + APP_TITLE, + MAX_UPLOAD_SIZE_MB, + OFFLINE_SMOKE_EVAL_RESULTS_PATH, + REAL_KB_EVAL_RESULTS_PATH, + SAMPLE_DOCS_DIR, + SAMPLE_QUESTIONS, + UPLOADS_DIR, + VECTORSTORE_DIR, + get_api_keys, +) +from document_generator import generate_all_documents +from rag_engine import ( + VECTORSTORE_LOCKED_MESSAGE, + get_vectorstore_stats, + ingest_documents, +) +from rfp_analyst.evals import load_eval_snapshot from rfp_analyst.exceptions import ( IngestionError, KnowledgeBaseNotReadyError, LLMProviderNotConfiguredError, - RFPAnalystError, + NoDocumentsFoundError, + RetrievalError, UnsupportedFileError, ) from rfp_analyst.health import get_app_health -from rfp_analyst.ui.helpers import format_latency_display, get_chat_avatar -from rfp_analyst.uploads import validate_uploaded_pdf - -APP_ROOT = Path(__file__).resolve().parent -EVAL_RESULTS_PATH = APP_ROOT / "evals" / "results.json" -LLM_CONFIGURATION_WARNING = ( - "No LLM provider is configured. Add GROQ_API_KEY or GOOGLE_API_KEY in your .env file locally, " - "or in Streamlit secrets on deployment." +from rfp_analyst.ui import get_chat_avatar +from rfp_analyst.uploads import persist_uploaded_pdf + +SCOPE_LABELS = { + "upload": "Uploaded documents only", + "sample": "Sample documents only", + "all": "All documents", +} +LABEL_TO_SCOPE = {label: scope for scope, label in SCOPE_LABELS.items()} +PENDING_UPLOADS_MESSAGE = ( + "Uploaded files are pending indexing. Click Ingest Documents before asking about them." ) -KNOWLEDGE_BASE_NOT_READY_MESSAGE = ( - "Knowledge base is not ready. Generate or upload PDFs and click Ingest Documents." + +st.set_page_config( + page_title="Internal RFP Analyst", + page_icon=":mag:", + layout="wide", + initial_sidebar_state="expanded", ) -st.markdown(""" +st.markdown( + """ -""", unsafe_allow_html=True) +""", + unsafe_allow_html=True, +) if "messages" not in st.session_state: st.session_state.messages = [] @@ -143,151 +150,411 @@ st.session_state.agent = None if "pending_query" not in st.session_state: st.session_state.pending_query = None +if "scope" not in st.session_state: + st.session_state.scope = "all" +if "upload_notice" not in st.session_state: + st.session_state.upload_notice = "" +if "ingestion_in_progress" not in st.session_state: + st.session_state.ingestion_in_progress = False +if "last_ingestion_error" not in st.session_state: + st.session_state.last_ingestion_error = "" +if "pending_uploads" not in st.session_state: + st.session_state.pending_uploads = False +if "setup_attempted" not in st.session_state: + st.session_state.setup_attempted = False +if "status_message" not in st.session_state: + st.session_state.status_message = "" + + +def clear_runtime_objects() -> None: + """Release cached app objects before rebuilding the vector store.""" + st.session_state.agent = None + for key in ("retriever", "vectorstore", "chroma_client"): + st.session_state.pop(key, None) + for cache_name in ("cache_resource", "cache_data"): + cache = getattr(st, cache_name, None) + if cache and hasattr(cache, "clear"): + cache.clear() + gc.collect() + + +def build_health_snapshot() -> tuple[dict, dict]: + """Return vector store stats and central health checks.""" + groq_api_key, google_api_key = get_api_keys() + stats = get_vectorstore_stats( + sample_dir=SAMPLE_DOCS_DIR, + uploads_dir=UPLOADS_DIR, + persist_dir=VECTORSTORE_DIR, + ) + health = get_app_health( + vectorstore_stats=stats, + data_dir=SAMPLE_DOCS_DIR, + vectorstore_dir=VECTORSTORE_DIR, + uploads_dir=UPLOADS_DIR, + groq_api_key=groq_api_key, + google_api_key=google_api_key, + ) + st.session_state.pending_uploads = health.get("pending_upload_count", 0) > 0 + return stats, health + + +def available_scope_options(health: dict) -> list[str]: + """Return the scope labels that make sense for the current indexed KB.""" + options = [] + if health.get("indexed_upload_document_count", 0) > 0: + options.append(SCOPE_LABELS["upload"]) + if health.get("indexed_sample_document_count", 0) > 0: + options.append(SCOPE_LABELS["sample"]) + if health.get("indexed_document_count", 0) > 0: + options.append(SCOPE_LABELS["all"]) + return options or [SCOPE_LABELS["all"]] + + +def resolve_scope(health: dict) -> str: + """Keep the current scope valid and default uploads when available.""" + available_scopes = [LABEL_TO_SCOPE[label] for label in available_scope_options(health)] + current_scope = st.session_state.scope + if current_scope in available_scopes: + return current_scope + if health.get("indexed_upload_document_count", 0) > 0: + st.session_state.scope = "upload" + elif "all" in available_scopes: + st.session_state.scope = "all" + else: + st.session_state.scope = available_scopes[0] + return st.session_state.scope -def _load_evaluation_snapshot(): - if not EVAL_RESULTS_PATH.exists(): - return None - try: - return json.loads(EVAL_RESULTS_PATH.read_text(encoding="utf-8")) - except Exception: - return None - +def render_reasoning(reasoning_trace, show_reasoning: bool): + """Render source trace details after a response.""" + if not reasoning_trace or not show_reasoning: + return -def _render_reasoning_trace(reasoning_trace: list): with st.expander("Sources Used", expanded=False): for step in reasoning_trace: if "tool" in step: - details = ", ".join( - f"{key}: {value}" for key, value in step.get("input", {}).items() - ) + tool_label = step.get("tool", "tool") + input_summary = step.get("input_summary") or step.get("input", {}).get("query", "") + output_summary = step.get("output_summary", "") + output_block = f"
{output_summary}" if output_summary else "" st.markdown( - f'
Tool: {step["tool"]}
{details}
', + ( + '
' + f"{tool_label}
" + f"{input_summary}" + f"{output_block}" + "
" + ), unsafe_allow_html=True, ) elif "tool_response" in step: st.markdown( - f'
{step["tool_response"]}
{step["snippet"][:150]}...
', + ( + '
' + f'{step["tool_response"]}
' + f'{step["snippet"][:150]}...' + "
" + ), unsafe_allow_html=True, ) -def _handle_uploads(uploaded_files): - if not uploaded_files: - return +def add_assistant_message(message: str, reasoning=None): + """Persist an assistant message to session state.""" + st.session_state.messages.append( + { + "role": "assistant", + "content": message, + "reasoning": reasoning or [], + } + ) + + +def show_generation_warning(message: str): + """Display a friendly warning and preserve it in chat history.""" + st.warning(message) + add_assistant_message(message) - DATA_DIR.mkdir(parents=True, exist_ok=True) - uploaded_count = 0 + +def handle_uploaded_files(uploaded_files) -> None: + """Validate and save uploaded PDF files without crashing the app.""" + saved_files = [] for uploaded_file in uploaded_files: try: - safe_name = validate_uploaded_pdf(uploaded_file) - save_path = DATA_DIR / safe_name - with open(save_path, "wb") as handle: - handle.write(uploaded_file.getbuffer()) - uploaded_count += 1 - except UnsupportedFileError as error: - st.warning(str(error)) - if uploaded_count: - st.success(f"Uploaded {uploaded_count} file(s). Click 'Ingest Documents' to index.") - - -provider_name = _get_provider_name() -health = get_app_health(provider_name) -llm_configured = health["llm_provider_configured"] -kb_ready = health["vectorstore_ready"] + save_path = persist_uploaded_pdf(uploaded_file, uploads_dir=UPLOADS_DIR) + saved_files.append(save_path.name) + except UnsupportedFileError as exc: + st.warning(str(exc)) + + if saved_files: + st.session_state.pending_uploads = True + st.session_state.setup_attempted = True + st.session_state.upload_notice = ( + f"Uploaded {len(saved_files)} file(s). Click Ingest Documents to index them." + ) + st.session_state.last_ingestion_error = "" + clear_runtime_objects() + + +def finish_ingestion(success: bool, message: str = "") -> None: + """Reset ingestion flags after a manual ingestion attempt.""" + st.session_state.ingestion_in_progress = False + if success: + st.session_state.last_ingestion_error = "" + st.session_state.pending_uploads = False + st.session_state.upload_notice = "" + st.session_state.status_message = message + else: + st.session_state.last_ingestion_error = message + st.session_state.status_message = "" -with st.sidebar: - st.markdown("### Knowledge Base") - st.markdown("---") - stats = get_vectorstore_stats() +def run_manual_ingestion() -> bool: + """Run ingestion only when the user explicitly clicks the button.""" + st.session_state.ingestion_in_progress = True + st.session_state.last_ingestion_error = "" + st.session_state.setup_attempted = True + clear_runtime_objects() - if stats["status"] == "ready": - st.markdown('Ready', unsafe_allow_html=True) - col1, col2 = st.columns(2) - with col1: - st.markdown( - f'
{stats["total_documents"]}
' - f'
Documents
', - unsafe_allow_html=True, - ) - with col2: - st.markdown( - f'
{stats["total_chunks"]}
' - f'
Chunks
', - unsafe_allow_html=True, + try: + with st.spinner("Processing documents..."): + ingest_documents( + sample_dir=SAMPLE_DOCS_DIR, + uploads_dir=UPLOADS_DIR, + persist_dir=VECTORSTORE_DIR, ) + except (NoDocumentsFoundError, KnowledgeBaseNotReadyError) as exc: + finish_ingestion(False, str(exc)) + return False + except IngestionError as exc: + finish_ingestion(False, str(exc)) + return False + except Exception as exc: + error_text = str(exc) + if "locked" in error_text.lower() or "winerror 5" in error_text.lower(): + finish_ingestion(False, VECTORSTORE_LOCKED_MESSAGE) + else: + finish_ingestion(False, f"Document ingestion failed: {exc}") + return False + + finish_ingestion(True, "Documents ingested successfully.") + return True + + +def render_eval_snapshot(): + """Show offline smoke and real KB evaluation snapshots separately.""" + st.markdown("### Evaluation Snapshot") + + st.caption("Offline Smoke Evaluation") + offline_snapshot = load_eval_snapshot( + OFFLINE_SMOKE_EVAL_RESULTS_PATH, + missing_message="No offline smoke evaluation run found", + ) + if offline_snapshot["status"] != "ready": + st.caption(offline_snapshot["message"]) else: - st.markdown('Not Initialized', unsafe_allow_html=True) - st.info(KNOWLEDGE_BASE_NOT_READY_MESSAGE) + payload = offline_snapshot["payload"] + st.caption(f"Latency: {offline_snapshot['latency_display']}") + for key in ("score", "pass_rate", "notes"): + if key in payload: + st.caption(f"{key.replace('_', ' ').title()}: {payload[key]}") + + st.caption("Real KB Evaluation") + real_snapshot = load_eval_snapshot( + REAL_KB_EVAL_RESULTS_PATH, + missing_message="No real KB evaluation run found", + ) + if real_snapshot["status"] != "ready": + st.caption(real_snapshot["message"]) + return - st.markdown("---") - st.markdown("### LLM Provider") - st.markdown(f'
{provider_name}
', unsafe_allow_html=True) - if not llm_configured: - st.warning(LLM_CONFIGURATION_WARNING) + payload = real_snapshot["payload"] + st.caption(f"Latency: {real_snapshot['latency_display']}") + for key in ("score", "pass_rate", "mode", "notes"): + if key in payload: + st.caption(f"{key.replace('_', ' ').title()}: {payload[key]}") - st.markdown("---") - st.markdown("### App Health") - st.caption(f"Vectorstore ready: {'Yes' if health['vectorstore_ready'] else 'No'}") - st.caption(f"Required directories ready: {'Yes' if all(item['exists'] for item in health['required_directories'].values()) else 'No'}") - evaluation_snapshot = _load_evaluation_snapshot() +def process_query(user_query: str, show_reasoning: bool, retrieval_scope: str): + """Process a user query with safe health checks and scoped streaming output.""" + stats, health = build_health_snapshot() + if st.session_state.ingestion_in_progress: + show_generation_warning("Document ingestion is in progress. Please wait for it to finish.") + return + if not health["llm_provider_configured"]: + show_generation_warning(LLM_CONFIG_WARNING) + return + if not health["vectorstore_ready"]: + show_generation_warning(KB_NOT_READY_MESSAGE) + return + if st.session_state.pending_uploads or health.get("pending_upload_count", 0) > 0: + show_generation_warning(PENDING_UPLOADS_MESSAGE) + return + if retrieval_scope in {"sample", "upload"} and int(stats.get("scope_chunk_counts", {}).get(retrieval_scope, 0) or 0) == 0: + show_generation_warning(NO_SCOPE_DOCUMENTS_MESSAGE) + return + + try: + prompt, reasoning_trace = prepare_query( + user_query, + chat_history=st.session_state.messages, + retrieval_scope=retrieval_scope, + vectorstore_stats=stats, + ) + + if st.session_state.agent is None: + st.session_state.agent = create_agent() + + with st.chat_message("assistant", avatar=get_chat_avatar("assistant")): + full_response = st.write_stream( + query_agent_stream(st.session_state.agent, prompt) + ) + render_reasoning(reasoning_trace, show_reasoning) + + add_assistant_message(full_response, reasoning_trace) + except (LLMProviderNotConfiguredError, KnowledgeBaseNotReadyError) as exc: + show_generation_warning(str(exc)) + except RetrievalError as exc: + show_generation_warning(str(exc)) + except Exception as exc: + safe_llm_error = format_llm_error(exc, debug=is_debug_mode_enabled()) + if safe_llm_error: + show_generation_warning(safe_llm_error) + return + + error_text = str(exc) + if "429" in error_text or "RESOURCE_EXHAUSTED" in error_text or "quota" in error_text.lower(): + show_generation_warning( + "Rate limit reached. Please wait a moment and try again. Consider adding a GROQ_API_KEY for faster, more reliable responses." + ) + else: + show_generation_warning(f"Request failed cleanly: {error_text}") + + +SAMPLE_DOCS_DIR.mkdir(parents=True, exist_ok=True) +UPLOADS_DIR.mkdir(parents=True, exist_ok=True) +stats, health = build_health_snapshot() +current_scope = resolve_scope(health) +interaction_locked = st.session_state.ingestion_in_progress +uploads_pending = st.session_state.pending_uploads or health.get("pending_upload_count", 0) > 0 +chat_disabled = interaction_locked or not (health["llm_provider_configured"] and health["vectorstore_ready"]) or uploads_pending +scope_options = available_scope_options(health) +current_scope_label = SCOPE_LABELS[current_scope] if SCOPE_LABELS[current_scope] in scope_options else scope_options[0] + +with st.sidebar: + st.markdown("### Knowledge Base") st.markdown("---") - st.markdown("### Evaluation Snapshot") - if evaluation_snapshot: - metrics = evaluation_snapshot.get("metrics", {}) + + if health["vectorstore_ready"]: st.markdown( - ( - '
' - f"Pass Rate: {evaluation_snapshot.get('passed_questions', 0)}/{evaluation_snapshot.get('total_questions', 0)}
" - f"Retrieval Hit Rate: {metrics.get('retrieval_hit_rate', 0):.2f}
" - f"Citation Coverage: {metrics.get('citation_coverage', 0):.2f}
" - f"Grounded Answer Score: {metrics.get('grounded_answer_score', 0):.2f}
" - f"Average Latency: {format_latency_display(metrics)}
" - f"Tool Call Count: {metrics.get('tool_call_count', 0):.2f}
" - f"Failure Rate: {metrics.get('failure_rate', 0):.2f}" - '
' - ), + 'Ready', unsafe_allow_html=True, ) else: - st.info("No evaluation run found") + st.markdown( + 'Not ready', + unsafe_allow_html=True, + ) + st.caption(KB_NOT_READY_MESSAGE) + + col1, col2 = st.columns(2) + with col1: + st.markdown( + f'
{health["document_count"]}
Documents
', + unsafe_allow_html=True, + ) + with col2: + st.markdown( + f'
{health["chunk_count"]}
Chunks
', + unsafe_allow_html=True, + ) + + if st.session_state.status_message: + st.success(st.session_state.status_message) + if st.session_state.upload_notice: + st.success(st.session_state.upload_notice) + if st.session_state.last_ingestion_error: + st.warning(st.session_state.last_ingestion_error) + if uploads_pending: + st.warning(PENDING_UPLOADS_MESSAGE) + + st.markdown("---") + st.markdown("### Document Scope") + selected_scope_label = st.selectbox( + "Document scope", + options=scope_options, + index=scope_options.index(current_scope_label), + label_visibility="collapsed", + disabled=interaction_locked, + ) + st.session_state.scope = LABEL_TO_SCOPE[selected_scope_label] + + st.markdown("---") + st.markdown("### LLM Provider") + st.markdown( + f'
{_get_provider_name()}
', + unsafe_allow_html=True, + ) + if not health["llm_provider_configured"]: + st.warning( + "Add GROQ_API_KEY or GOOGLE_API_KEY in .env locally, or in Streamlit secrets on deployment, to enable chat answers." + ) + + st.markdown("---") + st.markdown("### App Health") + st.caption(f"Vectorstore ready: {'Yes' if health['vectorstore_ready'] else 'No'}") + st.caption(f"Sample docs available: {health['sample_document_count']}") + st.caption(f"Uploaded docs available: {health['upload_document_count']}") + st.caption(f"Indexed uploaded docs: {health['indexed_upload_document_count']}") + st.caption( + f"Provider configured: {'Yes' if health['llm_provider_configured'] else 'No'}" + ) + for directory_name, exists in health["required_directories"].items(): + st.caption(f"{directory_name.title()} dir: {'OK' if exists else 'Missing'}") st.markdown("---") st.markdown("### Document Ingestion") + if st.button("Generate Sample PDFs", use_container_width=True, disabled=interaction_locked): + with st.spinner("Generating sample PDFs..."): + generate_all_documents() + st.session_state.setup_attempted = True + st.session_state.status_message = "Sample PDFs generated." + st.session_state.last_ingestion_error = "" + time.sleep(0.2) + st.rerun() - if st.button("Ingest Documents", use_container_width=True, type="primary"): - with st.spinner("Processing documents..."): - try: - ingest_documents() - st.success("Documents ingested successfully.") - st.session_state.agent = None - time.sleep(1) - st.rerun() - except (IngestionError, RFPAnalystError) as error: - st.warning(str(error)) - except Exception as error: - st.warning(f"Ingestion failed: {error}") + if st.button( + "Ingest Documents", + use_container_width=True, + type="primary", + disabled=interaction_locked, + ): + if run_manual_ingestion(): + time.sleep(0.2) + st.rerun() st.markdown("---") st.markdown("### Upload Custom PDFs") uploaded_files = st.file_uploader( - "Drop PDFs here", + f"Upload PDFs up to {MAX_UPLOAD_SIZE_MB} MB each", type=["pdf"], accept_multiple_files=True, label_visibility="collapsed", + disabled=interaction_locked, ) - _handle_uploads(uploaded_files) + if uploaded_files and not interaction_locked: + handle_uploaded_files(uploaded_files) st.markdown("---") - st.markdown("### Settings") - show_reasoning = st.toggle("Show Source Traces", value=True) + show_reasoning = st.toggle("Show Source Traces", value=True, disabled=interaction_locked) st.markdown("---") - if st.button("Clear Chat History", use_container_width=True): + render_eval_snapshot() + + st.markdown("---") + if st.button("Clear Chat History", use_container_width=True, disabled=interaction_locked): st.session_state.messages = [] - st.session_state.agent = None + clear_runtime_objects() st.rerun() st.markdown( @@ -300,21 +567,28 @@ def _handle_uploads(uploaded_files): unsafe_allow_html=True, ) -if not kb_ready: - st.info(KNOWLEDGE_BASE_NOT_READY_MESSAGE) -if not llm_configured: - st.warning(LLM_CONFIGURATION_WARNING) +if st.session_state.last_ingestion_error: + st.warning(st.session_state.last_ingestion_error) + +if not health["llm_provider_configured"]: + st.warning( + "Chat is disabled until you add GROQ_API_KEY or GOOGLE_API_KEY in .env locally, or in Streamlit secrets on deployment. Document upload and ingestion still work without an LLM." + ) +if uploads_pending: + st.warning(PENDING_UPLOADS_MESSAGE) +if not health["vectorstore_ready"]: + st.info(KB_NOT_READY_MESSAGE) if not st.session_state.messages: - st.markdown("#### Try asking:") - cols = st.columns(2) + st.markdown("#### Try asking") + columns = st.columns(2) for index, question in enumerate(SAMPLE_QUESTIONS[:6]): - with cols[index % 2]: + with columns[index % 2]: if st.button( question, key=f"sample_{index}", use_container_width=True, - disabled=(not llm_configured or not kb_ready), + disabled=chat_disabled, ): st.session_state.pending_query = question st.session_state.messages.append({"role": "user", "content": question}) @@ -323,57 +597,20 @@ def _handle_uploads(uploaded_files): for message in st.session_state.messages: with st.chat_message(message["role"], avatar=get_chat_avatar(message["role"])): st.markdown(message["content"]) - if message["role"] == "assistant" and message.get("reasoning") and show_reasoning: - _render_reasoning_trace(message["reasoning"]) - - -def _process_query(user_query: str): - try: - if not llm_configured: - raise LLMProviderNotConfiguredError(LLM_CONFIGURATION_WARNING) - if not kb_ready: - raise KnowledgeBaseNotReadyError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) - - if st.session_state.agent is None: - st.session_state.agent = create_agent() - - prompt, reasoning_trace = prepare_query(user_query, chat_history=st.session_state.messages) - - with st.chat_message("assistant", avatar=get_chat_avatar("assistant")): - full_response = st.write_stream(query_agent_stream(st.session_state.agent, prompt)) - if reasoning_trace and show_reasoning: - _render_reasoning_trace(reasoning_trace) - - st.session_state.messages.append( - {"role": "assistant", "content": full_response, "reasoning": reasoning_trace} - ) - except (LLMProviderNotConfiguredError, KnowledgeBaseNotReadyError, RFPAnalystError) as error: - friendly_message = str(error) - st.warning(friendly_message) - st.session_state.messages.append({"role": "assistant", "content": friendly_message, "reasoning": []}) - except Exception as error: - error_text = str(error) - if "429" in error_text or "RESOURCE_EXHAUSTED" in error_text or "quota" in error_text.lower(): - friendly_message = ( - "Rate limit reached. Please wait a moment and try again. " - "Consider adding a GROQ_API_KEY for faster, more reliable responses." - ) - else: - friendly_message = f"Error: {error_text}" - st.warning(friendly_message) - st.session_state.messages.append({"role": "assistant", "content": friendly_message, "reasoning": []}) - + if message["role"] == "assistant": + render_reasoning(message.get("reasoning", []), show_reasoning) pending_query = st.session_state.pending_query -if pending_query: +if pending_query and not interaction_locked: st.session_state.pending_query = None - _process_query(pending_query) + process_query(pending_query, show_reasoning, st.session_state.scope) -if user_input := st.chat_input( +user_input = st.chat_input( "Ask about past projects, tech stacks, proposals...", - disabled=(not llm_configured or not kb_ready), -): + disabled=chat_disabled, +) +if user_input: st.session_state.messages.append({"role": "user", "content": user_input}) with st.chat_message("user", avatar=get_chat_avatar("user")): st.markdown(user_input) - _process_query(user_input) + process_query(user_input, show_reasoning, st.session_state.scope) diff --git a/config.py b/config.py index d9e8d09..75baff5 100644 --- a/config.py +++ b/config.py @@ -1,43 +1,110 @@ -""" -Central Configuration for the Internal RFP Analyst Agent. -All paths, model settings, and pipeline parameters are defined here. -""" +"""Central configuration for the Internal RFP Analyst app.""" import os +import sys from pathlib import Path -from dotenv import load_dotenv - -load_dotenv() +from dotenv import dotenv_values, load_dotenv BASE_DIR = Path(__file__).parent +ENV_PATH = BASE_DIR / ".env" + + +def _is_pytest_runtime() -> bool: + return "PYTEST_CURRENT_TEST" in os.environ or "pytest" in sys.modules + + +def _allow_env_file_values() -> bool: + if os.getenv("RFP_ANALYST_ENABLE_DOTENV_IN_TESTS", "").lower() in {"1", "true", "yes"}: + return True + return not _is_pytest_runtime() + + +if _allow_env_file_values(): + load_dotenv(ENV_PATH) + DATA_DIR = BASE_DIR / "data" / "documents" +SAMPLE_DOCS_DIR = DATA_DIR +UPLOADS_DIR = BASE_DIR / "data" / "uploads" VECTORSTORE_DIR = BASE_DIR / "vectorstore" ASSETS_DIR = BASE_DIR / "assets" +EVALS_DIR = BASE_DIR / "evals" +OFFLINE_SMOKE_EVAL_RESULTS_PATH = EVALS_DIR / "offline_smoke_results.json" +REAL_KB_EVAL_RESULTS_PATH = EVALS_DIR / "real_kb_results.json" +EVAL_RESULTS_PATH = OFFLINE_SMOKE_EVAL_RESULTS_PATH + + +def _get_streamlit_secret(name: str) -> str: + try: + import streamlit as st -try: - import streamlit as st - GROQ_API_KEY = st.secrets.get("GROQ_API_KEY", os.getenv("GROQ_API_KEY", "")) - GOOGLE_API_KEY = st.secrets.get("GOOGLE_API_KEY", os.getenv("GOOGLE_API_KEY", "")) -except Exception: - GROQ_API_KEY = os.getenv("GROQ_API_KEY", "") - GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY", "") + return str(st.secrets.get(name, "") or "").strip() + except Exception: + return "" + + +def _get_env_file_value(name: str) -> str: + if not _allow_env_file_values(): + return "" + values = dotenv_values(ENV_PATH) + return str(values.get(name, "") or "").strip() + + +def _get_int_setting(name: str, default: int) -> int: + raw_value = os.getenv(name, "") or _get_env_file_value(name) + if not raw_value: + return default + try: + return int(raw_value) + except ValueError: + return default + + +def get_api_keys() -> tuple[str, str]: + """Resolve API keys dynamically so Streamlit reruns pick up .env changes.""" + groq_api_key = ( + _get_streamlit_secret("GROQ_API_KEY") + or os.getenv("GROQ_API_KEY", "") + or _get_env_file_value("GROQ_API_KEY") + ) + google_api_key = ( + _get_streamlit_secret("GOOGLE_API_KEY") + or os.getenv("GOOGLE_API_KEY", "") + or _get_env_file_value("GOOGLE_API_KEY") + ) + return groq_api_key.strip(), google_api_key.strip() + + +GROQ_API_KEY, GOOGLE_API_KEY = get_api_keys() GROQ_MODEL = "llama-3.3-70b-versatile" GEMINI_MODEL = "gemini-2.0-flash" + LLM_TEMPERATURE = 0.3 LLM_MAX_TOKENS = 2048 + EMBEDDING_MODEL = "BAAI/bge-small-en-v1.5" CHUNK_SIZE = 512 CHUNK_OVERLAP = 50 COLLECTION_NAME = "rfp_kb_v2" RETRIEVAL_K = 6 -MAX_UPLOAD_FILE_SIZE_BYTES = 25 * 1024 * 1024 -MAX_UPLOAD_PAGE_COUNT = 250 -AGENT_MODE = os.getenv("AGENT_MODE", "agentic").strip().lower() or "agentic" +MIN_RELEVANCE_SCORE = float(os.getenv("MIN_RELEVANCE_SCORE", "0.50")) +MAX_PROMPT_TOKENS = _get_int_setting("MAX_PROMPT_TOKENS", 6500) +RFP_ANALYSIS_MAX_OUTPUT_TOKENS = _get_int_setting("RFP_ANALYSIS_MAX_OUTPUT_TOKENS", 1200) +MAX_CONTEXT_CHARS_PER_CHUNK = _get_int_setting("MAX_CONTEXT_CHARS_PER_CHUNK", 1000) +MAX_HISTORY_MESSAGES = _get_int_setting("MAX_HISTORY_MESSAGES", 3) +MAX_TARGET_CHUNKS = _get_int_setting("MAX_TARGET_CHUNKS", 4) +MAX_CASE_STUDIES = _get_int_setting("MAX_CASE_STUDIES", 3) +MAX_CHUNKS_PER_CASE_STUDY = _get_int_setting("MAX_CHUNKS_PER_CASE_STUDY", 2) +MAX_UPLOAD_SIZE_MB = _get_int_setting("MAX_UPLOAD_SIZE_MB", 25) +MAX_UPLOAD_FILE_SIZE_BYTES = MAX_UPLOAD_SIZE_MB * 1024 * 1024 +MAX_UPLOAD_PAGE_COUNT = _get_int_setting("MAX_UPLOAD_PAGE_COUNT", 250) +AGENT_MODE = ( + os.getenv("AGENT_MODE", "") or _get_env_file_value("AGENT_MODE") or "agentic" +).strip().lower() or "agentic" -AGENT_SYSTEM_PROMPT = """You are the **Internal RFP Analyst**, an AI-powered knowledge agent +AGENT_SYSTEM_PROMPT = """You are the Internal RFP Analyst, an AI-powered knowledge agent for a global fintech consulting firm. Your job is to help internal teams quickly find information from past proposals, project outlines, RFP responses, and case studies. @@ -46,13 +113,13 @@ 2. Cite your sources clearly using [Source: , Page ] format. 3. If you cannot find relevant information, say so honestly. 4. When comparing projects, present information in a structured table format. -5. Be concise but thorough - consultants are busy people. +5. Be concise but thorough because consultants are busy people. 6. If the user's question is ambiguous, ask a clarifying question before answering. """ APP_TITLE = "Internal RFP Analyst" -APP_SUBTITLE = "AI-Powered Knowledge Agent for Fintech Consulting" -APP_ICON = "?" +APP_SUBTITLE = "AI-powered knowledge agent for fintech consulting" +APP_ICON = "??" SAMPLE_QUESTIONS = [ "What tech stack did we use for the last banking audit?", "Which projects used Azure services?", diff --git a/docs/AGENTIC_RAG.md b/docs/AGENTIC_RAG.md new file mode 100644 index 0000000..e665dcf --- /dev/null +++ b/docs/AGENTIC_RAG.md @@ -0,0 +1,134 @@ +# Agentic RAG + +## Why This System Is Agentic + +This repository does not use a single retrieve-and-generate chain as its primary execution model. The canonical runtime classifies intent, carries structured state across graph nodes, executes deterministic tools conditionally, separates target and case-study corpora, compacts evidence before model generation, and performs post-generation verification with an optional bounded repair pass. + +That combination of routing, tool execution, scoped retrieval, and verification is what makes the system agentic in practice. + +## Actual Tools + +The runtime uses the following deterministic tools: + +- `search_knowledge_base` +- `extract_rfp_requirements` +- `find_relevant_case_studies` +- `compare_projects` +- `generate_proposal_outline` +- `verify_answer_grounding` + +These tools operate on retrieved evidence and produce structured outputs that later nodes consume. + +## Graph State + +Key graph state values: + +- user request and resolved query +- chat history and resolved conversational entities +- retrieval scope and retrieval function +- vectorstore health/stats +- planned tools +- retrieved documents +- tool outputs +- prompt budget metadata +- answer mode, answer text, and verification results +- visible traces + +## Intents and Conditional Execution + +Supported intents: + +- `search` +- `compare` +- `proposal` +- `rfp_analysis` +- `previous_sources` +- `ambiguous` + +Conditional behavior: + +- `previous_sources` returns direct source recall without retrieval +- `ambiguous` returns a clarification prompt +- `rfp_analysis` forces upload-scoped target retrieval and sample-scoped case-study retrieval +- `compare` executes the comparison tool +- `proposal` and `rfp_analysis` execute requirements, case studies, and proposal helpers + +## Target and Sample Corpus Separation + +Target/sample separation is enforced by: + +- separate directories for generated samples and uploaded documents +- `document_origin` metadata +- scoped Chroma filters +- separate retrieval stages in `rfp_analysis` +- prompt compaction that keeps uploaded target evidence and sample case-study evidence in distinct sections + +This prevents internal sample documents from being mistaken for uploaded target requirements. + +## Structured Tool Outputs + +The runtime keeps full internal tool outputs for verification, but the prompt builder uses compact representations: + +- requirement summaries and inferred gaps +- ranked case-study summaries with fit scores and citations +- compact project comparison rows +- six-section proposal-outline bullets + +That allows the graph to stay grounded without sending raw nested objects to the model. + +## Verification and Repair + +The answer is generated first, then checked by `verify_answer_grounding`. If unsupported claims or vague citations remain: + +1. `grounding_verifier` records the first verification result +2. `answer_repair` removes or qualifies unsupported claims once +3. `final_grounding_verifier` verifies the repaired answer + +The repair pass is intentionally bounded and never loops indefinitely. + +## Agent Traces + +The UI exposes safe trace summaries only: + +- tool name +- short input summary +- short output summary +- selected sources +- prompt-budget data +- grounding status + +No private chain-of-thought is emitted. + +## Implementation Mapping + +| Capability | Implementation | +| --- | --- | +| Intent classification | `src/rfp_analyst/agent/graph.py::classify_intent` and `_classify_query` | +| Tool planning | `src/rfp_analyst/agent/graph.py::plan_tools` | +| Scoped retrieval | `src/rfp_analyst/agent/graph.py::execute_retrieval`, `rag_engine.py::similarity_search`, `src/rfp_analyst/retrieval/vector_store.py::VectorStoreManager.similarity_search` | +| Requirement extraction | `src/rfp_analyst/tools/rfp_gap_analyzer.py::extract_rfp_requirements` | +| Case-study search | `src/rfp_analyst/tools/rfp_gap_analyzer.py::find_relevant_case_studies` | +| Comparison | `src/rfp_analyst/tools/compare_projects.py::compare_projects` | +| Proposal generation | `src/rfp_analyst/tools/proposal_writer.py::generate_proposal_outline` | +| Prompt budgeting | `src/rfp_analyst/agent/graph.py::synthesize_prompt`, `_estimate_tokens`, `_build_compact_prompt_sections`, `compact_tool_outputs_for_prompt` | +| Grounding | `src/rfp_analyst/tools/source_verifier.py::verify_answer_grounding` and `src/rfp_analyst/agent/graph.py::_verify_generated_answer` | +| Previous-source recall | `src/rfp_analyst/agent/graph.py::_previous_answer_sources` and `_format_previous_sources` | + +## How This Differs from Basic RAG + +Basic RAG usually: + +- retrieves one mixed context set +- prompts once +- returns the answer + +This system instead: + +- routes by intent +- resolves conversational references +- separates target and internal corpora +- executes deterministic tools +- scores case studies explicitly +- compacts prompts to fit model limits +- verifies claims after generation +- performs one bounded repair pass when needed diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..cd7b64b --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,249 @@ +# Architecture + +## Overview + +Internal RFP Analyst has a narrow runtime surface: + +- [app.py](../app.py) renders the Streamlit UI and owns session-state behavior. +- [agent.py](../agent.py) selects the provider, exposes UI-facing wrappers, and normalizes provider errors. +- [src/rfp_analyst/agent/graph.py](../src/rfp_analyst/agent/graph.py) is the canonical orchestration runtime. +- [rag_engine.py](../rag_engine.py) handles ingestion, vectorstore statistics, and retrieval. +- [src/rfp_analyst/tools](../src/rfp_analyst/tools) contains deterministic task-specific helpers. + +## Component Architecture + +```mermaid +flowchart TD + UI["Streamlit UI"] --> Adapter["agent.py wrappers"] + Adapter --> Graph["LangGraph runtime"] + Graph --> Search["Scoped retrieval"] + Graph --> Tools["Deterministic tools"] + Search --> Store["ChromaDB"] + Tools --> Prompt["Prompt compaction"] + Prompt --> LLM["Groq / Gemini"] + LLM --> Verify["Grounding verification"] + Verify --> Repair["Optional bounded repair"] + Repair --> Output["Answer + traces"] +``` + +### UI layer + +The UI owns: + +- upload persistence +- manual ingestion triggering +- document-scope selection +- evaluation snapshot rendering +- grouped source traces +- session-state safety around ingestion and pending uploads + +### Adapter layer + +[agent.py](../agent.py) is intentionally thin. It: + +- resolves provider configuration +- constructs the LLM client +- converts canonical graph traces into UI-friendly reasoning traces +- formats known provider errors into safe user-facing messages + +### Graph layer + +The graph runtime is the source of truth for agentic behavior. It compiles a real `StateGraph` when `langgraph` is available and otherwise runs the same node functions in deterministic sequence. + +## Ingestion Architecture + +```mermaid +sequenceDiagram + participant User + participant App as app.py + participant Engine as rag_engine.py + participant Loader as ingestion/loaders.py + participant Chunker as ingestion/chunking.py + participant Store as retrieval/vector_store.py + + User->>App: Click "Ingest Documents" + App->>Engine: ingest_documents(sample_dir, uploads_dir, persist_dir) + Engine->>Loader: load_pdf_sources(sample) + Engine->>Loader: load_pdf_sources(upload) + Loader-->>Engine: LoadedSource[] + Engine->>Engine: deduplicate loaded sources by file hash + Engine->>Chunker: chunk_loaded_sources(unique sources) + Chunker-->>Engine: Document chunks with deterministic metadata + Engine->>Store: upsert_documents(unique chunks) + Store-->>Engine: persisted Chroma collection + Engine->>Engine: atomically swap temp vectorstore + Engine-->>App: stats + ingestion report +``` + +### File loading + +[src/rfp_analyst/ingestion/loaders.py](../src/rfp_analyst/ingestion/loaders.py): + +- sanitizes unsafe filenames +- validates file size and page count +- loads PDFs with `PyMuPDFLoader` +- enriches page documents with `source_file`, `source_path`, `file_hash`, `page`, `document_type`, and `document_origin` + +### Chunking + +[src/rfp_analyst/ingestion/chunking.py](../src/rfp_analyst/ingestion/chunking.py): + +- splits documents with `RecursiveCharacterTextSplitter` +- computes deterministic `chunk_id` from origin, source namespace, file hash, page, chunk index, and content hash +- preserves metadata required for scoped retrieval and source validation + +### Deduplication + +`rag_engine.py` deduplicates: + +- whole files by SHA256, preferring uploaded copies over sample copies +- chunks by `chunk_id` before Chroma upsert +- already-indexed chunks by existing Chroma IDs + +### Atomic replacement and Windows safety + +Ingestion builds a temporary vectorstore and swaps it into place. If Windows file locking blocks replacement, the app surfaces a friendly lock message rather than retrying automatically on rerun. + +## Retrieval Architecture + +[rag_engine.py](../rag_engine.py) and [src/rfp_analyst/retrieval/vector_store.py](../src/rfp_analyst/retrieval/vector_store.py) together provide: + +- `VectorStoreManager.load` +- `VectorStoreManager.upsert_documents` +- `VectorStoreManager.get_retriever` +- `VectorStoreManager.similarity_search` +- `get_vectorstore_stats` +- `similarity_search` + +Scoped retrieval uses `document_origin` filters: + +- `sample` +- `upload` +- `all` + +The graph can also inspect `indexed_sample_document_count`, `indexed_upload_document_count`, `indexed_*_files`, `pending_upload_files`, and `scope_chunk_counts`. + +## LangGraph Execution Path + +```mermaid +flowchart TD + START --> health_check + health_check --> classify_intent + classify_intent --> plan_tools + plan_tools --> execute_retrieval + execute_retrieval --> execute_specialized_tool + execute_specialized_tool --> synthesize_prompt + synthesize_prompt --> evidence_availability_check + evidence_availability_check --> final_response + final_response --> END +``` + +### State fields + +The graph state includes: + +- `user_query` +- `chat_history` +- `vectorstore_stats` +- `retrieval_k` +- `retrieval_scope` +- `retrieval_fn` +- `traces` +- `kb_ready` +- `intent` +- `planned_tools` +- `retrieved_documents` +- `retrieval_context` +- `specialized_notes` +- `tool_outputs` +- `prompt` +- `answer` +- `response_mode` +- `resolved_query` +- `resolved_entities` +- `grounded` +- `graph_backend` +- `prompt_budget` + +### Node descriptions + +- `health_check`: loads stats and determines whether the KB is actually ready. +- `classify_intent`: resolves conversational references and classifies the request. +- `plan_tools`: selects which deterministic tools will run. +- `execute_retrieval`: performs scoped search and target-context retrieval for `rfp_analysis`. +- `execute_specialized_tool`: runs comparison, requirements, case-study scoring, and proposal helpers. +- `synthesize_prompt`: compacts evidence and records prompt-budget trace data. +- `evidence_availability_check`: blocks LLM generation when there is no grounded evidence. +- `final_response`: final graph-side handoff before model generation. + +## Graph Routing and Cross-Corpus Flow + +### Intents + +Implemented intents: + +- `search` +- `compare` +- `proposal` +- `rfp_analysis` +- `previous_sources` +- `ambiguous` + +### Cross-corpus RFP workflow + +`rfp_analysis` has a distinct execution path: + +1. derive a target-focused upload query +2. retrieve uploaded target evidence +3. extract requirements and inferred gaps from uploaded evidence +4. retrieve sample case-study evidence +5. score and rank case studies +6. compare fit +7. generate a six-section proposal outline +8. compact the prompt +9. generate and verify the final answer + +Sample evidence is never treated as target requirements in this path. + +## Prompt-Budget Architecture + +Prompt compaction in [src/rfp_analyst/agent/graph.py](../src/rfp_analyst/agent/graph.py): + +- excludes raw Python object serialization +- excludes `Document` object representations +- excludes nested retrieval results, `page_content` fields in tool outputs, and debug payloads +- limits uploaded target chunks, case-study count, case-study chunks, and history depth +- estimates token usage conservatively +- records a visible `prompt_budget` trace + +When over budget, the runtime trims lower-value sample evidence and older history before touching required target evidence or the user’s current request. + +## Grounding and Repair Flow + +```mermaid +sequenceDiagram + participant Graph as graph.py + participant LLM + participant Verify as source_verifier.py + + Graph->>LLM: compact prompt + LLM-->>Graph: generated answer + Graph->>Verify: verify_answer_grounding(answer, retrieved_documents) + alt unsupported or vague citations + Graph->>Graph: answer_repair + Graph->>Verify: verify_answer_grounding(repaired_answer, retrieved_documents) + end + Graph-->>UI: final answer + traces +``` + +The repair pass is bounded to one attempt. Verification is heuristic rather than formal proof. + +## Error Boundaries + +The main error boundaries are: + +- upload validation in [src/rfp_analyst/uploads.py](../src/rfp_analyst/uploads.py) +- ingestion exceptions in [rag_engine.py](../rag_engine.py) +- provider/configuration handling in [agent.py](../agent.py) +- retrieval and grounding fallbacks in [src/rfp_analyst/agent/graph.py](../src/rfp_analyst/agent/graph.py) +- UI-facing warning surfaces in [app.py](../app.py) diff --git a/docs/FILE_MAP.md b/docs/FILE_MAP.md new file mode 100644 index 0000000..cc7c16b --- /dev/null +++ b/docs/FILE_MAP.md @@ -0,0 +1,133 @@ +# File Map + +This map covers the meaningful tracked project files and explains how they fit into the live application. Generated PDFs, local vectorstores, caches, upload artifacts, and `*.egg-info` metadata are intentionally excluded. + +## Root Application Files + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `app.py` | Streamlit UI entry point | `build_health_snapshot`, `handle_uploaded_files`, `run_manual_ingestion`, `process_query`, `render_eval_snapshot` | user interaction, session state, uploaded files | Streamlit views, warnings, chat responses | `agent.py`, `rag_engine.py`, `src/rfp_analyst/*` | Yes | Primary application entry point | +| `agent.py` | Provider selection and UI-facing wrappers | `get_llm`, `prepare_query`, `query_agent_stream`, `query_agent`, `format_llm_error` | provider keys, payloads, user query | LLM client, stream output, reasoning traces | `config.py`, graph runtime | Yes | Compatibility adapter over canonical graph runtime | +| `rag_engine.py` | Ingestion/retrieval facade | `ingest_documents`, `similarity_search`, `get_vectorstore_stats`, `get_retriever` | sample dir, uploads dir, persist dir, query | ingestion report, results, stats | ingestion modules, retrieval manager | Yes | Core KB management module | +| `config.py` | Global configuration | `get_api_keys`, `_get_int_setting` | env vars, `.env`, Streamlit secrets | constants and resolved keys | `dotenv`, `streamlit` | Yes | Source of truth for paths, limits, budgets | +| `document_generator.py` | Synthetic sample PDF generator | `generate_all_documents`, `PDFDocument` | static document definitions | generated sample PDFs | `fpdf2`, `config.py` | Yes | Used by UI and real KB eval setup | +| `Makefile` | Convenience commands | `install`, `test`, `lint`, `run`, `generate-docs` | shell target | local command execution | Python tooling | No | Helpful locally, not part of runtime | +| `pyproject.toml` | Packaging and tool config | project metadata, pytest/ruff config | package metadata | editable install config | setuptools | Indirect | Required for `pip install -e .` | +| `requirements.txt` | Runtime dependencies | package list | pip install | installed dependencies | pip | Indirect | Used by local setup and CI | +| `.env.example` | Example local configuration | environment variable template | developer edits | local `.env` seed | `config.py` | Indirect | Safe template only | +| `.gitignore` | Repository hygiene | ignore rules | git status/add | ignored runtime files | git | Indirect | Protects secrets and local artifacts | +| `.python-version` | Python version hint | `3.11` | pyenv/asdf style tools | version hint | local tooling | Indirect | Matches project requirement | +| `.streamlit/config.toml` | Streamlit theme/server config | theme and upload-size settings | Streamlit startup | UI theme / upload limit | Streamlit | Yes | Affects app appearance and file-upload cap | + +## Agent Modules + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `src/rfp_analyst/agent/graph.py` | Canonical orchestration runtime | `compile_query_graph`, `prepare_query_payload`, `execute_retrieval`, `execute_specialized_tool`, `synthesize_prompt`, `run_query`, `stream_query_response` | query, scope, chat history, stats, retrieval fn | prompt, answer, traces, verification state | LangGraph, tools, `rag_engine.py` | Yes | Main source of agentic behavior | +| `src/rfp_analyst/agent/runtime.py` | Compatibility runtime helpers | `prepare_simple_query`, `prepare_agentic_query`, deprecated wrappers | user query, chat history | compatibility payloads | graph runtime, prompts | Partial | Supports simple/agentic compatibility modes | +| `src/rfp_analyst/agent/state.py` | Legacy-compatible state container | `AgentState` | query, trace state | dataclass instances | stdlib dataclasses | Partial | Used by compatibility graph wrapper | +| `src/rfp_analyst/agent/prompts.py` | Prompt helpers for compatibility mode | `build_simple_prompt`, `build_agentic_prompt`, `classify_query_intent` | user query, tool outputs, stats | prompt text | `config.py` | Partial | Graph runtime is canonical; this remains for compatibility | +| `src/rfp_analyst/agent/__init__.py` | Package exports | graph/runtime exports | imports | module API | local agent modules | Indirect | Packaging convenience | + +## Ingestion Modules + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `src/rfp_analyst/ingestion/loaders.py` | PDF validation and loading | `sanitize_filename`, `validate_pdf`, `load_pdf_sources`, `sha256_file` | PDF paths | `LoadedSource[]` | PyMuPDF, config, exceptions | Yes | Enforces size and page-count limits | +| `src/rfp_analyst/ingestion/chunking.py` | Deterministic chunk generation | `build_chunk_id`, `chunk_loaded_sources` | `LoadedSource[]` | LangChain `Document[]` chunks | text splitter, schemas | Yes | Adds deterministic metadata | +| `src/rfp_analyst/ingestion/pipeline.py` | Shared ingestion helpers | project-specific utilities | internal module inputs | helper outputs | local ingestion modules | Indirect | Support code, not the main facade | +| `src/rfp_analyst/ingestion/registry.py` | Duplicate-prevention registry helpers | registry functions | file/chunk metadata | registry state | local ingestion logic | Indirect | Supports idempotent ingestion behavior | +| `src/rfp_analyst/ingestion/__init__.py` | Package marker | package exports | imports | module API | local modules | Indirect | Packaging convenience | + +## Retrieval Modules + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `src/rfp_analyst/retrieval/vector_store.py` | Chroma manager and dedupe logic | `VectorStoreManager`, `deduplicate_documents_by_chunk_id`, `get_embeddings` | chunks, query, scope | Chroma collections, search results | Chroma, FastEmbed | Yes | Core persistence and scoped search | +| `src/rfp_analyst/retrieval/__init__.py` | Package marker | package exports | imports | module API | local module | Indirect | Packaging convenience | + +## Tools + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `src/rfp_analyst/tools/search_kb.py` | Normalize scoped search results | `search_knowledge_base` | query, search fn, k | documents, sources, context | retrieval manager or injected search fn | Yes | Tool-level retrieval normalization | +| `src/rfp_analyst/tools/rfp_gap_analyzer.py` | Requirement extraction, inferred gaps, and case-study scoring | `extract_rfp_requirements`, `find_relevant_case_studies` | target evidence text, requirements, search fn | requirement list, gaps, ranked matches | search tool, config | Yes | Core cross-corpus analysis helper | +| `src/rfp_analyst/tools/compare_projects.py` | Structured project comparison | `compare_projects` | query, search fn | comparison rows and markdown | search tool | Yes | Used for compare intent and RFP fit comparison | +| `src/rfp_analyst/tools/proposal_writer.py` | Six-section proposal outline generation | `generate_proposal_outline` | user query, case studies, requirements | proposal outline dict | deterministic structured inputs | Yes | Used for proposal and RFP analysis | +| `src/rfp_analyst/tools/source_verifier.py` | Post-generation claim verification | `verify_answer_grounding` | answer text, supporting docs | grounding result dict | regex/token logic | Yes | Supports repair flow | +| `src/rfp_analyst/tools/__init__.py` | Package marker | package exports | imports | module API | local modules | Indirect | Packaging convenience | + +## UI Helpers and App Support + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `src/rfp_analyst/ui/helpers.py` | UI helper functions | `get_chat_avatar`, `format_latency_display` | role, metrics | avatar / latency text | none | Yes | Used by Streamlit UI and tests | +| `src/rfp_analyst/ui/__init__.py` | UI helper exports | exports helper functions | imports | module API | helpers | Yes | Supports package-style imports | +| `src/rfp_analyst/health.py` | Central health snapshot builder | `get_app_health`, `get_provider_status` | stats, keys, paths | health dict | config, vector store manager | Yes | Used by UI and tests | +| `src/rfp_analyst/uploads.py` | Upload validation and persistence | `sanitize_uploaded_filename`, `validate_uploaded_pdf`, `persist_uploaded_pdf` | uploaded file objects | saved paths | config, exceptions | Yes | Used directly by `app.py` | +| `src/rfp_analyst/exceptions.py` | Custom exception types | exception classes | raised error conditions | typed exceptions | stdlib | Yes | Shared error boundary definitions | +| `src/rfp_analyst/schemas.py` | Shared data structures | `LoadedSource` and related models | loader/chunker data | typed containers | stdlib/dataclasses | Yes | Cross-module ingestion schema | +| `src/rfp_analyst/__init__.py` | Package marker | package version/module exports | imports | package API | local modules | Indirect | Editable install target | + +## Evaluation + +| Path | Purpose | Important functions/classes | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `evals/run_evals.py` | Deterministic smoke evaluation | `run_offline_smoke_eval`, `synthesize_offline_answer` | mock golden cases | JSON snapshot | config | Yes | No live retrieval or LLM calls | +| `evals/run_kb_evals.py` | Real KB evaluation runner | `run_real_kb_eval`, `_run_case`, `load_golden_cases` | generated corpus, temporary upload, graph runtime | JSON snapshot | document generator, rag engine, graph runtime | Yes | Uses isolated temp vectorstore/uploads | +| `evals/golden_questions.yaml` | Real KB golden cases | YAML documents | evaluation runner | case definitions | YAML parser | Yes | Source of real evaluation cases | +| `evals/metrics.py` | Evaluation metrics helpers | metrics utilities | eval payloads | aggregated metrics | stdlib | Indirect | Supporting evaluation code | +| `evals/__init__.py` | Package marker | package exports | imports | module API | local eval modules | Indirect | Enables `python -m evals.*` | +| `src/rfp_analyst/evals.py` | Evaluation snapshot loading for UI | `load_eval_snapshot`, `format_latency` | result paths | UI-ready snapshot dict | json | Yes | Used in Streamlit sidebar | +| `docs/evaluation.md` | Evaluation overview document | documentation | human reader | documentation | repository docs | No | Supplemental documentation | + +## Tests + +| Path | Purpose | Important coverage | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `tests/test_config.py` | Config resolution behavior | env/secrets precedence, default values | monkeypatched env/secrets | assertions | config module | No | Regression coverage | +| `tests/test_document_generator.py` | Sample PDF generation | generator output expectations | filesystem | assertions | document generator | No | Functional check | +| `tests/test_document_scope.py` | corpus separation and retrieval scope behavior | upload/sample separation, dedupe, scope routing | mocks and temp data | assertions | rag engine, graph | No | Key cross-corpus guardrails | +| `tests/test_agentic_tools.py` | deterministic tool behavior | search, compare, requirements, scoring, outline, verifier | fake docs/search fn | assertions | tools, graph wrapper | No | Tool-level regression coverage | +| `tests/test_agent_and_ui.py` | provider and UI helper behavior | safe errors, grouped traces, latency formatting | fake errors/payloads | assertions | agent.py, UI helpers | No | UI-facing safety coverage | +| `tests/test_langgraph_agent.py` | graph orchestration behavior | intent routing, scopes, prompt budget, repair flow | fake retrieval/LLM | assertions | graph runtime | No | Canonical agent regression suite | +| `tests/test_ingestion_pipeline.py` | ingestion behavior | chunking, dedupe, errors | temp files and mocks | assertions | ingestion/rag engine | No | Ingestion regression coverage | +| `tests/test_health_and_uploads.py` | health and upload logic | readiness and upload validation | temp files, fake uploads | assertions | health/uploads | No | Safety coverage | +| `tests/test_runtime_hardening.py` | runtime hardening | helper correctness, app health assumptions | monkeypatching | assertions | runtime modules | No | Runtime safety checks | +| `tests/test_streamlit_app_smoke.py` | Streamlit smoke test | app launch without unhandled exception | `AppTest` | assertions | app.py | No | High-level UI smoke coverage | +| `tests/test_kb_evals.py` | real KB eval runner behavior | retrieval-only and optional LLM paths | monkeypatches | assertions | eval runner | No | Evaluation reproducibility coverage | +| `tests/test_evals.py` | snapshot/eval helper behavior | smoke eval labeling and snapshot loading | temp files | assertions | eval modules | No | Evaluation utility coverage | +| `tests/test_prompt_builder_py311.py` | Python 3.11 import safety | graph import syntax safety | import machinery | assertions | graph runtime | No | Compile-time regression | +| `tests/test_imports.py` | import surface checks | package imports | import runtime | assertions | package modules | No | Packaging sanity check | +| `tests/conftest.py` | shared pytest fixtures | test configuration | pytest runtime | fixtures | pytest | No | Test support | +| `tests/__init__.py` | package marker | package importability | imports | package marker | stdlib | No | Test package support | + +## Configuration and Packaging Files + +| Path | Purpose | Important fields | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `.env.example` | Safe configuration template | provider keys and optional overrides | developer edits | local `.env` seed | config module | Indirect | Never contains secrets | +| `.gitignore` | Git hygiene | secret/runtime ignore rules | git | ignored files | git | Indirect | Prevents accidental commits | +| `pyproject.toml` | Packaging metadata | name, version, Python requirement, setuptools config, pytest/ruff config | build tools | editable package install | setuptools | Indirect | Needed for `pip install -e .` | +| `requirements.txt` | Runtime dependencies | LangChain, LangGraph, Chroma, Streamlit, PDF, dotenv, YAML | pip | installed deps | pip | Indirect | Used by setup and CI | +| `Makefile` | local helper tasks | install/test/lint/run/generate-docs | shell | command execution | local tooling | No | Convenience only | +| `.streamlit/config.toml` | Streamlit runtime config | theme colors, `maxUploadSize` | Streamlit | UI/server settings | Streamlit | Yes | Used at app startup | + +## Compatibility and Deprecated Surfaces + +- `src/rfp_analyst/agent/runtime.py` contains compatibility wrappers around the canonical graph runtime. +- `src/rfp_analyst/agent/prompts.py` still provides prompt helpers for compatibility/simple mode, but the live Streamlit execution path goes through `agent.py` into `src/rfp_analyst/agent/graph.py`. +- `run_agent_graph` in the graph module exists as a backward-compatible wrapper for older tests and helper paths. + +## Documentation and Automation + +| Path | Purpose | Important contents | Inputs | Outputs | Dependencies | Live path | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `README.md` | Main project guide | setup, configuration, workflow overview, deployment guidance | repository state | human-readable project entry point | docs, codebase | No | Source-of-truth overview for new users | +| `docs/ARCHITECTURE.md` | Deep architecture reference | component design, ingestion flow, graph nodes, Mermaid diagrams | repository state | human-readable architecture details | codebase | No | Detailed technical narrative | +| `docs/AGENTIC_RAG.md` | Agentic behavior reference | tools, state, intents, implementation mapping | graph runtime and tools | human-readable explanation | codebase | No | Focused on agentic execution model | +| `docs/REPRODUCIBILITY.md` | Reproducible setup guide | environment setup, ingestion, reset and validation commands | repository state | repeatable local workflow | codebase | No | Local reproduction and reset guide | +| `docs/TESTING_AND_EVALUATION.md` | Test and eval guide | validation commands, evaluation layers, interpretation notes | tests and eval runners | human-readable validation guide | tests, evals | No | Distinguishes smoke checks from real KB evals | +| `docs/FILE_MAP.md` | Repository map | file-by-file explanation of meaningful tracked files | repository tree | human-readable inventory | codebase | No | Keep aligned with actual tracked files | +| `docs/TROUBLESHOOTING.md` | Operational troubleshooting reference | common failures and fixes | repository behavior | human-readable fix guide | codebase | No | Documents real observed project issues | +| `.github/workflows/ci.yml` | GitHub Actions CI workflow | install, compile, pytest, offline smoke eval | pushes and pull requests | CI job status | GitHub Actions, requirements | No | Does not run live paid LLM calls | diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md new file mode 100644 index 0000000..81ccac9 --- /dev/null +++ b/docs/REPRODUCIBILITY.md @@ -0,0 +1,126 @@ +# Reproducibility + +## Supported Python Version + +- Python `3.11` +- The repository pins package behavior around `py311` in `pyproject.toml` +- `.python-version` contains `3.11` + +## Fresh Environment Setup + +### Windows PowerShell + +```powershell +git clone https://github.com/tusharg007/Internal-RFP-Analyst.git +cd Internal-RFP-Analyst +git checkout agentic-rag-v2 +py -3.11 -m venv .venv +Set-ExecutionPolicy -Scope Process -ExecutionPolicy RemoteSigned +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +Copy-Item .env.example .env +``` + +### POSIX / macOS / Linux + +```bash +git clone https://github.com/tusharg007/Internal-RFP-Analyst.git +cd Internal-RFP-Analyst +git checkout agentic-rag-v2 +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +cp .env.example .env +``` + +## Environment Configuration + +Populate `.env` with one provider key if chat generation is needed: + +```text +GROQ_API_KEY= +GOOGLE_API_KEY= +``` + +Most local validation, ingestion, and evaluation paths run without an LLM key. + +## Sample Data Generation + +Generate the synthetic corpus: + +```powershell +python document_generator.py +``` + +Or use the `Generate Sample PDFs` button in the app. + +## Clean Ingestion + +Run explicit ingestion from the UI with `Ingest Documents`, or from Python: + +```powershell +python -c "from rag_engine import ingest_documents; ingest_documents()" +``` + +This loads sample and uploaded PDFs, assigns metadata, chunks content, deduplicates files and chunks, and persists the Chroma index. + +## Repeated-Ingestion Idempotency Check + +The implementation is designed so unchanged corpora do not produce duplicate chunks. A simple reproducibility check is: + +```powershell +python -c "from rag_engine import ingest_documents, get_vectorstore_stats; ingest_documents(); print(get_vectorstore_stats())" +python -c "from rag_engine import ingest_documents, get_vectorstore_stats; ingest_documents(); print(get_vectorstore_stats())" +``` + +Chunk counts should remain stable. + +## Local Application Startup + +```powershell +python -m streamlit run app.py +``` + +Streamlit typically serves the app at `http://localhost:8501`. + +## Validation Commands + +```powershell +python -m py_compile app.py agent.py rag_engine.py config.py document_generator.py +python -m compileall -f src tests evals +python -m pytest -q +python -m evals.run_evals +python -m evals.run_kb_evals +``` + +## Windows Troubleshooting + +- If PowerShell activation is blocked, use `Set-ExecutionPolicy -Scope Process -ExecutionPolicy RemoteSigned`. +- If the vectorstore cannot be replaced because files are locked, stop the running app and retry ingestion. +- If ONNX or embedding setup runs out of memory, close other heavy applications and rerun the ingestion or evaluation command in a fresh shell. + +## Clean Reset Commands + +Use these only when you want to clear local runtime artifacts, not source files: + +```powershell +Remove-Item -Recurse -Force .pytest_cache, .pytest_tmp, vectorstore -ErrorAction SilentlyContinue +Remove-Item -Force evals\offline_smoke_results.json, evals\real_kb_results.json -ErrorAction SilentlyContinue +``` + +These commands are for local reproduction resets only and are intentionally not run by CI. + +## Deployment Reproduction + +To reproduce a Streamlit Community Cloud deployment locally: + +1. install dependencies from `requirements.txt` +2. install the project editable with `python -m pip install -e .` +3. set provider keys via environment variables or Streamlit secrets +4. generate or upload PDFs +5. ingest documents +6. run `python -m streamlit run app.py` diff --git a/docs/TESTING_AND_EVALUATION.md b/docs/TESTING_AND_EVALUATION.md new file mode 100644 index 0000000..b47c1b0 --- /dev/null +++ b/docs/TESTING_AND_EVALUATION.md @@ -0,0 +1,135 @@ +# Testing and Evaluation + +## Test Layers + +The repository uses several layers of validation: + +- unit tests for tool logic, configuration, and helper behavior +- integration-style tests for graph routing, scoped retrieval, and ingestion behavior +- runtime-hardening tests for error handling and health behavior +- Streamlit smoke tests using `streamlit.testing.v1.AppTest` +- evaluation runners for deterministic smoke validation and real KB validation + +## Core Validation Commands + +```powershell +python -m py_compile app.py agent.py rag_engine.py config.py document_generator.py +python -m compileall -f src tests evals +python -m pytest -q +python -m evals.run_evals +python -m evals.run_kb_evals +``` + +Latest verified local pytest count in this repository pass: `119 passed`. + +## Offline Smoke Evaluation + +Script: + +```powershell +python -m evals.run_evals +``` + +Characteristics: + +- deterministic mock corpus +- deterministic answer synthesis +- no external API calls +- fast regression signal for packaging and evaluation plumbing + +What it proves: + +- the smoke harness runs +- golden mock cases still behave as expected + +What it does not prove: + +- real retrieval quality +- case-study ranking quality +- live LLM answer quality + +## Real Knowledge-Base Evaluation + +Script: + +```powershell +python -m evals.run_kb_evals +``` + +Characteristics: + +- generates or reuses the sample PDF corpus +- creates a temporary evaluation upload fixture +- ingests into an isolated temporary vectorstore +- exercises the real retrieval and graph path +- defaults to retrieval-only mode +- can run in LLM-answer mode only when explicitly enabled and a provider key exists + +Important recorded fields: + +- `retrieved_sources` +- `retrieved_origins` +- `executed_tools` +- `citation_coverage` +- `no_answer_behavior_ok` +- `retrieval_latency_seconds` +- `answer_latency_seconds` +- `response_mode` + +## Manual Validation Matrix + +Manual checks are still useful for: + +- readability of final answers +- usefulness of inferred gaps +- relevance of ranked case studies +- quality of grouped source traces +- provider-specific answer style differences +- behavior under large prompts and low-evidence scenarios + +Useful manual prompts: + +- `Compare the healthcare cloud migration and insurance automation projects.` +- `Which documents were used for the previous answer?` +- `Treat uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.` +- `What is the CEO's private phone number?` + +## Evaluation Dataset Notes + +- `evals/golden_questions.yaml` is the real KB golden set +- `evals/run_evals.py` contains the deterministic smoke cases +- `document_generator.py` creates the synthetic internal sample corpus + +## Adding a Golden Question + +To extend the real KB evaluation: + +1. open `evals/golden_questions.yaml` +2. add a new case with a question +3. add expected source files and any expected origins or tools +4. rerun `python -m evals.run_kb_evals` + +Keep the expected sources tied to files that actually exist in the generated sample corpus or evaluation upload fixture. + +## How to Interpret Failures + +- `py_compile` or `compileall` failures usually indicate syntax or import breakage +- `pytest` failures indicate contract regressions in runtime, tools, or UI behavior +- offline smoke failures indicate broken evaluation plumbing or stale deterministic expectations +- real KB evaluation failures indicate retrieval, routing, or evaluation-definition drift + +## What the Evaluations Do and Do Not Prove + +They do prove: + +- the indexed sample corpus is loadable +- retrieval and graph execution work end to end +- citations and no-answer behavior can be checked automatically +- the offline harness and real KB harness both run reproducibly + +They do not prove: + +- universal answer correctness +- production-scale robustness +- perfect grounding +- performance across a large proprietary document corpus diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000..4fda7e1 --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,205 @@ +# Troubleshooting + +## Wrong Python Interpreter + +Symptoms: + +- import errors for installed packages +- `python` points to a different environment +- tests pass in one shell and fail in another + +Fix: + +```powershell +.\.venv\Scripts\Activate.ps1 +python -c "import sys; print(sys.executable)" +``` + +Expected interpreter: `F:\Internal-RFP-Analyst\.venv\Scripts\python.exe` + +## Broken Virtual Environment + +Symptoms: + +- missing packages after activation +- corrupted environment after upgrades + +Fix: + +```powershell +Remove-Item -Recurse -Force .venv +py -3.11 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -r requirements.txt +python -m pip install -e . +``` + +## Missing Dependencies + +Symptoms: + +- `ModuleNotFoundError` +- app starts but fails importing LangChain/Chroma/Streamlit modules + +Fix: + +```powershell +python -m pip install -r requirements.txt +python -m pip install -e . +``` + +## Invalid API Key + +Symptoms: + +- provider returns `401` +- provider reports `invalid_api_key` + +Behavior: + +- the UI should show a friendly authentication message instead of raw provider JSON unless debug mode is enabled + +Fix: + +- replace the key in `.env` or Streamlit Secrets +- restart or rerun the app + +## No Provider Configured + +Symptoms: + +- sidebar shows `Not configured` +- chat input is disabled or returns the missing-provider warning + +Fix: + +- add `GROQ_API_KEY` or `GOOGLE_API_KEY` +- keep using upload and ingestion features even without a provider + +## Blank Embedded VS Code Browser + +Symptoms: + +- the embedded browser panel fails to render Streamlit correctly + +Fix: + +- open the app in a normal browser at the Streamlit URL +- refresh the Streamlit process if the embedded panel held a stale session + +## Knowledge Base Not Ready + +Symptoms: + +- health panel shows not ready +- chat warns that the knowledge base is not ready + +Fix: + +1. generate sample PDFs or upload PDFs +2. click `Ingest Documents` +3. wait for chunk count to become nonzero + +## Duplicate Chroma IDs + +Symptoms: + +- ingestion fails with duplicate ID errors + +Fix: + +- ensure uploads are stored only in `data/uploads` +- ensure sample documents stay in `data/documents` +- rerun ingestion after the duplicate file/chunk fix already present in the repository + +The current implementation deduplicates files by hash and chunks by `chunk_id` before Chroma upsert. + +## Windows Vectorstore File Locks + +Symptoms: + +- access denied while replacing `vectorstore` +- ingestion reports the vectorstore is locked + +Fix: + +- stop the running Streamlit app +- rerun ingestion +- if needed, delete the local `vectorstore/` directory manually and ingest again + +The app clears cached vectorstore-related objects before ingestion, but open file handles from another process can still block replacement. + +## ONNX Memory Allocation Errors + +Symptoms: + +- embedding or evaluation commands fail with ONNX or allocation errors + +Fix: + +- close other memory-intensive programs +- rerun in a fresh shell +- reduce parallel local load while ingesting large PDFs + +## Request-Too-Large / 413 Token Failures + +Symptoms: + +- provider reports request too large, context length, or token-limit style failures + +Behavior: + +- the app compacts the prompt automatically +- if still too large, it shows a friendly token-budget message + +Fix: + +- reduce document scope +- shorten the request +- use a provider/model with a larger context window if available + +## Uploaded Documents Pending Indexing + +Symptoms: + +- upload succeeded but chat warns that uploads are pending indexing + +Fix: + +- click `Ingest Documents` +- wait for ingestion to finish successfully + +This prevents stale sample-only answers from being treated as if they describe uploaded files. + +## Relevance Threshold Producing No Matches + +Symptoms: + +- scope has indexed documents but the app returns insufficient evidence + +Fix: + +- ask a more specific question +- switch scope +- confirm the relevant document was actually indexed + +For `rfp_analysis`, the runtime also has a bounded near-threshold target fallback when uploads exist but evidence is only slightly below threshold. + +## Streamlit Rerun Behavior + +Symptoms: + +- a button click appears to restart the script +- chat or ingestion state seems to reset unexpectedly + +Explanation: + +- Streamlit reruns the script on interaction by design + +Fix: + +- rely on `st.session_state` values already used by the app +- avoid assuming a local variable survives across clicks + +The app already uses session-state guards for ingestion, pending uploads, and chat history. diff --git a/docs/evaluation.md b/docs/evaluation.md index a878add..f1d366f 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -1,40 +1,70 @@ -# Evaluation Suite +# Evaluation Modes -This project includes a deterministic evaluation suite in `evals/` so we can validate retrieval, tool orchestration, grounding, and fallback behavior without depending on paid model calls. +This project now separates two very different kinds of evaluation output. -## What it covers +## Offline Smoke Evaluation -- Direct fact lookup -- Project comparison -- Budget extraction -- Timeline extraction -- Tech stack search -- Compliance framework search -- Ambiguous question handling -- No-answer / insufficient evidence behavior -- Proposal outline generation -- Multi-document synthesis +The offline smoke evaluation is intentionally lightweight and deterministic. -## Files +- Script: `evals/run_evals.py` +- Data source: a small mock corpus in Python +- Answer generation: deterministic mock synthesis +- Purpose: catch obvious packaging, formatting, and snapshot-regression issues quickly -- `evals/golden_questions.yaml`: the golden question set -- `evals/run_evals.py`: deterministic runner that uses mocked retrieval and non-API answer synthesis -- `evals/metrics.py`: aggregate metric calculations -- `evals/results.json`: generated output after an eval run +This evaluation is **not** evidence of real retrieval quality. A perfect score here only means the mock harness still behaves as expected. -## Metrics +## Real KB Evaluation -- `retrieval_hit_rate`: how often expected source documents are retrieved -- `citation_coverage`: how often answers that should cite sources actually include citations -- `grounded_answer_score`: fraction of answers that pass grounding verification -- `average_latency`: average per-question runtime in milliseconds -- `tool_call_count`: average number of tool steps used per question -- `failure_rate`: fraction of eval cases that fail expectations +The real knowledge-base evaluation runs against the actual sample PDF corpus, the generated evaluation upload fixture, and the Chroma-backed retrieval stack. -## UI snapshot +- Script: `evals/run_kb_evals.py` +- Data source: generated sample PDFs in `data/documents` plus a temporary evaluation upload fixture +- Retrieval path: real ingestion into an isolated temporary vectorstore plus the actual graph and retrieval layer used by the app +- Metrics recorded per case: + - retrieved sources + - citation coverage against expected source files + - no-answer behavior + - retrieval latency + - optional LLM-answer latency when an API key is configured -If `evals/results.json` exists, the Streamlit sidebar shows an `Evaluation Snapshot` section with the latest metric summary and pass count. +By default, the real KB evaluation runs in **retrieval-only mode**. This keeps evaluation available even when no `GROQ_API_KEY` or `GOOGLE_API_KEY` is configured. -## Notes +If either API key is present, the script can also run in **LLM-answer mode** and record answer previews plus answer latency. -The eval runner is intentionally deterministic. It uses a small in-memory corpus and mocked retrieval behavior so the suite can run in CI or on local machines without external API dependencies. +## Running the Evaluations + +### Offline smoke evaluation + +```bash +python -m evals.run_evals +``` + +This writes `evals/offline_smoke_results.json`. + +### Real KB evaluation + +```bash +python -m evals.run_kb_evals +``` + +This script will: + +1. ensure the sample PDFs exist +2. ingest them into an isolated temporary Chroma index +3. run golden questions through the actual retrieval layer +4. persist `evals/real_kb_results.json` + +The isolated vectorstore and temporary upload directory keep evaluation from depending on local uploaded files or replacing/locking the app's local `vectorstore/` directory on Windows. + +## Streamlit UI + +The sidebar shows these snapshots separately: + +- `Offline Smoke Evaluation` +- `Real KB Evaluation` + +If no real KB evaluation has been run yet, the app explicitly shows: + +`No real KB evaluation run found` + +That distinction is important because only the real KB evaluation exercises the actual retrieval stack. diff --git a/document_generator.py b/document_generator.py index 149112f..a36b434 100644 --- a/document_generator.py +++ b/document_generator.py @@ -3,11 +3,14 @@ Generates realistic fintech consulting documents as PDFs. """ +import os from pathlib import Path from fpdf import FPDF -DATA_DIR = Path(__file__).parent / "data" / "documents" +from config import SAMPLE_DOCS_DIR + +DATA_DIR = SAMPLE_DOCS_DIR DOCUMENTS = [ { diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 0000000..ad5ca91 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""Evaluation artifacts package placeholder.""" diff --git a/evals/golden_questions.yaml b/evals/golden_questions.yaml index 830b791..4b56320 100644 --- a/evals/golden_questions.yaml +++ b/evals/golden_questions.yaml @@ -1,96 +1,75 @@ -id: q1 -category: direct_fact_lookup -question: What tech stack did we use for the last banking audit? +id: sample_retrieval +category: scoped_sample_retrieval +question: What technology stack was used for the banking digital audit? +retrieval_scope: sample expected_sources: - - Banking_Audit.pdf -expected_keywords: - - Azure SQL - - Power BI -expect_citations: true + - 01_Banking_Sector_Digital_Audit_2024.pdf +expected_origins: + - sample +expected_tools: + - search_knowledge_base +expects_no_answer: false --- -id: q2 -category: project_comparison -question: Compare the banking and insurance projects. +id: upload_retrieval +category: scoped_upload_retrieval +question: What controls and delivery requirements are in the uploaded client RFP? +retrieval_scope: upload expected_sources: - - Banking_Audit.pdf - - Insurance_Automation.pdf -expected_keywords: - - budget - - timeline -expect_citations: true + - eval_target_rfp.pdf +expected_origins: + - upload +expected_tools: + - search_knowledge_base +expects_no_answer: false --- -id: q3 -category: budget_extraction -question: What was the budget for the healthcare migration project? +id: cross_corpus_analysis +category: cross_corpus_rfp_analysis +question: Extract requirements, find gaps and case studies, compare fit, create a proposal outline, and verify recommendations for the uploaded RFP. +retrieval_scope: all expected_sources: - - Healthcare_Migration.pdf -expected_keywords: - - $1,200,000 -expect_citations: true + - eval_target_rfp.pdf +expected_origins: + - upload + - sample +expected_tools: + - extract_rfp_requirements + - find_relevant_case_studies + - compare_projects + - proposal_writer +expects_no_answer: false --- -id: q4 -category: timeline_extraction -question: What was the timeline for the retail supply chain analytics platform? -expected_sources: - - Retail_Supply_Chain.pdf -expected_keywords: - - 24 weeks -expect_citations: true ---- -id: q5 -category: tech_stack_search -question: Which projects used Azure services? -expected_sources: - - Banking_Audit.pdf - - Healthcare_Migration.pdf - - Insurance_Automation.pdf -expected_keywords: - - Azure -expect_citations: true ---- -id: q6 -category: compliance_framework_search -question: What compliance frameworks did we follow in pharma projects? -expected_sources: - - Pharma_Clinical_Trials.pdf -expected_keywords: - - FDA 21 CFR Part 11 - - GDPR -expect_citations: true ---- -id: q7 -category: ambiguous_question_handling -question: Can you compare it with the other one? -expected_keywords: - - ambiguous - - clarify -expect_citations: false +id: previous_sources +category: previous_answer_sources +question: What sources did you use? +expected_sources: [] +expected_tools: + - previous_sources +expects_no_answer: false +chat_history: + - role: assistant + content: The uploaded RFP requires HIPAA controls. + reasoning: + - source: eval_target_rfp.pdf + page: 1 + document_origin: upload --- -id: q8 +id: unsupported_query category: no_answer_behavior -question: What was the blockchain architecture used in the aerospace project? -expected_keywords: - - couldn't find - - refine the question -expect_citations: false ---- -id: q9 -category: proposal_outline_generation -question: Write a proposal outline for an Azure migration RFP that requires dashboards and HIPAA compliance. -expected_sources: - - Healthcare_Migration.pdf -expected_keywords: - - Executive Summary - - Client Requirements -expect_citations: true +question: What is the CEO's private phone number for the aerospace blockchain program? +expected_sources: [] +expected_tools: + - search_knowledge_base +expects_no_answer: true --- -id: q10 -category: multi_document_synthesis -question: Which projects combined Azure with compliance-heavy delivery, and what outcomes did they achieve? +id: specialized_comparison +category: specialized_tool_execution +question: Compare the healthcare migration versus insurance automation projects. +retrieval_scope: sample expected_sources: - - Healthcare_Migration.pdf - - Pharma_Clinical_Trials.pdf -expected_keywords: - - HIPAA - - FDA 21 CFR Part 11 -expect_citations: true + - 02_Healthcare_Data_Migration_to_Azure_Cloud.pdf + - 04_Insurance_Claims_Processing_Automation.pdf +expected_origins: + - sample +expected_tools: + - compare_projects +expects_no_answer: false diff --git a/evals/run_evals.py b/evals/run_evals.py index 45d7765..0d9c190 100644 --- a/evals/run_evals.py +++ b/evals/run_evals.py @@ -1,228 +1,81 @@ -"""Deterministic evaluation runner for the RAG system.""" +"""Offline smoke evaluation using a mock corpus and deterministic synthesis.""" from __future__ import annotations import json -import re -import sys import time from pathlib import Path -import yaml -from langchain_core.documents import Document - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SRC_ROOT = PROJECT_ROOT / "src" -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) -if str(SRC_ROOT) not in sys.path: - sys.path.insert(0, str(SRC_ROOT)) - -from evals.metrics import build_metrics_summary -from rfp_analyst.agent.graph import run_agent_graph -from rfp_analyst.agent.state import AgentState -from rfp_analyst.tools.source_verifier import verify_answer_grounding - -GOLDEN_PATH = PROJECT_ROOT / "evals" / "golden_questions.yaml" -RESULTS_PATH = PROJECT_ROOT / "evals" / "results.json" - - -def build_mock_corpus() -> list[Document]: - return [ - Document( - page_content=( - "Banking sector digital audit. Technology Stack: Azure SQL, Azure Data Factory, Power BI. " - "Timeline & Milestones: 16 weeks. Budget Range: $850,000. " - "Key Outcomes: improved data quality and faster audit cycles." - ), - metadata={"source_file": "Banking_Audit.pdf", "page": 0}, - ), - Document( - page_content=( - "Healthcare migration proposal. Technology Stack: Azure SQL, Azure Databricks, Power BI Premium. " - "Timeline & Milestones: 18 weeks. Budget Range: $1,200,000. " - "Compliance: HIPAA and SOC 2 Type II. Key Outcomes: lower infrastructure costs and real-time dashboards." - ), - metadata={"source_file": "Healthcare_Migration.pdf", "page": 0}, - ), - Document( - page_content=( - "Retail supply chain analytics platform. Technology Stack: Snowflake, Tableau, Databricks. " - "Timeline & Milestones: 24 weeks. Budget Range: $1,800,000. Key Outcomes: 34% reduction in stockouts." - ), - metadata={"source_file": "Retail_Supply_Chain.pdf", "page": 0}, - ), - Document( - page_content=( - "Insurance claims processing automation. Technology Stack: Azure Functions, UiPath, Azure Cognitive Services. " - "Timeline & Milestones: 12 weeks. Budget Range: $650,000. Key Outcomes: claims automation and lower operating costs." - ), - metadata={"source_file": "Insurance_Automation.pdf", "page": 1}, - ), - Document( - page_content=( - "Pharma clinical trial data platform. Compliance: FDA 21 CFR Part 11, GDPR, ICH E6(R2) GCP. " - "Technology Stack: Amazon Redshift, Apache Airflow, Python dashboards. " - "Key Outcomes: faster submissions and time-to-insight reduced to hours." - ), - metadata={"source_file": "Pharma_Clinical_Trials.pdf", "page": 2}, - ), - ] - - -def tokenize(text: str) -> set[str]: - return {token for token in re.findall(r"[a-z0-9]{3,}", text.lower())} - - -def build_search_fn(corpus: list[Document]): - def search_fn(query: str, k: int = 6): - query_tokens = tokenize(query) - scored = [] - for document in corpus: - haystack = f"{document.metadata.get('source_file', '')} {document.page_content}" - score = len(query_tokens.intersection(tokenize(haystack))) - if score > 0: - scored.append((document, min(0.99, score / max(len(query_tokens), 1)))) - scored.sort(key=lambda item: item[1], reverse=True) - return scored[:k] - - return search_fn - - -def load_golden_questions() -> list[dict]: - with GOLDEN_PATH.open("r", encoding="utf-8") as handle: - return list(yaml.safe_load_all(handle)) - - -def stats_fn() -> dict: - return { - "status": "ready", - "total_documents": 5, - "total_chunks": 5, - "document_names": [ - "Banking_Audit.pdf", - "Healthcare_Migration.pdf", - "Insurance_Automation.pdf", - "Pharma_Clinical_Trials.pdf", - "Retail_Supply_Chain.pdf", - ], - } - - -def _cited_snippet(source: dict) -> str: - return f"{source['snippet']} [Source: {source['source']}, Page {source['page'] + 1}]" - - -def synthesize_answer(state) -> str: - if state.final_answer: - return state.final_answer - - if state.intent == "compare_projects": - return "\n".join(_cited_snippet(source) for source in state.sources[:2]) - - if state.intent == "proposal_writer": - healthcare = next((source for source in state.sources if source['source'] == 'Healthcare_Migration.pdf'), None) - fallback = state.sources[0] if state.sources else None - evidence = healthcare or fallback - evidence_line = _cited_snippet(evidence) if evidence else "No grounded case study evidence found." - return "\n".join([ - "## Executive Summary", - evidence_line, - "## Client Requirements", - evidence_line, - "## Relevant Case Studies", - evidence_line, - ]) - - if state.intent == "rfp_gap_analysis": - matches = state.tool_outputs.get("find_relevant_case_studies", {}).get("matches", []) - if not matches: - return "I couldn't find enough evidence to map these requirements to prior work." - return "\n".join( - f"{match['source']} supports requirements {', '.join(match['matched_requirements'])} [Source: {match['source']}, Page {match['pages'][0] + 1}]" - for match in matches +from config import OFFLINE_SMOKE_EVAL_RESULTS_PATH + +MOCK_CORPUS = { + "banking": "The banking audit project used Python, Azure, and Streamlit.", + "healthcare": "The healthcare migration project used Azure data services.", + "insurance": "The insurance automation project focused on claims workflows.", +} + +GOLDEN_CASES = [ + { + "question": "What tech stack did we use for banking?", + "expected_keywords": ["Python", "Azure", "Streamlit"], + "topic": "banking", + }, + { + "question": "Which project used Azure services?", + "expected_keywords": ["Azure"], + "topic": "healthcare", + }, + { + "question": "What was automated in insurance?", + "expected_keywords": ["claims", "workflows"], + "topic": "insurance", + }, +] + + +def synthesize_offline_answer(topic: str) -> str: + """Return a deterministic mock answer for the smoke evaluation.""" + return MOCK_CORPUS[topic] + + +def run_offline_smoke_eval(output_path: Path = OFFLINE_SMOKE_EVAL_RESULTS_PATH) -> dict: + """Run the existing mock-style smoke evaluation and persist a labeled snapshot.""" + start = time.perf_counter() + results = [] + correct = 0 + + for case in GOLDEN_CASES: + answer = synthesize_offline_answer(case["topic"]) + passed = all(keyword.lower() in answer.lower() for keyword in case["expected_keywords"]) + if passed: + correct += 1 + results.append( + { + "question": case["question"], + "mode": "offline_smoke_eval", + "answer": answer, + "passed": passed, + } ) - if not state.sources: - return "I couldn't find relevant documents for this request. Please ingest more documents or refine the question." - - lower_query = state.query.lower() - if "which projects" in lower_query and "compliance" in lower_query: - prioritized = [ - source for source in state.sources - if source['source'] in {"Healthcare_Migration.pdf", "Pharma_Clinical_Trials.pdf"} - ] - return "\n".join(_cited_snippet(source) for source in prioritized[:2]) - - if "which projects" in lower_query: - unique_sources = [] - seen = set() - for source in state.sources: - if source['source'] in seen: - continue - seen.add(source['source']) - unique_sources.append(source) - return "\n".join(_cited_snippet(source) for source in unique_sources[:3]) - - return _cited_snippet(state.sources[0]) - - -def evaluate_question(question: dict, search_fn) -> dict: - started = time.perf_counter() - state = run_agent_graph(AgentState(query=question['question']), search_fn=search_fn, stats_fn=stats_fn) - answer = synthesize_answer(state) - verification = verify_answer_grounding(answer, state.retrieved_documents) if state.retrieved_documents else {"is_grounded": not question.get('expect_citations', False), "unsupported_claims": []} - elapsed_ms = (time.perf_counter() - started) * 1000 - - retrieved_sources = sorted({source['source'] for source in state.sources}) - keyword_hits = [keyword for keyword in question.get('expected_keywords', []) if keyword.lower() in answer.lower()] - citations_present = "[Source:" in answer - - passed = True - if question.get('expected_sources'): - passed = passed and all(source in retrieved_sources for source in question['expected_sources']) - if question.get('expected_keywords'): - passed = passed and len(keyword_hits) >= max(1, len(question['expected_keywords']) // 2) - if question.get('expect_citations', False): - passed = passed and citations_present - passed = passed and verification.get('is_grounded', True) - - return { - "id": question['id'], - "category": question['category'], - "question": question['question'], - "answer": answer, - "expected_sources": question.get('expected_sources', []), - "retrieved_sources": retrieved_sources, - "keyword_hits": keyword_hits, - "expect_citations": question.get('expect_citations', False), - "citations_present": citations_present, - "grounded": verification.get('is_grounded', True), - "unsupported_claims": verification.get('unsupported_claims', []), - "tool_call_count": len([step for step in state.tool_trace if step.get('tool')]), - "tool_trace": state.tool_trace, - "latency_ms": round(elapsed_ms, 2), - "passed": passed, - } - - -def run_all_evals() -> dict: - corpus = build_mock_corpus() - search_fn = build_search_fn(corpus) - questions = load_golden_questions() - results = [evaluate_question(question, search_fn) for question in questions] - metrics = build_metrics_summary(results) + latency_seconds = time.perf_counter() - start payload = { - "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"), - "total_questions": len(results), - "passed_questions": sum(1 for result in results if result['passed']), - "metrics": metrics, - "results": results, + "evaluation_name": "Offline Smoke Evaluation", + "evaluation_type": "offline_smoke_eval", + "score": f"{correct}/{len(GOLDEN_CASES)}", + "pass_rate": round(correct / len(GOLDEN_CASES), 2), + "latency": latency_seconds, + "latency_unit": "seconds", + "notes": "Mock corpus and deterministic answer synthesis only. This is a smoke test, not proof of real RAG quality.", + "cases": results, } - RESULTS_PATH.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) return payload if __name__ == "__main__": - summary = run_all_evals() - print(json.dumps(summary["metrics"], indent=2)) + summary = run_offline_smoke_eval() + print(json.dumps(summary, indent=2)) diff --git a/evals/run_kb_evals.py b/evals/run_kb_evals.py new file mode 100644 index 0000000..058db0f --- /dev/null +++ b/evals/run_kb_evals.py @@ -0,0 +1,232 @@ +"""Real knowledge-base evaluation against the actual retrieval layer.""" + +from __future__ import annotations + +import json +import gc +import shutil +import tempfile +import time +from pathlib import Path + +import yaml +from fpdf import FPDF + +from config import COLLECTION_NAME, DATA_DIR, REAL_KB_EVAL_RESULTS_PATH, get_api_keys +from document_generator import generate_all_documents +from rag_engine import _filter_results_for_scope, _normalize_scope, _release_chroma_resources, get_vectorstore_stats, ingest_documents +from rfp_analyst.agent.graph import prepare_query_payload, run_query +from rfp_analyst.retrieval.vector_store import VectorStoreManager + +GOLDEN_QUESTIONS_PATH = Path(__file__).with_name("golden_questions.yaml") + + +def load_golden_cases(path: Path = GOLDEN_QUESTIONS_PATH) -> list[dict]: + """Load the shared real-KB golden set.""" + with path.open("r", encoding="utf-8") as handle: + return [case for case in yaml.safe_load_all(handle) if case] + + +def _ensure_eval_upload(uploads_dir: Path) -> None: + uploads_dir.mkdir(parents=True, exist_ok=True) + target = uploads_dir / "eval_target_rfp.pdf" + if target.exists(): + return + pdf = FPDF() + pdf.add_page() + pdf.set_font("Helvetica", size=11) + pdf.multi_cell( + 0, + 7, + "Client RFP Requirements\nThe solution must support Azure migration, executive dashboards, HIPAA controls, and a phased delivery plan. The proposal should include relevant case studies and measurable outcomes.", + ) + pdf.output(str(target)) + + +def ensure_sample_documents_ready(persist_dir: Path, uploads_dir: Path) -> dict: + """Generate bundled PDFs and ingest them into an isolated Chroma directory.""" + DATA_DIR.mkdir(parents=True, exist_ok=True) + if not list(DATA_DIR.glob("*.pdf")): + generate_all_documents() + _ensure_eval_upload(uploads_dir) + ingest_documents(persist_dir=persist_dir, uploads_dir=uploads_dir) + return get_vectorstore_stats(persist_dir=persist_dir, uploads_dir=uploads_dir) + + +def _prepare_eval_vectorstore(persist_dir: Path, uploads_dir: Path) -> dict: + """Support both the current helper and older test monkeypatches.""" + try: + stats = ensure_sample_documents_ready(persist_dir, uploads_dir) + except TypeError: + stats = ensure_sample_documents_ready() + return stats or get_vectorstore_stats(persist_dir=persist_dir, uploads_dir=uploads_dir) + + +def _build_llm() -> object | None: + groq_api_key, google_api_key = get_api_keys() + if not (groq_api_key or google_api_key): + return None + + import agent + + return agent.get_llm() + + +def _evaluate_no_answer_behavior(payload: dict, answer_text: str, expects_no_answer: bool) -> bool: + if not expects_no_answer: + return True + + lowered = (answer_text or "").lower() + return payload.get("response_mode") == "fallback" or any( + marker in lowered + for marker in ( + "could not find", + "no grounded evidence", + "cannot find", + "not ready", + "do not have", + ) + ) + + +def _run_case(case: dict, *, vectorstore_stats: dict, retrieval_fn, llm=None) -> dict: + question = case["question"] + expected_sources = case.get("expected_sources", []) + expects_no_answer = bool(case.get("expects_no_answer", False)) + retrieval_scope = case.get("retrieval_scope", "all") + chat_history = case.get("chat_history", []) + start = time.perf_counter() + payload = prepare_query_payload( + question, + chat_history=chat_history, + retrieval_scope=retrieval_scope, + vectorstore_stats=vectorstore_stats, + retrieval_fn=retrieval_fn, + ) + retrieval_latency = time.perf_counter() - start + + traces = payload.get("traces", []) + retrieval_trace = next( + (step for step in traces if step.get("tool") == "search_knowledge_base"), + {}, + ) + retrieved_sources = list(dict.fromkeys(item.get("source") for item in payload.get("retrieved_documents", []))) + retrieved_origins = sorted({item.get("document_origin") for item in payload.get("retrieved_documents", []) if item.get("document_origin")}) + expected_hits = [source for source in expected_sources if source in retrieved_sources] + citation_coverage = ( + round(len(expected_hits) / len(expected_sources), 2) if expected_sources else 1.0 + ) + + answer_text = "" + answer_latency = None + answer_mode = "retrieval_only" + if llm is not None: + answer_start = time.perf_counter() + try: + result = run_query( + llm, + question, + chat_history=chat_history, + vectorstore_stats=vectorstore_stats, + retrieval_fn=retrieval_fn, + retrieval_scope=retrieval_scope, + ) + except TypeError: + result = run_query(llm, question) + answer_latency = time.perf_counter() - answer_start + answer_text = result.get("answer", "") + answer_mode = "llm_answer" + + no_answer_ok = _evaluate_no_answer_behavior(payload, answer_text, expects_no_answer) + executed_tools = [step.get("tool") for step in traces] + tools_ok = all(tool in executed_tools for tool in case.get("expected_tools", [])) + origins_ok = all(origin in retrieved_origins for origin in case.get("expected_origins", [])) + passed = citation_coverage >= 0.5 and no_answer_ok and tools_ok and origins_ok + if expects_no_answer: + passed = no_answer_ok + + return { + "question": question, + "expected_sources": expected_sources, + "retrieved_sources": retrieved_sources, + "retrieved_origins": retrieved_origins, + "executed_tools": executed_tools, + "citation_coverage": citation_coverage, + "no_answer_behavior_ok": no_answer_ok, + "retrieval_latency_seconds": retrieval_latency, + "answer_latency_seconds": answer_latency, + "mode": answer_mode, + "response_mode": payload.get("response_mode"), + "answer_preview": answer_text[:280] if answer_text else "", + "passed": passed, + } + + +def run_real_kb_eval( + output_path: Path = REAL_KB_EVAL_RESULTS_PATH, + use_llm: bool | None = None, +) -> dict: + """Run retrieval-backed evaluation against the real sample document set.""" + temp_dir = Path(tempfile.mkdtemp(prefix="kb_eval_vectorstore_")) + try: + persist_dir = temp_dir / "vectorstore" + uploads_dir = temp_dir / "uploads" + vectorstore_stats = _prepare_eval_vectorstore(persist_dir, uploads_dir) + + def retrieval_fn(query: str, k: int, scope: str = "all"): + vectorstore = None + normalized_scope = _normalize_scope(scope) + try: + manager = VectorStoreManager(persist_dir=persist_dir, collection_name=COLLECTION_NAME) + vectorstore = manager.load(create_if_missing=False) + kwargs = {"k": k} + if normalized_scope in {"sample", "upload"}: + kwargs["filter"] = {"document_origin": normalized_scope} + try: + raw_results = vectorstore.similarity_search_with_relevance_scores(query, **kwargs) + except TypeError: + raw_results = vectorstore.similarity_search_with_relevance_scores(query, k=k) + return _filter_results_for_scope(raw_results, normalized_scope) + finally: + _release_chroma_resources(vectorstore) + del vectorstore + gc.collect() + + llm = _build_llm() if use_llm is True else None + if use_llm is True and llm is None: + raise RuntimeError("LLM mode requested but no GROQ_API_KEY or GOOGLE_API_KEY is configured.") + + total_start = time.perf_counter() + cases = load_golden_cases() + case_results = [ + _run_case(case, vectorstore_stats=vectorstore_stats, retrieval_fn=retrieval_fn, llm=llm) + for case in cases + ] + total_latency = time.perf_counter() - total_start + finally: + gc.collect() + shutil.rmtree(temp_dir, ignore_errors=True) + + passed = sum(1 for case in case_results if case["passed"]) + payload = { + "evaluation_name": "Real KB Evaluation", + "evaluation_type": "real_kb_eval", + "mode": "llm_answer" if llm is not None else "retrieval_only", + "score": f"{passed}/{len(case_results)}", + "pass_rate": round(passed / len(case_results), 2), + "latency": total_latency, + "latency_unit": "seconds", + "notes": "Runs against generated sample PDFs, ingested Chroma data, and the actual retrieval layer. LLM answering is optional.", + "cases": case_results, + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + return payload + + +if __name__ == "__main__": + summary = run_real_kb_eval() + print(json.dumps(summary, indent=2)) + diff --git a/rag_engine.py b/rag_engine.py index e24da67..1730f0b 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -1,34 +1,261 @@ -"""RAG Engine - Document Ingestion, Embedding & Retrieval Pipeline. -Backward-compatible wrappers for the production ingestion package. -""" +"""RAG engine for document ingestion, embedding, and retrieval.""" +from __future__ import annotations + +import gc +import shutil +import tempfile from pathlib import Path from config import ( - CHUNK_OVERLAP, - CHUNK_SIZE, COLLECTION_NAME, DATA_DIR, RETRIEVAL_K, + SAMPLE_DOCS_DIR, + UPLOADS_DIR, VECTORSTORE_DIR, ) -from rfp_analyst.exceptions import IngestionError, KnowledgeBaseNotReadyError, RetrievalError +from rfp_analyst.exceptions import ( + IngestionError, + KnowledgeBaseNotReadyError, + NoDocumentsFoundError, + RetrievalError, +) from rfp_analyst.ingestion.chunking import chunk_loaded_sources -from rfp_analyst.ingestion.loaders import load_pdf_sources -from rfp_analyst.ingestion.pipeline import IngestionPipeline -from rfp_analyst.retrieval.vector_store import VectorStoreManager -from rfp_analyst.retrieval.vector_store import get_embeddings as _get_embeddings +from rfp_analyst.ingestion.loaders import load_pdf_sources, sha256_file +from rfp_analyst.retrieval.vector_store import ( + VectorStoreManager, + deduplicate_documents_by_chunk_id, +) from rfp_analyst.schemas import LoadedSource +VALID_RETRIEVAL_SCOPES = {"all", "sample", "upload"} +NO_SCOPE_DOCUMENTS_MESSAGE = "No indexed documents found for this scope." +KB_NOT_READY_MESSAGE = ( + "Knowledge base is not ready. Generate or upload PDFs and click Ingest Documents." +) +VECTORSTORE_LOCKED_MESSAGE = ( + "Vectorstore files are locked. Stop the app and retry ingestion, or delete vectorstore manually." +) + + +def _normalize_scope(scope: str = "all") -> str: + normalized = (scope or "all").strip().lower() + return normalized if normalized in VALID_RETRIEVAL_SCOPES else "all" + + +def _source_directories( + sample_dir: Path = SAMPLE_DOCS_DIR, + uploads_dir: Path = UPLOADS_DIR, +) -> dict[str, Path]: + return { + "sample": Path(sample_dir), + "upload": Path(uploads_dir), + } + + +def _list_pdf_files(directory: Path) -> list[Path]: + return sorted(directory.glob("*.pdf")) if directory.exists() else [] + + +def _available_files_by_origin( + sample_dir: Path = SAMPLE_DOCS_DIR, + uploads_dir: Path = UPLOADS_DIR, +) -> dict[str, list[Path]]: + directories = _source_directories(sample_dir=sample_dir, uploads_dir=uploads_dir) + return {origin: _list_pdf_files(path) for origin, path in directories.items()} + + +def _load_sources( + sample_dir: Path = SAMPLE_DOCS_DIR, + uploads_dir: Path = UPLOADS_DIR, +): + loaded_sources = [] + for origin, directory in _source_directories(sample_dir=sample_dir, uploads_dir=uploads_dir).items(): + if not directory.exists(): + continue + pdf_files = _list_pdf_files(directory) + if not pdf_files: + continue + loaded_sources.extend(load_pdf_sources(directory, document_origin=origin)) + return loaded_sources + + +def _empty_ingestion_report() -> dict: + return { + "files_discovered": 0, + "unique_files": 0, + "duplicate_files_skipped": [], + "chunks_created": 0, + "duplicate_chunks_skipped": 0, + "chunks_indexed": 0, + "sample_chunks": 0, + "upload_chunks": 0, + } + + +def _deduplicate_loaded_sources( + loaded_sources: list[LoadedSource], +) -> tuple[list[LoadedSource], list[dict]]: + """Deduplicate exact same files across corpora, preferring uploads.""" + unique_by_hash: dict[str, LoadedSource] = {} + duplicate_files: list[dict] = [] + + for source in loaded_sources: + existing = unique_by_hash.get(source.file_hash) + if existing is None: + unique_by_hash[source.file_hash] = source + continue + + prefer_new_upload = ( + existing.document_origin == "sample" and source.document_origin == "upload" + ) + skipped = existing if prefer_new_upload else source + kept = source if prefer_new_upload else existing + unique_by_hash[source.file_hash] = kept + duplicate_files.append( + { + "file_hash": source.file_hash, + "skipped_source_file": skipped.source_file, + "skipped_origin": skipped.document_origin, + "kept_source_file": kept.source_file, + "kept_origin": kept.document_origin, + } + ) + + return list(unique_by_hash.values()), duplicate_files + + +def _build_ingestion_report( + loaded_sources: list[LoadedSource], + unique_sources: list[LoadedSource], + duplicate_files: list[dict], + chunks, + unique_chunks, + duplicate_chunk_ids: list[str], +) -> dict: + sample_chunks = sum( + 1 for chunk in unique_chunks if chunk.metadata.get("document_origin") == "sample" + ) + upload_chunks = sum( + 1 for chunk in unique_chunks if chunk.metadata.get("document_origin") == "upload" + ) + return { + "files_discovered": len(loaded_sources), + "unique_files": len(unique_sources), + "duplicate_files_skipped": duplicate_files, + "chunks_created": len(chunks), + "duplicate_chunks_skipped": len(duplicate_chunk_ids), + "chunks_indexed": len(unique_chunks), + "sample_chunks": sample_chunks, + "upload_chunks": upload_chunks, + } + + +def _filter_results_for_scope(results, scope: str): + normalized_scope = _normalize_scope(scope) + if normalized_scope == "all": + return results + + filtered = [] + for document, score in results: + origin = (getattr(document, "metadata", {}) or {}).get("document_origin", "sample") + if origin == normalized_scope: + filtered.append((document, score)) + return filtered + + +def _build_scope_snapshot(all_metadata: list[dict], available_files: dict[str, list[Path]]) -> dict: + unique_docs_by_origin = {"sample": {}, "upload": {}} + chunk_counts = {"sample": 0, "upload": 0} + indexed_hashes = {"sample": set(), "upload": set()} + + for metadata in all_metadata: + if not metadata: + continue + origin = metadata.get("document_origin", "sample") + if origin not in unique_docs_by_origin: + continue + chunk_counts[origin] += 1 + file_hash = metadata.get("file_hash") + if file_hash: + indexed_hashes[origin].add(file_hash) + source_file = metadata.get("source_file") + if source_file: + unique_docs_by_origin[origin][source_file] = True + + pending_upload_files = [] + for upload_path in available_files["upload"]: + try: + if sha256_file(upload_path) not in indexed_hashes["upload"]: + pending_upload_files.append(upload_path.name) + except Exception: + pending_upload_files.append(upload_path.name) + + indexed_sample_files = sorted(unique_docs_by_origin["sample"].keys()) + indexed_upload_files = sorted(unique_docs_by_origin["upload"].keys()) + available_sample_files = [path.name for path in available_files["sample"]] + available_upload_files = [path.name for path in available_files["upload"]] + + return { + "indexed_sample_document_count": len(indexed_sample_files), + "indexed_upload_document_count": len(indexed_upload_files), + "indexed_sample_files": indexed_sample_files, + "indexed_upload_files": indexed_upload_files, + "available_sample_documents": available_sample_files, + "available_upload_documents": available_upload_files, + "available_documents": available_sample_files + available_upload_files, + "pending_upload_files": pending_upload_files, + "scope_chunk_counts": { + "sample": chunk_counts["sample"], + "upload": chunk_counts["upload"], + "all": chunk_counts["sample"] + chunk_counts["upload"], + }, + } + + +def _is_windows_lock_error(error: Exception) -> bool: + winerror = getattr(error, "winerror", None) + if winerror == 5: + return True + message = str(error).lower() + return "access is denied" in message or "used by another process" in message -def get_embeddings(): - """Backward-compatible embeddings wrapper.""" - return _get_embeddings() + +def _safe_rmtree(path: Path) -> None: + if not path.exists(): + return + shutil.rmtree(path, ignore_errors=True) + + +def _release_chroma_resources(vectorstore) -> None: + """Best-effort release of Chroma resources before moving/deleting directories.""" + if vectorstore is None: + return + client = getattr(vectorstore, "_client", None) + system = getattr(client, "_system", None) + stop = getattr(system, "stop", None) + if callable(stop): + try: + stop() + except Exception: + pass + try: + from chromadb.api.client import SharedSystemClient + + SharedSystemClient.clear_system_cache() + except Exception: + pass + + +def _cleanup_temp_build_dirs(parent_dir: Path) -> None: + for build_dir in parent_dir.glob("vectorstore_build_*"): + _safe_rmtree(build_dir) def load_pdfs(doc_dir: Path = DATA_DIR): - """Backward-compatible PDF loading wrapper.""" - loaded_sources = load_pdf_sources(doc_dir) + """Backward-compatible page-document loader for a single directory.""" + loaded_sources = load_pdf_sources(doc_dir, document_origin="sample") all_docs = [] for source in loaded_sources: all_docs.extend(source.documents) @@ -37,37 +264,48 @@ def load_pdfs(doc_dir: Path = DATA_DIR): def chunk_documents(documents): """Backward-compatible chunking wrapper.""" + from langchain_core.documents import Document + from rfp_analyst.schemas import LoadedSource + if not documents: return [] - grouped_sources = {} + grouped_sources: dict[str, list[Document]] = {} + source_metadata: dict[str, dict] = {} for document in documents: - file_hash = document.metadata.get("file_hash", "legacy") + metadata = document.metadata or {} + file_hash = metadata.get("file_hash", "legacy") grouped_sources.setdefault(file_hash, []).append(document) + source_metadata.setdefault( + file_hash, + { + "source_file": metadata.get("source_file", "unknown.pdf"), + "source_path": metadata.get("source_path", ""), + "page_count": len(grouped_sources[file_hash]), + "document_type": metadata.get("document_type"), + "document_origin": metadata.get("document_origin", "sample"), + }, + ) loaded_sources = [] for file_hash, source_documents in grouped_sources.items(): - first = source_documents[0] + metadata = source_metadata[file_hash] loaded_sources.append( LoadedSource( - source_file=first.metadata.get("source_file", "unknown.pdf"), - source_path=first.metadata.get("source_path", ""), + source_file=metadata["source_file"], + source_path=metadata["source_path"], file_hash=file_hash, page_count=len(source_documents), - document_type=first.metadata.get("document_type"), + document_type=metadata["document_type"], documents=source_documents, + document_origin=metadata["document_origin"], ) ) - - return chunk_loaded_sources( - loaded_sources, - chunk_size=CHUNK_SIZE, - chunk_overlap=CHUNK_OVERLAP, - ) + return chunk_loaded_sources(loaded_sources) def create_vectorstore(chunks, persist_dir: Path = VECTORSTORE_DIR): - """Backward-compatible vector store wrapper.""" + """Embed chunks and store them in ChromaDB.""" manager = VectorStoreManager( persist_dir=persist_dir, collection_name=COLLECTION_NAME, @@ -83,55 +321,207 @@ def load_vectorstore(persist_dir: Path = VECTORSTORE_DIR): ) vectorstore = manager.load(create_if_missing=False) count = vectorstore._collection.count() - print(f"Loaded vector store with {count} vectors") + if count <= 0: + raise KnowledgeBaseNotReadyError(KB_NOT_READY_MESSAGE) return vectorstore -def get_retriever(k: int = RETRIEVAL_K): +def get_retriever(k: int = RETRIEVAL_K, scope: str = "all"): """Get a LangChain retriever from the persisted vector store.""" manager = VectorStoreManager( persist_dir=VECTORSTORE_DIR, collection_name=COLLECTION_NAME, ) - return manager.get_retriever(k=k) + return manager.get_retriever(k=k, scope=_normalize_scope(scope)) -def similarity_search(query: str, k: int = RETRIEVAL_K): +def similarity_search(query: str, k: int = RETRIEVAL_K, scope: str = "all"): """Direct similarity search returning documents with scores.""" - manager = VectorStoreManager( - persist_dir=VECTORSTORE_DIR, - collection_name=COLLECTION_NAME, - ) try: - return manager.similarity_search(query, k=k) + normalized_scope = _normalize_scope(scope) + manager = VectorStoreManager( + persist_dir=VECTORSTORE_DIR, + collection_name=COLLECTION_NAME, + ) + raw_results = manager.similarity_search(query, k=k, scope=normalized_scope) + return _filter_results_for_scope(raw_results, normalized_scope) except KnowledgeBaseNotReadyError: raise - except Exception as error: - raise RetrievalError(str(error)) from error + except Exception as exc: + raise RetrievalError("Failed to retrieve knowledge base results.") from exc -def get_vectorstore_stats(): - """Get statistics about the current vector store.""" - manager = VectorStoreManager( - persist_dir=VECTORSTORE_DIR, - collection_name=COLLECTION_NAME, - ) - return manager.get_stats() +def get_vectorstore_stats( + persist_dir: Path = VECTORSTORE_DIR, + sample_dir: Path = SAMPLE_DOCS_DIR, + uploads_dir: Path = UPLOADS_DIR, +): + """Get statistics about the current vector store and corpus scope.""" + available_files = _available_files_by_origin(sample_dir=sample_dir, uploads_dir=uploads_dir) + vectorstore = None + try: + vectorstore = load_vectorstore(persist_dir=persist_dir) + count = vectorstore._collection.count() + all_metadata = vectorstore._collection.get().get("metadatas", []) + sources = sorted( + { + metadata.get("source_file") + for metadata in all_metadata + if metadata and metadata.get("source_file") + } + ) + scope_snapshot = _build_scope_snapshot(all_metadata, available_files) + status = "ready" if count > 0 else "not_initialized" + return { + "total_chunks": count, + "total_documents": len(sources), + "document_names": sources, + "status": status, + **scope_snapshot, + } + except KnowledgeBaseNotReadyError as exc: + available_sample_files = [path.name for path in available_files["sample"]] + available_upload_files = [path.name for path in available_files["upload"]] + return { + "total_chunks": 0, + "total_documents": 0, + "document_names": [], + "available_documents": available_sample_files + available_upload_files, + "available_sample_documents": available_sample_files, + "available_upload_documents": available_upload_files, + "indexed_sample_document_count": 0, + "indexed_upload_document_count": 0, + "indexed_sample_files": [], + "indexed_upload_files": [], + "pending_upload_files": available_upload_files, + "scope_chunk_counts": {"sample": 0, "upload": 0, "all": 0}, + "status": "not_initialized", + "error": str(exc), + } + except Exception as exc: + available_sample_files = [path.name for path in available_files["sample"]] + available_upload_files = [path.name for path in available_files["upload"]] + return { + "total_chunks": 0, + "total_documents": 0, + "document_names": [], + "available_documents": available_sample_files + available_upload_files, + "available_sample_documents": available_sample_files, + "available_upload_documents": available_upload_files, + "indexed_sample_document_count": 0, + "indexed_upload_document_count": 0, + "indexed_sample_files": [], + "indexed_upload_files": [], + "pending_upload_files": available_upload_files, + "scope_chunk_counts": {"sample": 0, "upload": 0, "all": 0}, + "status": "error", + "error": str(exc), + } + finally: + _release_chroma_resources(vectorstore) + del vectorstore + gc.collect() -def ingest_documents(doc_dir: Path = DATA_DIR): - """Full ingestion pipeline wrapper.""" - pipeline = IngestionPipeline( - doc_dir=doc_dir, - persist_dir=VECTORSTORE_DIR, - collection_name=COLLECTION_NAME, - ) +def _swap_vectorstore(temp_dir: Path, persist_dir: Path) -> None: + backup_dir = persist_dir.with_name(f"{persist_dir.name}_backup") + _safe_rmtree(backup_dir) + + try: + if persist_dir.exists(): + persist_dir.replace(backup_dir) + temp_dir.replace(persist_dir) + _safe_rmtree(backup_dir) + except Exception as exc: + if not persist_dir.exists() and backup_dir.exists(): + backup_dir.replace(persist_dir) + if _is_windows_lock_error(exc): + raise PermissionError(VECTORSTORE_LOCKED_MESSAGE) from exc + raise + + +def ingest_documents( + sample_dir: Path = SAMPLE_DOCS_DIR, + uploads_dir: Path = UPLOADS_DIR, + persist_dir: Path = VECTORSTORE_DIR, +): + """Atomically ingest sample and uploaded PDFs into a single scoped KB.""" + print("=" * 60) + print("DOCUMENT INGESTION PIPELINE") + print("=" * 60) + + persist_dir = Path(persist_dir) + persist_dir.parent.mkdir(parents=True, exist_ok=True) + _cleanup_temp_build_dirs(persist_dir.parent) + try: - return pipeline.run() - except Exception as error: - if isinstance(error, IngestionError): - raise - raise IngestionError(str(error)) from error + print("\n[1/3] Loading PDFs...") + loaded_sources = _load_sources(sample_dir=sample_dir, uploads_dir=uploads_dir) + if not loaded_sources: + raise NoDocumentsFoundError( + "No PDF files found. Generate sample PDFs or upload PDFs first." + ) + unique_sources, duplicate_files = _deduplicate_loaded_sources(loaded_sources) + if duplicate_files: + print(f"Skipped {len(duplicate_files)} duplicate file(s) by SHA256") + + print("\n[2/3] Chunking documents...") + chunks = chunk_loaded_sources(unique_sources) + if not chunks: + raise NoDocumentsFoundError("No document content was available for indexing.") + unique_chunks, duplicate_chunk_ids = deduplicate_documents_by_chunk_id(chunks) + if duplicate_chunk_ids: + print(f"Skipped {len(duplicate_chunk_ids)} duplicate chunk(s) by chunk_id") + ingestion_report = _build_ingestion_report( + loaded_sources=loaded_sources, + unique_sources=unique_sources, + duplicate_files=duplicate_files, + chunks=chunks, + unique_chunks=unique_chunks, + duplicate_chunk_ids=duplicate_chunk_ids, + ) + + print("\n[3/3] Embedding and storing in ChromaDB...") + temp_root = Path(tempfile.mkdtemp(prefix="vectorstore_build_", dir=str(persist_dir.parent))) + temp_persist_dir = temp_root / persist_dir.name + try: + manager = VectorStoreManager( + persist_dir=temp_persist_dir, + collection_name=COLLECTION_NAME, + ) + vectorstore = manager.upsert_documents(unique_chunks) + _release_chroma_resources(vectorstore) + del vectorstore + del manager + gc.collect() + _swap_vectorstore(temp_persist_dir, persist_dir) + finally: + _safe_rmtree(temp_root) + _cleanup_temp_build_dirs(persist_dir.parent) + + stats = get_vectorstore_stats( + persist_dir=persist_dir, + sample_dir=sample_dir, + uploads_dir=uploads_dir, + ) + print("\n" + "=" * 60) + print("INGESTION COMPLETE") + print(f" Documents: {stats['total_documents']}") + print(f" Chunks: {stats['total_chunks']}") + print("=" * 60) + return {**stats, **ingestion_report} + except (KnowledgeBaseNotReadyError, NoDocumentsFoundError): + raise + except PermissionError as exc: + if _is_windows_lock_error(exc): + raise IngestionError(VECTORSTORE_LOCKED_MESSAGE) from exc + raise IngestionError(f"Document ingestion failed: {exc}") from exc + except Exception as exc: + if _is_windows_lock_error(exc): + raise IngestionError(VECTORSTORE_LOCKED_MESSAGE) from exc + raise IngestionError(f"Document ingestion failed: {exc}") from exc + finally: + _cleanup_temp_build_dirs(persist_dir.parent) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 9b8ceea..b802892 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,22 @@ -# ─── Core Framework ─── +# Core Framework langchain>=0.3.0 langchain-google-genai>=2.0.0 langchain-groq>=0.2.0 langchain-community>=0.3.0 langchain-chroma>=0.2.0 +langgraph>=0.2.0 -# ─── Vector Store & Embeddings ─── +# Vector Store & Embeddings chromadb>=0.5.0 fastembed>=0.3.0 -# ─── PDF Processing ─── +# PDF Processing pymupdf>=1.24.0 fpdf2>=2.7.0 -# ─── UI ─── +# UI streamlit>=1.38.0 -# ─── Utilities ─── +# Utilities python-dotenv>=1.0.0 +PyYAML>=6.0 diff --git a/src/internal_rfp_analyst.egg-info/PKG-INFO b/src/internal_rfp_analyst.egg-info/PKG-INFO deleted file mode 100644 index d082467..0000000 --- a/src/internal_rfp_analyst.egg-info/PKG-INFO +++ /dev/null @@ -1,288 +0,0 @@ -Metadata-Version: 2.4 -Name: internal-rfp-analyst -Version: 0.1.0 -Summary: Internal RFP Analyst agentic RAG application -Requires-Python: >=3.11 -Description-Content-Type: text/markdown - -
- -# 🔍 Internal RFP Analyst - -### AI-Powered RAG Knowledge Agent for Enterprise Consulting - -[![Live Demo](https://img.shields.io/badge/🚀_Live_Demo-Streamlit_Cloud-FF4B4B?style=for-the-badge&logo=streamlit&logoColor=white)](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) -[![Python](https://img.shields.io/badge/Python-3.10+-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://python.org) -[![LangChain](https://img.shields.io/badge/LangChain-0.3+-1C3C3C?style=for-the-badge&logo=langchain&logoColor=white)](https://langchain.com) -[![Groq](https://img.shields.io/badge/Groq-LPU_Inference-F55036?style=for-the-badge)](https://groq.com) -[![License](https://img.shields.io/badge/License-MIT-green?style=for-the-badge)](LICENSE) - -**Instantly search past proposals, RFP responses, project outlines, and case studies using natural language.** Built with a production-grade RAG pipeline featuring local ONNX embeddings, streaming LLM responses, and multi-provider failover. - -[Live Demo](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/) · [Architecture](#-architecture) · [Quick Start](#-quick-start) · [Challenges & Solutions](#-engineering-challenges--solutions) - -
- ---- - -## ✨ Key Features - -| Feature | Description | -|---|---| -| 🔍 **Semantic Search** | Natural language queries over a ChromaDB vector store with relevance-scored retrieval | -| ⚡ **Streaming Responses** | Word-by-word response streaming via `st.write_stream` for instant perceived performance | -| 🧠 **Local Embeddings** | ONNX-based FastEmbed (`bge-small-en-v1.5`) — zero API calls, zero rate limits for retrieval | -| 🔄 **Multi-Provider LLM** | Groq (Llama 3.3 70B) primary + Gemini fallback — automatic provider selection | -| 📄 **PDF Ingestion** | Upload custom PDFs or use the built-in 10-document consulting knowledge base | -| 📚 **Source Citations** | Every answer cites exact document name and page number | -| 💬 **Conversation Memory** | Chat history maintained in-session for contextual follow-ups | -| ☁️ **Zero-Config Deploy** | Auto-generates sample documents and ingests on first Streamlit Cloud boot | - ---- - -## 🏗️ Architecture - -```mermaid -flowchart TB - subgraph UI["🖥️ Streamlit UI"] - A["User Query"] --> B["Chat Interface"] - B --> C["st.write_stream"] - end - - subgraph RAG["⚡ RAG Pipeline"] - D["FastEmbed ONNX
bge-small-en-v1.5"] --> E["ChromaDB
Vector Store"] - E --> F["Top-K Retrieval
k=6 chunks"] - end - - subgraph LLM["🤖 LLM Layer"] - G{"Provider
Selection"} - G -->|"Primary"| H["Groq LPU
Llama 3.3 70B"] - G -->|"Fallback"| I["Google Gemini
2.0 Flash"] - end - - subgraph INGEST["📥 Ingestion Pipeline"] - J["PDF Documents"] --> K["PyMuPDF Loader"] - K --> L["Recursive Chunking
512 tokens, 50 overlap"] - L --> D - end - - A --> F - F --> |"Context + Prompt"| G - H --> C - I --> C - - style UI fill:#1a1a2e,stroke:#667eea,color:#fff - style RAG fill:#16213e,stroke:#0f3460,color:#fff - style LLM fill:#1a1a2e,stroke:#e94560,color:#fff - style INGEST fill:#16213e,stroke:#533483,color:#fff -``` - -### Request Flow (Single Query) - -```mermaid -sequenceDiagram - participant U as 👤 User - participant S as 🖥️ Streamlit - participant E as ⚡ FastEmbed (Local) - participant C as 🗄️ ChromaDB - participant L as 🤖 Groq/Gemini - - U->>S: "What tech stack did we use for banking?" - S->>E: Embed query (local, ~5ms) - E->>C: Similarity search (k=6) - C-->>S: Top 6 relevant chunks + metadata - S->>L: Single prompt with context - L-->>S: Streaming response tokens - S-->>U: Word-by-word answer with citations - - Note over E,C: Zero API calls for retrieval - Note over L: Single LLM call per query -``` - ---- - -## 🛠️ Technology Stack - -| Layer | Technology | Why This Choice | -|---|---|---| -| **LLM (Primary)** | Groq — Llama 3.3 70B | Fastest free inference (LPU), 30 RPM, sub-second latency | -| **LLM (Fallback)** | Google Gemini 2.0 Flash | Free tier backup, 15 RPM | -| **Embeddings** | FastEmbed (ONNX) — `bge-small-en-v1.5` | Local execution, no API calls, no rate limits | -| **Vector Store** | ChromaDB (persistent) | Lightweight, embedded, perfect for document-scale RAG | -| **RAG Framework** | LangChain 0.3+ | Industry-standard abstractions for retrieval chains | -| **PDF Processing** | PyMuPDF | Fastest Python PDF parser, preserves layout metadata | -| **UI** | Streamlit | Rapid prototyping with built-in streaming support | -| **Deployment** | Streamlit Community Cloud | Free hosting with GitHub auto-deploy | - ---- - -## 🚀 Quick Start - -### Option 1: Use the Live Demo -👉 **[app-rfp-analyst.streamlit.app](https://app-rfp-analyst-ne9xjgfqqdmtrrmgns8jfa.streamlit.app/)** — No setup required. The app auto-generates sample documents on first load. - -### Option 2: Run Locally - -#### 1. Get a Free API Key (Choose One) - -| Provider | Speed | Free Limit | Get Key | -|---|---|---|---| -| **Groq** ⭐ Recommended | ~100 tok/s | 30 RPM, 6000 RPD | [console.groq.com/keys](https://console.groq.com/keys) | -| Google Gemini | ~30 tok/s | 15 RPM | [aistudio.google.com/apikey](https://aistudio.google.com/apikey) | - -#### 2. Setup - -```bash -# Clone the repository -git clone https://github.com/tusharg007/Internal-RFP-Analyst.git -cd Internal-RFP-Analyst - -# Create virtual environment -python -m venv venv -venv\Scripts\activate # Windows -# source venv/bin/activate # Mac/Linux - -# Install dependencies -pip install -r requirements.txt - -# Configure API key -copy .env.example .env -# Edit .env → add your GROQ_API_KEY (or GOOGLE_API_KEY) -``` - -#### 3. Launch - -```bash -streamlit run app.py -``` - -The app will auto-generate 10 sample consulting documents and build the vector store on first launch. - ---- - -## 💬 Example Queries - -| Query | What It Tests | -|---|---| -| *"List all projects with their timelines"* | Full knowledge base traversal | -| *"What tech stack did we use for the banking audit?"* | Precise document retrieval | -| *"Compare the healthcare and insurance projects"* | Cross-document synthesis | -| *"Which projects used Azure services?"* | Multi-document filtering | -| *"What was the budget for the supply chain platform?"* | Specific fact extraction | -| *"What compliance frameworks did we follow in pharma?"* | Domain-specific retrieval | - ---- - -## 🧪 Engineering Challenges & Solutions - -### Challenge 1: Gemini API Rate Limits Killed the App - -**Problem:** The original architecture used Google Gemini for *both* embeddings and LLM generation. The free tier (100 embedding req/min, 15 LLM req/min) was exhausted within minutes, returning `429 RESOURCE_EXHAUSTED` errors. The multi-step ReAct agent made 3-5 LLM calls per query, compounding the problem. - -**Solution: Hybrid local + cloud architecture** - -```mermaid -flowchart LR - subgraph BEFORE["❌ Before — All API Calls"] - A1["Gemini Embeddings API"] -->|"Rate Limited"| B1["429 Error"] - A2["Gemini LLM x 3-5 calls"] -->|"Rate Limited"| B1 - end - - subgraph AFTER["✅ After — Minimal API Calls"] - C1["FastEmbed Local
Zero API calls"] -->|"Instant"| D1["Success"] - C2["Groq LLM x 1 call
30 RPM limit"] -->|"Sub-second"| D1 - end - - style BEFORE fill:#2d1117,stroke:#f85149,color:#fff - style AFTER fill:#0d1117,stroke:#3fb950,color:#fff -``` - -| Metric | Before | After | Improvement | -|---|---|---|---| -| API calls per query | 4-6 (embed + 3-5 LLM) | **1** (LLM only) | **83% reduction** | -| Embedding rate limits | 100/min (API) | **∞** (local) | **Eliminated** | -| LLM rate limits | 15 RPM (Gemini) | **30 RPM** (Groq) | **2x headroom** | - -### Challenge 2: 10+ Minute Response Times - -**Problem:** The ReAct agent architecture (LangGraph) made multiple sequential LLM calls — tool selection → execution → result processing → possibly more tools → final answer. Each call could trigger a rate-limit retry with exponential backoff (10s → 20s → 40s), compounding to 10+ minute waits. - -**Solution: Single-call RAG with streaming** - -- Replaced multi-step ReAct agent with a **single LLM call** architecture -- All context (retrieved chunks + project list + chat history) is assembled locally and sent in one prompt -- **Streaming responses** via `st.write_stream()` — text appears word-by-word, so the user sees output within 500ms even if full generation takes 3-5s - -| Metric | Before (ReAct) | After (Single-Call RAG) | -|---|---|---| -| LLM calls per query | 3-5 | **1** | -| Worst-case response time | 10+ minutes | **3-8 seconds** | -| Perceived latency | Full wait → wall of text | **~500ms** (streaming) | - -### Challenge 3: Sample Question Buttons Did Nothing - -**Problem:** Clicking a sample question button added the message to chat history and triggered `st.rerun()`, but after the rerun, only the `st.chat_input()` code path processed queries — sample button clicks were silently ignored. - -**Solution:** Introduced a `pending_query` session state flag. Button clicks set this flag before rerun. After rerun, a dedicated handler detects the pending query and routes it through the same processing pipeline as typed messages. - -### Challenge 4: Ephemeral Filesystem on Streamlit Cloud - -**Problem:** Streamlit Cloud's filesystem resets on every cold start, losing the vector store and requiring re-ingestion. - -**Solution:** Auto-setup pipeline — on first load, the app detects an empty vector store, generates 10 sample PDFs via `document_generator.py`, and ingests them automatically. With local embeddings, this entire process completes in **under 15 seconds** (vs. minutes with API-based embeddings). - ---- - -## 📁 Project Structure - -``` -Internal-RFP-Analyst/ -├── app.py # Streamlit UI with streaming chat -├── agent.py # RAG query engine (Groq/Gemini + retrieval) -├── rag_engine.py # Ingestion pipeline (FastEmbed + ChromaDB) -├── config.py # Central configuration & provider selection -├── document_generator.py # Generates 10 realistic consulting PDFs -├── requirements.txt # Python dependencies -├── .env.example # API key template -├── .streamlit/ -│ └── config.toml # Streamlit theme configuration -├── data/documents/ # PDF documents (auto-generated) -└── vectorstore/ # ChromaDB persistent storage -``` - -### Module Responsibilities - -| Module | Lines | Responsibility | -|---|---|---| -| `config.py` | ~75 | API keys, model selection, RAG parameters, system prompt | -| `rag_engine.py` | ~165 | PDF loading → chunking → local embedding → ChromaDB storage/retrieval | -| `agent.py` | ~155 | LLM provider selection, prompt assembly, streaming query execution | -| `app.py` | ~280 | Streamlit UI, session management, chat rendering, error handling | -| `document_generator.py` | ~550 | Generates 10 industry-specific consulting PDFs with realistic content | - ---- - -## 🔧 Configuration - -### Environment Variables - -| Variable | Required | Description | -|---|---|---| -| `GROQ_API_KEY` | ⭐ Recommended | Groq API key for fastest inference ([get free key](https://console.groq.com/keys)) | -| `GOOGLE_API_KEY` | Optional | Google Gemini key as fallback ([get free key](https://aistudio.google.com/apikey)) | - -### For Streamlit Cloud Deployment - -Add secrets in **Settings → Secrets**: - -```toml -GROQ_API_KEY = "gsk_your_key_here" -# Optional fallback: -# GOOGLE_API_KEY = "your_google_key_here" -``` - ---- - -## 📜 License - -This project is for educational and portfolio demonstration purposes. Built by [Tushar Ghosh](https://github.com/tusharg007). diff --git a/src/internal_rfp_analyst.egg-info/SOURCES.txt b/src/internal_rfp_analyst.egg-info/SOURCES.txt deleted file mode 100644 index 7adaf40..0000000 --- a/src/internal_rfp_analyst.egg-info/SOURCES.txt +++ /dev/null @@ -1,32 +0,0 @@ -README.md -pyproject.toml -src/internal_rfp_analyst.egg-info/PKG-INFO -src/internal_rfp_analyst.egg-info/SOURCES.txt -src/internal_rfp_analyst.egg-info/dependency_links.txt -src/internal_rfp_analyst.egg-info/top_level.txt -src/rfp_analyst/__init__.py -src/rfp_analyst/schemas.py -src/rfp_analyst/agent/__init__.py -src/rfp_analyst/agent/graph.py -src/rfp_analyst/agent/prompts.py -src/rfp_analyst/agent/runtime.py -src/rfp_analyst/agent/state.py -src/rfp_analyst/ingestion/__init__.py -src/rfp_analyst/ingestion/chunking.py -src/rfp_analyst/ingestion/loaders.py -src/rfp_analyst/ingestion/pipeline.py -src/rfp_analyst/ingestion/registry.py -src/rfp_analyst/retrieval/__init__.py -src/rfp_analyst/retrieval/vector_store.py -src/rfp_analyst/tools/__init__.py -src/rfp_analyst/tools/compare_projects.py -src/rfp_analyst/tools/proposal_writer.py -src/rfp_analyst/tools/rfp_gap_analyzer.py -src/rfp_analyst/tools/search_kb.py -src/rfp_analyst/tools/source_verifier.py -tests/test_agentic_tools.py -tests/test_config.py -tests/test_document_generator.py -tests/test_evals.py -tests/test_imports.py -tests/test_ingestion_pipeline.py \ No newline at end of file diff --git a/src/internal_rfp_analyst.egg-info/dependency_links.txt b/src/internal_rfp_analyst.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/internal_rfp_analyst.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/internal_rfp_analyst.egg-info/top_level.txt b/src/internal_rfp_analyst.egg-info/top_level.txt deleted file mode 100644 index 9bd1376..0000000 --- a/src/internal_rfp_analyst.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -rfp_analyst diff --git a/src/rfp_analyst.egg-info/PKG-INFO b/src/rfp_analyst.egg-info/PKG-INFO deleted file mode 100644 index e6bf909..0000000 --- a/src/rfp_analyst.egg-info/PKG-INFO +++ /dev/null @@ -1,3 +0,0 @@ -Metadata-Version: 2.4 -Name: rfp_analyst -Version: 0.0.0 diff --git a/src/rfp_analyst.egg-info/SOURCES.txt b/src/rfp_analyst.egg-info/SOURCES.txt deleted file mode 100644 index d7fc1d6..0000000 --- a/src/rfp_analyst.egg-info/SOURCES.txt +++ /dev/null @@ -1,32 +0,0 @@ -README.md -pyproject.toml -src/rfp_analyst/__init__.py -src/rfp_analyst/schemas.py -src/rfp_analyst.egg-info/PKG-INFO -src/rfp_analyst.egg-info/SOURCES.txt -src/rfp_analyst.egg-info/dependency_links.txt -src/rfp_analyst.egg-info/top_level.txt -src/rfp_analyst/agent/__init__.py -src/rfp_analyst/agent/graph.py -src/rfp_analyst/agent/prompts.py -src/rfp_analyst/agent/runtime.py -src/rfp_analyst/agent/state.py -src/rfp_analyst/ingestion/__init__.py -src/rfp_analyst/ingestion/chunking.py -src/rfp_analyst/ingestion/loaders.py -src/rfp_analyst/ingestion/pipeline.py -src/rfp_analyst/ingestion/registry.py -src/rfp_analyst/retrieval/__init__.py -src/rfp_analyst/retrieval/vector_store.py -src/rfp_analyst/tools/__init__.py -src/rfp_analyst/tools/compare_projects.py -src/rfp_analyst/tools/proposal_writer.py -src/rfp_analyst/tools/rfp_gap_analyzer.py -src/rfp_analyst/tools/search_kb.py -src/rfp_analyst/tools/source_verifier.py -tests/test_agentic_tools.py -tests/test_config.py -tests/test_document_generator.py -tests/test_evals.py -tests/test_imports.py -tests/test_ingestion_pipeline.py \ No newline at end of file diff --git a/src/rfp_analyst.egg-info/dependency_links.txt b/src/rfp_analyst.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/rfp_analyst.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/rfp_analyst.egg-info/top_level.txt b/src/rfp_analyst.egg-info/top_level.txt deleted file mode 100644 index 9bd1376..0000000 --- a/src/rfp_analyst.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -rfp_analyst diff --git a/src/rfp_analyst/__init__.py b/src/rfp_analyst/__init__.py index ce8278c..501a621 100644 --- a/src/rfp_analyst/__init__.py +++ b/src/rfp_analyst/__init__.py @@ -1 +1,21 @@ -"""Internal RFP Analyst package.""" +"""Internal RFP Analyst support package.""" + +from .exceptions import ( + IngestionError, + KnowledgeBaseNotReadyError, + LLMProviderNotConfiguredError, + NoDocumentsFoundError, + RetrievalError, + RFPAnalystError, + UnsupportedFileError, +) + +__all__ = [ + "IngestionError", + "KnowledgeBaseNotReadyError", + "LLMProviderNotConfiguredError", + "NoDocumentsFoundError", + "RetrievalError", + "RFPAnalystError", + "UnsupportedFileError", +] diff --git a/src/rfp_analyst/agent/__init__.py b/src/rfp_analyst/agent/__init__.py index c84763c..c82f80c 100644 --- a/src/rfp_analyst/agent/__init__.py +++ b/src/rfp_analyst/agent/__init__.py @@ -1 +1,17 @@ -"""Agent orchestration package.""" +"""Agentic RAG orchestration package.""" + +from .graph import ( + compile_query_graph, + prepare_query_payload, + run_agent_graph, + run_query, + stream_query_response, +) + +__all__ = [ + "compile_query_graph", + "prepare_query_payload", + "run_agent_graph", + "run_query", + "stream_query_response", +] diff --git a/src/rfp_analyst/agent/graph.py b/src/rfp_analyst/agent/graph.py index 6d16af9..e78151a 100644 --- a/src/rfp_analyst/agent/graph.py +++ b/src/rfp_analyst/agent/graph.py @@ -1,86 +1,1547 @@ -"""Deterministic agent graph for orchestration.""" +"""LangGraph-backed query workflow with deterministic fallback.""" from __future__ import annotations -from rag_engine import get_vectorstore_stats -from rfp_analyst.agent.prompts import ( - build_agentic_prompt, - build_ambiguous_question_message, - build_no_documents_message, - classify_query_intent, - is_ambiguous_query, +import re +from functools import lru_cache +from typing import Any, Callable, TypedDict + +from config import ( + AGENT_SYSTEM_PROMPT, + MAX_CASE_STUDIES, + MAX_CHUNKS_PER_CASE_STUDY, + MAX_CONTEXT_CHARS_PER_CHUNK, + MAX_HISTORY_MESSAGES, + MAX_PROMPT_TOKENS, + MAX_TARGET_CHUNKS, + MIN_RELEVANCE_SCORE, + RETRIEVAL_K, + RFP_ANALYSIS_MAX_OUTPUT_TOKENS, +) +from rag_engine import ( + KB_NOT_READY_MESSAGE, + NO_SCOPE_DOCUMENTS_MESSAGE, + get_vectorstore_stats, + similarity_search, ) -from rfp_analyst.agent.state import AgentState from rfp_analyst.tools.compare_projects import compare_projects from rfp_analyst.tools.proposal_writer import generate_proposal_outline -from rfp_analyst.tools.rfp_gap_analyzer import extract_rfp_requirements, find_relevant_case_studies -from rfp_analyst.tools.search_kb import search_knowledge_base +from rfp_analyst.tools.rfp_gap_analyzer import ( + extract_rfp_requirements, + find_relevant_case_studies, +) +from rfp_analyst.tools.source_verifier import verify_answer_grounding + +CLARIFICATION_MESSAGE = ( + "Could you clarify which project, document, comparison, or proposal you want me to work on?" +) +FOLLOWUP_CLARIFICATION_MESSAGE = ( + "Which previously discussed document or project should I use for this follow-up?" +) +UNSUPPORTED_CLAIM_MESSAGE = "The retrieved evidence does not support this claim." +INSUFFICIENT_EVIDENCE_MESSAGE = ( + "I could not find sufficiently relevant evidence in the selected document scope." +) +MISSING_UPLOAD_CONTEXT_MESSAGE = ( + "No indexed uploaded documents were found. Upload and ingest the target RFP before running cross-corpus analysis." +) +NO_RELEVANT_UPLOAD_TARGET_MESSAGE = ( + "Uploaded documents are indexed, but no sufficiently relevant target evidence matched this analysis request. " + "Select a specific uploaded document or make the target requirements more explicit." +) +VAGUE_REFERENCE_PATTERN = re.compile( + r"\b(here|this|that|it|those|these|above|the above project|the project|same one)\b", + re.IGNORECASE, +) +PLURAL_REFERENCE_PATTERN = re.compile(r"\b(those|these|projects|documents|the above)\b", re.IGNORECASE) +RFP_ANALYSIS_ORCHESTRATION_PHRASES = ( + "find case studies", + "compare fit", + "generate proposal", + "verify recommendations", + "proposal outline", + "return technical requirements", + "return requirements", + "three case studies", + "case studies", +) +RFP_TARGET_FOCUS_AREAS = ( + "requirements", + "business problem", + "architecture", + "technologies", + "security/compliance", + "timeline", + "risks", + "measurable outcomes", +) +RFP_TARGET_MAX_CHUNKS = 4 +RFP_TARGET_FALLBACK_MARGIN = 0.15 + +try: + from langgraph.graph import END, START, StateGraph + + LANGGRAPH_AVAILABLE = True +except Exception: + END = "__end__" + START = "__start__" + StateGraph = None + LANGGRAPH_AVAILABLE = False + + +class QueryState(TypedDict, total=False): + user_query: str + chat_history: list[dict[str, Any]] + vectorstore_stats: dict[str, Any] + retrieval_k: int + retrieval_scope: str + retrieval_fn: Callable[[str, int, str], list] + traces: list[dict[str, Any]] + kb_ready: bool + intent: str + planned_tools: list[str] + retrieved_documents: list[dict[str, Any]] + retrieval_context: str + specialized_notes: str + tool_outputs: dict[str, Any] + prompt: str + answer: str + response_mode: str + resolved_query: str + resolved_entities: list[dict[str, Any]] + grounded: bool + graph_backend: str + prompt_budget: dict[str, Any] + + +class DeterministicCompiledGraph: + """Fallback runner that mirrors the LangGraph node flow.""" + + def __init__(self): + self.node_order = [ + "health_check", + "classify_intent", + "plan_tools", + "execute_retrieval", + "execute_specialized_tool", + "synthesize_prompt", + "evidence_availability_check", + "final_response", + ] + + def invoke(self, state: QueryState) -> QueryState: + current = dict(state) + current["graph_backend"] = "deterministic-fallback" + for node_name in self.node_order: + node_fn = NODE_FUNCTIONS[node_name] + updates = node_fn(current) + if updates: + current.update(updates) + return current + + +@lru_cache(maxsize=1) +def compile_query_graph(): + """Compile the query graph, using a deterministic fallback when needed.""" + if not LANGGRAPH_AVAILABLE: + return DeterministicCompiledGraph() + + workflow = StateGraph(QueryState) + workflow.add_node("health_check", health_check) + workflow.add_node("classify_intent", classify_intent) + workflow.add_node("plan_tools", plan_tools) + workflow.add_node("execute_retrieval", execute_retrieval) + workflow.add_node("execute_specialized_tool", execute_specialized_tool) + workflow.add_node("synthesize_prompt", synthesize_prompt) + workflow.add_node("evidence_availability_check", evidence_availability_check) + workflow.add_node("final_response", final_response) + + workflow.add_edge(START, "health_check") + workflow.add_edge("health_check", "classify_intent") + workflow.add_edge("classify_intent", "plan_tools") + workflow.add_edge("plan_tools", "execute_retrieval") + workflow.add_edge("execute_retrieval", "execute_specialized_tool") + workflow.add_edge("execute_specialized_tool", "synthesize_prompt") + workflow.add_edge("synthesize_prompt", "evidence_availability_check") + workflow.add_edge("evidence_availability_check", "final_response") + workflow.add_edge("final_response", END) + + return workflow.compile() + + +def _append_trace(state: QueryState, step: str, details: dict[str, Any]) -> list[dict[str, Any]]: + trace = list(state.get("traces", [])) + trace.append({"step": step, **details}) + return trace + + +def _is_ambiguous_query(query: str) -> bool: + lowered = query.lower().strip() + if len(lowered) < 12: + return True + ambiguous_phrases = { + "help me", + "tell me more", + "what about that", + "can you help", + "do it", + "continue", + } + return lowered in ambiguous_phrases + + +def _is_vague_followup(query: str) -> bool: + return bool(VAGUE_REFERENCE_PATTERN.search(query or "")) + + +def _dedupe_entities(entities: list[dict[str, Any]]) -> list[dict[str, Any]]: + seen = set() + unique = [] + for entity in entities: + source = entity.get("source") + if not source: + continue + key = (source, entity.get("document_origin", "sample")) + if key in seen: + continue + seen.add(key) + unique.append(entity) + return unique + + +def _extract_recent_entities( + chat_history: list[dict[str, Any]] | None, + scope: str, +) -> list[dict[str, Any]]: + """Extract recent retrieved source entities from persisted UI traces.""" + if not chat_history: + return [] + + entities: list[dict[str, Any]] = [] + for message in reversed(chat_history[-8:]): + reasoning = message.get("reasoning") or [] + for step in reversed(reasoning): + source = step.get("source") + if not source and step.get("tool_response"): + source = str(step["tool_response"]).split(" (Page ", 1)[0] + if not source: + continue + origin = step.get("document_origin", "sample") + if scope in {"sample", "upload"} and origin != scope: + continue + entities.append( + { + "source": source, + "document_origin": origin, + "page": step.get("page"), + } + ) + if entities: + break + return _dedupe_entities(entities) + + +def _resolve_conversational_query( + query: str, + chat_history: list[dict[str, Any]] | None, + scope: str, +) -> tuple[str, list[dict[str, Any]], str]: + """Resolve vague follow-up references using recent retrieved entities.""" + if not _is_vague_followup(query): + return query, [], "not_needed" + + entities = _extract_recent_entities(chat_history, scope) + if not entities: + return query, [], "ambiguous" + + if len(entities) > 1 and not PLURAL_REFERENCE_PATTERN.search(query): + return query, entities, "ambiguous" + + entity_names = ", ".join(entity["source"] for entity in entities[:4]) + resolved_query = f"{query}\n\nResolved follow-up target document(s): {entity_names}" + return resolved_query, entities, "resolved" + + +def _build_rfp_target_query(query: str) -> str: + """Focus upload retrieval on target requirements, not orchestration verbs.""" + cleaned = (query or "").replace("\n", " ") + for phrase in RFP_ANALYSIS_ORCHESTRATION_PHRASES: + cleaned = re.sub(rf"\b{re.escape(phrase)}\b", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s+", " ", cleaned).strip(" ,.;:-") + focus = ", ".join(RFP_TARGET_FOCUS_AREAS) + if cleaned: + return f"{cleaned}. Focus on: {focus}." + return f"Focus on: {focus}." + + +def _dedupe_documents_by_source_page(documents: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + unique: list[dict[str, Any]] = [] + seen: set[tuple[str, int]] = set() + for item in documents: + key = (item.get("source", "Unknown"), int(item.get("page", 0) or 0)) + if key in seen: + continue + seen.add(key) + unique.append(item) + if len(unique) >= limit: + break + return unique + + +def _classify_query(query: str) -> str: + lowered = query.lower().strip() + if any( + phrase in lowered + for phrase in ( + "what sources did you use", + "which documents were used", + "show the citations from your last response", + "sources for the previous answer", + ) + ): + return "previous_sources" + if _is_ambiguous_query(lowered): + return "ambiguous" + analysis_signals = ( + "extract requirements", + "find gaps", + "find case studies", + "compare fit", + "verify recommendations", + "proposal outline", + ) + if any(signal in lowered for signal in analysis_signals): + return "rfp_analysis" + if any(token in lowered for token in ("proposal", "draft", "write", "respond to rfp", "rfp response")): + return "proposal" + if any(token in lowered for token in ("compare", "difference", "versus", " vs ", "contrast")): + return "compare" + return "search" + + +def _previous_answer_sources(chat_history: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + """Return the prior assistant answer's displayed sources without retrieval.""" + for message in reversed(chat_history or []): + if message.get("role") != "assistant": + continue + sources = [] + seen = set() + for step in message.get("reasoning") or []: + source = step.get("source") + page = step.get("page") + if not source or page is None: + continue + key = (source, int(page), step.get("document_origin", "sample")) + if key in seen: + continue + seen.add(key) + sources.append( + { + "source_file": source, + "page": int(page), + "document_origin": step.get("document_origin", "sample"), + } + ) + return sources + return [] + + +def _format_previous_sources(sources: list[dict[str, Any]]) -> str: + if not sources: + return "The previous answer did not contain any source citations." + return "\n".join( + f"- {item['source_file']}, Page {item['page']}, origin: {item['document_origin']}" + for item in sources + ) + + +def _format_sources(results: list[dict[str, Any]]) -> str: + parts = [] + for item in results: + page_number = item["page"] + 1 if isinstance(item["page"], int) else item["page"] + parts.append( + f"[Source: {item['source']}, Page {page_number}]\n{item['content']}" + ) + return "\n\n---\n\n".join(parts) if parts else "No relevant documents found." + + +def _build_history_text(chat_history: list[dict[str, Any]] | None) -> str: + if not chat_history: + return "" + recent = [ + message + for message in chat_history[-6:] + if message.get("role") in {"user", "assistant"} + ] + lines = [] + for message in recent: + role = "User" if message["role"] == "user" else "Assistant" + lines.append(f"{role}: {message.get('content', '')[:300]}") + return "\n".join(lines) + + +def _estimate_tokens(text: str) -> int: + return max(1, (len(text or "") + 3) // 4) + +def _truncate_text(text: str, limit: int = MAX_CONTEXT_CHARS_PER_CHUNK) -> str: + compact = " ".join(str(text or "").split()) + if len(compact) <= limit: + return compact + return compact[: max(limit - 3, 0)].rstrip() + "..." -def _add_source_trace(state: AgentState, sources: list[dict]) -> None: - for source in sources[:5]: - state.tool_trace.append( + +def _format_compact_citation(source: str, page: int, origin: str) -> str: + return f"{source} (Page {int(page) + 1}, origin={origin})" + + +def _compact_document_entry(document: dict[str, Any]) -> dict[str, Any]: + return { + "source": document.get("source", "Unknown"), + "page": int(document.get("page", 0) or 0), + "score": float(document.get("score", 0.0) or 0.0), + "chunk_id": document.get("chunk_id", ""), + "document_origin": document.get("document_origin", "sample"), + "content": _truncate_text(document.get("content", "")), + } + + +def _group_case_study_documents(documents: list[dict[str, Any]]) -> list[dict[str, Any]]: + grouped: dict[str, dict[str, Any]] = {} + for document in documents: + if document.get("document_origin") != "sample": + continue + source = document.get("source", "Unknown") + group = grouped.setdefault( + source, + { + "source": source, + "score": float(document.get("score", 0.0) or 0.0), + "documents": [], + }, + ) + group["score"] = max(group["score"], float(document.get("score", 0.0) or 0.0)) + group["documents"].append(document) + + normalized = [] + for source, group in grouped.items(): + docs = sorted( + group["documents"], + key=lambda item: (-float(item.get("score", 0.0) or 0.0), item.get("page", 0), item.get("chunk_id", "")), + ) + docs = _dedupe_documents_by_source_page(docs, MAX_CHUNKS_PER_CASE_STUDY) + normalized.append({"source": source, "score": group["score"], "documents": docs}) + normalized.sort(key=lambda item: (-item["score"], item["source"])) + return normalized[:MAX_CASE_STUDIES] + + +def _dedupe_documents_by_content_signature(documents: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + unique: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in documents: + signature = re.sub(r"\d+", "", item.get("content", "").lower())[:120] + if signature in seen: + continue + seen.add(signature) + unique.append(item) + if len(unique) >= limit: + break + return unique + + +def compact_tool_outputs_for_prompt(tool_outputs: dict[str, Any]) -> dict[str, Any]: + requirements = [] + for item in (tool_outputs.get("extract_rfp_requirements", {}) or {}).get("requirements", [])[:8]: + requirements.append( + { + "id": item.get("id", ""), + "text": _truncate_text(item.get("text", ""), 220), + "source_file": item.get("source_file", ""), + "page": item.get("page"), + "document_origin": item.get("document_origin", ""), + } + ) + + gaps = [] + for item in (tool_outputs.get("extract_rfp_requirements", {}) or {}).get("gaps", [])[:10]: + gaps.append( { - "tool_response": f"{source['source']} (Page {source['page'] + 1})", - "snippet": source["snippet"], + "requirement_id": item.get("requirement_id", ""), + "detail": _truncate_text(item.get("detail", ""), 220), + "status": item.get("status", "absent_or_ambiguous"), } ) + case_matches = [] + for match in (tool_outputs.get("find_relevant_case_studies", {}) or {}).get("matches", [])[:MAX_CASE_STUDIES]: + pages = [int(page) + 1 for page in match.get("pages", [])[:2]] + case_matches.append( + { + "source": match.get("source", "Unknown"), + "fit_reason": _truncate_text(" ; ".join(match.get("snippets", [])[:2]), 220), + "fit_score": match.get("fit_score", len(match.get("matched_requirements", []))), + "matched_requirements": match.get("matched_requirements", []), + "missing_coverage": match.get("missing_coverage", []), + "citations": [f"Page {page}" for page in pages], + } + ) -def run_agent_graph(state: AgentState, search_fn=None, stats_fn=None) -> AgentState: - """Run the agent workflow from intent classification to synthesis prompt creation.""" - state.stats = stats_fn() if stats_fn else get_vectorstore_stats() - if state.stats.get("status") != "ready" or state.stats.get("total_documents", 0) == 0: - state.tool_trace.append({"tool": "knowledge_base_status", "input": {"status": "not_initialized"}}) - state.final_answer = build_no_documents_message(state.query) - return state - - if is_ambiguous_query(state.query): - state.tool_trace.append({"tool": "ambiguity_check", "input": {"status": "ambiguous"}}) - state.final_answer = build_ambiguous_question_message(state.query) - return state - - state.intent = classify_query_intent(state.query) - state.tool_trace.append({"tool": "classify_intent", "input": {"query": state.query, "intent": state.intent}}) - - if state.intent == "compare_projects": - comparison = compare_projects(state.query, search_fn=search_fn) - state.tool_outputs["compare_projects"] = comparison - state.retrieved_documents = comparison["documents"] - state.sources = comparison["sources"] - state.tool_trace.append({"tool": "compare_projects", "input": {"query": state.query}}) - _add_source_trace(state, state.sources) - elif state.intent in {"rfp_gap_analysis", "proposal_writer"}: - requirements = extract_rfp_requirements(state.query) - state.tool_outputs["extract_rfp_requirements"] = requirements - state.tool_trace.append({"tool": "extract_rfp_requirements", "input": {"count": len(requirements['requirements'])}}) - - case_studies = find_relevant_case_studies(requirements["requirements"], search_fn=search_fn) - state.tool_outputs["find_relevant_case_studies"] = case_studies - state.retrieved_documents = case_studies["documents"] - state.sources = [ - {"source": match["source"], "page": match["pages"][0] if match["pages"] else 0, "snippet": match["snippets"][0] if match["snippets"] else ""} - for match in case_studies["matches"] + comparison_rows = [] + for row in (tool_outputs.get("compare_projects", {}) or {}).get("rows", [])[:3]: + comparison_rows.append( + { + "source": row.get("source", "Unknown"), + "timeline": _truncate_text(row.get("timeline", ""), 120), + "budget": _truncate_text(row.get("budget", ""), 80), + "tech_stack": _truncate_text(row.get("tech_stack", ""), 140), + "outcomes": _truncate_text(row.get("outcomes", ""), 140), + } + ) + + proposal = tool_outputs.get("proposal_writer", {}) or {} + proposal_headings = [] + for line in str(proposal.get("outline", "") or "").splitlines(): + stripped = line.strip() + if stripped.startswith("## "): + proposal_headings.append(stripped[3:]) + elif stripped.startswith("- "): + proposal_headings.append(_truncate_text(stripped[2:], 160)) + if len(proposal_headings) >= 8: + break + if not proposal_headings and proposal.get("outline"): + proposal_headings.append(_truncate_text(proposal.get("outline", ""), 220)) + + return { + "requirements": requirements, + "gaps": gaps, + "selected_case_studies": case_matches, + "comparison": comparison_rows, + "proposal": proposal_headings, + } + + +def _render_compact_tool_outputs(tool_outputs: dict[str, Any], verbose: bool = True) -> str: + compact = compact_tool_outputs_for_prompt(tool_outputs) + lines = [] + + requirements = compact.get("requirements", []) + if requirements: + lines.append("-- Requirement Summary --") + for item in requirements[:6]: + citation = "" + if item.get("source_file") and item.get("page") is not None: + citation = f" [{item['source_file']}, Page {int(item['page']) + 1}]" + lines.append(f"- {item.get('id', '')}: {item.get('text', '')}{citation}") + + gaps = compact.get("gaps", []) + if gaps: + lines.append("-- Inferred Requirement Gaps --") + for item in gaps[:8]: + lines.append(f"- {item.get('requirement_id', '')}: {item.get('detail', '')}") + + case_studies = compact.get("selected_case_studies", []) + if case_studies: + lines.append("-- Selected Case Studies --") + case_limit = MAX_CASE_STUDIES if verbose else min(MAX_CASE_STUDIES, 2) + for item in case_studies[:case_limit]: + detail = f": {item.get('fit_reason', '')}" if verbose and item.get("fit_reason") else "" + citations = ", ".join(item.get("citations", [])[:2]) + fit_score = item.get("fit_score") + score_text = f" fit={fit_score}" if fit_score is not None else "" + matched = f"; matched={', '.join(item.get('matched_requirements', []))}" if item.get("matched_requirements") else "" + missing = f"; missing={', '.join(item.get('missing_coverage', []))}" if item.get("missing_coverage") else "" + citation_text = f" [{citations}]" if citations else "" + lines.append(f"- {item.get('source', 'Unknown')}{score_text}{matched}{missing}{citation_text}{detail}") + + comparison_rows = compact.get("comparison", []) + if comparison_rows: + lines.append("-- Comparison Summary --") + for row in comparison_rows[:3]: + if verbose: + lines.append( + f"- {row['source']}: timeline={row['timeline']}; budget={row['budget']}; " + f"stack={row['tech_stack']}; outcomes={row['outcomes']}" + ) + else: + lines.append(f"- {row['source']}: stack={row['tech_stack']}; outcomes={row['outcomes']}") + + proposal = compact.get("proposal", []) + if proposal: + lines.append("-- Proposal Outline --") + proposal_limit = 8 if verbose else 4 + for item in proposal[:proposal_limit]: + lines.append(f"- {item}") + + return "\n".join(lines).strip() + + +def _build_compact_prompt_sections( + state: QueryState, + history_messages: list[dict[str, Any]], + target_documents: list[dict[str, Any]], + sample_case_groups: list[dict[str, Any]], + include_verbose_tools: bool, +) -> dict[str, str]: + stats = state.get("vectorstore_stats", {}) + scope_label = state.get("retrieval_scope", "all") + project_list = "\n".join(f" - {name}" for name in stats.get("document_names", [])[:12]) or " No documents ingested yet." + tool_block = _render_compact_tool_outputs(state.get("tool_outputs", {}), verbose=include_verbose_tools) + + target_lines = [] + for document in target_documents: + target_lines.append( + f"- {_format_compact_citation(document['source'], document['page'], document['document_origin'])}: " + f"{document['content']}" + ) + target_block = "\n".join(target_lines) if target_lines else "- No uploaded target evidence selected." + + sample_lines = [] + for group in sample_case_groups: + sample_lines.append(f"- {group['source']}") + for document in group.get("documents", []): + sample_lines.append( + f" * Page {int(document['page']) + 1}: {document['content']}" + ) + sample_block = "\n".join(sample_lines) if sample_lines else "- No sample case-study evidence selected." + + history_lines = [] + for message in history_messages: + role = "User" if message.get("role") == "user" else "Assistant" + history_lines.append(f"{role}: {_truncate_text(message.get('content', ''), 240)}") + history_block = "\n".join(history_lines) + + sections = [ + AGENT_SYSTEM_PROMPT.strip(), + "\n-- Workflow Intent --", + state.get("intent", "search"), + "\n-- Planned Tools --", + ", ".join(state.get("planned_tools", [])) or "none", + "\n-- Document Scope --", + scope_label, + "\n-- Available Documents --", + f"{project_list}\nTotal: {stats.get('total_documents', 0)} documents, {stats.get('total_chunks', 0)} chunks", + "\n-- Uploaded Target Evidence --", + target_block, + "\n-- Sample Case Study Evidence --", + sample_block, + ] + if tool_block: + sections.extend(["\n-- Compact Tool Outputs --", tool_block]) + if history_block: + sections.extend(["\n-- Recent Conversation --", history_block]) + sections.extend( + [ + "\n-- Question --", + state.get("user_query", ""), + "\nAnswer thoroughly with source citations. If the evidence is incomplete, say so explicitly.", + 'Never write "[Source: None]". If a claim is unsupported, write: The retrieved evidence does not support this claim.', ] - state.tool_trace.append({"tool": "find_relevant_case_studies", "input": {"matches": len(case_studies['matches'])}}) - _add_source_trace(state, state.sources) + ) + return { + "prompt": "\n".join(section for section in sections if section), + "tool_block": tool_block, + } - if state.intent == "proposal_writer": - outline = generate_proposal_outline(state.query, case_studies, requirements["requirements"]) - state.tool_outputs["generate_proposal_outline"] = outline - state.tool_trace.append({"tool": "generate_proposal_outline", "input": {"sections": 5}}) - else: - search_result = search_knowledge_base(state.query, search_fn=search_fn) - state.tool_outputs["search_knowledge_base"] = search_result - state.retrieved_documents = search_result["documents"] - state.sources = search_result["sources"] - state.tool_trace.append({"tool": "search_knowledge_base", "input": {"query": state.query}}) - _add_source_trace(state, state.sources) - - if not state.retrieved_documents and not state.sources: - state.final_answer = "I couldn't find relevant documents for this request. Please ingest more documents or refine the question." - return state - - state.prompt = build_agentic_prompt(state) - state.tool_trace.append({"tool": "verify_answer_grounding", "input": {"status": "planned"}}) + +def health_check(state: QueryState) -> QueryState: + stats = dict(state.get("vectorstore_stats") or get_vectorstore_stats()) + kb_ready = stats.get("status") == "ready" and int(stats.get("total_chunks", 0) or 0) > 0 + return { + "vectorstore_stats": stats, + "kb_ready": kb_ready, + "traces": _append_trace( + state, + "health_check", + { + "tool": "health_check", + "kb_ready": kb_ready, + "document_count": int(stats.get("total_documents", 0) or 0), + "chunk_count": int(stats.get("total_chunks", 0) or 0), + }, + ), + } + + +def classify_intent(state: QueryState) -> QueryState: + retrieval_scope = state.get("retrieval_scope", "all") + resolved_query, resolved_entities, resolution_status = _resolve_conversational_query( + state.get("user_query", ""), + state.get("chat_history"), + retrieval_scope, + ) + intent = _classify_query(resolved_query if resolution_status == "resolved" else state.get("user_query", "")) + if intent == "previous_sources": + sources = _previous_answer_sources(state.get("chat_history")) + return { + "intent": intent, + "response_mode": "direct", + "answer": _format_previous_sources(sources), + "tool_outputs": {"previous_sources": sources}, + "resolved_query": resolved_query, + "resolved_entities": [], + "traces": _append_trace( + state, + "classify_intent", + { + "tool": "previous_sources", + "intent": intent, + "input_summary": state.get("user_query", "")[:120], + "output_summary": f"Returned {len(sources)} source citation(s) from the previous answer.", + }, + ), + } + if resolution_status == "ambiguous": + intent = "ambiguous" + response_mode = "clarification" if intent == "ambiguous" else "llm" + answer = FOLLOWUP_CLARIFICATION_MESSAGE if resolution_status == "ambiguous" else "" + answer = answer or (CLARIFICATION_MESSAGE if intent == "ambiguous" else "") + return { + "intent": intent, + "response_mode": response_mode, + "answer": answer, + "resolved_query": resolved_query, + "resolved_entities": resolved_entities, + "traces": _append_trace( + state, + "classify_intent", + { + "tool": "intent_classifier", + "intent": intent, + "input_summary": state.get("user_query", "")[:120], + "resolution_status": resolution_status, + "resolved_entities": [entity["source"] for entity in resolved_entities], + }, + ), + } + + +def plan_tools(state: QueryState) -> QueryState: + intent = state.get("intent", "search") + planned_tools: list[str] = [] + if intent == "search": + planned_tools = ["search_knowledge_base"] + elif intent == "compare": + planned_tools = ["search_knowledge_base", "compare_projects"] + elif intent == "proposal": + planned_tools = [ + "search_knowledge_base", + "extract_rfp_requirements", + "find_relevant_case_studies", + "proposal_writer", + ] + elif intent == "rfp_analysis": + planned_tools = [ + "search_knowledge_base", + "extract_rfp_requirements", + "find_relevant_case_studies", + "compare_projects", + "proposal_writer", + ] + + return { + "planned_tools": planned_tools, + "traces": _append_trace( + state, + "plan_tools", + { + "tool": "tool_planner", + "planned_tools": planned_tools, + "output_summary": f"Selected {len(planned_tools)} tool(s): {', '.join(planned_tools) or 'none'}", + }, + ), + } + + +def execute_retrieval(state: QueryState) -> QueryState: + planned_tools = state.get("planned_tools", []) + is_rfp_analysis = state.get("intent") == "rfp_analysis" + retrieval_scope = "upload" if is_rfp_analysis else state.get("retrieval_scope", "all") + if "search_knowledge_base" not in planned_tools: + return { + "retrieved_documents": [], + "retrieval_context": "", + } + + if not state.get("kb_ready"): + traces = _append_trace( + state, + "execute_retrieval", + { + "tool": "search_knowledge_base", + "status": "skipped", + "reason": KB_NOT_READY_MESSAGE, + "scope": retrieval_scope, + }, + ) + return { + "retrieved_documents": [], + "retrieval_context": "", + "response_mode": "fallback", + "answer": KB_NOT_READY_MESSAGE, + "traces": traces, + } + + retrieval_fn = state.get("retrieval_fn") or similarity_search + retrieval_k = int(state.get("retrieval_k", RETRIEVAL_K)) + base_query = state.get("resolved_query") or state.get("user_query", "") + retrieval_query = _build_rfp_target_query(base_query) if is_rfp_analysis else base_query + try: + raw_results = retrieval_fn(retrieval_query, retrieval_k, retrieval_scope) + except TypeError: + raw_results = retrieval_fn(retrieval_query, retrieval_k) + + documents = [] + for doc, score in raw_results: + metadata = getattr(doc, "metadata", {}) or {} + page = metadata.get("page", 0) + try: + page = int(page) + except Exception: + page = 0 + documents.append( + { + "source": metadata.get("source_file", "Unknown"), + "page": page, + "score": float(score), + "content": getattr(doc, "page_content", ""), + "chunk_id": metadata.get("chunk_id", ""), + "document_origin": metadata.get("document_origin", "sample"), + } + ) + + if is_rfp_analysis: + documents = [item for item in documents if item["document_origin"] == "upload"] + + scope_chunk_counts = state.get("vectorstore_stats", {}).get("scope_chunk_counts", {}) + indexed_upload_chunk_count = int(scope_chunk_counts.get("upload", 0) or 0) + indexed_upload_document_count = int(state.get("vectorstore_stats", {}).get("indexed_upload_document_count", 0) or 0) + indexed_upload_files = list(state.get("vectorstore_stats", {}).get("indexed_upload_files", []) or []) + relevant_documents = [item for item in documents if item["score"] >= MIN_RELEVANCE_SCORE] + below_threshold_count = len(documents) - len(relevant_documents) + selected_documents = relevant_documents + target_fallback_used = False + + if is_rfp_analysis: + selected_documents = _dedupe_documents_by_source_page(selected_documents, RFP_TARGET_MAX_CHUNKS) + if not selected_documents and indexed_upload_chunk_count > 0: + near_threshold = [ + item for item in documents if item["score"] >= max(MIN_RELEVANCE_SCORE - RFP_TARGET_FALLBACK_MARGIN, 0.0) + ] + if near_threshold: + target_fallback_used = True + selected_documents = _dedupe_documents_by_source_page(near_threshold, RFP_TARGET_MAX_CHUNKS) + + documents = selected_documents + + traces = _append_trace( + state, + "execute_retrieval", + { + "tool": "search_knowledge_base", + "status": "completed", + "scope": retrieval_scope, + "input": { + "query": retrieval_query, + "original_query": state.get("user_query", ""), + "k": retrieval_k, + "scope": retrieval_scope, + }, + "input_summary": f"Search scope={retrieval_scope}; k={retrieval_k}", + "output_summary": ( + f"Retrieved {len(documents)} relevant chunk(s); " + f"filtered {below_threshold_count} below threshold {MIN_RELEVANCE_SCORE:.2f}" + ), + "documents": [ + { + "source": item["source"], + "page": item["page"] + 1, + "score": f"{item['score']:.2f}", + "chunk_id": item["chunk_id"], + "document_origin": item["document_origin"], + } + for item in documents[:6] + ], + }, + ) + + if is_rfp_analysis: + traces = _append_trace( + {"traces": traces}, + "execute_retrieval", + { + "tool": "target_context_retrieval", + "status": "completed", + "indexed_upload_chunk_count": indexed_upload_chunk_count, + "indexed_upload_document_count": indexed_upload_document_count, + "indexed_upload_files": indexed_upload_files, + "raw_upload_hits": len(raw_results), + "qualifying_upload_hits": len(relevant_documents), + "relevance_threshold": MIN_RELEVANCE_SCORE, + "target_fallback_used": target_fallback_used, + "selected_upload_source_files": sorted({item["source"] for item in documents}), + "input_summary": "Upload-only target retrieval for cross-corpus analysis.", + "output_summary": ( + f"Indexed upload chunks={indexed_upload_chunk_count}; " + f"selected {len(documents)} upload chunk(s)" + + (" using bounded fallback." if target_fallback_used else ".") + ), + }, + ) + + if is_rfp_analysis and indexed_upload_chunk_count == 0: + return { + "retrieved_documents": [], + "retrieval_context": "", + "response_mode": "fallback", + "answer": MISSING_UPLOAD_CONTEXT_MESSAGE, + "traces": traces, + } + if is_rfp_analysis and not documents: + return { + "retrieved_documents": [], + "retrieval_context": "", + "response_mode": "fallback", + "answer": NO_RELEVANT_UPLOAD_TARGET_MESSAGE, + "traces": traces, + } + if not documents and retrieval_scope in {"sample", "upload"} and int(scope_chunk_counts.get(retrieval_scope, 0) or 0) == 0: + return { + "retrieved_documents": [], + "retrieval_context": "", + "response_mode": "fallback", + "answer": NO_SCOPE_DOCUMENTS_MESSAGE, + "traces": traces, + } + + if not documents: + return { + "retrieved_documents": [], + "retrieval_context": "", + "response_mode": "fallback", + "answer": INSUFFICIENT_EVIDENCE_MESSAGE, + "traces": traces, + } + + return { + "retrieved_documents": documents, + "retrieval_context": _format_sources(documents), + "traces": traces, + } + + +def _scoped_search(state: QueryState, scope: str) -> Callable[[str, int], list]: + retrieval_fn = state.get("retrieval_fn") or similarity_search + + def search(query: str, k: int = RETRIEVAL_K): + try: + return retrieval_fn(query, k, scope) + except TypeError: + return retrieval_fn(query, k) + + return search + + +def _documents_to_text(documents: list[dict[str, Any]]) -> str: + return "\n\n".join(item.get("content", "") for item in documents if item.get("content")) + + +def _normalize_tool_documents(documents: list[object]) -> list[dict[str, Any]]: + normalized = [] + for document in documents: + metadata = getattr(document, "metadata", {}) or {} + normalized.append( + { + "source": metadata.get("source_file", "Unknown"), + "page": int(metadata.get("page", 0) or 0), + "score": 1.0, + "content": getattr(document, "page_content", ""), + "chunk_id": metadata.get("chunk_id", ""), + "document_origin": metadata.get("document_origin", "sample"), + } + ) + return normalized + + +def _attach_requirement_source_metadata(requirements: list[dict[str, Any]], documents: list[dict[str, Any]]) -> list[dict[str, Any]]: + target_documents = [document for document in documents if document.get("document_origin") == "upload"] + if not target_documents: + return requirements + fallback = sorted( + target_documents, + key=lambda item: (-float(item.get("score", 0.0) or 0.0), item.get("source", ""), item.get("page", 0)), + )[0] + enriched = [] + for requirement in requirements: + item = dict(requirement) + item.setdefault("source_file", fallback.get("source", "Unknown")) + item.setdefault("page", int(fallback.get("page", 0) or 0)) + item.setdefault("document_origin", fallback.get("document_origin", "upload")) + enriched.append(item) + return enriched + + +def execute_specialized_tool(state: QueryState) -> QueryState: + intent = state.get("intent", "search") + if state.get("response_mode") in {"fallback", "clarification", "direct"}: + return {"specialized_notes": "", "tool_outputs": state.get("tool_outputs", {})} + + query = state.get("resolved_query") or state.get("user_query", "") + outputs = dict(state.get("tool_outputs", {})) + traces = list(state.get("traces", [])) + notes = [] + + if intent == "compare": + result = compare_projects( + query, + search_fn=_scoped_search(state, state.get("retrieval_scope", "all")), + k=int(state.get("retrieval_k", RETRIEVAL_K)), + ) + outputs["compare_projects"] = result + summary = f"Compared {len(result.get('rows', []))} projects across 4 dimensions" + notes.append(result.get("comparison_markdown", "")) + traces.append( + { + "step": "execute_specialized_tool", + "tool": "compare_projects", + "status": "completed", + "input_summary": f"Comparison scope={state.get('retrieval_scope', 'all')}", + "output_summary": summary, + } + ) + + if intent in {"proposal", "rfp_analysis"}: + requirements_result = extract_rfp_requirements( + _documents_to_text(state.get("retrieved_documents", [])) + ) + requirements = _attach_requirement_source_metadata( + requirements_result.get("requirements", []), + state.get("retrieved_documents", []), + ) + requirements_result = {**requirements_result, "requirements": requirements} + outputs["extract_rfp_requirements"] = requirements_result + traces.append( + { + "step": "execute_specialized_tool", + "tool": "extract_rfp_requirements", + "status": "completed", + "input_summary": "Uploaded target evidence" if intent == "rfp_analysis" else "Retrieved target evidence", + "output_summary": requirements_result.get("summary", "Extracted requirements."), + } + ) + + case_scope = "sample" if intent == "rfp_analysis" else state.get("retrieval_scope", "all") + case_studies = find_relevant_case_studies( + requirements, + search_fn=_scoped_search(state, case_scope), + k=int(state.get("retrieval_k", RETRIEVAL_K)), + ) + outputs["find_relevant_case_studies"] = case_studies + selected_names = [item.get("source", "Unknown") for item in case_studies.get("matches", [])[:5]] + traces.append( + { + "step": "execute_specialized_tool", + "tool": "find_relevant_case_studies", + "status": "completed", + "input_summary": f"Case-study scope={case_scope}", + "output_summary": f"Selected {len(selected_names)} case study match(es): {', '.join(selected_names) or 'none'}", + } + ) + + if intent == "rfp_analysis": + comparison = compare_projects( + "Compare fit for: " + " ".join(item.get("text", "") for item in requirements), + search_fn=_scoped_search(state, "sample"), + k=int(state.get("retrieval_k", RETRIEVAL_K)), + ) + outputs["compare_projects"] = comparison + traces.append( + { + "step": "execute_specialized_tool", + "tool": "compare_projects", + "status": "completed", + "input_summary": "Compare sample case-study fit against uploaded requirements", + "output_summary": f"Compared {len(comparison.get('rows', []))} projects across 4 dimensions", + } + ) + + proposal = generate_proposal_outline(query, case_studies, requirements) + outputs["proposal_writer"] = proposal + notes.extend( + [ + requirements_result.get("summary", ""), + f"Selected case studies: {', '.join(selected_names) or 'none'}", + proposal.get("outline", ""), + ] + ) + traces.append( + { + "step": "execute_specialized_tool", + "tool": "proposal_writer", + "status": "completed", + "input_summary": f"{len(requirements)} requirements and {len(selected_names)} case study match(es)", + "output_summary": "Generated a structured proposal outline from executed tool outputs.", + } + ) + + supporting = _normalize_tool_documents(case_studies.get("documents", [])) + existing_ids = {item.get("chunk_id") for item in state.get("retrieved_documents", [])} + supporting = [item for item in supporting if not item.get("chunk_id") or item.get("chunk_id") not in existing_ids] + return { + "specialized_notes": "\n\n".join(item for item in notes if item), + "tool_outputs": outputs, + "retrieved_documents": [*state.get("retrieved_documents", []), *supporting], + "retrieval_context": _format_sources([*state.get("retrieved_documents", []), *supporting]), + "traces": traces, + } + + return { + "specialized_notes": "\n\n".join(item for item in notes if item), + "tool_outputs": outputs, + "traces": traces, + } + + +def synthesize_prompt(state: QueryState) -> QueryState: + response_mode = state.get("response_mode", "llm") + if response_mode in {"clarification", "fallback", "direct"}: + return { + "prompt": "", + "traces": _append_trace( + state, + "synthesize_prompt", + { + "tool": "prompt_synthesizer", + "status": "skipped", + "reason": response_mode, + "output_summary": f"Skipped because response_mode={response_mode}", + }, + ), + } + + intent = state.get("intent", "search") + retrieved_documents = [_compact_document_entry(item) for item in state.get("retrieved_documents", [])] + target_documents = [ + item for item in retrieved_documents if item.get("document_origin") == "upload" + ] + target_documents = sorted( + target_documents, + key=lambda item: (-float(item.get("score", 0.0) or 0.0), item.get("source", ""), item.get("page", 0)), + ) + target_documents = _dedupe_documents_by_source_page(target_documents, MAX_TARGET_CHUNKS) + if intent == "rfp_analysis": + target_documents = _dedupe_documents_by_content_signature(target_documents, MAX_TARGET_CHUNKS) + + sample_groups = _group_case_study_documents(retrieved_documents) + history_messages = [ + message + for message in (state.get("chat_history") or []) + if message.get("role") in {"user", "assistant"} + ][-MAX_HISTORY_MESSAGES:] + + verbose_tools = True + chunks_dropped = 0 + sample_case_groups = sample_groups + + sections = _build_compact_prompt_sections( + state, + history_messages=history_messages, + target_documents=target_documents, + sample_case_groups=sample_case_groups, + include_verbose_tools=verbose_tools, + ) + prompt = sections["prompt"] + estimated_input_tokens = _estimate_tokens(prompt) + projected_total_tokens = estimated_input_tokens + RFP_ANALYSIS_MAX_OUTPUT_TOKENS + + while estimated_input_tokens > MAX_PROMPT_TOKENS: + dropped = False + + for group in reversed(sample_case_groups): + if len(group.get("documents", [])) > 1: + group["documents"].pop() + chunks_dropped += 1 + dropped = True + break + + if not dropped and len(sample_case_groups) > 1: + sample_case_groups.pop() + chunks_dropped += 1 + dropped = True + + if not dropped and len(history_messages) > 1: + history_messages.pop(0) + dropped = True + + if not dropped and verbose_tools: + verbose_tools = False + dropped = True + + if not dropped and len(sample_case_groups) > 0: + sample_case_groups.pop() + chunks_dropped += 1 + dropped = True + + if not dropped: + break + + sections = _build_compact_prompt_sections( + state, + history_messages=history_messages, + target_documents=target_documents, + sample_case_groups=sample_case_groups, + include_verbose_tools=verbose_tools, + ) + prompt = sections["prompt"] + estimated_input_tokens = _estimate_tokens(prompt) + projected_total_tokens = estimated_input_tokens + RFP_ANALYSIS_MAX_OUTPUT_TOKENS + + prompt_budget = { + "estimated_input_tokens": estimated_input_tokens, + "reserved_output_tokens": RFP_ANALYSIS_MAX_OUTPUT_TOKENS, + "projected_total_tokens": projected_total_tokens, + "budget_limit": MAX_PROMPT_TOKENS, + "target_chunks_included": len(target_documents), + "sample_chunks_included": sum(len(group.get("documents", [])) for group in sample_case_groups), + "chunks_dropped": chunks_dropped, + "history_messages_included": len(history_messages), + "status": "within_budget" if estimated_input_tokens <= MAX_PROMPT_TOKENS else "over_budget", + } + return { + "prompt": prompt, + "prompt_budget": prompt_budget, + "traces": _append_trace( + {"traces": _append_trace( + state, + "synthesize_prompt", + { + "tool": "prompt_synthesizer", + "status": "completed", + "intent": intent, + "scope": state.get("retrieval_scope", "all"), + "output_summary": "Built grounded answer prompt from compact retrieved evidence and conversation context.", + }, + )}, + "synthesize_prompt", + {"tool": "prompt_budget", **prompt_budget}, + ), + } + + +def evidence_availability_check(state: QueryState) -> QueryState: + response_mode = state.get("response_mode", "llm") + documents = state.get("retrieved_documents", []) + grounded = bool(documents) if response_mode == "llm" else response_mode == "clarification" + answer = state.get("answer", "") + if response_mode == "llm" and not documents: + grounded = False + answer = "I could not find grounded evidence for that request in the current knowledge base." + response_mode = "fallback" + if "[Source: None]" in answer: + answer = answer.replace("[Source: None]", UNSUPPORTED_CLAIM_MESSAGE) + + return { + "grounded": grounded, + "response_mode": response_mode, + "answer": answer, + "traces": _append_trace( + state, + "verify_grounding", + { + "tool": "evidence_availability_check", + "grounded": grounded, + "response_mode": response_mode, + "verification_status": "available" if grounded else "unavailable", + "output_summary": "Checked whether sufficient evidence is available before answer generation.", + }, + ), + } + + +def final_response(state: QueryState) -> QueryState: + response_mode = state.get("response_mode", "llm") + traces = _append_trace( + state, + "final_response", + { + "tool": "final_response", + "response_mode": response_mode, + "graph_backend": state.get( + "graph_backend", + "langgraph" if LANGGRAPH_AVAILABLE else "deterministic-fallback", + ), + "output_summary": f"Prepared {response_mode} response.", + }, + ) + return { + "traces": traces, + } + + +NODE_FUNCTIONS = { + "health_check": health_check, + "classify_intent": classify_intent, + "plan_tools": plan_tools, + "execute_retrieval": execute_retrieval, + "execute_specialized_tool": execute_specialized_tool, + "synthesize_prompt": synthesize_prompt, + "evidence_availability_check": evidence_availability_check, + "final_response": final_response, +} + + +def prepare_query_payload( + user_query: str, + chat_history: list[dict[str, Any]] | None = None, + vectorstore_stats: dict[str, Any] | None = None, + retrieval_fn: Callable[[str, int, str], list] | None = None, + retrieval_k: int = RETRIEVAL_K, + retrieval_scope: str = "all", +) -> dict[str, Any]: + """Run the orchestration graph and return the full query payload.""" + graph = compile_query_graph() + initial_state: QueryState = { + "user_query": user_query, + "chat_history": chat_history or [], + "vectorstore_stats": vectorstore_stats or {}, + "retrieval_fn": retrieval_fn or similarity_search, + "retrieval_k": retrieval_k, + "retrieval_scope": retrieval_scope, + "traces": [], + "response_mode": "llm", + "resolved_query": user_query, + "resolved_entities": [], + "tool_outputs": {}, + "graph_backend": "langgraph" if LANGGRAPH_AVAILABLE else "deterministic-fallback", + } + final_state = graph.invoke(initial_state) + return dict(final_state) + + +def run_agent_graph(state, search_fn=None, stats_fn=None): + """Backward-compatible AgentState wrapper around the current query graph.""" + from rfp_analyst.agent.prompts import build_no_documents_message + + stats = stats_fn() if stats_fn else None + payload = prepare_query_payload( + user_query=state.query, + chat_history=state.chat_history, + vectorstore_stats=stats, + retrieval_fn=search_fn, + ) + + intent_map = { + "search": "search", + "compare": "compare_projects", + "proposal": "proposal_writer", + "ambiguous": "ambiguous", + } + state.intent = intent_map.get(payload.get("intent", "search"), payload.get("intent", "search")) + state.stats = payload.get("vectorstore_stats", {}) + state.prompt = payload.get("prompt", "") + state.final_answer = payload.get("answer", "") + if payload.get("response_mode") == "fallback" and "Please ingest documents first" not in state.final_answer: + state.final_answer = build_no_documents_message(state.query) + + state.tool_trace = payload.get("traces", []) + state.retrieved_documents = payload.get("retrieved_documents", []) + state.sources = [ + { + "source": document.get("source", "Unknown"), + "page": document.get("page", 0), + "snippet": document.get("content", ""), + } + for document in state.retrieved_documents + ] + state.tool_outputs = { + "planned_tools": payload.get("planned_tools", []), + "retrieval_context": payload.get("retrieval_context", ""), + } return state + + +def _human_message(content: str): + from langchain_core.messages import HumanMessage + + return HumanMessage(content=content) + + +_BAD_CITATION_PATTERN = re.compile( + r"\[Source:\s*(None|respective documents?|the documents?|various documents?)\s*(?:,\s*Page\s*\d+)?\]", + re.IGNORECASE, +) +_GENERIC_SOURCE_PATTERN = re.compile(r"\[Source:\s*[^,\]]+\]", re.IGNORECASE) + + +def _sanitize_answer_text(answer: str) -> str: + sanitized = str(answer or "").replace("[Source: None]", UNSUPPORTED_CLAIM_MESSAGE) + sanitized = _BAD_CITATION_PATTERN.sub(UNSUPPORTED_CLAIM_MESSAGE, sanitized) + sanitized = _GENERIC_SOURCE_PATTERN.sub(UNSUPPORTED_CLAIM_MESSAGE, sanitized) + return sanitized + + +def _has_vague_or_invalid_citations(answer: str) -> bool: + if _BAD_CITATION_PATTERN.search(answer or ""): + return True + for citation in re.findall(r"\[Source:[^\]]+\]", answer or "", flags=re.IGNORECASE): + if not re.search(r"\.pdf\s*,\s*Page\s+\d+", citation): + return True + return False + + +def _repair_unsupported_answer(answer: str, unsupported_claims: list[str]) -> str: + unsupported = [claim.strip() for claim in unsupported_claims if claim.strip()] + repaired_lines = [] + changed = False + for raw_line in _sanitize_answer_text(answer).splitlines(): + line = raw_line.rstrip() + if not line: + repaired_lines.append(line) + continue + if _has_vague_or_invalid_citations(line): + repaired_lines.append(UNSUPPORTED_CLAIM_MESSAGE) + changed = True + continue + if any(claim in line for claim in unsupported): + if line.lstrip().startswith("-"): + repaired_lines.append("- The retrieved evidence does not support this claim.") + else: + repaired_lines.append(UNSUPPORTED_CLAIM_MESSAGE) + changed = True + continue + repaired_lines.append(line) + if not changed and unsupported: + repaired_lines.append(UNSUPPORTED_CLAIM_MESSAGE) + return "\n".join(repaired_lines).strip() + + +def _verify_generated_answer(payload: dict[str, Any], answer: str) -> tuple[str, dict[str, Any]]: + """Verify a completed generated answer and append the canonical visible trace.""" + sanitized_answer = _sanitize_answer_text(answer) + verification = verify_answer_grounding(sanitized_answer, payload.get("retrieved_documents", [])) + trace = { + "step": "verify_grounding", + "tool": "grounding_verifier", + "status": "completed", + "verification_status": "grounded" if verification["is_grounded"] else "unsupported", + "input_summary": "Complete generated answer and retrieved source metadata", + "output_summary": ( + f"Verified {len(verification['checked_claims'])} claim(s); " + f"identified {len(verification['unsupported_claims'])} unsupported claim(s)." + ), + "unsupported_claims": verification["unsupported_claims"][:3], + } + payload.setdefault("traces", []).append(trace) + + final_answer = sanitized_answer + final_verification = verification + repair_needed = (not verification["is_grounded"]) or _has_vague_or_invalid_citations(sanitized_answer) + if repair_needed: + repaired_answer = _repair_unsupported_answer(sanitized_answer, verification["unsupported_claims"]) + payload.setdefault("traces", []).append( + { + "step": "answer_repair", + "tool": "answer_repair", + "status": "completed", + "input_summary": "Unsupported or vague-cited claims from first grounding pass", + "output_summary": "Removed or qualified unsupported claims once.", + "claims_repaired": len(verification["unsupported_claims"]), + } + ) + final_answer = repaired_answer + final_verification = verify_answer_grounding(final_answer, payload.get("retrieved_documents", [])) + + payload.setdefault("traces", []).append( + { + "step": "verify_grounding", + "tool": "final_grounding_verifier", + "status": "completed", + "verification_status": "grounded" if final_verification["is_grounded"] else "unsupported", + "input_summary": "Final answer after optional bounded repair", + "output_summary": ( + f"Verified {len(final_verification['checked_claims'])} final claim(s); " + f"identified {len(final_verification['unsupported_claims'])} unsupported claim(s)." + ), + "unsupported_claims": final_verification["unsupported_claims"][:3], + } + ) + + ui_trace = payload.get("_ui_reasoning_trace") + if isinstance(ui_trace, list): + ui_trace.append( + { + "tool": "grounding_verifier", + "input_summary": trace["input_summary"], + "output_summary": trace["output_summary"], + } + ) + if repair_needed: + ui_trace.append( + { + "tool": "answer_repair", + "input_summary": "Unsupported or vague-cited claims", + "output_summary": "Removed or qualified unsupported claims once.", + } + ) + ui_trace.append( + { + "tool": "final_grounding_verifier", + "input_summary": "Final repaired answer", + "output_summary": "Completed final grounding check.", + } + ) + return final_answer, final_verification + + +def stream_query_response(llm, payload: dict[str, Any]): + """Stream either a direct fallback response or an LLM completion.""" + if payload.get("response_mode") in {"clarification", "fallback", "direct"}: + yield _sanitize_answer_text(payload.get("answer", "")) + return + + prompt = payload.get("prompt", "") + complete_answer = "" + for chunk in llm.stream([_human_message(prompt)]): + if chunk.content: + sanitized = _sanitize_answer_text(chunk.content) + complete_answer += sanitized + yield sanitized + repaired_answer, verification = _verify_generated_answer(payload, complete_answer) + if repaired_answer != complete_answer: + yield "\n\nGrounding repair:\n" + repaired_answer + elif not verification["is_grounded"]: + yield "\n\nGrounding check: the retrieved evidence does not support every generated claim." + + +def run_query( + llm, + user_query: str, + thread_id: str = "default", + chat_history: list[dict[str, Any]] | None = None, + vectorstore_stats: dict[str, Any] | None = None, + retrieval_fn: Callable[[str, int, str], list] | None = None, + retrieval_scope: str = "all", +) -> dict[str, Any]: + """Execute the graph and optionally call the LLM when grounding is ready.""" + del thread_id + payload = prepare_query_payload( + user_query=user_query, + chat_history=chat_history, + vectorstore_stats=vectorstore_stats, + retrieval_fn=retrieval_fn, + retrieval_scope=retrieval_scope, + ) + + if payload.get("response_mode") in {"clarification", "fallback", "direct"}: + answer = _sanitize_answer_text(payload.get("answer", "")) + else: + response = llm.invoke([_human_message(payload.get("prompt", ""))]) + answer, verification = _verify_generated_answer(payload, response.content) + if not verification["is_grounded"]: + answer += "\n\nGrounding check: some claims are not supported by the retrieved evidence." + + return { + "answer": answer, + "reasoning_trace": payload.get("traces", []), + "all_messages": [], + "payload": payload, + } diff --git a/src/rfp_analyst/agent/runtime.py b/src/rfp_analyst/agent/runtime.py index 2a7dd48..df031fc 100644 --- a/src/rfp_analyst/agent/runtime.py +++ b/src/rfp_analyst/agent/runtime.py @@ -6,7 +6,12 @@ from config import AGENT_MODE from rag_engine import get_vectorstore_stats -from rfp_analyst.agent.graph import run_agent_graph +from rfp_analyst.agent.graph import ( + prepare_query_payload as graph_prepare_query_payload, + run_agent_graph, + run_query as graph_run_query, + stream_query_response as graph_stream_query_response, +) from rfp_analyst.agent.prompts import build_simple_prompt from rfp_analyst.agent.state import AgentState from rfp_analyst.exceptions import KnowledgeBaseNotReadyError, RetrievalError @@ -79,60 +84,17 @@ def prepare_agentic_query(user_query: str, chat_history: list | None = None) -> def prepare_query_payload(user_query: str, chat_history: list | None = None) -> dict: - if get_agent_mode() == "simple": - return prepare_simple_query(user_query, chat_history) - return prepare_agentic_query(user_query, chat_history) + """Deprecated compatibility wrapper for the canonical graph runtime.""" + return graph_prepare_query_payload(user_query, chat_history) def stream_query_response(llm, payload): + """Deprecated compatibility wrapper for the canonical graph runtime.""" if isinstance(payload, str): - for chunk in llm.stream([HumanMessage(content=payload)]): - if chunk.content: - yield chunk.content - return - - prebuilt_answer = payload.get("prebuilt_answer") - if prebuilt_answer: - yield from _stream_text(prebuilt_answer) - return - - prompt = payload.get("prompt", "") - if not prompt: - raise KnowledgeBaseNotReadyError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) - - full_response = "" - for chunk in llm.stream([HumanMessage(content=prompt)]): - if chunk.content: - full_response += chunk.content - yield chunk.content - - verification = verify_answer_grounding(full_response, payload.get("documents", [])) - if not verification["is_grounded"]: - warning_lines = ["\n\nGrounding check: some claims may not be fully supported:"] - warning_lines.extend(f"- {claim}" for claim in verification["unsupported_claims"][:3]) - yield "\n".join(warning_lines) + payload = {"prompt": payload, "response_mode": "llm", "retrieved_documents": []} + yield from graph_stream_query_response(llm, payload) def run_query(llm, user_query: str, chat_history: list | None = None) -> dict: - payload = prepare_query_payload(user_query, chat_history) - if payload.get("prebuilt_answer"): - return { - "answer": payload["prebuilt_answer"], - "reasoning_trace": payload["reasoning_trace"], - "all_messages": [], - } - - prompt = payload.get("prompt", "") - if not prompt: - raise RetrievalError(KNOWLEDGE_BASE_NOT_READY_MESSAGE) - - response = llm.invoke([HumanMessage(content=prompt)]) - answer = response.content - verification = verify_answer_grounding(answer, payload.get("documents", [])) - if not verification["is_grounded"]: - answer += "\n\nGrounding check: some claims may not be fully supported." - return { - "answer": answer, - "reasoning_trace": payload["reasoning_trace"], - "all_messages": [], - } + """Deprecated compatibility wrapper for the canonical graph runtime.""" + return graph_run_query(llm, user_query, chat_history=chat_history) diff --git a/src/rfp_analyst/evals.py b/src/rfp_analyst/evals.py new file mode 100644 index 0000000..54aadf4 --- /dev/null +++ b/src/rfp_analyst/evals.py @@ -0,0 +1,40 @@ +"""Helpers for rendering evaluation snapshots safely.""" + +import json +from pathlib import Path + + +def format_latency(value, unit: str | None = None) -> str: + """Format latency consistently, converting seconds to ms when appropriate.""" + if value is None: + return "N/A" + + numeric_value = float(value) + normalized_unit = (unit or "").lower() + if normalized_unit in {"s", "sec", "secs", "second", "seconds"} or ( + not normalized_unit and numeric_value < 10 + ): + milliseconds = numeric_value * 1000 + if milliseconds < 1000: + return f"{milliseconds:.0f} ms" + return f"{numeric_value:.2f} s" + + return f"{numeric_value:.2f} ms" + + +def load_eval_snapshot(results_path: Path, missing_message: str = "No evaluation run found") -> dict: + """Load a local evaluation snapshot if one exists.""" + if not results_path.exists(): + return {"status": "missing", "message": missing_message} + + with results_path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + + latency_value = payload.get("latency") + latency_unit = payload.get("latency_unit") + return { + "status": "ready", + "message": "Evaluation snapshot loaded", + "payload": payload, + "latency_display": format_latency(latency_value, latency_unit), + } diff --git a/src/rfp_analyst/exceptions.py b/src/rfp_analyst/exceptions.py index 8706084..456cb0a 100644 --- a/src/rfp_analyst/exceptions.py +++ b/src/rfp_analyst/exceptions.py @@ -1,22 +1,20 @@ -"""Project-specific exceptions.""" - -from __future__ import annotations +"""Custom exceptions for the Internal RFP Analyst app.""" class RFPAnalystError(Exception): - """Base application exception.""" + """Base exception for app-specific failures.""" class LLMProviderNotConfiguredError(RFPAnalystError): - """Raised when no LLM provider credentials are configured.""" + """Raised when no supported LLM provider is configured.""" class KnowledgeBaseNotReadyError(RFPAnalystError): - """Raised when retrieval is attempted before the knowledge base is ready.""" + """Raised when retrieval is attempted before the vector store is ready.""" class NoDocumentsFoundError(RFPAnalystError): - """Raised when no source documents are available for ingestion.""" + """Raised when ingestion is attempted without any source PDFs.""" class IngestionError(RFPAnalystError): @@ -24,7 +22,7 @@ class IngestionError(RFPAnalystError): class RetrievalError(RFPAnalystError): - """Raised when retrieval fails.""" + """Raised when knowledge base retrieval fails.""" class UnsupportedFileError(RFPAnalystError): diff --git a/src/rfp_analyst/health.py b/src/rfp_analyst/health.py index 4733bfa..b5b76d4 100644 --- a/src/rfp_analyst/health.py +++ b/src/rfp_analyst/health.py @@ -1,33 +1,94 @@ -"""App health checks.""" - -from __future__ import annotations +"""Health checks for app readiness and configuration.""" from pathlib import Path -from config import ASSETS_DIR, DATA_DIR, VECTORSTORE_DIR +from config import DATA_DIR, UPLOADS_DIR, VECTORSTORE_DIR from rfp_analyst.retrieval.vector_store import VectorStoreManager - REQUIRED_DIRECTORIES = { "data_dir": DATA_DIR, "vectorstore_dir": VECTORSTORE_DIR, - "assets_dir": ASSETS_DIR, } -def get_app_health(llm_provider_name: str = "Not configured") -> dict: - """Return high-level application health without raising UI-breaking errors.""" - directories = { +def get_provider_status(groq_api_key: str = "", google_api_key: str = "") -> dict: + """Return provider readiness and display information.""" + if groq_api_key: + return {"configured": True, "provider": "Groq"} + if google_api_key: + return {"configured": True, "provider": "Gemini"} + return {"configured": False, "provider": "Not configured"} + + +def _legacy_health_snapshot(provider_name: str) -> dict: + stats = VectorStoreManager().get_stats() + required_directories = { name: {"path": str(path), "exists": Path(path).exists()} for name, path in REQUIRED_DIRECTORIES.items() } + chunk_count = int(stats.get("total_chunks", 0) or 0) + document_count = int(stats.get("total_documents", 0) or 0) + return { + "vectorstore_ready": stats.get("status") == "ready" and chunk_count > 0, + "document_count": document_count, + "indexed_document_count": document_count, + "chunk_count": chunk_count, + "llm_provider_configured": provider_name != "Not configured", + "provider_name": provider_name, + "required_directories": required_directories, + "missing_directories": [ + name for name, entry in required_directories.items() if not entry["exists"] + ], + "vectorstore_status": stats.get("status", "not_initialized"), + "document_names": stats.get("document_names", []), + } + + +def get_app_health( + vectorstore_stats: dict | str | None = None, + data_dir: Path | None = None, + vectorstore_dir: Path | None = None, + assets_dir: Path | None = None, + uploads_dir: Path | None = None, + groq_api_key: str = "", + google_api_key: str = "", +) -> dict: + """Build a central snapshot of app health for the UI and query flow.""" + if isinstance(vectorstore_stats, str): + return _legacy_health_snapshot(vectorstore_stats) + + stats = vectorstore_stats or {} + provider_status = get_provider_status(groq_api_key, google_api_key) + data_path = data_dir or REQUIRED_DIRECTORIES["data_dir"] + vectorstore_path = vectorstore_dir or REQUIRED_DIRECTORIES["vectorstore_dir"] + uploads_path = uploads_dir or UPLOADS_DIR + sample_files = sorted(data_path.glob("*.pdf")) if data_path.exists() else [] + upload_files = sorted(uploads_path.glob("*.pdf")) if uploads_path.exists() else [] + required_directories = { + "documents": data_path.exists(), + "vectorstore": vectorstore_path.exists(), + } + chunk_count = int(stats.get("total_chunks", 0) or 0) + indexed_document_count = int(stats.get("total_documents", 0) or 0) + vectorstore_ready = stats.get("status") == "ready" and chunk_count > 0 - stats = VectorStoreManager(persist_dir=VECTORSTORE_DIR).get_stats() return { - "vectorstore_ready": stats.get("status") == "ready", - "document_count": stats.get("total_documents", 0), - "chunk_count": stats.get("total_chunks", 0), - "llm_provider_configured": llm_provider_name != "Not configured", - "llm_provider_name": llm_provider_name, - "required_directories": directories, + "vectorstore_ready": vectorstore_ready, + "document_count": max(indexed_document_count, len(sample_files) + len(upload_files)), + "indexed_document_count": indexed_document_count, + "chunk_count": chunk_count, + "llm_provider_configured": provider_status["configured"], + "provider_name": provider_status["provider"], + "required_directories": required_directories, + "missing_directories": [ + name for name, exists in required_directories.items() if not exists + ], + "vectorstore_status": stats.get("status", "not_initialized"), + "document_names": stats.get("document_names", []), + "sample_document_count": len(sample_files), + "upload_document_count": len(upload_files), + "indexed_sample_document_count": int(stats.get("indexed_sample_document_count", 0) or 0), + "indexed_upload_document_count": int(stats.get("indexed_upload_document_count", 0) or 0), + "pending_upload_files": stats.get("pending_upload_files", []), + "pending_upload_count": len(stats.get("pending_upload_files", [])), } diff --git a/src/rfp_analyst/ingestion/chunking.py b/src/rfp_analyst/ingestion/chunking.py index c4850a9..17d4f7a 100644 --- a/src/rfp_analyst/ingestion/chunking.py +++ b/src/rfp_analyst/ingestion/chunking.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +from pathlib import Path from langchain_core.documents import Document from langchain_text_splitters import RecursiveCharacterTextSplitter @@ -11,9 +12,35 @@ from rfp_analyst.schemas import LoadedSource -def build_chunk_id(file_hash: str, page: int, chunk_index: int, content: str) -> str: - """Build a deterministic chunk ID from stable source data.""" - fingerprint = f"{file_hash}:{page}:{chunk_index}:{content}".encode("utf-8") +def _normalize_source_namespace(source_path: str, source_file: str) -> str: + """Return a stable path namespace for deterministic chunk IDs.""" + if source_path: + return Path(source_path).as_posix().lower() + return source_file.lower() + + +def build_chunk_id( + document_origin: str, + source_path: str, + source_file: str, + file_hash: str, + page: int, + chunk_index: int, + content: str, +) -> str: + """Build a deterministic chunk ID from source identity and chunk position.""" + content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + fingerprint = "|".join( + [ + document_origin, + _normalize_source_namespace(source_path, source_file), + file_hash, + source_file, + str(page), + str(chunk_index), + content_hash, + ] + ).encode("utf-8") return hashlib.sha256(fingerprint).hexdigest() @@ -30,12 +57,16 @@ def chunk_loaded_sources( separators=["\n\n", "\n", ". ", " ", ""], ) + chunks: list[Document] = [] for source in loaded_sources: source_chunks = splitter.split_documents(source.documents) for chunk_index, chunk in enumerate(source_chunks): page = int(chunk.metadata.get("page", 0)) chunk_id = build_chunk_id( + document_origin=source.document_origin, + source_path=source.source_path, + source_file=source.source_file, file_hash=source.file_hash, page=page, chunk_index=chunk_index, @@ -47,8 +78,10 @@ def chunk_loaded_sources( "source_path": source.source_path, "file_hash": source.file_hash, "page": page, + "chunk_index": chunk_index, "chunk_id": chunk_id, "document_type": source.document_type, + "document_origin": source.document_origin, } ) chunks.append(chunk) diff --git a/src/rfp_analyst/ingestion/loaders.py b/src/rfp_analyst/ingestion/loaders.py index 33b22eb..0a22236 100644 --- a/src/rfp_analyst/ingestion/loaders.py +++ b/src/rfp_analyst/ingestion/loaders.py @@ -1,4 +1,4 @@ -"""Validated document loading utilities.""" +"""Validated document loading utilities.""" from __future__ import annotations @@ -79,7 +79,10 @@ def validate_pdf(pdf_path: Path) -> int: return page_count -def load_pdf_sources(doc_dir: Path = DATA_DIR) -> list[LoadedSource]: +def load_pdf_sources( + doc_dir: Path = DATA_DIR, + document_origin: str = "sample", +) -> list[LoadedSource]: """Load, validate, and enrich all PDFs in a directory.""" pdf_files = sorted(doc_dir.glob("*.pdf")) if not pdf_files: @@ -102,6 +105,7 @@ def load_pdf_sources(doc_dir: Path = DATA_DIR) -> list[LoadedSource]: document.metadata["file_hash"] = file_hash document.metadata["page"] = int(document.metadata.get("page", page_index)) document.metadata["document_type"] = document_type + document.metadata["document_origin"] = document_origin loaded_sources.append( LoadedSource( @@ -111,9 +115,10 @@ def load_pdf_sources(doc_dir: Path = DATA_DIR) -> list[LoadedSource]: page_count=page_count, document_type=document_type, documents=documents, + document_origin=document_origin, ) ) - print(f" Loaded: {pdf_path.name} ({page_count} pages)") + print(f" Loaded: {pdf_path.name} ({page_count} pages, origin={document_origin})") except NoDocumentsFoundError: raise except Exception as error: diff --git a/src/rfp_analyst/retrieval/vector_store.py b/src/rfp_analyst/retrieval/vector_store.py index 40da3ed..45ef28a 100644 --- a/src/rfp_analyst/retrieval/vector_store.py +++ b/src/rfp_analyst/retrieval/vector_store.py @@ -1,4 +1,4 @@ -"""Vector store management for ingestion and retrieval.""" +"""Vector store management for ingestion and retrieval.""" from __future__ import annotations @@ -11,12 +11,27 @@ from config import COLLECTION_NAME, EMBEDDING_MODEL, RETRIEVAL_K, VECTORSTORE_DIR from rfp_analyst.exceptions import KnowledgeBaseNotReadyError +VALID_SCOPES = {"all", "sample", "upload"} + def get_embeddings(): """Initialize local embeddings for ingestion and search.""" return FastEmbedEmbeddings(model_name=EMBEDDING_MODEL) +def deduplicate_documents_by_chunk_id(documents: list[Document]) -> tuple[list[Document], list[str]]: + """Return documents with unique chunk IDs, preserving first occurrence.""" + unique_chunks_by_id: dict[str, Document] = {} + duplicate_ids: list[str] = [] + for document in documents: + chunk_id = document.metadata["chunk_id"] + if chunk_id in unique_chunks_by_id: + duplicate_ids.append(chunk_id) + continue + unique_chunks_by_id[chunk_id] = document + return list(unique_chunks_by_id.values()), duplicate_ids + + class VectorStoreManager: """Encapsulate Chroma persistence and deduplicated upserts.""" @@ -48,10 +63,13 @@ def upsert_documents(self, documents: list[Document]) -> Chroma: """Add only new chunk IDs into the collection.""" vectorstore = self.load(create_if_missing=True) existing_ids = set(vectorstore._collection.get().get("ids", [])) + unique_documents, duplicate_ids = deduplicate_documents_by_chunk_id(documents) + if duplicate_ids: + print(f"Skipped {len(duplicate_ids)} duplicate chunk ID(s) before Chroma upsert") new_documents = [] new_ids = [] - for document in documents: + for document in unique_documents: chunk_id = document.metadata["chunk_id"] if chunk_id in existing_ids: continue @@ -59,22 +77,34 @@ def upsert_documents(self, documents: list[Document]) -> Chroma: new_ids.append(chunk_id) if new_documents: + if len(new_ids) != len(set(new_ids)): + raise ValueError("Duplicate chunk IDs detected before Chroma upsert") vectorstore.add_documents(new_documents, ids=new_ids) print(f"Vector store contains {vectorstore._collection.count()} vectors") print(f"Persisted to: {self.persist_dir}") return vectorstore - def get_retriever(self, k: int = RETRIEVAL_K): + def get_retriever(self, k: int = RETRIEVAL_K, scope: str = "all"): """Get a retriever for similarity search.""" + search_kwargs = {"k": k} + if scope in {"sample", "upload"}: + search_kwargs["filter"] = {"document_origin": scope} return self.load(create_if_missing=False).as_retriever( search_type="similarity", - search_kwargs={"k": k}, + search_kwargs=search_kwargs, ) - def similarity_search(self, query: str, k: int = RETRIEVAL_K): + def similarity_search(self, query: str, k: int = RETRIEVAL_K, scope: str = "all"): """Run similarity search with relevance scores.""" - return self.load(create_if_missing=False).similarity_search_with_relevance_scores(query, k=k) + vectorstore = self.load(create_if_missing=False) + kwargs = {"k": k} + if scope in {"sample", "upload"}: + kwargs["filter"] = {"document_origin": scope} + try: + return vectorstore.similarity_search_with_relevance_scores(query, **kwargs) + except TypeError: + return vectorstore.similarity_search_with_relevance_scores(query, k=k) def get_stats(self) -> dict: """Return collection stats for the UI.""" diff --git a/src/rfp_analyst/schemas.py b/src/rfp_analyst/schemas.py index 21b774a..ca5d3b8 100644 --- a/src/rfp_analyst/schemas.py +++ b/src/rfp_analyst/schemas.py @@ -16,6 +16,7 @@ class IngestionRecord: file_hash: str page_count: int document_type: str | None = None + document_origin: str = "sample" chunk_ids: list[str] = field(default_factory=list) def to_dict(self) -> dict: @@ -36,3 +37,4 @@ class LoadedSource: page_count: int document_type: str | None documents: list[Document] + document_origin: str = "sample" diff --git a/src/rfp_analyst/tools/proposal_writer.py b/src/rfp_analyst/tools/proposal_writer.py index 208c7e2..cd2a831 100644 --- a/src/rfp_analyst/tools/proposal_writer.py +++ b/src/rfp_analyst/tools/proposal_writer.py @@ -9,35 +9,54 @@ def generate_proposal_outline(user_query: str, case_studies: dict, requirements: matches = case_studies.get("matches", []) requirement_lines = "\n".join( - f"- {requirement['id']}: {requirement['text']}" for requirement in requirements + f"- {requirement['id']}: {requirement['text']}" + + ( + f" [Source: {requirement['source_file']}, Page {int(requirement['page']) + 1}]" + if requirement.get("source_file") and requirement.get("page") is not None + else "" + ) + for requirement in requirements ) or "- No structured requirements were extracted." + gap_lines = [] + for requirement in requirements: + for gap in requirement.get("inferred_gaps", [])[:3]: + gap_lines.append( + f"- {requirement['id']}: {gap['detail']}" + ) + gap_block = "\n".join(gap_lines[:8]) or "- No inferred gaps were identified from the available target evidence." + evidence_lines = "\n".join( - f"- {match['source']} [Source: {match['source']}, Page {match['pages'][0] + 1}]" - for match in matches[:5] + ( + f"- {match['source']}: fit score {match.get('fit_score', 0)}/5; " + f"matched {', '.join(match.get('matched_requirements', [])) or 'no explicit requirements'}; " + f"missing {', '.join(match.get('missing_coverage', [])) or 'no scored coverage gaps'} " + f"[Source: {match['source']}, Page {match['pages'][0] + 1}]" + ) + for match in matches[:3] ) or "- No matching case studies found." outline = f"""## Executive Summary -Address the request: {user_query} +- Address the request with a proposal grounded in the uploaded target requirements and the strongest cited case-study evidence. +- Treat unprovided target details as assumptions to confirm before pricing or delivery commitment. -## Client Requirements +## Requirement Understanding and Gaps {requirement_lines} +{gap_block} -## Relevant Case Studies -{evidence_lines} +## Proposed Architecture and Security +- Map Azure, integration, and control decisions directly to the confirmed requirements before final architecture selection. +- Confirm absent or ambiguous security/compliance controls before representing them as delivery facts. -## Proposed Approach -- Reuse proven delivery patterns from the cited projects. -- Map each workstream to the structured client requirements. -- Highlight measurable outcomes supported by prior engagements. +## Dashboard and Analytics Workstream +- Define dashboard users, KPI definitions, refresh cadence, and acceptance criteria before build planning. +- Use cited analytics case-study patterns only where the supporting evidence matches the target requirement. -## Delivery Plan -- Discovery and requirements confirmation -- Solution design and implementation -- Validation, rollout, and stakeholder enablement +## Phased Delivery and Risk Management +- Convert confirmed requirements into discovery, design, implementation, validation, and rollout milestones. +- Track missing timelines, dependencies, SLAs, staffing, and risk ownership as proposal assumptions. -## Risks and Mitigations -- Call out delivery risks only when supported by cited prior work. -- Add compliance and implementation assumptions explicitly. +## Case Studies, Outcomes and Success Metrics +{evidence_lines} """ return {"outline": outline, "requirements": requirements, "matches": matches} diff --git a/src/rfp_analyst/tools/rfp_gap_analyzer.py b/src/rfp_analyst/tools/rfp_gap_analyzer.py index d06683e..00d8157 100644 --- a/src/rfp_analyst/tools/rfp_gap_analyzer.py +++ b/src/rfp_analyst/tools/rfp_gap_analyzer.py @@ -7,6 +7,50 @@ from config import RETRIEVAL_K from rfp_analyst.tools.search_kb import search_knowledge_base +GAP_DIMENSIONS = { + "scope and source systems": ("scope", "source system", "source systems"), + "data volume": ("data volume", "records", "terabyte", "gb", "rows"), + "integrations": ("integration", "integrations", "api", "connector"), + "security/compliance controls": ("security", "compliance", "hipaa", "control", "controls"), + "users and dashboard KPIs": ("user", "users", "dashboard", "kpi", "analytics"), + "timelines and milestones": ("timeline", "milestone", "phase", "phased", "weeks", "months"), + "SLAs and acceptance criteria": ("sla", "acceptance", "criteria", "uptime"), + "budget and staffing": ("budget", "staffing", "team", "fte", "cost"), + "measurable success metrics": ("metric", "metrics", "outcome", "outcomes", "success"), + "risks and dependencies": ("risk", "risks", "dependency", "dependencies"), +} + +FIT_DIMENSIONS = { + "Azure migration": ("azure", "migration", "cloud"), + "regulatory/HIPAA alignment": ("hipaa", "regulatory", "compliance", "controls", "audit"), + "dashboards and analytics": ("dashboard", "dashboards", "analytics", "power bi", "reporting"), + "phased delivery": ("phase", "phased", "timeline", "milestone", "weeks"), + "measurable outcomes": ("outcome", "outcomes", "improved", "reduced", "increased", "measurable"), +} + + +def _contains_any(text: str, keywords: tuple[str, ...]) -> bool: + lowered = text.lower() + return any(keyword in lowered for keyword in keywords) + + +def _infer_requirement_gaps(requirement_text: str) -> list[dict]: + gaps = [] + for label, keywords in GAP_DIMENSIONS.items(): + if _contains_any(requirement_text, keywords): + continue + gaps.append( + { + "dimension": label, + "detail": ( + f"Inferred gap: the target requirement does not specify {label}; " + "treat this as absent or ambiguous information until confirmed." + ), + "status": "absent_or_ambiguous", + } + ) + return gaps + def extract_rfp_requirements(rfp_text: str) -> dict: """Turn raw RFP text into a simple structured requirements list.""" @@ -24,14 +68,44 @@ def extract_rfp_requirements(rfp_text: str) -> dict: requirements = [] for index, text in enumerate(candidates, start=1): - requirements.append({"id": f"REQ-{index:02d}", "text": text}) + requirements.append( + { + "id": f"REQ-{index:02d}", + "text": text, + "inferred_gaps": _infer_requirement_gaps(text), + } + ) + + gaps = [ + {"requirement_id": requirement["id"], **gap} + for requirement in requirements + for gap in requirement.get("inferred_gaps", []) + ] return { "requirements": requirements, + "gaps": gaps, "summary": f"Extracted {len(requirements)} requirement(s).", } +def _score_case_study(match: dict) -> dict: + evidence_text = " ".join(match.get("snippets", [])) + dimension_scores = {} + missing_coverage = [] + for dimension, keywords in FIT_DIMENSIONS.items(): + score = 1 if _contains_any(evidence_text, keywords) else 0 + dimension_scores[dimension] = score + if not score: + missing_coverage.append(dimension) + total_score = sum(dimension_scores.values()) + return { + "fit_score": total_score, + "dimension_scores": dimension_scores, + "missing_coverage": missing_coverage, + } + + def find_relevant_case_studies(requirements: list[dict], search_fn=None, k: int = RETRIEVAL_K) -> dict: """Find matching prior work for a set of requirements.""" matches = {} @@ -41,26 +115,43 @@ def find_relevant_case_studies(requirements: list[dict], search_fn=None, k: int for source in result["sources"]: entry = matches.setdefault( source["source"], - {"source": source["source"], "pages": set(), "matched_requirements": [], "snippets": []}, + { + "source": source["source"], + "pages": set(), + "matched_requirements": [], + "snippets": [], + "citations": set(), + }, ) entry["pages"].add(source["page"]) entry["matched_requirements"].append(requirement["id"]) entry["snippets"].append(source["snippet"]) + entry["citations"].add((source["source"], source["page"])) supporting_documents.extend(result["documents"]) normalized_matches = [] for item in matches.values(): + scoring = _score_case_study(item) normalized_matches.append( { "source": item["source"], "pages": sorted(item["pages"]), "matched_requirements": sorted(set(item["matched_requirements"])), "snippets": item["snippets"][:3], + "fit_score": scoring["fit_score"], + "dimension_scores": scoring["dimension_scores"], + "missing_coverage": scoring["missing_coverage"], + "citations": [ + {"source": source, "page": page} + for source, page in sorted(item["citations"], key=lambda value: (value[0], value[1]))[:2] + ], } ) - normalized_matches.sort(key=lambda item: (-len(item["matched_requirements"]), item["source"])) + normalized_matches.sort( + key=lambda item: (-item["fit_score"], -len(item["matched_requirements"]), item["source"]) + ) return { - "matches": normalized_matches, + "matches": normalized_matches[:3], "documents": supporting_documents, } diff --git a/src/rfp_analyst/tools/source_verifier.py b/src/rfp_analyst/tools/source_verifier.py index a42c14d..b80489d 100644 --- a/src/rfp_analyst/tools/source_verifier.py +++ b/src/rfp_analyst/tools/source_verifier.py @@ -16,8 +16,16 @@ def verify_answer_grounding(answer: str, supporting_documents: list[object]) -> """Check whether the answer's major claims are backed by cited evidence.""" source_index = {} for document in supporting_documents: - source_name = document.metadata.get("source_file", "Unknown") - source_index.setdefault(source_name, []).append(document.page_content) + if isinstance(document, dict): + source_name = document.get("source", "Unknown") + page = int(document.get("page", 0) or 0) + 1 + content = document.get("content", "") + else: + metadata = document.metadata or {} + source_name = metadata.get("source_file", "Unknown") + page = int(metadata.get("page", 0) or 0) + 1 + content = document.page_content + source_index.setdefault((source_name, page), []).append(content) unsupported_claims = [] checked_claims = [] @@ -25,8 +33,13 @@ def verify_answer_grounding(answer: str, supporting_documents: list[object]) -> line = raw_line.strip() if len(line) <= 20 or line.startswith("##"): continue + if "retrieved evidence does not support this claim" in line.lower(): + continue - line_citations = [match.group("source").strip() for match in _CITATION_PATTERN.finditer(line)] + line_citations = [ + (match.group("source").strip(), int(match.group("page"))) + for match in _CITATION_PATTERN.finditer(line) + ] line_without_citations = _CITATION_PATTERN.sub("", line) sentence_candidates = [segment.strip() for segment in re.split(r"(?<=[.!?])\s+", line_without_citations) if segment.strip()] @@ -40,10 +53,13 @@ def verify_answer_grounding(answer: str, supporting_documents: list[object]) -> candidate_sources = line_citations or list(source_index) supported = False - for source_name in candidate_sources: - combined_source_text = " ".join(source_index.get(source_name, [])) + for source_key in candidate_sources: + combined_source_text = " ".join(source_index.get(source_key, [])) source_tokens = _tokenize(combined_source_text) - if len(claim_tokens.intersection(source_tokens)) >= 2: + numeric_claims = set(re.findall(r"\b\d[\d,.%$-]*\b", claim)) + numeric_evidence = set(re.findall(r"\b\d[\d,.%$-]*\b", combined_source_text)) + numbers_supported = not numeric_claims or numeric_claims.issubset(numeric_evidence) + if len(claim_tokens.intersection(source_tokens)) >= 2 and numbers_supported: supported = True break diff --git a/src/rfp_analyst/ui/__init__.py b/src/rfp_analyst/ui/__init__.py new file mode 100644 index 0000000..e4adef8 --- /dev/null +++ b/src/rfp_analyst/ui/__init__.py @@ -0,0 +1,5 @@ +"""UI helper package exports.""" + +from .helpers import format_latency_display, get_chat_avatar + +__all__ = ["format_latency_display", "get_chat_avatar"] diff --git a/src/rfp_analyst/uploads.py b/src/rfp_analyst/uploads.py index df11018..49ad1e0 100644 --- a/src/rfp_analyst/uploads.py +++ b/src/rfp_analyst/uploads.py @@ -1,31 +1,55 @@ -"""Upload validation helpers.""" +"""Upload validation helpers for user-provided PDFs.""" from __future__ import annotations +import re from pathlib import Path -from config import MAX_UPLOAD_FILE_SIZE_BYTES -from rfp_analyst.exceptions import UnsupportedFileError -from rfp_analyst.ingestion.loaders import sanitize_filename +from config import MAX_UPLOAD_SIZE_MB, UPLOADS_DIR +from .exceptions import UnsupportedFileError -VALID_PDF_MIME_TYPES = {"application/pdf", "application/x-pdf"} +MAX_UPLOAD_SIZE_BYTES = MAX_UPLOAD_SIZE_MB * 1024 * 1024 +ALLOWED_MIME_TYPES = {"application/pdf", "application/x-pdf"} -def validate_uploaded_pdf(uploaded_file) -> str: - """Validate upload metadata and return a sanitized filename.""" - original_suffix = Path(uploaded_file.name).suffix.lower() - if original_suffix != ".pdf": +def sanitize_uploaded_filename(filename: str) -> str: + """Normalize uploaded filenames and strip unsafe path segments.""" + raw_name = Path(filename or "").name.strip() + if not raw_name: + raise UnsupportedFileError("Uploaded file must have a filename.") + + stem = re.sub(r"[^A-Za-z0-9._-]+", "_", Path(raw_name).stem).strip("._") + suffix = Path(raw_name).suffix.lower() + if not stem: + stem = "document" + if suffix != ".pdf": raise UnsupportedFileError("Only PDF files are supported.") + return f"{stem}{suffix}" + - sanitized_name = sanitize_filename(uploaded_file.name) - file_type = getattr(uploaded_file, "type", "") or "" - if file_type and file_type not in VALID_PDF_MIME_TYPES: - raise UnsupportedFileError("The uploaded file does not appear to be a valid PDF.") +def validate_uploaded_pdf(uploaded_file) -> str: + """Validate a Streamlit uploaded file and return a safe filename.""" + sanitized_name = sanitize_uploaded_filename(getattr(uploaded_file, "name", "")) + mime_type = getattr(uploaded_file, "type", "") + if mime_type and mime_type not in ALLOWED_MIME_TYPES: + raise UnsupportedFileError("Uploaded file must be a PDF.") file_size = getattr(uploaded_file, "size", None) - if file_size is not None and int(file_size) > MAX_UPLOAD_FILE_SIZE_BYTES: + if file_size is None: + file_size = len(uploaded_file.getbuffer()) + if file_size > MAX_UPLOAD_SIZE_BYTES: raise UnsupportedFileError( - f"Uploaded PDF exceeds the max size of {MAX_UPLOAD_FILE_SIZE_BYTES} bytes." + f"Uploaded file exceeds the {MAX_UPLOAD_SIZE_MB} MB limit." ) return sanitized_name + + +def persist_uploaded_pdf(uploaded_file, uploads_dir: Path = UPLOADS_DIR) -> Path: + """Persist a validated uploaded file into the dedicated uploads directory.""" + sanitized_name = validate_uploaded_pdf(uploaded_file) + uploads_dir.mkdir(parents=True, exist_ok=True) + target_path = uploads_dir / sanitized_name + with target_path.open("wb") as handle: + handle.write(uploaded_file.getbuffer()) + return target_path diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..cdfd58c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for Internal RFP Analyst.""" diff --git a/tests/test_agent_and_ui.py b/tests/test_agent_and_ui.py new file mode 100644 index 0000000..48f0a4e --- /dev/null +++ b/tests/test_agent_and_ui.py @@ -0,0 +1,130 @@ +import pytest + +import agent +from rfp_analyst.exceptions import LLMProviderNotConfiguredError +from rfp_analyst.ui import get_chat_avatar + + +def test_no_llm_provider_state(monkeypatch): + monkeypatch.setattr(agent, "get_api_keys", lambda: ("", "")) + + assert agent._get_provider_name() == "Not configured" + with pytest.raises(LLMProviderNotConfiguredError): + agent.get_llm() + + +def test_avatar_helper_uses_valid_emojis(): + assert get_chat_avatar("user") == "👤" + assert get_chat_avatar("assistant") == "🤖" + assert get_chat_avatar("anything-else") not in {"User", "AI"} + +def test_llm_auth_error_format_hides_provider_json_by_default(): + provider_error = RuntimeError( + 'Error code: 401 - {"error":{"type":"invalid_api_key","message":"bad key"}}' + ) + + message = agent.format_llm_error(provider_error) + + assert message == agent.LLM_AUTH_ERROR_MESSAGE + assert "invalid_api_key" not in message + assert "bad key" not in message + + +def test_llm_auth_error_format_includes_details_in_debug_mode(): + provider_error = RuntimeError( + 'Error code: 401 - {"error":{"type":"invalid_api_key","message":"bad key"}}' + ) + + message = agent.format_llm_error(provider_error, debug=True) + + assert agent.LLM_AUTH_ERROR_MESSAGE in message + assert "invalid_api_key" in message + assert "bad key" in message + + +def test_llm_token_budget_error_format_hides_provider_json_by_default(): + provider_error = RuntimeError( + 'Error code: 413 - {"error":{"type":"rate_limit_exceeded","message":"request too large"}}' + ) + + message = agent.format_llm_error(provider_error) + + assert message == agent.LLM_TOKEN_BUDGET_ERROR_MESSAGE + assert "rate_limit_exceeded" not in message + assert "request too large" not in message + + +def test_query_agent_stream_returns_friendly_token_budget_error(): + class FakeLLM: + def stream(self, _messages): + raise RuntimeError( + 'Error code: 413 - {"error":{"type":"rate_limit_exceeded","message":"request too large"}}' + ) + + output = "".join(agent.query_agent_stream(FakeLLM(), {"prompt": "hello", "response_mode": "llm"})) + + assert output == agent.LLM_TOKEN_BUDGET_ERROR_MESSAGE + + +def test_duplicate_source_cards_are_grouped_with_one_based_pages(): + payload = { + "user_query": "what stack?", + "traces": [ + { + "tool": "search_knowledge_base", + "input": {"query": "what stack?"}, + "documents": [ + { + "source": "client_profile.pdf", + "page": 1, + "score": "0.85", + "chunk_id": "def", + "document_origin": "upload", + }, + { + "source": "client_profile.pdf", + "page": 1, + "score": "0.95", + "chunk_id": "abc", + "document_origin": "upload", + }, + ], + } + ], + } + + trace = agent._payload_to_reasoning_trace(payload) + source_cards = [item for item in trace if item.get("tool_response")] + + assert len(source_cards) == 1 + assert source_cards[0]["tool_response"] == "client_profile.pdf (Page 1)" + assert source_cards[0]["page"] == 1 + assert source_cards[0]["match_count"] == 2 + assert source_cards[0]["score"] == "0.95" + assert set(source_cards[0]["chunk_ids"]) == {"abc", "def"} + + +def test_source_none_never_appears_after_sanitization(): + answer = agent.sanitize_answer_text("Uses React [Source: None]") + + assert "[Source: None]" not in answer + assert "The retrieved evidence does not support this claim." in answer + + +def test_tool_trace_contains_summaries_not_hidden_reasoning(): + payload = { + "user_query": "write proposal", + "traces": [ + { + "tool": "proposal_writer", + "input_summary": "proposal request", + "output_summary": "Draft proposal outline from retrieved evidence.", + } + ], + } + + trace = agent._payload_to_reasoning_trace(payload) + + assert trace[0]["input_summary"] == "proposal request" + assert "output_summary" in trace[0] + assert "chain-of-thought" not in str(trace).lower() diff --git a/tests/test_agentic_tools.py b/tests/test_agentic_tools.py index 9a74613..5a41f62 100644 --- a/tests/test_agentic_tools.py +++ b/tests/test_agentic_tools.py @@ -52,6 +52,9 @@ def test_extract_rfp_requirements(): result = extract_rfp_requirements("The solution must support Azure. The vendor should include dashboards.") assert len(result["requirements"]) >= 2 assert result["requirements"][0]["id"].startswith("REQ-") + assert result["gaps"] + assert all("Inferred gap" in gap["detail"] for gap in result["gaps"]) + assert all(gap["status"] == "absent_or_ambiguous" for gap in result["gaps"]) def test_find_relevant_case_studies_with_mocked_retriever(): @@ -59,15 +62,77 @@ def test_find_relevant_case_studies_with_mocked_retriever(): result = find_relevant_case_studies(requirements, search_fn=fake_search_fn) assert result["matches"] assert result["matches"][0]["source"] == "Banking_Audit.pdf" + assert result["matches"][0]["fit_score"] >= result["matches"][-1]["fit_score"] + assert result["matches"][0]["citations"][0]["source"] == "Banking_Audit.pdf" + + +def test_case_studies_rank_by_requirement_coverage(): + def ranked_search_fn(query: str, k: int = 6): + return [ + ( + make_doc( + "Strong_Case.pdf", + 0, + "Azure migration with HIPAA compliance controls, Power BI dashboards, phased milestones, and measurable outcomes.", + ), + 0.82, + ), + ( + make_doc("Weak_Case.pdf", 0, "Generic analytics reporting project."), + 0.99, + ), + ] + + requirements = [{"id": "REQ-01", "text": "Azure HIPAA dashboards phased outcomes"}] + result = find_relevant_case_studies(requirements, search_fn=ranked_search_fn) + + assert result["matches"][0]["source"] == "Strong_Case.pdf" + assert result["matches"][0]["fit_score"] == 5 + assert "regulatory/HIPAA alignment" not in result["matches"][0]["missing_coverage"] def test_generate_proposal_outline(): case_studies = { - "matches": [{"source": "Banking_Audit.pdf", "pages": [0], "snippets": ["Azure SQL and Power BI"]}] + "matches": [ + { + "source": "Banking_Audit.pdf", + "pages": [0], + "snippets": ["Azure SQL and Power BI"], + "fit_score": 4, + "matched_requirements": ["REQ-01"], + "missing_coverage": ["phased delivery"], + } + ] } - outline = generate_proposal_outline("Write a proposal", case_studies, [{"id": "REQ-01", "text": "Azure"}]) + outline = generate_proposal_outline( + "Write a proposal", + case_studies, + [ + { + "id": "REQ-01", + "text": "Azure", + "source_file": "Target.pdf", + "page": 0, + "inferred_gaps": [ + { + "detail": "Inferred gap: the target requirement does not specify timeline; treat this as absent or ambiguous information until confirmed." + } + ], + } + ], + ) assert "## Executive Summary" in outline["outline"] assert "[Source: Banking_Audit.pdf, Page 1]" in outline["outline"] + headings = [line for line in outline["outline"].splitlines() if line.startswith("## ")] + assert headings == [ + "## Executive Summary", + "## Requirement Understanding and Gaps", + "## Proposed Architecture and Security", + "## Dashboard and Analytics Workstream", + "## Phased Delivery and Risk Management", + "## Case Studies, Outcomes and Success Metrics", + ] + assert "absent or ambiguous" in outline["outline"] def test_verify_answer_grounding_catches_unsupported_claims(): diff --git a/tests/test_document_scope.py b/tests/test_document_scope.py new file mode 100644 index 0000000..2e1b49d --- /dev/null +++ b/tests/test_document_scope.py @@ -0,0 +1,482 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest +from langchain_core.documents import Document +from streamlit.testing.v1 import AppTest + +import agent +import document_generator +import rag_engine +from rfp_analyst.agent.graph import prepare_query_payload +from rfp_analyst.exceptions import IngestionError +from rfp_analyst.ingestion.chunking import build_chunk_id, chunk_loaded_sources +from rfp_analyst.ingestion import loaders +from rfp_analyst.retrieval.vector_store import VectorStoreManager +from rfp_analyst.schemas import LoadedSource +from rfp_analyst.uploads import persist_uploaded_pdf + + +class FakeUpload: + def __init__(self, name: str, data: bytes = b"pdf-bytes", file_type: str = "application/pdf"): + self.name = name + self._data = data + self.type = file_type + self.size = len(data) + + def getbuffer(self): + return self._data + + +class FakeLoader: + def __init__(self, _path: str): + self.path = _path + + def load(self): + return [Document(page_content="hello", metadata={"page": 0})] + + +class FakeVectorStoreManager: + def similarity_search(self, _query: str, k: int = 6, scope: str = "all"): + docs = [ + (Document(page_content="sample", metadata={"document_origin": "sample", "source_file": "sample.pdf", "page": 0, "chunk_id": "s1"}), 0.9), + (Document(page_content="upload", metadata={"document_origin": "upload", "source_file": "upload.pdf", "page": 0, "chunk_id": "u1"}), 0.8), + ] + return docs[:k] + + +class FakeIngestionManager: + seen_documents = [] + + def __init__(self, persist_dir, collection_name): + self.persist_dir = persist_dir + self.collection_name = collection_name + + def upsert_documents(self, documents): + self.__class__.seen_documents = list(documents) + self.persist_dir.mkdir(parents=True, exist_ok=True) + (self.persist_dir / "manifest.txt").write_text(str(len(documents)), encoding="utf-8") + return object() + + +def make_loaded_source(origin: str = "sample"): + return LoadedSource( + source_file=f"{origin}.pdf", + source_path=f"C:/tmp/{origin}.pdf", + file_hash=f"hash-{origin}", + page_count=1, + document_type="Proposal", + documents=[Document(page_content=f"{origin} content", metadata={"page": 0})], + document_origin=origin, + ) + + +def test_identical_content_from_different_origins_has_different_chunk_ids(): + sample = LoadedSource( + source_file="client_profile.pdf", + source_path="C:/samples/client_profile.pdf", + file_hash="same-hash", + page_count=1, + document_type=None, + documents=[Document(page_content="same content", metadata={"page": 0})], + document_origin="sample", + ) + upload = LoadedSource( + source_file="client_profile.pdf", + source_path="C:/uploads/client_profile.pdf", + file_hash="same-hash", + page_count=1, + document_type=None, + documents=[Document(page_content="same content", metadata={"page": 0})], + document_origin="upload", + ) + + chunks = chunk_loaded_sources([sample, upload], chunk_size=100, chunk_overlap=0) + chunk_ids = [chunk.metadata["chunk_id"] for chunk in chunks] + + assert len(chunk_ids) == 2 + assert len(set(chunk_ids)) == 2 + + +def test_chunk_ids_differ_by_page_and_chunk_index(): + first = build_chunk_id( + document_origin="upload", + source_path="C:/uploads/client.pdf", + source_file="client.pdf", + file_hash="hash", + page=0, + chunk_index=0, + content="same", + ) + different_page = build_chunk_id( + document_origin="upload", + source_path="C:/uploads/client.pdf", + source_file="client.pdf", + file_hash="hash", + page=1, + chunk_index=0, + content="same", + ) + different_index = build_chunk_id( + document_origin="upload", + source_path="C:/uploads/client.pdf", + source_file="client.pdf", + file_hash="hash", + page=0, + chunk_index=1, + content="same", + ) + + assert first != different_page + assert first != different_index + + +def test_uploaded_pdfs_are_saved_to_uploads_dir(tmp_path: Path): + uploads_dir = tmp_path / "uploads" + sample_dir = tmp_path / "samples" + sample_dir.mkdir() + + saved_path = persist_uploaded_pdf(FakeUpload("Client Stack.pdf"), uploads_dir=uploads_dir) + + assert saved_path.parent == uploads_dir + assert saved_path.exists() + assert not list(sample_dir.glob("*.pdf")) + + +def test_sample_generator_writes_only_to_sample_docs_dir(monkeypatch, tmp_path: Path): + sample_dir = tmp_path / "samples" + upload_dir = tmp_path / "uploads" + upload_dir.mkdir() + monkeypatch.setattr(document_generator, "DATA_DIR", sample_dir) + + document_generator.generate_all_documents() + + assert list(sample_dir.glob("*.pdf")) + assert not list(upload_dir.glob("*.pdf")) + + +@pytest.mark.parametrize("origin", ["upload", "sample"]) +def test_loaded_docs_get_document_origin_metadata(monkeypatch, tmp_path: Path, origin: str): + pdf_path = tmp_path / f"{origin}.pdf" + pdf_path.write_bytes(b"fake") + + monkeypatch.setattr(loaders, "ensure_safe_pdf_path", lambda path: path) + monkeypatch.setattr(loaders, "validate_pdf", lambda _path: 1) + monkeypatch.setattr(loaders, "sha256_file", lambda _path: f"hash-{origin}") + monkeypatch.setattr(loaders, "PyMuPDFLoader", FakeLoader) + + sources = loaders.load_pdf_sources(tmp_path, document_origin=origin) + + assert len(sources) == 1 + assert sources[0].document_origin == origin + assert sources[0].documents[0].metadata["document_origin"] == origin + + +def test_retrieval_scope_upload_excludes_sample_docs(monkeypatch): + monkeypatch.setattr(rag_engine, "VectorStoreManager", lambda **_kwargs: FakeVectorStoreManager()) + + results = rag_engine.similarity_search("tech stack", scope="upload") + + assert results + assert all(doc.metadata["document_origin"] == "upload" for doc, _score in results) + + +def test_retrieval_scope_sample_excludes_upload_docs(monkeypatch): + monkeypatch.setattr(rag_engine, "VectorStoreManager", lambda **_kwargs: FakeVectorStoreManager()) + + results = rag_engine.similarity_search("tech stack", scope="sample") + + assert results + assert all(doc.metadata["document_origin"] == "sample" for doc, _score in results) + + +def test_source_trace_is_deduplicated_by_source_page_and_chunk_id(): + payload = { + "user_query": "what is my stack", + "traces": [ + { + "tool": "search_knowledge_base", + "input": {"query": "what is my stack", "scope": "upload"}, + "documents": [ + {"source": "upload.pdf", "page": 0, "score": "0.91", "chunk_id": "abc", "document_origin": "upload"}, + {"source": "upload.pdf", "page": 0, "score": "0.91", "chunk_id": "abc", "document_origin": "upload"}, + ], + } + ] + } + + trace = agent._payload_to_reasoning_trace(payload) + + source_entries = [item for item in trace if item.get("tool_response")] + assert len(source_entries) == 1 + + +def test_scope_with_zero_chunks_returns_clean_message(): + payload = prepare_query_payload( + user_query="what is my stack", + retrieval_scope="upload", + vectorstore_stats={ + "status": "ready", + "total_chunks": 5, + "total_documents": 1, + "scope_chunk_counts": {"upload": 0, "sample": 5, "all": 5}, + "document_names": ["sample.pdf"], + }, + retrieval_fn=lambda _query, _k, _scope: [], + ) + + assert payload["response_mode"] == "fallback" + assert payload["answer"] == rag_engine.NO_SCOPE_DOCUMENTS_MESSAGE + + +def test_all_documents_scope_still_returns_results(): + captured = {} + + def retrieval_fn(_query, _k, scope): + captured["scope"] = scope + return [ + (Document(page_content="sample", metadata={"document_origin": "sample", "source_file": "sample.pdf", "page": 0, "chunk_id": "s1"}), 0.9), + (Document(page_content="upload", metadata={"document_origin": "upload", "source_file": "upload.pdf", "page": 0, "chunk_id": "u1"}), 0.8), + ] + + payload = prepare_query_payload( + user_query="what is my stack", + retrieval_scope="all", + vectorstore_stats={ + "status": "ready", + "total_chunks": 2, + "total_documents": 2, + "scope_chunk_counts": {"upload": 1, "sample": 1, "all": 2}, + "document_names": ["sample.pdf", "upload.pdf"], + }, + retrieval_fn=retrieval_fn, + ) + + assert captured["scope"] == "all" + assert len(payload["retrieved_documents"]) == 2 + + +def test_pending_uploads_warn_before_answering(): + app_path = Path(__file__).resolve().parents[1] / "app.py" + + pending_stats = { + "status": "ready", + "total_documents": 1, + "total_chunks": 3, + "document_names": ["sample.pdf"], + "indexed_sample_document_count": 1, + "indexed_upload_document_count": 0, + "pending_upload_files": ["client.pdf"], + "scope_chunk_counts": {"sample": 3, "upload": 0, "all": 3}, + } + + with ( + patch("config.get_api_keys", return_value=("groq", "")), + patch("rag_engine.get_vectorstore_stats", return_value=pending_stats), + patch("rag_engine.ingest_documents"), + ): + at = AppTest.from_file(str(app_path)).run(timeout=10) + + assert not at.exception + assert any("pending indexing" in str(element.value).lower() for element in at.warning) + assert at.chat_input[0].disabled is True + + +def test_ingestion_success_clears_in_progress_and_pending_state(): + app_path = Path(__file__).resolve().parents[1] / "app.py" + not_ready = { + "status": "not_initialized", + "total_documents": 1, + "total_chunks": 0, + "document_names": [], + "indexed_sample_document_count": 0, + "indexed_upload_document_count": 0, + "pending_upload_files": ["client.pdf"], + "scope_chunk_counts": {"sample": 0, "upload": 0, "all": 0}, + } + ready = { + "status": "ready", + "total_documents": 2, + "total_chunks": 4, + "document_names": ["sample.pdf", "client.pdf"], + "indexed_sample_document_count": 1, + "indexed_upload_document_count": 1, + "pending_upload_files": [], + "scope_chunk_counts": {"sample": 2, "upload": 2, "all": 4}, + } + stats_calls = {"count": 0} + seen_flag = {"value": None} + + def fake_stats(*_args, **_kwargs): + stats_calls["count"] += 1 + return not_ready if stats_calls["count"] < 2 else ready + + def fake_ingest(*_args, **_kwargs): + import streamlit as st + + seen_flag["value"] = st.session_state.ingestion_in_progress + return None + + with ( + patch("config.get_api_keys", return_value=("groq", "")), + patch("rag_engine.get_vectorstore_stats", side_effect=fake_stats), + patch("rag_engine.ingest_documents", side_effect=fake_ingest), + ): + at = AppTest.from_file(str(app_path)).run(timeout=10) + at.button[7].click().run(timeout=10) + + assert seen_flag["value"] is True + assert at.session_state.filtered_state["ingestion_in_progress"] is False + assert at.session_state.filtered_state["pending_uploads"] is False + assert at.session_state.filtered_state["last_ingestion_error"] == "" + + +def test_windows_lock_failure_is_friendly_and_cleans_temp_dirs(monkeypatch, tmp_path: Path): + sample_dir = tmp_path / "sample" + upload_dir = tmp_path / "uploads" + persist_dir = tmp_path / "vectorstore" + sample_dir.mkdir() + upload_dir.mkdir() + + monkeypatch.setattr(rag_engine, "_load_sources", lambda **_kwargs: [make_loaded_source("sample")]) + monkeypatch.setattr( + rag_engine, + "chunk_loaded_sources", + lambda _sources: [Document(page_content="chunk", metadata={"chunk_id": "abc", "source_file": "sample.pdf", "document_origin": "sample", "file_hash": "hash-sample", "page": 0})], + ) + monkeypatch.setattr(rag_engine, "VectorStoreManager", FakeIngestionManager) + monkeypatch.setattr( + rag_engine, + "_swap_vectorstore", + lambda _temp, _persist: (_ for _ in ()).throw(PermissionError(5, "Access is denied")), + ) + + with pytest.raises(IngestionError, match="Vectorstore files are locked"): + rag_engine.ingest_documents(sample_dir=sample_dir, uploads_dir=upload_dir, persist_dir=persist_dir) + + assert not list(tmp_path.glob("vectorstore_build_*")) + + +def test_successful_ingestion_cleans_temp_dirs(monkeypatch, tmp_path: Path): + sample_dir = tmp_path / "sample" + upload_dir = tmp_path / "uploads" + persist_dir = tmp_path / "vectorstore" + sample_dir.mkdir() + upload_dir.mkdir() + + monkeypatch.setattr(rag_engine, "_load_sources", lambda **_kwargs: [make_loaded_source("sample")]) + monkeypatch.setattr( + rag_engine, + "chunk_loaded_sources", + lambda _sources: [Document(page_content="chunk", metadata={"chunk_id": "abc", "source_file": "sample.pdf", "document_origin": "sample", "file_hash": "hash-sample", "page": 0})], + ) + monkeypatch.setattr(rag_engine, "VectorStoreManager", FakeIngestionManager) + monkeypatch.setattr( + rag_engine, + "get_vectorstore_stats", + lambda **_kwargs: { + "status": "ready", + "total_documents": 1, + "total_chunks": 1, + "document_names": ["sample.pdf"], + "indexed_sample_document_count": 1, + "indexed_upload_document_count": 0, + "pending_upload_files": [], + "scope_chunk_counts": {"sample": 1, "upload": 0, "all": 1}, + }, + ) + + result = rag_engine.ingest_documents(sample_dir=sample_dir, uploads_dir=upload_dir, persist_dir=persist_dir) + + assert result["total_chunks"] == 1 + assert not list(tmp_path.glob("vectorstore_build_*")) + + +def test_duplicate_file_hash_prefers_upload_deterministically(): + sample = make_loaded_source("sample") + upload = make_loaded_source("upload") + sample = LoadedSource(**{**sample.__dict__, "file_hash": "same-hash"}) + upload = LoadedSource(**{**upload.__dict__, "file_hash": "same-hash"}) + + unique_sources, duplicate_files = rag_engine._deduplicate_loaded_sources([sample, upload]) + + assert [source.document_origin for source in unique_sources] == ["upload"] + assert duplicate_files[0]["skipped_origin"] == "sample" + assert duplicate_files[0]["kept_origin"] == "upload" + + +def test_same_pdf_in_sample_and_upload_does_not_crash_ingestion(monkeypatch, tmp_path: Path): + sample_dir = tmp_path / "sample" + upload_dir = tmp_path / "uploads" + persist_dir = tmp_path / "vectorstore" + sample_dir.mkdir() + upload_dir.mkdir() + + sample = make_loaded_source("sample") + upload = make_loaded_source("upload") + sample = LoadedSource(**{**sample.__dict__, "source_file": "client_profile.pdf", "file_hash": "same-hash"}) + upload = LoadedSource(**{**upload.__dict__, "source_file": "client_profile.pdf", "file_hash": "same-hash"}) + FakeIngestionManager.seen_documents = [] + + monkeypatch.setattr(rag_engine, "_load_sources", lambda **_kwargs: [sample, upload]) + monkeypatch.setattr(rag_engine, "VectorStoreManager", FakeIngestionManager) + monkeypatch.setattr( + rag_engine, + "get_vectorstore_stats", + lambda **_kwargs: { + "status": "ready", + "total_documents": 1, + "total_chunks": len(FakeIngestionManager.seen_documents), + "document_names": ["client_profile.pdf"], + "indexed_sample_document_count": 0, + "indexed_upload_document_count": 1, + "pending_upload_files": [], + "scope_chunk_counts": { + "sample": 0, + "upload": len(FakeIngestionManager.seen_documents), + "all": len(FakeIngestionManager.seen_documents), + }, + }, + ) + + result = rag_engine.ingest_documents(sample_dir=sample_dir, uploads_dir=upload_dir, persist_dir=persist_dir) + + assert result["files_discovered"] == 2 + assert result["unique_files"] == 1 + assert result["duplicate_files_skipped"][0]["skipped_origin"] == "sample" + assert result["sample_chunks"] == 0 + assert result["upload_chunks"] > 0 + assert all(doc.metadata["document_origin"] == "upload" for doc in FakeIngestionManager.seen_documents) + + +def test_vectorstore_upsert_sends_unique_ids_to_chroma(monkeypatch, tmp_path: Path): + class FakeCollection: + def __init__(self): + self.ids = [] + + def get(self): + return {"ids": self.ids, "metadatas": []} + + def count(self): + return len(self.ids) + + class FakeChroma: + def __init__(self): + self._collection = FakeCollection() + + def add_documents(self, documents, ids): + assert len(ids) == len(set(ids)) + self._collection.ids.extend(ids) + + fake_chroma = FakeChroma() + monkeypatch.setattr(VectorStoreManager, "load", lambda self, create_if_missing=True: fake_chroma) + manager = VectorStoreManager(persist_dir=tmp_path, embedding_function=object()) + docs = [ + Document(page_content="first", metadata={"chunk_id": "duplicate"}), + Document(page_content="second", metadata={"chunk_id": "duplicate"}), + Document(page_content="third", metadata={"chunk_id": "unique"}), + ] + + manager.upsert_documents(docs) + + assert fake_chroma._collection.ids == ["duplicate", "unique"] diff --git a/tests/test_evals.py b/tests/test_evals.py index 6c5ff0d..04e9e7c 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,36 +1,34 @@ -from evals.metrics import build_metrics_summary - - -def test_build_metrics_summary(): - results = [ - { - "expected_sources": ["A.pdf"], - "retrieved_sources": ["A.pdf"], - "expect_citations": True, - "answer": "Fact [Source: A.pdf, Page 1]", - "grounded": True, - "latency_ms": 10, - "tool_call_count": 2, - "passed": True, - }, - { - "expected_sources": ["B.pdf"], - "retrieved_sources": [], - "expect_citations": False, - "answer": "No answer", - "grounded": False, - "latency_ms": 30, - "tool_call_count": 1, - "passed": False, - }, - ] - - metrics = build_metrics_summary(results) - - assert metrics["retrieval_hit_rate"] == 0.5 - assert metrics["citation_coverage"] == 1.0 - assert metrics["grounded_answer_score"] == 0.5 - assert metrics["average_latency"] == 20.0 - assert metrics["average_latency_ms"] == 20.0 - assert metrics["tool_call_count"] == 1.5 - assert metrics["failure_rate"] == 0.5 +from pathlib import Path + +from evals.run_evals import run_offline_smoke_eval +from rfp_analyst.evals import format_latency, load_eval_snapshot + + +def test_eval_latency_formatting_from_seconds(): + assert format_latency(0.23, "seconds") == "230 ms" + assert format_latency(1.5, "seconds") == "1.50 s" + + +def test_eval_snapshot_missing_file(tmp_path: Path): + snapshot = load_eval_snapshot(tmp_path / "results.json") + + assert snapshot["status"] == "missing" + assert snapshot["message"] == "No evaluation run found" + + +def test_eval_snapshot_missing_real_kb_message(tmp_path: Path): + snapshot = load_eval_snapshot( + tmp_path / "real_kb_results.json", + missing_message="No real KB evaluation run found", + ) + + assert snapshot["status"] == "missing" + assert snapshot["message"] == "No real KB evaluation run found" + + +def test_offline_smoke_eval_is_labeled(tmp_path: Path): + payload = run_offline_smoke_eval(tmp_path / "offline_smoke_results.json") + + assert payload["evaluation_name"] == "Offline Smoke Evaluation" + assert payload["evaluation_type"] == "offline_smoke_eval" + assert "mock" in payload["notes"].lower() diff --git a/tests/test_health_and_uploads.py b/tests/test_health_and_uploads.py new file mode 100644 index 0000000..b7bdafa --- /dev/null +++ b/tests/test_health_and_uploads.py @@ -0,0 +1,85 @@ +from pathlib import Path + +import pytest + +import rag_engine +from rfp_analyst.exceptions import KnowledgeBaseNotReadyError, UnsupportedFileError +from rfp_analyst.health import get_app_health +from rfp_analyst.uploads import sanitize_uploaded_filename, validate_uploaded_pdf + + +class FakeUpload: + def __init__(self, name: str, mime_type: str = "application/pdf", size: int = 128): + self.name = name + self.type = mime_type + self.size = size + + def getbuffer(self): + return b"x" * self.size + + +def test_missing_vectorstore_is_graceful(tmp_path: Path): + missing_dir = tmp_path / "missing-vectorstore" + + with pytest.raises(KnowledgeBaseNotReadyError): + rag_engine.load_vectorstore(missing_dir) + + stats = rag_engine.get_vectorstore_stats() + assert stats["status"] in {"ready", "not_initialized", "error"} + + +def test_invalid_upload_filename_sanitization(): + assert sanitize_uploaded_filename("../../bad name!!.pdf") == "bad_name.pdf" + with pytest.raises(UnsupportedFileError): + sanitize_uploaded_filename("malware.exe") + + +def test_invalid_upload_mime_rejected(): + with pytest.raises(UnsupportedFileError): + validate_uploaded_pdf(FakeUpload("notes.pdf", mime_type="text/plain")) + + +def test_app_health_check_function(tmp_path: Path): + data_dir = tmp_path / "documents" + vectorstore_dir = tmp_path / "vectorstore" + assets_dir = tmp_path / "assets" + data_dir.mkdir() + assets_dir.mkdir() + (data_dir / "sample.pdf").write_bytes(b"pdf") + + health = get_app_health( + vectorstore_stats={ + "status": "not_initialized", + "total_documents": 0, + "total_chunks": 0, + "document_names": [], + }, + data_dir=data_dir, + vectorstore_dir=vectorstore_dir, + assets_dir=assets_dir, + uploads_dir=tmp_path / "uploads", + groq_api_key="", + google_api_key="", + ) + + assert health["vectorstore_ready"] is False + assert health["document_count"] == 1 + assert health["chunk_count"] == 0 + assert health["llm_provider_configured"] is False + assert health["required_directories"]["documents"] is True + assert health["required_directories"]["vectorstore"] is False + + +def test_vectorstore_is_not_ready_when_chunk_count_is_zero(): + health = get_app_health( + vectorstore_stats={ + "status": "ready", + "total_documents": 4, + "total_chunks": 0, + "document_names": ["a.pdf"], + }, + groq_api_key="", + google_api_key="", + ) + + assert health["vectorstore_ready"] is False diff --git a/tests/test_kb_evals.py b/tests/test_kb_evals.py new file mode 100644 index 0000000..657bf08 --- /dev/null +++ b/tests/test_kb_evals.py @@ -0,0 +1,74 @@ +from types import SimpleNamespace + +from evals import run_kb_evals + + +class FakeDoc: + def __init__(self, content: str, source: str, page: int = 0): + self.page_content = content + self.metadata = {"source_file": source, "page": page} + + +class FakeLLM: + def invoke(self, _messages): + return SimpleNamespace(content="I could not find grounded evidence for that request.") + + +def test_real_kb_eval_defaults_to_retrieval_only(monkeypatch, tmp_path): + monkeypatch.setattr(run_kb_evals, "ensure_sample_documents_ready", lambda: None) + monkeypatch.setattr(run_kb_evals, "_build_llm", lambda: None) + + monkeypatch.setattr(run_kb_evals, "load_golden_cases", lambda: [ + {"question": "banking", "expected_sources": ["01_Banking_Sector_Digital_Audit_2024.pdf"]}, + {"question": "Mars", "expected_sources": [], "expects_no_answer": True}, + ]) + + def fake_prepare_query_payload(question: str, **kwargs): + source = "01_Banking_Sector_Digital_Audit_2024.pdf" + if "Compare" in question: + docs = [ + {"source": "02_Healthcare_Data_Migration_to_Azure_Cloud.pdf", "page": 0, "score": "0.91"}, + {"source": "04_Insurance_Claims_Processing_Automation.pdf", "page": 0, "score": "0.88"}, + ] + elif "Mars" in question: + docs = [] + else: + docs = [{"source": source, "page": 0, "score": "0.95"}] + return { + "response_mode": "fallback" if "Mars" in question else "llm", + "traces": [{"tool": "search_knowledge_base", "documents": docs}], + } + + monkeypatch.setattr(run_kb_evals, "prepare_query_payload", fake_prepare_query_payload) + + payload = run_kb_evals.run_real_kb_eval(tmp_path / "real_kb_results.json") + + assert payload["evaluation_name"] == "Real KB Evaluation" + assert payload["mode"] == "retrieval_only" + assert len(payload["cases"]) == 2 + + +def test_real_kb_eval_can_use_llm_mode(monkeypatch, tmp_path): + monkeypatch.setattr(run_kb_evals, "ensure_sample_documents_ready", lambda: None) + monkeypatch.setattr(run_kb_evals, "_build_llm", lambda: FakeLLM()) + monkeypatch.setattr( + run_kb_evals, + "prepare_query_payload", + lambda question, **kwargs: { + "response_mode": "llm", + "traces": [{"tool": "search_knowledge_base", "documents": [{"source": "01_Banking_Sector_Digital_Audit_2024.pdf", "page": 0, "score": "0.95"}]}], + }, + ) + monkeypatch.setattr( + run_kb_evals, + "run_query", + lambda llm, question: {"answer": "Grounded answer with citations."}, + ) + + payload = run_kb_evals.run_real_kb_eval(tmp_path / "real_kb_results.json", use_llm=True) + + assert payload["mode"] == "llm_answer" + assert all(case["mode"] == "llm_answer" for case in payload["cases"]) + monkeypatch.setattr(run_kb_evals, "load_golden_cases", lambda: [ + {"question": "banking", "expected_sources": ["01_Banking_Sector_Digital_Audit_2024.pdf"]} + ]) diff --git a/tests/test_langgraph_agent.py b/tests/test_langgraph_agent.py new file mode 100644 index 0000000..7a9cbbf --- /dev/null +++ b/tests/test_langgraph_agent.py @@ -0,0 +1,769 @@ +from types import SimpleNamespace + +import importlib +import re + +from config import MAX_PROMPT_TOKENS, RFP_ANALYSIS_MAX_OUTPUT_TOKENS + +from rfp_analyst.agent.graph import ( + INSUFFICIENT_EVIDENCE_MESSAGE, + MISSING_UPLOAD_CONTEXT_MESSAGE, + NO_RELEVANT_UPLOAD_TARGET_MESSAGE, + compile_query_graph, + prepare_query_payload, + run_query, +) + +graph_module = importlib.import_module("rfp_analyst.agent.graph") + + +class FakeDoc: + def __init__(self, content: str, source: str, page: int = 0, origin: str = "sample"): + self.page_content = content + self.metadata = { + "source_file": source, + "page": page, + "document_origin": origin, + "chunk_id": f"{origin}-{source}-{page}", + } + + +def fake_retrieval(query: str, k: int): + return [ + ( + FakeDoc( + content=f"Evidence for: {query}", + source="banking_case_study.pdf", + page=0, + ), + 0.91, + ) + ][:k] + + +def ready_stats(): + return { + "status": "ready", + "total_documents": 3, + "total_chunks": 9, + "document_names": [ + "banking_case_study.pdf", + "healthcare_migration.pdf", + "insurance_automation.pdf", + ], + } + + +def not_ready_stats(): + return { + "status": "not_initialized", + "total_documents": 0, + "total_chunks": 0, + "document_names": [], + } + + +def test_graph_compiles(): + graph = compile_query_graph() + assert graph is not None + assert hasattr(graph, "invoke") + + +def test_search_intent_routes_to_search_knowledge_base(): + payload = prepare_query_payload( + user_query="What tech stack did we use for the banking audit?", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert payload["intent"] == "search" + assert payload["planned_tools"] == ["search_knowledge_base"] + assert any(step.get("tool") == "search_knowledge_base" for step in payload["traces"]) + + +def test_compare_intent_routes_to_compare_projects(): + payload = prepare_query_payload( + user_query="Compare the healthcare and insurance projects", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert payload["intent"] == "compare" + assert "compare_projects" in payload["planned_tools"] + assert any(step.get("tool") == "compare_projects" for step in payload["traces"]) + + +def test_proposal_intent_routes_to_proposal_writer(): + payload = prepare_query_payload( + user_query="Write a proposal response for a banking modernization RFP", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert payload["intent"] == "proposal" + assert "proposal_writer" in payload["planned_tools"] + assert any(step.get("tool") == "proposal_writer" for step in payload["traces"]) + + +def test_no_kb_returns_graceful_fallback(): + payload = prepare_query_payload( + user_query="What projects used Azure?", + vectorstore_stats=not_ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert payload["response_mode"] == "fallback" + assert "Knowledge base is not ready" in payload["answer"] + assert any( + step.get("tool") == "search_knowledge_base" and step.get("status") == "skipped" + for step in payload["traces"] + ) + + +def test_ambiguous_query_returns_clarification(): + payload = prepare_query_payload( + user_query="help me", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert payload["intent"] == "ambiguous" + assert payload["response_mode"] == "clarification" + assert "clarify" in payload["answer"].lower() + + +def test_vague_followup_uses_prior_entities(): + captured = {} + history = [ + { + "role": "assistant", + "content": "The uploaded client profile uses React.", + "reasoning": [ + { + "tool_response": "client_profile.pdf (Page 1)", + "source": "client_profile.pdf", + "page": 1, + "chunk_id": "client-profile-1", + "document_origin": "upload", + } + ], + } + ] + + def retrieval(query: str, k: int, scope: str): + captured["query"] = query + captured["scope"] = scope + return fake_retrieval(query, k) + + payload = prepare_query_payload( + user_query="what is tech stack here?", + chat_history=history, + vectorstore_stats=ready_stats(), + retrieval_fn=retrieval, + retrieval_scope="upload", + ) + + assert payload["response_mode"] == "llm" + assert "client_profile.pdf" in captured["query"] + assert captured["scope"] == "upload" + assert payload["resolved_entities"][0]["source"] == "client_profile.pdf" + + +def test_ambiguous_followup_asks_for_clarification(): + payload = prepare_query_payload( + user_query="what is tech stack here?", + chat_history=[], + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + retrieval_scope="all", + ) + + assert payload["response_mode"] == "clarification" + assert "which previously discussed" in payload["answer"].lower() + + +def test_scope_is_retained_across_followups(): + captured = {} + history = [ + { + "role": "assistant", + "content": "Resume answer", + "reasoning": [ + { + "tool_response": "client_profile.pdf (Page 1)", + "source": "client_profile.pdf", + "page": 1, + "chunk_id": "client-profile-1", + "document_origin": "upload", + } + ], + } + ] + + def retrieval(query: str, k: int, scope: str): + captured["scope"] = scope + return fake_retrieval(query, k) + + prepare_query_payload( + user_query="what about this?", + chat_history=history, + vectorstore_stats=ready_stats(), + retrieval_fn=retrieval, + retrieval_scope="upload", + ) + + assert captured["scope"] == "upload" + + +def test_complex_rfp_query_invokes_multiple_tools_with_summaries(): + payload = prepare_query_payload( + user_query="Write a proposal response for a banking modernization RFP", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + tools = [step.get("tool") for step in payload["traces"]] + assert "intent_classifier" in tools + assert "search_knowledge_base" in tools + assert "proposal_writer" in tools + assert "evidence_availability_check" in tools + assert all("thought" not in str(step).lower() for step in payload["traces"]) + assert any(step.get("output_summary") for step in payload["traces"]) + + +def test_compare_tool_is_actually_called(monkeypatch): + called = {} + + def fake_compare(query, search_fn=None, k=6): + called["query"] = query + called["search_fn"] = search_fn + return {"rows": [{"source": "a.pdf"}, {"source": "b.pdf"}], "comparison_markdown": "comparison"} + + monkeypatch.setattr(graph_module, "compare_projects", fake_compare) + payload = prepare_query_payload( + "Compare project A versus project B", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + retrieval_scope="sample", + ) + + assert called["query"] + assert callable(called["search_fn"]) + assert len(payload["tool_outputs"]["compare_projects"]["rows"]) == 2 + assert any("Compared 2 projects across 4 dimensions" in step.get("output_summary", "") for step in payload["traces"]) + + +def test_proposal_tools_are_actually_called(monkeypatch): + calls = [] + monkeypatch.setattr(graph_module, "extract_rfp_requirements", lambda text: calls.append("extract") or {"requirements": [{"id": "REQ-01", "text": "Must use Azure"}], "summary": "Extracted 1 requirement(s)."}) + monkeypatch.setattr(graph_module, "find_relevant_case_studies", lambda requirements, search_fn=None, k=6: calls.append("cases") or {"matches": [{"source": "banking.pdf", "pages": [0]}], "documents": []}) + monkeypatch.setattr(graph_module, "generate_proposal_outline", lambda query, cases, requirements=None: calls.append("outline") or {"outline": "Outline naming banking.pdf"}) + + payload = prepare_query_payload( + "Write a proposal response for an Azure RFP", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert calls == ["extract", "cases", "outline"] + assert "Outline naming banking.pdf" in payload["prompt"] + + +def test_cross_corpus_rfp_analysis_uses_upload_target_and_sample_cases(monkeypatch): + calls = [] + + def scoped_retrieval(query, k, scope): + calls.append({"query": query, "scope": scope}) + source = "target_rfp.pdf" if scope == "upload" else "banking_case.pdf" + return [(FakeDoc("Must support Azure dashboards.", source, origin=scope), 0.92)] + + payload = prepare_query_payload( + "Extract requirements, find case studies, compare fit, and create proposal outline", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 1, + "indexed_upload_files": ["target_rfp.pdf"], + "scope_chunk_counts": {"upload": 2, "sample": 7}, + }, + retrieval_fn=scoped_retrieval, + retrieval_scope="all", + ) + + assert payload["intent"] == "rfp_analysis" + assert calls[0]["scope"] == "upload" + assert "sample" in [call["scope"] for call in calls[1:]] + assert "compare fit" not in calls[0]["query"].lower() + assert "case studies" not in calls[0]["query"].lower() + assert "requirements" in calls[0]["query"].lower() + assert "architecture" in calls[0]["query"].lower() + assert all(item["document_origin"] == "upload" for item in payload["retrieved_documents"] if item["source"] == "target_rfp.pdf") + assert "banking_case.pdf" in payload["prompt"] + target_trace = next(step for step in payload["traces"] if step.get("tool") == "target_context_retrieval") + assert target_trace["indexed_upload_chunk_count"] == 2 + assert target_trace["target_fallback_used"] is False + assert target_trace["selected_upload_source_files"] == ["target_rfp.pdf"] + + +def test_rfp_analysis_without_upload_context_returns_clean_fallback(): + def no_upload(query, k, scope): + assert scope == "upload" + return [] + + payload = prepare_query_payload( + "Extract requirements and find case studies for my uploaded RFP", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 0, + "indexed_upload_files": [], + "scope_chunk_counts": {"upload": 0, "sample": 9}, + }, + retrieval_fn=no_upload, + ) + + assert payload["response_mode"] == "fallback" + assert payload["answer"] == MISSING_UPLOAD_CONTEXT_MESSAGE + + +def test_rfp_analysis_indexed_uploads_without_qualifying_hits_returns_target_message(): + def low_scores(query, k, scope): + assert scope == "upload" + return [ + (FakeDoc("Weakly related content", "target_rfp.pdf", page=0, origin="upload"), 0.12), + (FakeDoc("Another weakly related note", "target_rfp.pdf", page=1, origin="upload"), 0.08), + ] + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 1, + "indexed_upload_files": ["target_rfp.pdf"], + "scope_chunk_counts": {"upload": 8, "sample": 9, "all": 17}, + }, + retrieval_fn=low_scores, + retrieval_scope="all", + ) + + assert payload["response_mode"] == "fallback" + assert payload["answer"] == NO_RELEVANT_UPLOAD_TARGET_MESSAGE + assert payload["answer"] != MISSING_UPLOAD_CONTEXT_MESSAGE + + +def test_rfp_analysis_uses_bounded_upload_fallback_when_hits_are_slightly_below_threshold(): + def near_threshold(query, k, scope): + assert scope == "upload" + return [ + (FakeDoc("Requirement A", "target_rfp.pdf", page=0, origin="upload"), 0.47), + (FakeDoc("Requirement A duplicate chunk", "target_rfp.pdf", page=0, origin="upload"), 0.46), + (FakeDoc("Requirement B", "target_rfp.pdf", page=1, origin="upload"), 0.45), + (FakeDoc("Requirement C", "target_rfp_2.pdf", page=0, origin="upload"), 0.44), + (FakeDoc("Requirement D", "target_rfp_3.pdf", page=0, origin="upload"), 0.43), + ] + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 3, + "indexed_upload_files": ["target_rfp.pdf", "target_rfp_2.pdf", "target_rfp_3.pdf"], + "scope_chunk_counts": {"upload": 10, "sample": 9, "all": 19}, + }, + retrieval_fn=near_threshold, + retrieval_scope="all", + ) + + assert payload["response_mode"] == "llm" + upload_docs = [item for item in payload["retrieved_documents"] if item["document_origin"] == "upload"] + assert 2 <= len(upload_docs) <= 4 + assert len({(item["source"], item["page"]) for item in upload_docs}) == len(upload_docs) + target_trace = next(step for step in payload["traces"] if step.get("tool") == "target_context_retrieval") + assert target_trace["target_fallback_used"] is True + assert target_trace["qualifying_upload_hits"] == 0 + + +def test_rfp_analysis_target_evidence_stays_upload_only(): + def scoped_retrieval(query, k, scope): + if scope == "upload": + return [ + (FakeDoc("Uploaded requirement", "target_rfp.pdf", page=0, origin="upload"), 0.91), + (FakeDoc("Mislabeled sample should be dropped", "wrong.pdf", page=0, origin="sample"), 0.93), + ] + return [(FakeDoc("Sample case study", "case_study.pdf", page=0, origin="sample"), 0.95)] + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 1, + "indexed_upload_files": ["target_rfp.pdf"], + "scope_chunk_counts": {"upload": 4, "sample": 7, "all": 11}, + }, + retrieval_fn=scoped_retrieval, + retrieval_scope="all", + ) + + upload_docs = [item for item in payload["retrieved_documents"] if item["source"] == "target_rfp.pdf"] + assert upload_docs + assert all(item["document_origin"] == "upload" for item in upload_docs) + assert all(item["source"] != "wrong.pdf" for item in payload["retrieved_documents"]) + + +def test_rfp_analysis_case_study_evidence_stays_sample_only(): + def scoped_retrieval(query, k, scope): + source = "target_rfp.pdf" if scope == "upload" else "sample_case.pdf" + origin = "upload" if scope == "upload" else "sample" + return [(FakeDoc("Evidence", source, page=0, origin=origin), 0.93)] + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 1, + "indexed_upload_files": ["target_rfp.pdf"], + "scope_chunk_counts": {"upload": 2, "sample": 6, "all": 8}, + }, + retrieval_fn=scoped_retrieval, + retrieval_scope="all", + ) + + case_docs = [item for item in payload["retrieved_documents"] if item["source"] == "sample_case.pdf"] + assert case_docs + assert all(item["document_origin"] == "sample" for item in case_docs) + + +def test_rfp_analysis_target_prompt_budget_is_bounded(): + def many_upload_hits(query, k, scope): + assert scope == "upload" + return [ + (FakeDoc(f"Requirement {index}", f"target_{index}.pdf", page=0, origin="upload"), 0.9 - (index * 0.01)) + for index in range(6) + ] + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 6, + "indexed_upload_files": [f"target_{index}.pdf" for index in range(6)], + "scope_chunk_counts": {"upload": 12, "sample": 9, "all": 21}, + }, + retrieval_fn=many_upload_hits, + retrieval_scope="all", + ) + + upload_docs = [item for item in payload["retrieved_documents"] if item["document_origin"] == "upload"] + assert len(upload_docs) <= 4 + + +def test_non_rfp_queries_do_not_get_upload_target_fallback(): + def near_threshold(query, k, scope): + return [(FakeDoc("Weakly related upload note", "target_rfp.pdf", page=0, origin="upload"), 0.47)] + + payload = prepare_query_payload( + "What is the CEO phone number?", + vectorstore_stats={**ready_stats(), "scope_chunk_counts": {"upload": 3, "sample": 9, "all": 12}}, + retrieval_fn=near_threshold, + retrieval_scope="upload", + ) + + assert payload["response_mode"] == "fallback" + assert payload["answer"] == INSUFFICIENT_EVIDENCE_MESSAGE + assert all(step.get("tool") != "target_context_retrieval" for step in payload["traces"]) + + +def test_previous_sources_does_not_retrieve(): + def forbidden_retrieval(*args): + raise AssertionError("retrieval must not run") + + history = [{"role": "assistant", "content": "answer", "reasoning": [{"source": "client_profile.pdf", "page": 2, "document_origin": "upload"}]}] + payload = prepare_query_payload( + "What sources did you use?", + chat_history=history, + vectorstore_stats=ready_stats(), + retrieval_fn=forbidden_retrieval, + ) + + assert payload["intent"] == "previous_sources" + assert payload["tool_outputs"]["previous_sources"] == [{"source_file": "client_profile.pdf", "page": 2, "document_origin": "upload"}] + assert "client_profile.pdf" in payload["answer"] + + +def test_low_relevance_results_return_insufficient_evidence(): + def unrelated(query, k, scope="all"): + return [(FakeDoc("Unrelated project text", "unrelated.pdf"), 0.1)] + + for question in ("What is the CEO phone number?", "Describe the aerospace blockchain architecture"): + payload = prepare_query_payload(question, vectorstore_stats=ready_stats(), retrieval_fn=unrelated) + assert payload["response_mode"] == "fallback" + assert "sufficiently relevant evidence" in payload["answer"] + assert "unrelated.pdf" not in payload["answer"] + + +def test_generated_answer_is_verified_after_llm_generation(): + class FakeLLM: + def invoke(self, messages): + return SimpleNamespace(content="Azure is used. [Source: banking_case_study.pdf, Page 1]") + + result = run_query( + FakeLLM(), + "What technology was used?", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + tools = [step.get("tool") for step in result["payload"]["traces"]] + assert tools.index("grounding_verifier") > tools.index("final_response") + + +def test_rfp_analysis_prompt_is_compact_and_within_budget(monkeypatch): + phrase = "Azure dashboards with HIPAA controls and phased rollout milestones." + + def scoped_retrieval(query, k, scope): + if scope == "upload": + return [ + (FakeDoc(f"{phrase} Requirement chunk {index}", f"target_{index}.pdf", page=index, origin="upload"), 0.95 - (index * 0.02)) + for index in range(6) + ] + return [(FakeDoc("Sample retrieval evidence", "sample.pdf", page=0, origin="sample"), 0.9)] + + monkeypatch.setattr( + graph_module, + "extract_rfp_requirements", + lambda _text: { + "requirements": [ + { + "id": "REQ-01", + "text": phrase, + "source_file": "target_0.pdf", + "page": 0, + "document_origin": "upload", + } + ], + "summary": "Extracted 1 requirement.", + }, + ) + + def fake_cases(requirements, search_fn=None, k=6): + assert search_fn is not None + docs = [] + matches = [] + for case_index in range(4): + source = f"case_{case_index}.pdf" + matches.append( + { + "source": source, + "pages": [0, 1, 2], + "matched_requirements": ["REQ-01"], + "snippets": [f"Fit reason {case_index}"] * 3, + } + ) + for page in range(3): + docs.append(FakeDoc(f"Case {case_index} page {page} tech stack evidence", source, page=page, origin="sample")) + return {"matches": matches, "documents": docs} + + monkeypatch.setattr(graph_module, "find_relevant_case_studies", fake_cases) + monkeypatch.setattr( + graph_module, + "compare_projects", + lambda *args, **kwargs: { + "rows": [ + {"source": "case_0.pdf", "timeline": "12 weeks", "budget": "$100k", "tech_stack": "Azure", "outcomes": "Dashboard launch"}, + {"source": "case_1.pdf", "timeline": "16 weeks", "budget": "$120k", "tech_stack": "Python", "outcomes": "Workflow automation"}, + {"source": "case_2.pdf", "timeline": "20 weeks", "budget": "$140k", "tech_stack": "Snowflake", "outcomes": "Reporting uplift"}, + {"source": "case_3.pdf", "timeline": "24 weeks", "budget": "$160k", "tech_stack": "Kubernetes", "outcomes": "Platform rebuild"}, + ] + }, + ) + monkeypatch.setattr( + graph_module, + "generate_proposal_outline", + lambda query, case_studies, requirements=None: { + "outline": "\n".join( + [ + "## Executive Summary", + "- Map Azure migration to client requirements.", + "## Delivery Plan", + "- Run phased rollout.", + "## Risks", + "- Track HIPAA controls.", + ] + ) + }, + ) + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 6, + "indexed_upload_files": [f"target_{index}.pdf" for index in range(6)], + "scope_chunk_counts": {"upload": 12, "sample": 12, "all": 24}, + }, + retrieval_fn=scoped_retrieval, + retrieval_scope="all", + ) + + prompt = payload["prompt"] + budget = payload["prompt_budget"] + + assert "Document(" not in prompt + assert "page_content" not in prompt + assert prompt.count(phrase) <= 2 + assert budget["estimated_input_tokens"] <= MAX_PROMPT_TOKENS + assert budget["projected_total_tokens"] == budget["estimated_input_tokens"] + RFP_ANALYSIS_MAX_OUTPUT_TOKENS + assert budget["target_chunks_included"] <= 4 + assert budget["status"] == "within_budget" + assert any(step.get("tool") == "prompt_budget" for step in payload["traces"]) + + upload_lines = [line for line in prompt.splitlines() if "origin=upload" in line] + assert 1 <= len(upload_lines) <= 4 + assert "-- Uploaded Target Evidence --" in prompt + assert "-- Sample Case Study Evidence --" in prompt + + case_names = sorted(set(re.findall(r"case_[0-9]\\.pdf", prompt))) + assert len(case_names) <= 3 + for name in case_names: + assert prompt.count(name) <= 3 + assert "REQ-01" in prompt + assert "target_0.pdf, Page 1" in prompt + + +def test_prompt_budget_drops_lowest_scoring_evidence_first(monkeypatch): + monkeypatch.setattr(graph_module, "MAX_PROMPT_TOKENS", 250) + + def scoped_retrieval(query, k, scope): + if scope == "upload": + return [(FakeDoc("Target evidence", "target.pdf", page=0, origin="upload"), 0.95)] + return [(FakeDoc("Sample fallback", "sample.pdf", page=0, origin="sample"), 0.9)] + + monkeypatch.setattr( + graph_module, + "extract_rfp_requirements", + lambda _text: {"requirements": [{"id": "REQ-01", "text": "Must support Azure"}], "summary": "Extracted."}, + ) + monkeypatch.setattr( + graph_module, + "find_relevant_case_studies", + lambda requirements, search_fn=None, k=6: { + "matches": [ + {"source": "alpha.pdf", "pages": [0], "matched_requirements": ["REQ-01"], "snippets": ["strong fit"]}, + {"source": "beta.pdf", "pages": [0], "matched_requirements": ["REQ-01"], "snippets": ["medium fit"]}, + {"source": "gamma.pdf", "pages": [0], "matched_requirements": ["REQ-01"], "snippets": ["lower fit"]}, + {"source": "delta.pdf", "pages": [0], "matched_requirements": ["REQ-01"], "snippets": ["lowest fit"]}, + ], + "documents": [ + FakeDoc("alpha evidence" * 40, "alpha.pdf", page=0, origin="sample"), + FakeDoc("beta evidence" * 40, "beta.pdf", page=0, origin="sample"), + FakeDoc("gamma evidence" * 40, "gamma.pdf", page=0, origin="sample"), + FakeDoc("delta evidence" * 40, "delta.pdf", page=0, origin="sample"), + ], + }, + ) + monkeypatch.setattr( + graph_module, + "compare_projects", + lambda *args, **kwargs: {"rows": [{"source": "alpha.pdf", "timeline": "12 weeks", "budget": "$100k", "tech_stack": "Azure", "outcomes": "Launch"}]}, + ) + monkeypatch.setattr(graph_module, "generate_proposal_outline", lambda *args, **kwargs: {"outline": "## Summary\n- concise"}) + + payload = prepare_query_payload( + "Use uploaded documents as target requirements and numbered PDFs as internal case studies. Return technical requirements, gaps, three case studies and a proposal outline.", + vectorstore_stats={ + **ready_stats(), + "indexed_upload_document_count": 1, + "indexed_upload_files": ["target.pdf"], + "scope_chunk_counts": {"upload": 2, "sample": 8, "all": 10}, + }, + retrieval_fn=scoped_retrieval, + retrieval_scope="all", + ) + + assert payload["prompt_budget"]["chunks_dropped"] >= 1 + assert "alpha.pdf" in payload["prompt"] + assert "delta.pdf" not in payload["prompt"] + + +def test_complete_documents_remain_available_for_post_generation_verification(monkeypatch): + captured = {} + + def fake_verify(answer, documents): + captured["documents"] = documents + return {"is_grounded": True, "checked_claims": ["ok"], "unsupported_claims": []} + + monkeypatch.setattr(graph_module, "verify_answer_grounding", fake_verify) + + class FakeLLM: + def invoke(self, messages): + return SimpleNamespace(content="Grounded answer. [Source: banking_case_study.pdf, Page 1]") + + result = run_query( + FakeLLM(), + "What technology was used?", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert result["answer"].startswith("Grounded answer") + assert captured["documents"] + assert captured["documents"][0]["content"].startswith("Evidence for:") + + +def test_unsupported_claims_trigger_one_repair_pass(monkeypatch): + calls = {"count": 0} + + def fake_verify(answer, documents): + calls["count"] += 1 + if calls["count"] == 1: + return { + "is_grounded": False, + "checked_claims": ["Unsupported Kubernetes claim"], + "unsupported_claims": ["Unsupported Kubernetes claim"], + } + return {"is_grounded": True, "checked_claims": [], "unsupported_claims": []} + + monkeypatch.setattr(graph_module, "verify_answer_grounding", fake_verify) + + class FakeLLM: + def invoke(self, messages): + return SimpleNamespace( + content=( + "Supported Azure claim [Source: banking_case_study.pdf, Page 1].\n" + "Unsupported Kubernetes claim [Source: respective documents]." + ) + ) + + result = run_query( + FakeLLM(), + "What technology was used?", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + tools = [step.get("tool") for step in result["payload"]["traces"]] + assert calls["count"] == 2 + assert tools.count("answer_repair") == 1 + assert "final_grounding_verifier" in tools + assert "[Source: respective documents]" not in result["answer"] + assert "Unsupported Kubernetes claim" not in result["answer"] + + +def test_vague_citations_are_removed_by_repair(monkeypatch): + def fake_verify(answer, documents): + return {"is_grounded": True, "checked_claims": [], "unsupported_claims": []} + + monkeypatch.setattr(graph_module, "verify_answer_grounding", fake_verify) + + class FakeLLM: + def invoke(self, messages): + return SimpleNamespace(content="Azure is relevant [Source: respective documents].") + + result = run_query( + FakeLLM(), + "What technology was used?", + vectorstore_stats=ready_stats(), + retrieval_fn=fake_retrieval, + ) + + assert "[Source: respective documents]" not in result["answer"] + assert "[Source: None]" not in result["answer"] + assert "The retrieved evidence does not support this claim." in result["answer"] + diff --git a/tests/test_prompt_builder_py311.py b/tests/test_prompt_builder_py311.py new file mode 100644 index 0000000..c4e1110 --- /dev/null +++ b/tests/test_prompt_builder_py311.py @@ -0,0 +1,25 @@ +from rfp_analyst.agent.graph import prepare_query_payload + + +class FakeDoc: + page_content = "Evidence supporting the banking audit answer." + metadata = {"source_file": "banking_case_study.pdf", "page": 0} + + +def test_prompt_builder_imports_and_builds_prompt_on_py311(): + payload = prepare_query_payload( + user_query="What tech stack did we use for the banking audit?", + chat_history=[{"role": "user", "content": "Earlier context"}], + vectorstore_stats={ + "status": "ready", + "total_documents": 1, + "total_chunks": 1, + "document_names": ["banking_case_study.pdf"], + }, + retrieval_fn=lambda _query, _k: [(FakeDoc(), 0.92)], + ) + + assert payload["response_mode"] == "llm" + assert "-- Planned Tools --" in payload["prompt"] + assert "-- Recent Conversation --" in payload["prompt"] + assert "banking_case_study.pdf" in payload["prompt"] diff --git a/tests/test_streamlit_app_smoke.py b/tests/test_streamlit_app_smoke.py new file mode 100644 index 0000000..dc097dd --- /dev/null +++ b/tests/test_streamlit_app_smoke.py @@ -0,0 +1,84 @@ +from pathlib import Path +from unittest.mock import patch + +from streamlit.testing.v1 import AppTest + + +def ready_stats(): + return { + "status": "ready", + "total_documents": 1, + "total_chunks": 2, + "document_names": ["sample.pdf"], + "available_documents": ["sample.pdf"], + "indexed_sample_document_count": 1, + "indexed_upload_document_count": 0, + "pending_upload_files": [], + "scope_chunk_counts": {"sample": 2, "upload": 0, "all": 2}, + } + + +def not_ready_stats(): + return { + "status": "not_initialized", + "total_documents": 0, + "total_chunks": 0, + "document_names": [], + "available_documents": [], + "indexed_sample_document_count": 0, + "indexed_upload_document_count": 0, + "pending_upload_files": [], + "scope_chunk_counts": {"sample": 0, "upload": 0, "all": 0}, + } + + +def test_app_smoke_has_no_unhandled_streamlit_exception(): + app_path = Path(__file__).resolve().parents[1] / "app.py" + + with ( + patch("config.get_api_keys", return_value=("", "")), + patch("rag_engine.get_vectorstore_stats", return_value=ready_stats()), + patch("rag_engine.ingest_documents"), + patch("document_generator.generate_all_documents"), + patch("agent.create_agent"), + patch("agent.query_agent_stream", return_value=iter([])), + ): + at = AppTest.from_file(str(app_path)).run(timeout=10) + + assert not at.exception + + +def test_app_does_not_auto_ingest_when_kb_is_missing(): + app_path = Path(__file__).resolve().parents[1] / "app.py" + + with ( + patch("config.get_api_keys", return_value=("", "")), + patch("rag_engine.get_vectorstore_stats", return_value=not_ready_stats()), + patch("rag_engine.ingest_documents") as ingest_documents, + patch("document_generator.generate_all_documents") as generate_documents, + ): + at = AppTest.from_file(str(app_path)) + at.run(timeout=10) + at.run(timeout=10) + + assert not at.exception + assert ingest_documents.call_count == 0 + assert generate_documents.call_count == 0 + assert any("Knowledge base is not ready" in str(item.value) for item in at.info) + + +def test_failed_ingestion_does_not_retry_on_next_rerun(): + app_path = Path(__file__).resolve().parents[1] / "app.py" + + with ( + patch("config.get_api_keys", return_value=("groq", "")), + patch("rag_engine.get_vectorstore_stats", return_value=not_ready_stats()), + patch("rag_engine.ingest_documents", side_effect=RuntimeError("boom")) as ingest_documents, + ): + at = AppTest.from_file(str(app_path)).run(timeout=10) + at.button[7].click().run(timeout=10) + at.run(timeout=10) + + assert ingest_documents.call_count == 1 + assert at.session_state.filtered_state["ingestion_in_progress"] is False + assert "boom" in at.session_state.filtered_state["last_ingestion_error"] From e30a68528110bc6671aa8d6a26cc742969f24f9f Mon Sep 17 00:00:00 2001 From: Tushar Ghosh Date: Sun, 12 Jul 2026 06:10:06 +0530 Subject: [PATCH 3/3] fix: install test dependencies in CI --- .github/workflows/ci.yml | 2 +- pyproject.toml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30d6ea2..1e40ec1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r requirements.txt - python -m pip install -e . + python -m pip install -e ".[dev]" - name: Compile critical entry points run: python -m py_compile app.py agent.py rag_engine.py config.py document_generator.py diff --git a/pyproject.toml b/pyproject.toml index 852739e..e2cc738 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,11 @@ description = "Internal RFP Analyst agentic RAG application" readme = "README.md" requires-python = ">=3.11" +[project.optional-dependencies] +dev = [ + "pytest>=8.0,<10", +] + [tool.setuptools] package-dir = {"" = "src"}